crawlex 1.0.4

Stealth crawler with Chrome-perfect TLS/H2 fingerprint, render pool, hooks, persistent queue
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101

use std::path::PathBuf;

pub use self::installation::BrowserFetcherInstallation;
pub use self::options::BrowserFetcherOptions;
use crate::render::chrome_fetcher::error::{FetcherError, Result};
use crate::render::chrome_fetcher::{
    BrowserHost, BrowserKind, BrowserVersion, BuildInfo, Platform, Runtime,
};

mod installation;
mod options;

/// A [`BrowserFetcher`] used to download and install a version of chromium.
pub struct BrowserFetcher {
    host: BrowserHost,
    path: PathBuf,
    platform: Platform,
    kind: BrowserKind,
    version: BrowserVersion,
}

impl BrowserFetcher {
    pub fn new(options: BrowserFetcherOptions) -> Self {
        Self {
            host: options.host,
            path: options.path,
            platform: options.platform,
            kind: options.kind,
            version: options.version,
        }
    }

    /// Fetches the browser revision, either locally if it was previously
    /// installed or remotely. If fetching remotely, the method can take a long
    /// time to resolve.
    ///
    /// This fails if the download or installation fails. The fetcher doesn't
    /// retry on network errors during download. If the installation fails,
    /// it might leave the cache in a bad state and it is advised to wipe it.
    ///
    /// If providing a custom host, make sure files are in the same places as
    /// the official builds otherwise the installation will succeed but the runtime
    /// will fail.
    pub async fn fetch(&self) -> Result<BrowserFetcherInstallation> {
        let build_info = self
            .version
            .resolve(self.kind, self.platform, &self.host)
            .await?;

        tracing::debug!(target: "crate::render::chrome_fetcher::fetch", build_info = ?build_info, "Build info");
        if !self.local(&build_info).await {
            self.download(&build_info).await?;
        }

        Ok(self.installation(build_info))
    }

    async fn local(&self, build_info: &BuildInfo) -> bool {
        let folder_path = self.folder_path(build_info);
        let executable_path = self
            .kind
            .executable(self.platform, build_info, &folder_path);
        Runtime::exists(&executable_path).await
    }

    async fn download(&self, build_info: &BuildInfo) -> Result<()> {
        let url = self
            .kind
            .download_url(self.platform, build_info, &self.host);
        let folder_path = self.folder_path(build_info);
        let archive_path = folder_path.with_extension("zip");

        tracing::debug!(target: "crate::render::chrome_fetcher::download", url = %url, archive_path = ?archive_path, "Downloading browser");
        Runtime::download_file(&url, &archive_path)
            .await
            .map_err(FetcherError::DownloadFailed)?;
        Runtime::unzip(archive_path, folder_path)
            .await
            .map_err(FetcherError::InstallFailed)?;

        Ok(())
    }

    fn folder_path(&self, build_info: &BuildInfo) -> PathBuf {
        let mut folder_path = self.path.clone();
        folder_path.push(self.platform.folder_name(build_info));
        folder_path
    }

    fn installation(&self, build_info: BuildInfo) -> BrowserFetcherInstallation {
        let folder_path = self.folder_path(&build_info);
        let executable_path = self
            .kind
            .executable(self.platform, &build_info, &folder_path);
        BrowserFetcherInstallation {
            folder_path,
            executable_path,
            build_info,
        }
    }
}