crawlex 1.0.4

Stealth crawler with Chrome-perfect TLS/H2 fingerprint, render pool, hooks, persistent queue
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37

use crate::render::chrome_fetcher::BrowserKind;

/// Host for downloading browsers and metadata.
pub struct BrowserHost {
    pub(crate) object: String,
    pub(crate) metadata: String,
}

impl BrowserHost {
    pub fn new(object: &str, metadata: &str) -> Self {
        Self {
            object: object.to_string(),
            metadata: metadata.to_string(),
        }
    }

    pub fn single(host: &str) -> Self {
        Self {
            object: host.to_string(),
            metadata: host.to_string(),
        }
    }

    #[doc(hidden)] // internal API
    pub fn current(kind: BrowserKind) -> Self {
        match kind {
            BrowserKind::Chromium => Self {
                object: "https://storage.googleapis.com".to_string(),
                metadata: "https://storage.googleapis.com".to_string(),
            },
            BrowserKind::Chrome | BrowserKind::ChromeHeadlessShell => Self {
                object: "https://storage.googleapis.com".to_string(),
                metadata: "https://googlechromelabs.github.io/".to_string(),
            },
        }
    }
}