spider 2.51.39

A web crawler and scraper, building blocks for data curation workloads.
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124

/// Chrome WebDriver arguments for headless mode.
#[cfg(not(feature = "webdriver_headed"))]
pub(crate) static CHROME_WEBDRIVER_ARGS: &[&str] = &[
    "--headless",
    "--disable-gpu",
    "--no-sandbox",
    "--disable-dev-shm-usage",
    "--disable-extensions",
    "--disable-background-networking",
    "--disable-sync",
    "--disable-default-apps",
    "--mute-audio",
    "--no-first-run",
    "--disable-popup-blocking",
    "--disable-hang-monitor",
    "--disable-prompt-on-repost",
    "--disable-client-side-phishing-detection",
    "--disable-component-update",
    "--disable-backgrounding-occluded-windows",
    "--disable-renderer-backgrounding",
    "--disable-background-timer-throttling",
    "--disable-ipc-flooding-protection",
    "--password-store=basic",
    "--use-mock-keychain",
    "--metrics-recording-only",
];

/// Chrome WebDriver arguments for headed mode.
#[cfg(feature = "webdriver_headed")]
pub(crate) static CHROME_WEBDRIVER_ARGS: &[&str] = &[
    "--disable-gpu",
    "--no-sandbox",
    "--disable-dev-shm-usage",
    "--disable-extensions",
    "--disable-background-networking",
    "--disable-sync",
    "--disable-default-apps",
    "--mute-audio",
    "--no-first-run",
    "--disable-popup-blocking",
    "--disable-hang-monitor",
    "--disable-prompt-on-repost",
    "--disable-client-side-phishing-detection",
    "--disable-component-update",
    "--disable-backgrounding-occluded-windows",
    "--disable-renderer-backgrounding",
    "--disable-background-timer-throttling",
    "--disable-ipc-flooding-protection",
    "--password-store=basic",
    "--use-mock-keychain",
    "--metrics-recording-only",
];

/// Firefox WebDriver arguments for headless mode.
#[cfg(not(feature = "webdriver_headed"))]
pub(crate) static FIREFOX_WEBDRIVER_ARGS: &[&str] = &["-headless", "-no-remote", "-new-instance"];

/// Firefox WebDriver arguments for headed mode.
#[cfg(feature = "webdriver_headed")]
pub(crate) static FIREFOX_WEBDRIVER_ARGS: &[&str] = &["-no-remote", "-new-instance"];

/// Edge WebDriver arguments for headless mode.
#[cfg(not(feature = "webdriver_headed"))]
pub(crate) static EDGE_WEBDRIVER_ARGS: &[&str] = &[
    "--headless",
    "--disable-gpu",
    "--no-sandbox",
    "--disable-dev-shm-usage",
    "--disable-extensions",
    "--disable-background-networking",
    "--disable-sync",
    "--disable-default-apps",
    "--mute-audio",
    "--no-first-run",
    "--disable-popup-blocking",
    "--disable-hang-monitor",
    "--disable-prompt-on-repost",
    "--disable-client-side-phishing-detection",
    "--disable-component-update",
    "--disable-backgrounding-occluded-windows",
    "--disable-renderer-backgrounding",
    "--disable-background-timer-throttling",
    "--disable-ipc-flooding-protection",
    "--password-store=basic",
    "--use-mock-keychain",
    "--metrics-recording-only",
];

/// Edge WebDriver arguments for headed mode.
#[cfg(feature = "webdriver_headed")]
pub(crate) static EDGE_WEBDRIVER_ARGS: &[&str] = &[
    "--disable-gpu",
    "--no-sandbox",
    "--disable-dev-shm-usage",
    "--disable-extensions",
    "--disable-background-networking",
    "--disable-sync",
    "--disable-default-apps",
    "--mute-audio",
    "--no-first-run",
    "--disable-popup-blocking",
    "--disable-hang-monitor",
    "--disable-prompt-on-repost",
    "--disable-client-side-phishing-detection",
    "--disable-component-update",
    "--disable-backgrounding-occluded-windows",
    "--disable-renderer-backgrounding",
    "--disable-background-timer-throttling",
    "--disable-ipc-flooding-protection",
    "--password-store=basic",
    "--use-mock-keychain",
    "--metrics-recording-only",
];

/// Get the default arguments for a browser type.
pub(crate) fn get_browser_args(
    browser: &super::webdriver_common::WebDriverBrowser,
) -> &'static [&'static str] {
    match browser {
        super::webdriver_common::WebDriverBrowser::Chrome => CHROME_WEBDRIVER_ARGS,
        super::webdriver_common::WebDriverBrowser::Firefox => FIREFOX_WEBDRIVER_ARGS,
        super::webdriver_common::WebDriverBrowser::Edge => EDGE_WEBDRIVER_ARGS,
    }
}