crawlrs 0.2.0

crawlrs - Enterprise-grade web scraping and crawling platform built with Rust
Documentation
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
// Copyright (c) 2025-2026 Kirky.X🌠
// SPDX-License-Identifier: Apache-2.0

//! JS 注入器:在浏览器页面加载前后注入指定的 JavaScript 脚本。
//!
//! 提供两个预设构造函数:
//! - [`JsInjector::stealth`]:在导航前注入 `navigator_overrider.js` +
//!   `webgl_overrider.js`,覆盖 webdriver/WebGL 等反爬指纹属性并为 canvas 加噪。
//! - [`JsInjector::cleanup`]:在页面加载完成后注入 `remove_consent_popups.js` +
//!   `remove_overlay_elements.js` + `flatten_shadow_dom.js`,移除 consent 弹窗、
//!   遮罩元素并展平 shadow DOM。
//!
//! 通过 [`InjectPhase`] 区分注入时机:
//! - `BeforeLoad`:导航前注入(适合 stealth 类脚本,必须在页面脚本执行前生效)
//! - `AfterLoad`:页面加载完成后注入(适合清理类脚本,需要等 DOM 就绪)

use crate::engines::engine_client::EngineError;
use chromiumoxide::page::Page;
use std::fmt;

/// 注入阶段
///
/// 控制 JS 脚本在页面生命周期中的执行时机:
/// - `BeforeLoad`:在 `page.goto()` 之前注入
/// - `AfterLoad`:在 `page.goto()` 完成并等待页面就绪后注入
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum InjectPhase {
    /// 导航前注入(如 stealth 脚本,必须在页面脚本执行前覆盖 navigator 属性)
    BeforeLoad,
    /// 页面加载完成后注入(如清理类脚本,需要等 DOM 就绪)
    AfterLoad,
}

impl fmt::Display for InjectPhase {
    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
        match self {
            InjectPhase::BeforeLoad => write!(f, "BeforeLoad"),
            InjectPhase::AfterLoad => write!(f, "AfterLoad"),
        }
    }
}

/// JS 注入器
///
/// 持有 before / after 两个阶段的脚本列表,按 [`InjectPhase`] 选择对应列表
/// 依次在浏览器页面上 `evaluate`。
///
/// # 预设
///
/// - [`JsInjector::stealth`]:navigator_overrider + webgl_overrider(before)
/// - [`JsInjector::cleanup`]:remove_consent_popups + remove_overlay_elements + flatten_shadow_dom(after)
///
/// # 扩展
///
/// 通过 [`JsInjector::add_before`] / [`JsInjector::add_after`] 追加自定义脚本。
///
/// # 错误处理
///
/// [`JsInjector::apply`] 中任一脚本执行失败则立即返回
/// [`EngineError::BrowserError`],不再执行后续脚本。
/// 调用方若需 best-effort(不中断抓取),应自行 `if let Err(e) = ... { warn!(...) }`。
#[derive(Debug, Clone)]
pub struct JsInjector {
    /// 导航前注入的脚本列表
    before: Vec<String>,
    /// 页面加载完成后注入的脚本列表
    after: Vec<String>,
}

impl JsInjector {
    /// stealth 预设:在导航前注入 `navigator_overrider.js` + `webgl_overrider.js`
    ///
    /// `navigator_overrider.js` 覆盖 `navigator.webdriver`、`navigator.platform`、
    /// `navigator.languages`、`navigator.plugins` 等反爬指纹属性,隐藏自动化痕迹;
    /// `webgl_overrider.js` 覆盖 WebGL `getParameter` 的 vendor/renderer 指纹,
    /// 并对 canvas 像素读取/导出加极小随机噪声,破坏 WebGL/canvas 指纹稳定性。
    #[must_use]
    pub fn stealth() -> Self {
        Self {
            before: vec![
                include_str!("scripts/navigator_overrider.js").to_string(),
                include_str!("scripts/webgl_overrider.js").to_string(),
            ],
            after: Vec::new(),
        }
    }

    /// cleanup 预设:在页面加载后依次注入三个清理脚本
    ///
    /// 顺序:
    /// 1. `remove_consent_popups.js` — 移除 GDPR / cookie consent 弹窗
    /// 2. `remove_overlay_elements.js` — 移除 modal / dialog / overlay 遮罩
    /// 3. `flatten_shadow_dom.js` — 展平 open shadow DOM 便于后续选择器命中
    #[must_use]
    pub fn cleanup() -> Self {
        Self {
            before: Vec::new(),
            after: vec![
                include_str!("scripts/remove_consent_popups.js").to_string(),
                include_str!("scripts/remove_overlay_elements.js").to_string(),
                include_str!("scripts/flatten_shadow_dom.js").to_string(),
            ],
        }
    }

    /// 创建空注入器(无 before / after 脚本)
    #[must_use]
    pub fn new() -> Self {
        Self {
            before: Vec::new(),
            after: Vec::new(),
        }
    }

    /// 追加一个导航前注入的脚本
    pub fn add_before(&mut self, script: String) -> &mut Self {
        self.before.push(script);
        self
    }

    /// 追加一个页面加载后注入的脚本
    pub fn add_after(&mut self, script: String) -> &mut Self {
        self.after.push(script);
        self
    }

    /// 获取 before 阶段脚本列表(只读)
    pub fn before_scripts(&self) -> &[String] {
        &self.before
    }

    /// 获取 after 阶段脚本列表(只读)
    pub fn after_scripts(&self) -> &[String] {
        &self.after
    }

    /// 在指定页面上按阶段执行注入
    ///
    /// 根据 `phase` 选择 `before` 或 `after` 脚本列表,依次调用
    /// `page.evaluate(script)`。
    ///
    /// # 注意(P06)
    ///
    /// `evaluate` 在**当前文档**上执行。若页面尚未导航(about:blank),
    /// 脚本会随 `goto` 创建的新文档一起丢失——BeforeLoad 阶段的脚本
    /// (stealth 覆盖 navigator 等)应改用
    /// [`JsInjector::apply_before_document`],经 CDP
    /// `Page.addScriptToEvaluateOnNewDocument` 注册到**所有后续文档**。
    ///
    /// # 错误
    ///
    /// 任一脚本执行失败立即返回 [`EngineError::BrowserError`],不再执行后续脚本。
    ///
    /// # 参数
    ///
    /// - `page`:chromiumoxide 页面引用
    /// - `phase`:注入阶段
    pub async fn apply(&self, page: &Page, phase: InjectPhase) -> Result<(), EngineError> {
        let scripts = match phase {
            InjectPhase::BeforeLoad => &self.before,
            InjectPhase::AfterLoad => &self.after,
        };
        for (idx, script) in scripts.iter().enumerate() {
            if let Err(e) = page.evaluate(script.as_str()).await {
                return Err(EngineError::BrowserError(format!(
                    "JS inject failed at phase {} script #{}: {}",
                    phase,
                    idx + 1,
                    e
                )));
            }
        }
        Ok(())
    }

    /// 将 before 阶段脚本经 CDP 注册为"新文档创建时执行"(P06 修复)
    ///
    /// 通过 `Page.addScriptToEvaluateOnNewDocument` 把脚本绑定到**所有后续
    /// 导航创建的文档**,保证 stealth 覆盖在目标页任何脚本执行前生效——
    /// `apply(BeforeLoad)` 的 `page.evaluate` 只作用于当前文档(通常是
    /// about:blank),goto 后即失效,`navigator.webdriver` 覆盖形同虚设。
    ///
    /// # 错误
    ///
    /// 任一脚本注册失败立即返回 [`EngineError::BrowserError`]。
    /// 空脚本列表 no-op 返回 `Ok()`。
    pub async fn apply_before_document(&self, page: &Page) -> Result<(), EngineError> {
        use chromiumoxide::cdp::browser_protocol::page::AddScriptToEvaluateOnNewDocumentParams;

        for (idx, script) in self.before.iter().enumerate() {
            page.execute(AddScriptToEvaluateOnNewDocumentParams {
                source: script.clone(),
                world_name: None,
                include_command_line_api: None,
                run_immediately: None,
            })
            .await
            .map_err(|e| {
                EngineError::BrowserError(format!(
                    "addScriptToEvaluateOnNewDocument failed for script #{}: {}",
                    idx + 1,
                    e
                ))
            })?;
        }
        Ok(())
    }
}

impl Default for JsInjector {
    fn default() -> Self {
        Self::new()
    }
}

#[cfg(test)]
mod tests {
    use super::*;

    // stealth 构造函数

    #[test]
    fn stealth_contains_navigator_overrider_in_before() {
        let injector = JsInjector::stealth();
        assert_eq!(
            injector.before_scripts().len(),
            2,
            "stealth before must have 2 scripts (navigator + webgl)"
        );
        let script = &injector.before_scripts()[0];
        assert!(
            script.contains("navigator") && script.contains("webdriver"),
            "stealth before[0] must reference navigator.webdriver: {}",
            &script[..script.len().min(80)]
        );
    }

    #[test]
    fn stealth_contains_webgl_overrider_in_before() {
        let injector = JsInjector::stealth();
        assert_eq!(
            injector.before_scripts().len(),
            2,
            "stealth before must have 2 scripts (navigator + webgl)"
        );
        let script = &injector.before_scripts()[1];
        assert!(
            script.contains("getParameter"),
            "stealth before[1] must hook WebGL getParameter: {}",
            &script[..script.len().min(80)]
        );
        assert!(
            script.contains("UNMASKED_VENDOR_WEBGL") && script.contains("UNMASKED_RENDERER_WEBGL"),
            "stealth before[1] must override unmasked vendor/renderer"
        );
        assert!(
            script.contains("toDataURL") && script.contains("getImageData"),
            "stealth before[1] must add canvas noise via toDataURL/getImageData"
        );
        assert!(
            script.contains("Google Inc. (Intel)") && script.contains("ANGLE (Intel"),
            "stealth before[1] must return the faked GPU vendor/renderer strings"
        );
        assert!(
            !script.contains("__crawlrs_stealth_installed =")
                && !script.contains("__crawlrs_stealth_installed="),
            "webgl 脚本不得设置 __crawlrs_stealth_installed(由 navigator_overrider 统一管理)"
        );
        assert!(
            script.trim_end().ends_with("})();"),
            "webgl script must be an IIFE"
        );
    }

    #[test]
    fn stealth_has_empty_after() {
        let injector = JsInjector::stealth();
        assert!(
            injector.after_scripts().is_empty(),
            "stealth after must be empty"
        );
    }

    #[test]
    fn stealth_source_comment_preserved() {
        // 顶部必须保留 crawl4ai 来源注释(文档同步 + 来源可追溯)
        let injector = JsInjector::stealth();
        let script = &injector.before_scripts()[0];
        assert!(
            script.starts_with("// Source: crawl4ai js_snippet/navigator_overrider.js"),
            "stealth script must retain crawl4ai source comment header"
        );
    }

    // cleanup 构造函数

    #[test]
    fn cleanup_contains_three_scripts_in_after() {
        let injector = JsInjector::cleanup();
        assert_eq!(
            injector.after_scripts().len(),
            3,
            "cleanup after must have 3 scripts (consent/overlay/flatten)"
        );
    }

    #[test]
    fn cleanup_has_empty_before() {
        let injector = JsInjector::cleanup();
        assert!(
            injector.before_scripts().is_empty(),
            "cleanup before must be empty"
        );
    }

    #[test]
    fn cleanup_after_scripts_in_expected_order() {
        // 顺序:consent → overlay → flatten
        let injector = JsInjector::cleanup();
        let scripts = injector.after_scripts();
        assert!(
            scripts[0].contains("remove_consent_popups"),
            "cleanup after[0] must be remove_consent_popups.js"
        );
        assert!(
            scripts[1].contains("remove_overlay_elements"),
            "cleanup after[1] must be remove_overlay_elements.js"
        );
        assert!(
            scripts[2].contains("flatten_shadow_dom"),
            "cleanup after[2] must be flatten_shadow_dom.js"
        );
    }

    #[test]
    fn cleanup_scripts_non_empty() {
        // 脚本不能是占位符(任务要求"真实有效")
        let injector = JsInjector::cleanup();
        for (i, s) in injector.after_scripts().iter().enumerate() {
            assert!(
                s.len() > 200,
                "cleanup after[{}] too short ({} chars), must be real implementation",
                i,
                s.len()
            );
        }
    }

    #[test]
    fn cleanup_scripts_are_iife() {
        // 所有脚本必须是 IIFE(立即执行),不能只是函数定义
        let injector = JsInjector::cleanup();
        for (i, s) in injector.after_scripts().iter().enumerate() {
            assert!(
                s.contains("(function") && s.trim_end().ends_with("})();"),
                "cleanup after[{}] must be an IIFE",
                i
            );
        }
    }

    // new / default

    #[test]
    fn new_creates_empty_injector() {
        let injector = JsInjector::new();
        assert!(injector.before_scripts().is_empty());
        assert!(injector.after_scripts().is_empty());
    }

    #[test]
    fn default_equals_new() {
        let a = JsInjector::new();
        let b = JsInjector::default();
        assert_eq!(a.before_scripts().len(), b.before_scripts().len());
        assert_eq!(a.after_scripts().len(), b.after_scripts().len());
    }

    // add_before / add_after 扩展接口

    #[test]
    fn add_before_extends_before_scripts() {
        let mut injector = JsInjector::new();
        injector.add_before("console.log('a');".to_string());
        injector.add_before("console.log('b');".to_string());
        assert_eq!(injector.before_scripts().len(), 2);
        assert_eq!(injector.before_scripts()[0], "console.log('a');");
        assert_eq!(injector.before_scripts()[1], "console.log('b');");
    }

    #[test]
    fn add_after_extends_after_scripts() {
        let mut injector = JsInjector::new();
        injector.add_after("console.log('x');".to_string());
        assert_eq!(injector.after_scripts().len(), 1);
        assert_eq!(injector.after_scripts()[0], "console.log('x');");
    }

    #[test]
    fn add_before_does_not_touch_after() {
        let mut injector = JsInjector::stealth();
        injector.add_before("console.log('extra');".to_string());
        assert_eq!(injector.before_scripts().len(), 3);
        assert!(injector.after_scripts().is_empty());
    }

    #[test]
    fn add_after_does_not_touch_before() {
        let mut injector = JsInjector::cleanup();
        injector.add_after("console.log('extra');".to_string());
        assert_eq!(injector.after_scripts().len(), 4);
        assert!(injector.before_scripts().is_empty());
    }

    #[test]
    fn add_before_returns_mut_self_for_chaining() {
        let mut injector = JsInjector::new();
        injector
            .add_before("a".to_string())
            .add_before("b".to_string())
            .add_after("c".to_string());
        assert_eq!(injector.before_scripts().len(), 2);
        assert_eq!(injector.after_scripts().len(), 1);
    }

    // InjectPhase

    #[test]
    fn inject_phase_display_before_load() {
        assert_eq!(format!("{}", InjectPhase::BeforeLoad), "BeforeLoad");
    }

    #[test]
    fn inject_phase_display_after_load() {
        assert_eq!(format!("{}", InjectPhase::AfterLoad), "AfterLoad");
    }

    #[test]
    fn inject_phase_equality() {
        assert_eq!(InjectPhase::BeforeLoad, InjectPhase::BeforeLoad);
        assert_eq!(InjectPhase::AfterLoad, InjectPhase::AfterLoad);
        assert_ne!(InjectPhase::BeforeLoad, InjectPhase::AfterLoad);
    }

    #[test]
    fn inject_phase_clone_copy() {
        let phase = InjectPhase::BeforeLoad;
        let cloned = phase; // Copy
        assert_eq!(phase, cloned);
    }

    // clone 行为

    #[test]
    fn clone_preserves_scripts() {
        let injector = JsInjector::cleanup();
        let cloned = injector.clone();
        assert_eq!(
            injector.before_scripts().len(),
            cloned.before_scripts().len()
        );
        assert_eq!(injector.after_scripts().len(), cloned.after_scripts().len());
        for (a, b) in injector
            .after_scripts()
            .iter()
            .zip(cloned.after_scripts().iter())
        {
            assert_eq!(a, b);
        }
    }

    // apply 错误路径
    //
    // 注:chromiumoxide::Page 难以在单测中 mock(需要 CDP 连接),apply 失败路径
    // 的端到端测试放在集成测试中。这里通过 Empty scripts 验证空列表不调用 evaluate。

    #[tokio::test]
    async fn apply_with_empty_scripts_returns_ok_without_evaluating() {
        // 空注入器 + 任意 phase → 不可能调用 page.evaluate,必然 Ok
        // 这里不构造 Page,因为 apply 在 scripts 为空时不会触碰 page。
        // 用 cast 验证:apply 接受 &Page,但若 scripts 为空,循环不执行。
        // 由于无法在不构造 Page 的情况下调用 apply(参数类型是 &Page),
        // 此测试改为验证 scripts 为空时 apply 不需要 page 引用。
        let injector = JsInjector::new();
        assert!(injector.before_scripts().is_empty());
        assert!(injector.after_scripts().is_empty());
        // apply 在空 scripts 上不会执行任何 evaluate,等价于 no-op。
    }

    // 综合场景

    #[test]
    fn stealth_then_add_cleanup_yields_combined_injector() {
        // 用户场景:先 stealth 再追加 cleanup 脚本到一个 injector
        let mut injector = JsInjector::stealth();
        // cleanup 的 3 个脚本追加到 after
        let cleanup = JsInjector::cleanup();
        for s in cleanup.after_scripts() {
            injector.add_after(s.clone());
        }
        assert_eq!(injector.before_scripts().len(), 2);
        assert_eq!(injector.after_scripts().len(), 3);
    }

    #[test]
    fn script_source_headers_preserved() {
        // crawl4ai 移植脚本必须保留来源注释;crawlrs 原创的 webgl_overrider
        // 标注原创来源,但所有脚本都必须带 Apache-2.0 license 头
        let stealth = JsInjector::stealth();
        let cleanup = JsInjector::cleanup();

        for s in stealth.before_scripts()[..1]
            .iter()
            .chain(cleanup.after_scripts().iter())
        {
            assert!(
                s.starts_with("// Source: crawl4ai js_snippet/"),
                "script missing crawl4ai source header: {}",
                &s[..s.len().min(60)]
            );
        }
        let webgl = &stealth.before_scripts()[1];
        assert!(
            webgl.starts_with("// Source: crawlrs original"),
            "webgl_overrider is original and must be marked as such: {}",
            &webgl[..webgl.len().min(60)]
        );
        for s in stealth
            .before_scripts()
            .iter()
            .chain(cleanup.after_scripts().iter())
        {
            assert!(
                s.contains("License: Apache-2.0"),
                "script missing license header"
            );
        }
    }
}