crawlex 1.0.4

Stealth crawler with Chrome-perfect TLS/H2 fingerprint, render pool, hooks, persistent queue
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50

//! Scroll scheduler — bursts with Pareto dwells (B.1).

#![cfg(feature = "cdp-backend")]

use crawlex::render::motion::scroll::{schedule, ScrollParams};
use crawlex::render::motion::MotionProfile;

#[test]
fn balanced_scroll_has_bursts_and_dwells() {
    let p = ScrollParams::for_profile(MotionProfile::Balanced);
    let ticks = schedule(2_000.0, &p, 7);
    let dwells = ticks.iter().filter(|t| t.delta_y == 0.0).count();
    let moves = ticks.iter().filter(|t| t.delta_y.abs() > 0.0).count();
    assert!(dwells >= 1, "expected reading dwells between bursts");
    assert!(moves > dwells, "moves should outnumber dwells");
}

#[test]
fn fast_profile_preserves_throughput() {
    // Fast schedule: no zero-delta dwells, tick deltas large (≤240 px).
    let p = ScrollParams::for_profile(MotionProfile::Fast);
    let ticks = schedule(1_000.0, &p, 1);
    for t in &ticks {
        assert!(t.delta_y.abs() > 1.0, "fast must not insert dwells");
        assert!(t.delta_y.abs() <= 260.0);
    }
}

#[test]
fn sum_of_deltas_is_dy() {
    let p = ScrollParams::for_profile(MotionProfile::Human);
    let dy = 3_000.0;
    let ticks = schedule(dy, &p, 99);
    let sum: f64 = ticks.iter().map(|t| t.delta_y).sum();
    assert!((sum - dy).abs() <= p.peak_tick_px, "sum={sum}, dy={dy}");
}

#[test]
fn determinism_across_profiles() {
    for prof in [
        MotionProfile::Balanced,
        MotionProfile::Human,
        MotionProfile::Paranoid,
    ] {
        let p = ScrollParams::for_profile(prof);
        let a = schedule(600.0, &p, 17);
        let b = schedule(600.0, &p, 17);
        assert_eq!(a.len(), b.len());
    }
}