void_crawl_core 0.5.0

Rust-native CDP browser automation core — stealth-patched headless Chrome, profile leasing, captcha detection
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417

//! Integration tests for screen recording (`Page::record`,
//! `Page::start_recording`).
//!
//! The load-bearing tests here are the occlusion ones. Chrome composites only
//! the frontmost tab of a window, so a shared-window tab must be foregrounded
//! to record, while a tab in its **own** window records at full rate
//! concurrently — which is what `foreground: None` auto-detects.
//!
//! These assert a frame *rate*, never merely "some frames": a screencast
//! always emits one initial frame even from a fully occluded tab, so a `> 0`
//! assertion passes even when capture is completely broken. An earlier version
//! of this file made exactly that mistake and concluded the opposite.
//!
//! Requires a real Chromium/Chrome binary. Run serially:
//!
//!     cargo test -p void_crawl_core --test recording -- --test-threads=1
#![allow(clippy::expect_used, clippy::unwrap_used, clippy::panic)]

use std::time::{Duration, Instant};

use tokio::time::{sleep, timeout};
use void_crawl_core::{
    BrowserSession, Page, RecordingOptions, SelectorEntry, SelectorKind, VoidCrawlError,
};

/// A page that repaints continuously, so Chrome has a reason to emit frames.
/// A static page legitimately produces almost none — see the module docs on
/// paint-driven delivery.
const ANIMATED: &str = r#"
<html><body style="margin:0">
  <div id="box" style="width:200px;height:120px;background:#c00"></div>
  <div id="other" style="width:150px;height:90px;background:#0c0"></div>
  <script>
    let t = 0;
    function tick() {
      t += 4;
      document.getElementById('box').style.background =
        'hsl(' + (t % 360) + ',80%,50%)';
      document.getElementById('other').style.transform =
        'translateX(' + (t % 50) + 'px)';
      requestAnimationFrame(tick);
    }
    tick();
  </script>
</body></html>
"#;

async fn headless_session() -> BrowserSession {
    BrowserSession::builder()
        .headless()
        .no_sandbox()
        .launch()
        .await
        .expect("failed to launch headless browser")
}

fn data_url(html: &str) -> String {
    let encoded = html
        .replace('%', "%25")
        .replace('"', "%22")
        .replace('#', "%23")
        .replace('<', "%3C")
        .replace('>', "%3E")
        .replace(' ', "%20")
        .replace('\n', "%0A");
    format!("data:text/html,{encoded}")
}

async fn animated_page(session: &BrowserSession) -> Page {
    session.new_page(&data_url(ANIMATED)).await.expect("new_page failed")
}

fn css(value: &str) -> SelectorEntry {
    SelectorEntry {
        kind:  SelectorKind::Css,
        value: value.to_string(),
        regex: None,
        name:  None,
        nth:   None,
        x:     None,
        y:     None,
    }
}

fn opts_for(secs: u64) -> RecordingOptions {
    RecordingOptions::default().with_max_duration(Duration::from_secs(secs)).with_fps(10)
}

#[tokio::test]
async fn records_the_viewport() {
    let session = headless_session().await;
    let page = animated_page(&session).await;

    let rec = page.record(opts_for(2)).await.expect("record failed");

    assert_eq!(rec.regions.len(), 1, "no crop requested → exactly one region");
    assert_eq!(rec.regions[0].label, "viewport");
    assert!(rec.regions[0].bbox.is_none());
    assert!(rec.frames_captured > 0, "an animating page must yield frames");
    assert_eq!(rec.regions[0].frames.len(), rec.frames_captured);
    assert!(
        rec.regions[0].frames.iter().all(|f| !f.data.is_empty()),
        "every frame must carry image bytes"
    );
    // Offsets are real elapsed times, so they must be non-decreasing and
    // bounded by the recording duration.
    let offsets: Vec<Duration> = rec.regions[0].frames.iter().map(|f| f.offset).collect();
    assert!(offsets.windows(2).all(|w| w[0] <= w[1]), "offsets must be monotonic");
    assert!(offsets.iter().all(|o| *o <= rec.duration + Duration::from_millis(500)));
}

/// Why a shared-window tab must be foregrounded.
///
/// A tab sharing a window with others stops painting the moment a sibling
/// takes focus, so an unforegrounded recording on such a tab collects
/// essentially nothing. Asserted so that flipping the default silently would
/// fail loudly here.
#[tokio::test]
async fn shared_window_tab_stalls_when_not_foregrounded() {
    let session = headless_session().await;
    let recorded = animated_page(&session).await;
    let stealer = animated_page(&session).await;

    // `screenshot` is the focus-stealer of record: it brings its own tab to
    // front before capturing (see `Page::screenshot`), which is exactly the
    // contention this reproduces.
    stealer.screenshot_png().await.expect("stealer screenshot failed");

    let rec = recorded
        .record(RecordingOptions { foreground: Some(false), ..opts_for(3) })
        .await
        .expect("record failed");

    assert!(
        rec.effective_fps() < 2.0,
        "a shared-window tab is expected to stall while occluded, but captured {} frames \
         ({:.1} fps). If Chrome has changed here, RecordingOptions::foreground could default \
         to false and the module docs need updating.",
        rec.frames_captured,
        rec.effective_fps()
    );
}

/// The escape hatch: a tab in its own window records at full rate while
/// another window holds the foreground, so no capture lock is needed.
///
/// Note the construction order — `new_page` puts its tab in the most
/// recently active window, so the sibling must be created *before* the
/// recorded page gets its own window, or it lands inside it.
#[tokio::test]
async fn own_window_tab_records_while_another_window_is_focused() {
    let session = headless_session().await;
    // Order matters: `new_page` lands in the most recently active window, so
    // the stealer must exist before the recorded page claims its own window.
    let stealer = animated_page(&session).await;
    let recorded =
        session.new_page_in_window(&data_url(ANIMATED)).await.expect("new_page_in_window failed");
    assert!(
        recorded.alone_in_window().await.expect("alone_in_window failed"),
        "test setup: the recorded page is not alone in its window"
    );

    stealer.screenshot_png().await.expect("stealer screenshot failed");

    let handle = recorded
        .start_recording(RecordingOptions { foreground: Some(false), ..opts_for(4) })
        .await
        .expect("start_recording failed");
    sleep(Duration::from_secs(2)).await;
    // Steal focus again mid-recording — the recorded window must not care.
    stealer.screenshot_png().await.expect("stealer screenshot failed");
    sleep(Duration::from_secs(1)).await;
    let rec = handle.stop(&recorded).await.expect("stop failed");

    assert!(
        rec.effective_fps() > 3.0,
        "an own-window tab must keep recording at rate while another window is focused, \
         but captured {} frames ({:.1} fps)",
        rec.frames_captured,
        rec.effective_fps()
    );
}

/// The concrete benefit: with the recorded tab in its own window and
/// `foreground: false`, a screenshot elsewhere isn't blocked by the
/// in-flight recording.
#[tokio::test]
async fn own_window_recording_does_not_block_sibling_screenshots() {
    let session = headless_session().await;
    // Order matters — see the sibling test.
    let sibling = animated_page(&session).await;
    let recorded =
        session.new_page_in_window(&data_url(ANIMATED)).await.expect("new_page_in_window failed");
    assert!(
        recorded.alone_in_window().await.expect("alone_in_window failed"),
        "test setup: the recorded page is not alone in its window"
    );

    let handle = recorded
        .start_recording(RecordingOptions { foreground: Some(false), ..opts_for(10) })
        .await
        .expect("start failed");

    // Well under the recording's 10s duration: if the capture lock were held,
    // this would block until the recording ended.
    let shot = timeout(Duration::from_secs(4), sibling.screenshot_png())
        .await
        .expect("sibling screenshot blocked on the in-flight recording")
        .expect("sibling screenshot failed");
    assert!(!shot.is_empty());

    // Keep recording after the sibling stole focus, so the assertion below is
    // about frames captured *while occluded* rather than about the handful
    // that arrived before the screenshot returned.
    sleep(Duration::from_secs(2)).await;
    let rec = handle.stop(&recorded).await.expect("stop failed");

    let after_steal =
        rec.regions[0].frames.iter().filter(|f| f.offset > Duration::from_secs(1)).count();
    assert!(
        after_steal > 3,
        "recording must continue at rate after a sibling window took focus, but only \
         {after_steal} of {} frames arrived after it",
        rec.frames_captured
    );
}

#[tokio::test]
async fn records_multiple_selector_regions_from_one_screencast() {
    let session = headless_session().await;
    let page = animated_page(&session).await;

    let rec = page
        .record(opts_for(2).with_selector(css("#box")).with_selector(css("#other")))
        .await
        .expect("record failed");

    assert_eq!(rec.regions.len(), 2, "one region per selector");
    assert_eq!(rec.regions[0].label, "0_box");
    assert_eq!(rec.regions[1].label, "1_other");
    for region in &rec.regions {
        assert!(region.bbox.is_some(), "a selector region must carry its resolved rect");
        assert_eq!(
            region.frames.len(),
            rec.frames_captured,
            "every region is cut from the same frame sequence"
        );
    }
    // The two regions have different source rects, so their cropped frames
    // must differ — proof the crop actually happened per region.
    let (a, b) = (&rec.regions[0], &rec.regions[1]);
    assert_ne!(a.bbox, b.bbox);
    if let (Some(fa), Some(fb)) = (a.frames.first(), b.frames.first()) {
        assert_ne!(fa.data, fb.data, "distinct regions must produce distinct crops");
    }
}

#[tokio::test]
async fn rejects_bbox_and_selectors_together() {
    let session = headless_session().await;
    let page = animated_page(&session).await;

    let err = page
        .record(
            opts_for(1)
                .with_bbox(void_crawl_core::Bbox { x: 0, y: 0, width: 10, height: 10 })
                .with_selector(css("#box")),
        )
        .await
        .expect_err("bbox + selectors must be rejected");
    assert!(matches!(err, VoidCrawlError::RecordingError(_)), "got {err:?}");
}

/// A selector that matches nothing must fail *before* the screencast runs,
/// not after burning the full duration and returning empty regions.
#[tokio::test]
async fn unresolvable_selector_fails_fast() {
    let session = headless_session().await;
    let page = animated_page(&session).await;

    let started = Instant::now();
    let err = page
        .record(opts_for(30).with_selector(css("#nope")))
        .await
        .expect_err("a selector matching nothing must error");
    assert!(matches!(err, VoidCrawlError::ElementNotVisible(_)), "got {err:?}");
    assert!(
        started.elapsed() < Duration::from_secs(10),
        "must fail before the 30s recording duration elapses, took {:?}",
        started.elapsed()
    );
}

/// The one-shot viewport override must not leak, exactly as it doesn't for
/// `screenshot` — a recording lives long enough for a leak to matter more.
#[tokio::test]
async fn viewport_override_is_restored() {
    use void_crawl_core::Viewport;

    let session = headless_session().await;
    let page = animated_page(&session).await;
    let before = page.current_viewport();

    let rec = page
        .record(opts_for(1).with_viewport(Viewport::custom(500, 400)))
        .await
        .expect("record failed");
    assert!(rec.frames_captured > 0);

    assert_eq!(page.current_viewport(), before, "viewport override leaked past the recording");
}

/// `fps` is a ceiling: frames closer together than 1/fps are dropped, and
/// the drop is reported rather than hidden.
#[tokio::test]
async fn fps_ceiling_is_enforced_and_reported() {
    let session = headless_session().await;
    let page = animated_page(&session).await;

    let rec = page.record(opts_for(2).with_fps(2)).await.expect("record failed");

    assert!(rec.frames_captured > 0);
    // 2s at a 2fps ceiling can't produce more than ~5 frames.
    assert!(rec.frames_captured <= 6, "captured {} frames at 2fps", rec.frames_captured);
    assert!(
        rec.effective_fps() <= 3.0,
        "effective fps {} exceeded the ceiling",
        rec.effective_fps()
    );
    let gaps: Vec<Duration> =
        rec.regions[0].frames.windows(2).map(|w| w[1].offset.saturating_sub(w[0].offset)).collect();
    assert!(
        gaps.iter().all(|g| *g >= Duration::from_millis(400)),
        "frames arrived closer together than the fps ceiling allows: {gaps:?}"
    );
}

/// `foreground: None` must detect which situation it's in, so neither the
/// safe case nor the concurrent case requires the caller to know anything
/// about Chrome's per-window compositing.
#[tokio::test]
async fn foreground_is_auto_detected_from_window_placement() {
    let session = headless_session().await;
    // Sibling first: `new_page` targets the most recently active window, so
    // creating it after `new_page_in_window` would put it in that window.
    let shared = animated_page(&session).await;
    let owned =
        session.new_page_in_window(&data_url(ANIMATED)).await.expect("new_page_in_window failed");

    assert!(!shared.alone_in_window().await.expect("alone_in_window failed"));
    assert!(owned.alone_in_window().await.expect("alone_in_window failed"));

    let shared_rec = shared.record(opts_for(2)).await.expect("record failed");
    assert!(shared_rec.foregrounded, "a shared-window tab must be foregrounded automatically");
    assert!(shared_rec.effective_fps() > 3.0, "{} frames", shared_rec.frames_captured);

    let owned_rec = owned.record(opts_for(2)).await.expect("record failed");
    assert!(
        !owned_rec.foregrounded,
        "a tab alone in its window must record without taking the capture lock"
    );
    assert!(owned_rec.effective_fps() > 3.0, "{} frames", owned_rec.frames_captured);
}

/// GIF encoding is feature-gated; when the feature is on it must produce a
/// real file, and when a region is cropped the artifact must be per-region.
#[cfg(feature = "encode-gif")]
#[tokio::test]
async fn encodes_regions_to_gif() {
    use std::fs;

    use void_crawl_core::Encoding;

    let session = headless_session().await;
    let page = animated_page(&session).await;
    let dir = tempfile::tempdir().expect("tempdir");

    let rec = page
        .record(
            opts_for(2)
                .with_selector(css("#box"))
                .with_selector(css("#other"))
                .with_dir(dir.path())
                .with_encoding(Encoding::Gif),
        )
        .await
        .expect("record failed");

    assert_eq!(rec.regions.len(), 2);
    for region in &rec.regions {
        assert_eq!(region.outputs.len(), 1, "one artifact per region");
        let path = &region.outputs[0];
        let meta = fs::metadata(path).expect("gif was not written");
        assert!(meta.len() > 0, "empty gif at {}", path.display());
        let header = fs::read(path).expect("read gif");
        assert_eq!(&header[..6], b"GIF89a", "not a GIF at {}", path.display());
    }
}

/// Asking for an encoding whose feature is off must fail loudly and keep the
/// frames, rather than silently producing nothing.
#[cfg(not(feature = "encode-gif"))]
#[tokio::test]
async fn gif_without_the_feature_is_an_actionable_error() {
    use void_crawl_core::Encoding;

    let session = headless_session().await;
    let page = animated_page(&session).await;
    let dir = tempfile::tempdir().expect("tempdir");

    let err = page
        .record(opts_for(2).with_dir(dir.path()).with_encoding(Encoding::Gif))
        .await
        .expect_err("must error without the encode-gif feature");
    assert!(matches!(err, VoidCrawlError::RecordingEncodeError(_)), "got {err:?}");
    assert!(err.to_string().contains("encode-gif"), "error must name the feature: {err}");
}