browser_oxide 0.1.3

Stealth headless browser engine in Rust: real HTML/CSS/DOM/JS, V8 via deno_core, own BoringSSL TLS/JA4 fingerprint, no Chromium, no CDP — for anti-bot web scraping, archival, and AI agents
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408

//! CSP enforcement integration test.
//!
//! Locks in the load-bearing behavior: when a page declares
//! `script-src 'self' 'strict-dynamic' 'nonce-XXX'`, the parser-injected
//! `<script src="...">` *without* a matching nonce must NOT be fetched.
//! This is exactly what real Chrome does on walmart.com — and what we
//! were failing to do before, causing browser_oxide to issue
//! `/akam/13/...` requests that Chrome never makes.

use browser_oxide::stealth::presets::chrome_148_macos;
use browser_oxide::Page;

/// A miniaturized Walmart-style CSP. The page declares strict-dynamic
/// + nonce, then includes one parser-injected script with a matching
/// nonce (allowed) and one without (blocked).
const HTML: &str = r#"<!doctype html>
<html><head>
<meta http-equiv="Content-Security-Policy"
      content="script-src 'self' 'strict-dynamic' 'nonce-MRjHHgrLk9lNoNBv'">
<title>csp test</title>
</head><body>
<script nonce="MRjHHgrLk9lNoNBv">
  globalThis.__legitimate_inline_ran = true;
</script>
<!--
  Parser-injected without a nonce — must be blocked under strict-dynamic.
  We point it at a never-resolvable host so a fetch attempt would surface
  as a network error in the runtime; if the engine respects CSP, the
  fetch is never attempted at all and the page reaches DOMContentLoaded
  cleanly.
-->
<script src="https://blocked-by-csp.invalid./payload.js"></script>
</body></html>"#;

/// End-to-end: load the strict-dynamic page, confirm the inline script
/// with matching nonce ran, and confirm the engine reports a CSP block
/// for the parser-injected script (proving the gate fires before the
/// fetch is attempted).
#[tokio::test]
async fn parser_injected_script_without_nonce_is_blocked() {
    use std::sync::{Arc, Mutex};

    // Capture stderr-equivalent: the engine emits
    // `[csp] Refused to load the script '...'` via eprintln!. We can't
    // intercept stderr from inside cargo test, but we can run the
    // navigation from a Page::from_html and additionally directly
    // exercise the check_csp() function with the parsed policy.
    use browser_oxide::csp_collector::collect_csp;
    use browser_oxide::html_parser::parse_html;
    use browser_oxide::js_runtime::extensions::fetch_ext as csp_state;
    use browser_oxide::net::csp::Directive;
    use url::Url;

    // Install the policy as the engine would.
    let dom = parse_html(HTML);
    let policy = collect_csp(&[], &dom);
    csp_state::set_csp_policy(
        Arc::new(policy),
        Url::parse("https://example.com/").unwrap(),
        true,
    );

    // The blocked-by-csp.invalid script must trip CSP.
    let blocked_url = Url::parse("https://blocked-by-csp.invalid./payload.js").unwrap();
    let block_decision = csp_state::check_csp(
        Directive::ScriptSrcElem,
        &blocked_url,
        None, // no nonce → blocked under strict-dynamic
        true, // parser_inserted
    );
    assert!(
        block_decision.is_err(),
        "parser-injected, no-nonce script must trip CSP"
    );
    assert_eq!(block_decision.unwrap_err(), "script-src");

    // The same URL with a matching nonce must NOT trip CSP.
    let allowed_decision = csp_state::check_csp(
        Directive::ScriptSrcElem,
        &blocked_url,
        Some("MRjHHgrLk9lNoNBv"),
        true,
    );
    assert!(
        allowed_decision.is_ok(),
        "matching nonce must clear CSP under strict-dynamic"
    );

    // Now actually load the page — the inline script with nonce should
    // run, the parser-injected blocked-by-csp.invalid script must be
    // skipped without a network attempt.
    let captured_log: Arc<Mutex<Vec<String>>> = Arc::new(Mutex::new(Vec::new()));
    let _captured_for_log = captured_log.clone();
    let mut page = Page::from_html(HTML, Some(chrome_148_macos()))
        .await
        .unwrap();

    let inline_ran = page
        .evaluate("String(globalThis.__legitimate_inline_ran)")
        .unwrap();
    assert_eq!(
        inline_ran, "true",
        "inline script with matching nonce must execute"
    );

    // Cleanup so the next test sees a fresh state.
    csp_state::clear_csp_policy();
}

/// A page with NO CSP at all must still load both scripts (parser-
/// injected, no nonce, no policy → no enforcement).
#[tokio::test]
async fn no_csp_does_not_block_anything() {
    const NO_CSP_HTML: &str = r#"<!doctype html>
<html><head><title>no csp</title></head><body>
<script>globalThis.__inline_ran = "yes";</script>
</body></html>"#;
    let mut page = Page::from_html(NO_CSP_HTML, Some(chrome_148_macos()))
        .await
        .unwrap();
    let inline = page.evaluate("globalThis.__inline_ran").unwrap();
    assert_eq!(inline.trim_matches('"'), "yes");
}

/// `securitypolicyviolation` event must fire on `document` (and bubble
/// to `window`) when a fetch is blocked. Verifies the full pipeline:
/// page-level meta-CSP install → queued violation from a Rust gate →
/// JS-side dispatcher → event listener with correct fields.
#[tokio::test]
#[ignore = "not yet implemented: CSP violation event delivery to document listeners"]
async fn securitypolicyviolation_event_fires_on_block() {
    use browser_oxide::js_runtime::extensions::fetch_ext as csp_state;
    use browser_oxide::net::csp::Directive;
    use url::Url;

    // Page sets its own CSP via meta-tag and installs a listener BEFORE
    // anything fires. After page init we trip the gate from Rust then
    // explicitly drain so the test is deterministic (doesn't rely on
    // setTimeout timing).
    const HTML: &str = r#"<html><head>
<meta http-equiv="Content-Security-Policy" content="connect-src 'self'">
<title>spv</title>
</head><body>
<script>
  globalThis.__spv_events = [];
  document.addEventListener('securitypolicyviolation', (e) => {
    globalThis.__spv_events.push({
      blockedURI: e.blockedURI,
      effectiveDirective: e.effectiveDirective,
      violatedDirective: e.violatedDirective,
      disposition: e.disposition,
      typeOk: typeof SecurityPolicyViolationEvent === 'function' && (e instanceof SecurityPolicyViolationEvent),
    });
  });
</script>
</body></html>"#;

    let mut page = Page::from_html(HTML, Some(chrome_148_macos()))
        .await
        .unwrap();

    // Trip the gate AFTER Page is up so the violation queue is fresh
    // and the policy hasn't been re-set since the listener registered.
    let _ = csp_state::check_csp(
        Directive::ConnectSrc,
        &Url::parse("https://collector.example/api").unwrap(),
        None,
        false,
    );

    // Force an explicit drain — deterministic; doesn't depend on the
    // background setTimeout timeline.
    let _ = page
        .evaluate("globalThis.__drainCspViolations && globalThis.__drainCspViolations()")
        .unwrap();

    let n = page
        .evaluate("String(globalThis.__spv_events.length)")
        .unwrap();
    let n_clean = n.trim_matches('"');
    assert!(
        n_clean.parse::<i64>().unwrap_or(0) >= 1,
        "at least one securitypolicyviolation event must have fired, got {n_clean}"
    );
    let blocked = page
        .evaluate("globalThis.__spv_events[0].blockedURI")
        .unwrap();
    assert!(
        blocked.contains("collector.example"),
        "blockedURI must point at the blocked URL, got {blocked}"
    );
    let directive = page
        .evaluate("globalThis.__spv_events[0].effectiveDirective")
        .unwrap();
    assert!(
        directive.contains("connect-src"),
        "effectiveDirective must echo 'connect-src', got {directive}"
    );
    let type_ok = page
        .evaluate("String(globalThis.__spv_events[0].typeOk)")
        .unwrap();
    assert_eq!(
        type_ok, "true",
        "the event must be a SecurityPolicyViolationEvent instance"
    );
    let disposition = page
        .evaluate("globalThis.__spv_events[0].disposition")
        .unwrap();
    assert!(
        disposition.contains("enforce"),
        "disposition must be 'enforce' for this policy, got {disposition}"
    );

    csp_state::clear_csp_policy();
}

/// CSP `connect-src` enforcement: when the policy doesn't whitelist a
/// host, `window.fetch()` and XHR must short-circuit with a network-error
/// response (status 0, ok=false). Real Chrome behavior. We exercise the
/// `check_csp` API directly here since the connect-src gate fires at
/// the Rust op layer, before the request hits the network.
#[tokio::test]
async fn connect_src_blocks_disallowed_hosts() {
    use browser_oxide::csp_collector::collect_csp;
    use browser_oxide::html_parser::parse_html;
    use browser_oxide::js_runtime::extensions::fetch_ext as csp_state;
    use browser_oxide::net::csp::Directive;
    use std::sync::Arc;
    use url::Url;

    const CSP_HTML: &str = r#"<!doctype html><html><head>
<meta http-equiv="Content-Security-Policy"
      content="connect-src 'self' https://api.example.com">
</head><body></body></html>"#;
    let dom = parse_html(CSP_HTML);
    let policy = collect_csp(&[], &dom);
    csp_state::set_csp_policy(
        Arc::new(policy),
        Url::parse("https://example.com/").unwrap(),
        true,
    );

    // Same origin → allowed.
    let allowed = csp_state::check_csp(
        Directive::ConnectSrc,
        &Url::parse("https://example.com/api/v1/data").unwrap(),
        None,
        false,
    );
    assert!(allowed.is_ok(), "same-origin connect must be allowed");

    // Whitelisted host → allowed.
    let api = csp_state::check_csp(
        Directive::ConnectSrc,
        &Url::parse("https://api.example.com/feed").unwrap(),
        None,
        false,
    );
    assert!(api.is_ok(), "whitelisted host must be allowed");

    // Off-policy host → blocked.
    let bad = csp_state::check_csp(
        Directive::ConnectSrc,
        &Url::parse("https://collector-pxu6b0qd2s.px-cloud.net/api/v2/collector").unwrap(),
        None,
        false,
    );
    assert!(bad.is_err(), "off-policy connect-src must be blocked");
    assert_eq!(bad.unwrap_err(), "connect-src");

    csp_state::clear_csp_policy();
}

/// `frame-src` enforcement: iframe navigations against off-policy hosts
/// must be refused. The `child-src` and `default-src` fallback chain
/// applies — if `frame-src` is absent, those govern instead.
#[tokio::test]
async fn frame_src_blocks_disallowed_iframe() {
    use browser_oxide::csp_collector::collect_csp;
    use browser_oxide::html_parser::parse_html;
    use browser_oxide::js_runtime::extensions::fetch_ext as csp_state;
    use browser_oxide::net::csp::Directive;
    use std::sync::Arc;
    use url::Url;

    let dom = parse_html(
        r#"<html><head>
<meta http-equiv="Content-Security-Policy" content="frame-src 'self' https://www.youtube.com">
</head></html>"#,
    );
    let policy = collect_csp(&[], &dom);
    csp_state::set_csp_policy(
        Arc::new(policy),
        Url::parse("https://example.com/").unwrap(),
        true,
    );

    // YouTube whitelist match → allowed.
    let yt = csp_state::check_csp(
        Directive::FrameSrc,
        &Url::parse("https://www.youtube.com/embed/abc").unwrap(),
        None,
        false,
    );
    assert!(yt.is_ok());

    // Off-policy iframe target → blocked.
    let bad = csp_state::check_csp(
        Directive::FrameSrc,
        &Url::parse("https://attacker.example/").unwrap(),
        None,
        false,
    );
    assert!(bad.is_err());
    assert_eq!(bad.unwrap_err(), "frame-src");

    csp_state::clear_csp_policy();
}

/// `frame-src` falls back to `child-src` then `default-src`. Verify
/// the chain works end-to-end so a default-src-only policy still gates
/// iframe navigation.
#[tokio::test]
async fn frame_src_falls_back_through_child_src_to_default_src() {
    use browser_oxide::csp_collector::collect_csp;
    use browser_oxide::html_parser::parse_html;
    use browser_oxide::js_runtime::extensions::fetch_ext as csp_state;
    use browser_oxide::net::csp::Directive;
    use std::sync::Arc;
    use url::Url;

    // No frame-src or child-src — falls back to default-src.
    let dom = parse_html(
        r#"<html><head>
<meta http-equiv="Content-Security-Policy" content="default-src 'self'">
</head></html>"#,
    );
    let policy = collect_csp(&[], &dom);
    csp_state::set_csp_policy(
        Arc::new(policy),
        Url::parse("https://example.com/").unwrap(),
        true,
    );

    let same_origin = csp_state::check_csp(
        Directive::FrameSrc,
        &Url::parse("https://example.com/iframe.html").unwrap(),
        None,
        false,
    );
    assert!(
        same_origin.is_ok(),
        "default-src 'self' allows same-origin iframe"
    );

    let cross = csp_state::check_csp(
        Directive::FrameSrc,
        &Url::parse("https://other.example/iframe.html").unwrap(),
        None,
        false,
    );
    assert!(
        cross.is_err(),
        "default-src 'self' blocks cross-origin iframe via fallback chain"
    );
    assert_eq!(cross.unwrap_err(), "default-src");

    csp_state::clear_csp_policy();
}

/// `BROWSER_OXIDE_CSP_BYPASS=1` env var must turn off enforcement entirely
/// without touching the policy parser. Useful for benchmarking and
/// for sites where the policy is overly tight on us specifically.
#[tokio::test]
async fn bypass_env_var_disables_enforcement() {
    use browser_oxide::csp_collector::collect_csp;
    use browser_oxide::html_parser::parse_html;
    use browser_oxide::js_runtime::extensions::fetch_ext as csp_state;
    use browser_oxide::net::csp::Directive;
    use std::sync::Arc;
    use url::Url;

    let dom = parse_html(
        r#"<html><head>
<meta http-equiv="Content-Security-Policy" content="connect-src 'none'">
</head></html>"#,
    );
    let policy = collect_csp(&[], &dom);

    // enforce=false simulates BROWSER_OXIDE_CSP_BYPASS=1.
    csp_state::set_csp_policy(
        Arc::new(policy),
        Url::parse("https://example.com/").unwrap(),
        false,
    );

    let any = csp_state::check_csp(
        Directive::ConnectSrc,
        &Url::parse("https://anywhere.test/x").unwrap(),
        None,
        false,
    );
    assert!(
        any.is_ok(),
        "bypass=true must allow even 'none' policy fetches"
    );
    csp_state::clear_csp_policy();
}