browser_control/cli/
fetch.rs

1//! `browser-control fetch` — run an HTTP request from the page's context.
2//!
3//! The request is executed by injecting [`crate::dom::scripts::FETCH_JS`] into
4//! the active page via the engine-agnostic [`PageSession`] and parsing the
5//! `{status, statusText, headers, body}` envelope it returns.
6//!
7//! Output mirrors `curl`:
8//! - `--include` prepends `HTTP/1.1 <code> <text>\r\n` and response headers.
9//! - `--output PATH` writes the body to PATH (and `chmod 0600` on Unix).
10//! - Without `--output`, the body is written to stdout.
11//!
12//! Transport errors (script failure, attach failure) exit non-zero; HTTP
13//! status is reported verbatim and does not change the exit code.
14
15use std::collections::BTreeMap;
16use std::path::{Path, PathBuf};
17use std::time::Duration;
18
19use anyhow::{anyhow, bail, Context, Result};
20use serde::Deserialize;
21use serde_json::{json, Map, Value};
22
23use crate::cli::route;
24use crate::cli::trace::CommandTrace;
25use crate::dom::scripts::FETCH_JS;
26use crate::session::freshness;
27use crate::session::{evaluate_for_origin_with_recover_once, PageSession};
28
29// The default fetch timeout (60 000 ms) is set on the CLI flag in
30// `main.rs`. Bounded so a wedged renderer fails fast instead of dragging
31// out the upstream protocol timeout; tunable per invocation via
32// `--timeout-ms` for slow-network workloads.
33
34#[allow(clippy::too_many_arguments)]
35pub async fn run(
36    browser: Option<String>,
37    url: String,
38    method: String,
39    headers: Vec<String>,
40    data: Option<String>,
41    target: Option<String>,
42    include: bool,
43    output: Option<PathBuf>,
44    timeout_ms: u64,
45    max_age: String,
46) -> Result<()> {
47    let mut trace = CommandTrace::new("fetch");
48    let result = run_inner(
49        browser, url, method, headers, data, target, include, output, timeout_ms, max_age,
50        &mut trace,
51    )
52    .await;
53    trace.finish(result)
54}
55
56#[allow(clippy::too_many_arguments)]
57async fn run_inner(
58    browser: Option<String>,
59    url: String,
60    method: String,
61    headers: Vec<String>,
62    data: Option<String>,
63    target: Option<String>,
64    include: bool,
65    output: Option<PathBuf>,
66    timeout_ms: u64,
67    max_age: String,
68    trace: &mut CommandTrace,
69) -> Result<()> {
70    let header_map = parse_headers(&headers)?;
71    let fetch_timeout = Duration::from_millis(timeout_ms);
72    let script_timeout_ms = script_fetch_timeout_ms(fetch_timeout);
73    let expr = build_fetch_expr(
74        &url,
75        &method,
76        &header_map,
77        data.as_deref(),
78        script_timeout_ms,
79    )?;
80    let max_age = freshness::parse_max_age(&max_age)?;
81
82    // Path-syntax parsing: `<browser>` or `<browser>/<tab>` in the
83    // positional. `--target <regex>` and `/<tab>` are mutually exclusive.
84    // The preamble (parse, mutual-exclusion, resolve, registry, BiDi lock) is
85    // shared with `eval`/`storage`; see `crate::cli::route`.
86    let r = route::preamble(browser, target.as_deref(), trace).await?;
87    let resolved = &r.resolved;
88
89    let result = match (r.tab_name.clone(), target.as_deref()) {
90        // Path 1: <browser>/<tab> — fetch against the named tab.
91        // Engine-agnostic via TabBackend + recover-once on dead tabs.
92        (Some(name), None) => {
93            trace.route("named-tab").tab_name(&name);
94            let expr = expr.clone();
95            route::run_named_tab(
96                &r,
97                &name,
98                "named tabs (`<browser>/<name>`) require a registered browser",
99                move |b, target_id| {
100                    let expr = expr.clone();
101                    async move {
102                        b.ensure_fresh(&target_id, max_age).await?;
103                        b.evaluate(&target_id, &expr, true, fetch_timeout).await
104                    }
105                },
106            )
107            .await?
108        }
109        // Path 2: --target regex (legacy).
110        (None, Some(regex)) => {
111            trace.route("target-regex");
112            let session =
113                PageSession::attach(&resolved.endpoint, resolved.engine, Some(regex)).await?;
114            session.ensure_fresh(max_age).await?;
115            let value = session
116                .evaluate_with_timeout(&expr, true, Some(fetch_timeout))
117                .await;
118            session.close().await;
119            value?
120        }
121        // Path 3: bare browser → attach_for_origin (current default).
122        // Auth-inheritance: the request runs from the URL's origin, so
123        // cookies/credentials propagate. Scratch routing (which would use
124        // about:blank) would lose this; we deliberately keep
125        // attach_for_origin here instead.
126        //
127        // Recover-once lives in the shared session helper so CLI fetch and
128        // wait-for-cookie validation cannot drift on the origin-bound contract.
129        (None, None) => {
130            trace.route("attach-for-origin");
131            evaluate_for_origin_with_recover_once(
132                &resolved.endpoint,
133                resolved.engine,
134                &url,
135                &expr,
136                true,
137                fetch_timeout,
138                max_age,
139            )
140            .await?
141        }
142        _ => unreachable!("mutex was checked above"),
143    };
144
145    let envelope = parse_envelope(&result)?;
146
147    let mut bytes = Vec::new();
148    if include {
149        bytes.extend_from_slice(format_status_and_headers(&envelope).as_bytes());
150    }
151    bytes.extend_from_slice(envelope.body.as_bytes());
152
153    match output {
154        Some(path) => {
155            write_file(&path, &bytes)?;
156            tracing::info!(
157                target = "fetch",
158                "wrote {} bytes to {}",
159                bytes.len(),
160                path.display()
161            );
162            eprintln!("wrote {} bytes to {}", bytes.len(), path.display());
163        }
164        None => {
165            use std::io::Write;
166            let mut out = std::io::stdout().lock();
167            out.write_all(&bytes)?;
168        }
169    }
170    Ok(())
171}
172
173/// Parsed `{status, statusText, headers, body}` envelope from `FETCH_JS`.
174#[derive(Debug, Clone, PartialEq)]
175struct FetchEnvelope {
176    status: u16,
177    status_text: String,
178    headers: Vec<(String, String)>,
179    body: String,
180}
181
182/// Wire shape of the `FETCH_JS` envelope. `headers` is a `BTreeMap` so
183/// iteration is key-sorted, matching the previous `serde_json::Map` iteration
184/// order.
185#[derive(Deserialize)]
186struct RawFetchEnvelope {
187    #[serde(default)]
188    ok: Option<bool>,
189    #[serde(default)]
190    status: Option<u16>,
191    #[serde(default, rename = "statusText")]
192    status_text: String,
193    #[serde(default)]
194    headers: BTreeMap<String, String>,
195    #[serde(default)]
196    body: String,
197    #[serde(default)]
198    error: Option<String>,
199    #[serde(default, rename = "errorName")]
200    error_name: Option<String>,
201}
202
203fn parse_headers(headers: &[String]) -> Result<Map<String, Value>> {
204    let mut map = Map::new();
205    for raw in headers {
206        let (k, v) = raw
207            .split_once(':')
208            .ok_or_else(|| anyhow!("malformed header `{raw}`: expected `Key: Value`"))?;
209        let key = k.trim();
210        if key.is_empty() {
211            bail!("malformed header `{raw}`: empty key");
212        }
213        // Per RFC 7230, header names are tokens; reject whitespace/control in name.
214        if key.chars().any(|c| c.is_whitespace() || c.is_control()) {
215            bail!("malformed header `{raw}`: invalid character in name");
216        }
217        let value = v.trim();
218        map.insert(key.to_string(), Value::String(value.to_string()));
219    }
220    Ok(map)
221}
222
223/// Build the JS expression that invokes `FETCH_JS` with a JSON-encoded arg
224/// string. All user-controlled fields are JSON-encoded twice (once inside the
225/// args object, once when we embed the args string as a JS string literal)
226/// so the page can't be tricked into evaluating arbitrary expressions.
227fn build_fetch_expr(
228    url: &str,
229    method: &str,
230    headers: &Map<String, Value>,
231    body: Option<&str>,
232    timeout_ms: u64,
233) -> Result<String> {
234    let args = json!({
235        "url": url,
236        "method": method,
237        "headers": Value::Object(headers.clone()),
238        "body": body,
239        "timeoutMs": timeout_ms,
240    });
241    let args_json = serde_json::to_string(&args)?;
242    let args_literal = serde_json::to_string(&args_json)?;
243    Ok(format!("({FETCH_JS})({args_literal})"))
244}
245
246pub(crate) fn script_fetch_timeout_ms(timeout: Duration) -> u64 {
247    let ms = u64::try_from(timeout.as_millis()).unwrap_or(u64::MAX);
248    ms.saturating_sub(1_000).max(1)
249}
250
251/// `FETCH_JS` returns `JSON.stringify({...})`, so the evaluator hands us a
252/// JSON value of *type string*. Decode the inner JSON.
253fn parse_envelope(v: &Value) -> Result<FetchEnvelope> {
254    let s = v
255        .as_str()
256        .ok_or_else(|| anyhow!("fetch script returned non-string value: {v}"))?;
257    let raw: RawFetchEnvelope = serde_json::from_str(s)
258        .with_context(|| format!("fetch script returned invalid JSON: {s}"))?;
259    if raw.ok == Some(false) {
260        let mut msg = raw.error.unwrap_or_else(|| "fetch failed".to_string());
261        if let Some(name) = raw.error_name {
262            if !name.is_empty() {
263                msg.push_str(&format!(" ({name})"));
264            }
265        }
266        bail!(msg);
267    }
268    let status = raw
269        .status
270        .ok_or_else(|| anyhow!("fetch script response missing status: {s}"))?;
271    Ok(FetchEnvelope {
272        status,
273        status_text: raw.status_text,
274        headers: raw.headers.into_iter().collect(),
275        body: raw.body,
276    })
277}
278
279fn format_status_and_headers(env: &FetchEnvelope) -> String {
280    let mut s = format!("HTTP/1.1 {} {}\r\n", env.status, env.status_text);
281    for (k, v) in &env.headers {
282        s.push_str(&format!("{k}: {v}\r\n"));
283    }
284    s.push_str("\r\n");
285    s
286}
287
288fn write_file(path: &Path, body: &[u8]) -> Result<()> {
289    std::fs::write(path, body).with_context(|| format!("failed to write {}", path.display()))?;
290    #[cfg(unix)]
291    {
292        use std::os::unix::fs::PermissionsExt;
293        std::fs::set_permissions(path, std::fs::Permissions::from_mode(0o600))
294            .with_context(|| format!("failed to chmod 600 {}", path.display()))?;
295    }
296    Ok(())
297}
298
299#[cfg(test)]
300mod tests {
301    use super::*;
302
303    #[test]
304    fn parse_headers_basic() {
305        let m = parse_headers(&[
306            "Accept: application/json".to_string(),
307            "X-Token: abc".to_string(),
308        ])
309        .unwrap();
310        assert_eq!(m.get("Accept").unwrap(), &json!("application/json"));
311        assert_eq!(m.get("X-Token").unwrap(), &json!("abc"));
312    }
313
314    #[test]
315    fn parse_headers_trims_extra_spaces() {
316        let m = parse_headers(&["  Accept   :   text/plain  ".to_string()]).unwrap();
317        assert_eq!(m.get("Accept").unwrap(), &json!("text/plain"));
318    }
319
320    #[test]
321    fn parse_headers_value_with_colon_kept_intact() {
322        // Only the first `:` separates key/value; the value may contain colons.
323        let m = parse_headers(&["Authorization: Bearer a:b:c".to_string()]).unwrap();
324        assert_eq!(m.get("Authorization").unwrap(), &json!("Bearer a:b:c"));
325    }
326
327    #[test]
328    fn parse_headers_rejects_missing_colon() {
329        let err = parse_headers(&["NoColonHere".to_string()]).unwrap_err();
330        assert!(err.to_string().contains("malformed header"));
331    }
332
333    #[test]
334    fn parse_headers_rejects_empty_key() {
335        let err = parse_headers(&[": value".to_string()]).unwrap_err();
336        assert!(err.to_string().contains("empty key"));
337    }
338
339    #[test]
340    fn parse_headers_rejects_whitespace_in_name() {
341        let err = parse_headers(&["bad name: v".to_string()]).unwrap_err();
342        assert!(err.to_string().contains("invalid character"));
343    }
344
345    #[test]
346    fn build_expr_json_escapes_url_and_body() {
347        let mut h = Map::new();
348        h.insert("X".to_string(), json!("y"));
349        // Body and URL contain quotes / backslashes / newlines that would
350        // break naive string interpolation.
351        let url = "https://x.test/?q=\"hi\"";
352        let body = "line1\n\"line2\"\\end";
353        let expr = build_fetch_expr(url, "POST", &h, Some(body), 59_750).unwrap();
354        // The expression must wrap a single JSON-encoded string argument.
355        let prefix = format!("({FETCH_JS})(");
356        let inner = expr
357            .strip_prefix(&prefix)
358            .unwrap()
359            .strip_suffix(')')
360            .unwrap();
361        // No raw user-controlled quote or newline can appear unescaped at the
362        // top level — the literal is JSON, so quotes inside are `\"` and the
363        // string contains no real newline byte.
364        assert!(!inner.contains('\n'));
365        // Decode the literal back twice and confirm round-trip equality.
366        let args_str: String = serde_json::from_str(inner).unwrap();
367        let args: Value = serde_json::from_str(&args_str).unwrap();
368        assert_eq!(args["url"], url);
369        assert_eq!(args["body"], body);
370        assert_eq!(args["method"], "POST");
371        assert_eq!(args["timeoutMs"], 59_750);
372    }
373
374    #[test]
375    fn build_expr_method_and_headers_round_trip() {
376        let mut h = Map::new();
377        h.insert("Accept".to_string(), json!("*/*"));
378        let expr = build_fetch_expr("https://x.test/", "GET", &h, None, 59_750).unwrap();
379        // Extract the JSON-string literal argument and decode twice.
380        let prefix = format!("({FETCH_JS})(");
381        let inner = expr
382            .strip_prefix(&prefix)
383            .unwrap()
384            .strip_suffix(')')
385            .unwrap();
386        let args_str: String = serde_json::from_str(inner).unwrap();
387        let args: Value = serde_json::from_str(&args_str).unwrap();
388        assert_eq!(args["url"], "https://x.test/");
389        assert_eq!(args["method"], "GET");
390        assert_eq!(args["headers"]["Accept"], "*/*");
391        assert!(args["body"].is_null());
392        assert_eq!(args["timeoutMs"], 59_750);
393    }
394
395    #[test]
396    fn parse_envelope_decodes_inner_json() {
397        let inner = json!({
398            "ok": true,
399            "status": 200,
400            "statusText": "OK",
401            "headers": {"content-type": "text/plain"},
402            "body": "hello"
403        });
404        let v = Value::String(inner.to_string());
405        let env = parse_envelope(&v).unwrap();
406        assert_eq!(env.status, 200);
407        assert_eq!(env.status_text, "OK");
408        assert_eq!(env.body, "hello");
409        assert_eq!(
410            env.headers,
411            vec![("content-type".to_string(), "text/plain".to_string())]
412        );
413    }
414
415    #[test]
416    fn parse_envelope_rejects_non_string() {
417        let v = json!({"status": 200});
418        assert!(parse_envelope(&v).is_err());
419    }
420
421    #[test]
422    fn parse_envelope_reports_fetch_error() {
423        let inner = json!({
424            "ok": false,
425            "error": "fetch timed out after 9750ms",
426            "errorName": "AbortError"
427        });
428        let v = Value::String(inner.to_string());
429        let err = parse_envelope(&v).unwrap_err();
430        assert!(err.to_string().contains("fetch timed out after 9750ms"));
431        assert!(err.to_string().contains("AbortError"));
432    }
433
434    #[test]
435    fn script_timeout_leaves_outer_timeout_margin() {
436        assert_eq!(
437            script_fetch_timeout_ms(Duration::from_millis(10_000)),
438            9_000
439        );
440        assert_eq!(script_fetch_timeout_ms(Duration::from_millis(100)), 1);
441    }
442
443    #[test]
444    fn format_include_emits_status_and_headers() {
445        let env = FetchEnvelope {
446            status: 404,
447            status_text: "Not Found".to_string(),
448            headers: vec![
449                ("content-type".to_string(), "text/plain".to_string()),
450                ("x-trace".to_string(), "abc".to_string()),
451            ],
452            body: "missing".to_string(),
453        };
454        let s = format_status_and_headers(&env);
455        assert_eq!(
456            s,
457            "HTTP/1.1 404 Not Found\r\n\
458             content-type: text/plain\r\n\
459             x-trace: abc\r\n\
460             \r\n"
461        );
462    }
463
464    #[test]
465    fn write_file_chmods_0600_on_unix() {
466        let dir = std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR"))
467            .join("target")
468            .join("fetch-test-scratch");
469        std::fs::create_dir_all(&dir).unwrap();
470        let p = dir.join(format!("out-{}.bin", std::process::id()));
471        write_file(&p, b"hello").unwrap();
472        assert_eq!(std::fs::read(&p).unwrap(), b"hello");
473        #[cfg(unix)]
474        {
475            use std::os::unix::fs::PermissionsExt;
476            let mode = std::fs::metadata(&p).unwrap().permissions().mode() & 0o777;
477            assert_eq!(mode, 0o600);
478        }
479        let _ = std::fs::remove_file(&p);
480        let _ = std::fs::remove_dir(&dir);
481    }
482}
browser_control/cli/fetch.rs

browser_control/cli/
fetch.rs
