agent_first_http/cli/cmd/
fetch.rs

1//! `afhttp fetch` subcommand.
2
3use std::path::PathBuf;
4
5use clap::Args as ClapArgs;
6use clap::ValueEnum;
7
8use crate::cli::output;
9use crate::host::bootstrap::BrowserChoice;
10use crate::sdk::fetch::{FetchCookie, NetworkBodies, RenderMode, Wait};
11use crate::sdk::{Client, InlineConfig};
12use crate::shared::artifacts::Artifact;
13use crate::shared::error::{Error, ErrorCode};
14use crate::shared::ids::TabId;
15use crate::shared::time::parse_duration;
16
17#[derive(ValueEnum, Debug, Clone, Copy, Default)]
18pub enum NetworkBodiesArg {
19    #[default]
20    Off,
21    Xhr,
22    All,
23}
24
25impl From<NetworkBodiesArg> for NetworkBodies {
26    fn from(v: NetworkBodiesArg) -> Self {
27        match v {
28            NetworkBodiesArg::Off => NetworkBodies::Off,
29            NetworkBodiesArg::Xhr => NetworkBodies::Xhr,
30            NetworkBodiesArg::All => NetworkBodies::All,
31        }
32    }
33}
34
35impl std::fmt::Display for NetworkBodiesArg {
36    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
37        f.write_str(match self {
38            Self::Off => "off",
39            Self::Xhr => "xhr",
40            Self::All => "all",
41        })
42    }
43}
44
45#[derive(ValueEnum, Debug, Clone, Copy, Default)]
46pub enum NetworkRedactArg {
47    #[default]
48    On,
49    Off,
50}
51
52impl std::fmt::Display for NetworkRedactArg {
53    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
54        f.write_str(match self {
55            Self::On => "on",
56            Self::Off => "off",
57        })
58    }
59}
60
61#[derive(ClapArgs, Debug)]
62pub struct Args {
63    /// URL to fetch.
64    pub url: String,
65    /// CDP endpoint of a running host. Omit to spawn an inline ephemeral host
66    /// for this one fetch.
67    #[arg(long = "endpoint-url", help_heading = "Connection")]
68    pub endpoint: Option<String>,
69    /// Bearer token, if the host was started with `--token-secret`.
70    #[arg(long = "token-secret", help_heading = "Connection")]
71    pub token: Option<String>,
72    /// Browser backend for the inline host: auto, chromium, chrome,
73    /// chrome_shell, fingerprint-chromium, edge, brave, lightpanda, camoufox.
74    /// Ignored when --endpoint-url is set (the host owns its browser).
75    #[arg(long, default_value = "auto", help_heading = "Connection")]
76    pub browser: String,
77    /// Browser binary path for the inline host, for when auto-discovery can't
78    /// find one. Ignored when --endpoint-url is set.
79    #[arg(long = "browser-bin", value_name = "PATH", help_heading = "Connection")]
80    pub browser_bin: Option<PathBuf>,
81    /// Render strategy: none (HTTP fast path, no browser), auto (HTTP first,
82    /// escalate to the browser on failure), or always (browser only).
83    #[arg(long, default_value = "auto", help_heading = "Rendering")]
84    pub render: String,
85    /// Tab target to use. "new" allocates a temporary target and closes it
86    /// after fetch; an id reuses that target and leaves it open.
87    #[arg(
88        long,
89        default_value = "new",
90        value_name = "new|<id>",
91        help_heading = "Connection"
92    )]
93    pub tab: String,
94    /// Readiness signal before capture on the browser path:
95    /// load | idle | selector:<css> | selector-visible:<css> | ms:<n>.
96    #[arg(long, default_value = "load", help_heading = "Rendering")]
97    pub wait: String,
98    /// Add a request header (repeatable). Format: `Name: value`.
99    #[arg(long = "header", value_name = "K:V", help_heading = "Request")]
100    pub headers: Vec<String>,
101    /// Add a request cookie (repeatable). Format: `name=value`.
102    #[arg(long = "cookie", value_name = "name=value", help_heading = "Request")]
103    pub cookies: Vec<String>,
104    /// Override the User-Agent header for this fetch.
105    #[arg(long, help_heading = "Request")]
106    pub user_agent: Option<String>,
107    /// JavaScript to evaluate after the wait condition resolves (repeatable).
108    /// Runs in page context before artifacts are captured.
109    #[arg(long, value_name = "js", help_heading = "Rendering")]
110    pub evaluate_after_wait: Vec<String>,
111    /// Artifacts to capture, comma-separated. Omit for all of: body,
112    /// rendered_html, text, screenshot, network, console, observation
113    /// (storage is opt-in only).
114    #[arg(long, value_delimiter = ',', help_heading = "Rendering")]
115    pub want: Vec<String>,
116    /// HTTP method. Common values: POST, PUT, PATCH, DELETE.
117    #[arg(long, default_value = "GET", help_heading = "Request")]
118    pub method: String,
119    /// Request body as a string. Prefix with `@` to read from a file path
120    /// (e.g. `--data @payload.json`). Mutually exclusive with `--form`.
121    #[arg(long, help_heading = "Request")]
122    pub data: Option<String>,
123    /// Request body from a file path. Mutually exclusive with `--form`.
124    #[arg(long, help_heading = "Request")]
125    pub data_file: Option<PathBuf>,
126    /// Add a form field (repeatable). Sends body as
127    /// `application/x-www-form-urlencoded`. Mutually exclusive with `--data`.
128    /// Format: `key=value`.
129    #[arg(long = "form", value_name = "key=value", help_heading = "Request")]
130    pub form: Vec<String>,
131    /// Capture response bodies for network requests: off, xhr (XHR/fetch
132    /// only), or all.
133    #[arg(long, default_value_t = NetworkBodiesArg::Off, help_heading = "Network capture")]
134    pub network_bodies: NetworkBodiesArg,
135    /// Per-body cap for captured network bodies, in bytes.
136    #[arg(long, default_value_t = 1_048_576, help_heading = "Network capture")]
137    pub network_body_max_bytes: u64,
138    /// Redact sensitive values in network.json: on or off. On by default;
139    /// off writes raw Authorization/Cookie headers and token-bearing query
140    /// params to the artifact — only disable for trusted local debugging.
141    #[arg(long, default_value_t = NetworkRedactArg::On, help_heading = "Network capture")]
142    pub network_redact: NetworkRedactArg,
143    /// Directory to write artifacts into. Defaults to an `afhttp-out`
144    /// subdirectory of the working directory.
145    #[arg(long, help_heading = "Output")]
146    pub out: Option<PathBuf>,
147    /// Override the cookie-jar path. The default — derived from the host's
148    /// `GET /profile` — places the jar at `<profile-dir>/cookies.jar.json`.
149    /// This override is rejected with `invalid_argument` if it does not
150    /// match the host's profile path; the flag exists for tests and
151    /// forensic tooling, not production sessions. Honors
152    /// `AFHTTP_COOKIE_JAR` when omitted (same validation applies).
153    #[arg(long, help_heading = "Cookies")]
154    pub cookie_jar: Option<PathBuf>,
155    /// Opt out of cookie-jar persistence for this fetch. No cookies are
156    /// replayed from the jar and no `Set-Cookie` responses are merged
157    /// back.
158    #[arg(long, help_heading = "Cookies")]
159    pub no_cookie_jar: bool,
160    /// Upper bound on the browser-path wait for the main document network
161    /// event, in milliseconds. Raise for slow networks or low-end machines.
162    #[arg(long, default_value_t = 500, help_heading = "Rendering")]
163    pub observe_main_wait_ms: u64,
164    /// Upper bound on the HTTP-path response body, in bytes. Default
165    /// 1 GiB (`1073741824`). `0` disables the cap entirely. When the
166    /// cap is hit, the fetch returns successfully with a
167    /// `network_body_truncated` warning and the prefix bytes that
168    /// were collected.
169    #[arg(long, default_value_t = 1_073_741_824, help_heading = "HTTP transport")]
170    pub max_response_bytes: u64,
171    /// Number of additional attempts after the first. Retries fire
172    /// only when the error has `retryable: true` (e.g.
173    /// `host_unreachable`, `cdp_timeout`); non-retryable failures
174    /// (`tls_error`, `wait_selector_unmatched`, etc.) short-circuit.
175    /// Default 0 = single attempt.
176    #[arg(long, default_value_t = 0, help_heading = "Retry")]
177    pub retry: u32,
178    /// Fixed delay between retries, in milliseconds.
179    #[arg(long, default_value_t = 250, help_heading = "Retry")]
180    pub backoff_ms: u64,
181    /// Per-fetch upstream HTTP/HTTPS proxy for the HTTP fast path.
182    /// The SDK never honors `HTTP_PROXY` from the environment; this
183    /// flag is the only way to route an HTTP-path fetch through one.
184    /// Format: `http://user:pass@host:port` or `socks5://host:port`.
185    #[arg(long = "proxy-url", help_heading = "HTTP transport")]
186    pub proxy: Option<String>,
187    /// Path to a PEM file containing extra root CAs to trust for
188    /// this fetch's HTTP path. Useful for self-signed staging or
189    /// corporate MITM CAs.
190    #[arg(long, help_heading = "HTTP transport")]
191    pub ca_cert: Option<PathBuf>,
192    /// Disable TLS certificate verification for this fetch's HTTP
193    /// path. Dangerous; leaves the connection open to MITM. Use only
194    /// against known-self-signed environments.
195    #[arg(long, help_heading = "HTTP transport")]
196    pub tls_insecure: bool,
197    /// Overall fetch timeout (e.g. `30s`, `1500ms`).
198    #[arg(long, default_value = "30s", help_heading = "HTTP transport")]
199    pub timeout: String,
200    /// Capture WebSocket frame payloads to network-bodies/<id>.frames.jsonl.
201    /// Frames may carry bearer tokens, session IDs, and message content —
202    /// treat the artifact as sensitive.
203    #[arg(long, help_heading = "Network capture")]
204    pub capture_ws: bool,
205    /// Capture SSE event payloads to network-bodies/<id>.frames.jsonl. Events
206    /// may carry PII; treat the artifact as sensitive.
207    #[arg(long, help_heading = "Network capture")]
208    pub capture_sse: bool,
209}
210
211pub async fn run(args: Args) -> Result<(), Error> {
212    let render = RenderMode::parse(&args.render)?;
213    let wait = Wait::parse(&args.wait)?;
214    let timeout = parse_duration(&args.timeout)?;
215    let network_bodies = NetworkBodies::from(args.network_bodies);
216    let network_redact = matches!(args.network_redact, NetworkRedactArg::On);
217
218    // Resolve the request body from --data / --data-file / --form.
219    if args.data.is_some() && args.data_file.is_some() {
220        return Err(Error::new(
221            ErrorCode::InvalidArgument,
222            "--data and --data-file are mutually exclusive",
223        ));
224    }
225    if (args.data.is_some() || args.data_file.is_some()) && !args.form.is_empty() {
226        return Err(Error::new(
227            ErrorCode::InvalidArgument,
228            "--data/--data-file and --form are mutually exclusive",
229        ));
230    }
231    let body_bytes: Option<Vec<u8>> = if let Some(data) = &args.data {
232        if let Some(path) = data.strip_prefix('@') {
233            Some(
234                tokio::fs::read(path)
235                    .await
236                    .map_err(|e| Error::new(ErrorCode::IoError, format!("--data @{path}: {e}")))?,
237            )
238        } else {
239            Some(data.as_bytes().to_vec())
240        }
241    } else if let Some(path) = &args.data_file {
242        Some(tokio::fs::read(path).await.map_err(|e| {
243            Error::new(
244                ErrorCode::IoError,
245                format!("--data-file {}: {e}", path.display()),
246            )
247        })?)
248    } else {
249        None
250    };
251
252    let want: std::collections::BTreeSet<Artifact> = if args.want.is_empty() {
253        Artifact::ALL.iter().copied().collect()
254    } else {
255        let mut s = std::collections::BTreeSet::new();
256        for token in &args.want {
257            let a = parse_artifact(token)?;
258            s.insert(a);
259        }
260        s
261    };
262
263    let client = match args.endpoint.as_deref() {
264        Some(ep) => {
265            let mut c = Client::connect(ep)?;
266            if let Some(t) = args.token.as_deref() {
267                c = c.with_token(t);
268            }
269            c
270        }
271        None if matches!(render, RenderMode::None) => Client::http_only()?,
272        None => {
273            let browser = args
274                .browser
275                .parse::<BrowserChoice>()
276                .map_err(|e| Error::new(ErrorCode::InvalidArgument, format!("--browser: {e}")))?;
277            let cfg = InlineConfig {
278                browser,
279                browser_bin: args.browser_bin.clone(),
280            };
281            if matches!(render, RenderMode::Auto) {
282                Client::inline_ephemeral_lazy(cfg).await?
283            } else {
284                Client::inline_ephemeral_with(cfg).await?
285            }
286        }
287    };
288
289    let mut builder = client
290        .fetch(args.url.clone())
291        .render(render)
292        .wait(wait)
293        .timeout(timeout)
294        .want(want)
295        .network_bodies(network_bodies)
296        .network_body_max_bytes(args.network_body_max_bytes)
297        .network_redact(network_redact)
298        .method(args.method);
299    if let Some(bytes) = body_bytes {
300        builder = builder.body(bytes);
301    }
302    for raw in &args.form {
303        let (k, v) = raw.split_once('=').ok_or_else(|| {
304            Error::new(
305                ErrorCode::InvalidArgument,
306                format!("--form: expected key=value, got {raw:?}"),
307            )
308        })?;
309        builder = builder.form_field(k, v);
310    }
311    for raw in args.headers {
312        let (name, value) = parse_header_arg(&raw)?;
313        builder = builder.header(name, value);
314    }
315    for raw in args.cookies {
316        builder = builder.cookie_full(parse_cookie_arg(&raw)?);
317    }
318    if let Some(user_agent) = args.user_agent {
319        builder = builder.user_agent(user_agent);
320    }
321    for js in args.evaluate_after_wait {
322        builder = builder.evaluate_after_wait(js);
323    }
324    if args.tab != "new" {
325        builder = builder.tab(TabId::new(args.tab));
326    }
327    if let Some(out) = args.out {
328        builder = builder.out_dir(out);
329    }
330    builder = builder.observe_main_wait_ms(args.observe_main_wait_ms);
331    builder = builder.max_response_bytes(args.max_response_bytes);
332    builder = builder.retry(args.retry).backoff_ms(args.backoff_ms);
333    if let Some(url) = args.proxy {
334        builder = builder.proxy(url);
335    }
336    if let Some(path) = args.ca_cert {
337        builder = builder.ca_cert(path);
338    }
339    if args.tls_insecure {
340        builder = builder.tls_insecure(true);
341    }
342    if args.capture_ws {
343        builder = builder.capture_ws(true);
344    }
345    if args.capture_sse {
346        builder = builder.capture_sse(true);
347    }
348    if args.no_cookie_jar {
349        builder = builder.no_cookie_jar();
350    } else {
351        let cookie_jar = args.cookie_jar.or_else(|| {
352            std::env::var_os("AFHTTP_COOKIE_JAR")
353                .filter(|v| !v.is_empty())
354                .map(PathBuf::from)
355        });
356        if let Some(jar) = cookie_jar {
357            builder = builder.cookie_jar(jar);
358        }
359    }
360
361    let result = builder.send().await?;
362    output::emit("fetch", &result)
363}
364
365fn parse_artifact(token: &str) -> Result<Artifact, Error> {
366    Ok(match token {
367        "body" => Artifact::Body,
368        "rendered_html" => Artifact::RenderedHtml,
369        "text" => Artifact::Text,
370        "screenshot" => Artifact::Screenshot,
371        "network" => Artifact::Network,
372        "console" => Artifact::Console,
373        "observation" => Artifact::Observation,
374        "storage" => Artifact::Storage,
375        other => {
376            return Err(Error::new(
377                ErrorCode::InvalidArgument,
378                format!("--want: unknown artifact {other:?}"),
379            ));
380        }
381    })
382}
383
384fn parse_header_arg(raw: &str) -> Result<(String, String), Error> {
385    let (name, value) = raw.split_once(':').ok_or_else(|| {
386        Error::new(
387            ErrorCode::InvalidArgument,
388            format!("--header: expected K:V, got {raw:?}"),
389        )
390    })?;
391    let name = name.trim();
392    if name.is_empty() {
393        return Err(Error::new(
394            ErrorCode::InvalidArgument,
395            format!("--header: header name must not be empty in {raw:?}"),
396        ));
397    }
398    Ok((name.to_string(), value.trim_start().to_string()))
399}
400
401fn parse_cookie_arg(raw: &str) -> Result<FetchCookie, Error> {
402    if !raw.contains('=') {
403        return Err(Error::new(
404            ErrorCode::InvalidArgument,
405            format!("--cookie: expected Set-Cookie style name=value, got {raw:?}"),
406        ));
407    }
408    let cookie = FetchCookie::parse(raw.to_string())
409        .map_err(|e| Error::new(ErrorCode::InvalidArgument, format!("--cookie: {e}")))?
410        .into_owned();
411    if cookie.name().trim().is_empty() {
412        return Err(Error::new(
413            ErrorCode::InvalidArgument,
414            format!("--cookie: cookie name must not be empty in {raw:?}"),
415        ));
416    }
417    Ok(cookie)
418}
419
420#[cfg(test)]
421mod tests {
422    use super::*;
423
424    #[test]
425    fn header_arg_accepts_colon_separator() {
426        assert_eq!(
427            parse_header_arg("X-Test: yes").unwrap(),
428            ("X-Test".to_string(), "yes".to_string())
429        );
430    }
431
432    #[test]
433    fn header_arg_rejects_missing_colon() {
434        let err = parse_header_arg("X-Test").err().unwrap();
435        assert_eq!(err.error_code, ErrorCode::InvalidArgument);
436    }
437
438    #[test]
439    fn cookie_arg_accepts_equals_separator() {
440        let cookie = parse_cookie_arg("sid=abc=def").unwrap();
441        assert_eq!(cookie.name_value(), ("sid", "abc=def"));
442    }
443
444    #[test]
445    fn cookie_arg_accepts_full_set_cookie_attributes() {
446        let cookie = parse_cookie_arg("sid=abc; Path=/; Secure; HttpOnly; SameSite=Lax").unwrap();
447        assert_eq!(cookie.name_value(), ("sid", "abc"));
448        assert_eq!(cookie.path(), Some("/"));
449        assert_eq!(cookie.secure(), Some(true));
450        assert_eq!(cookie.http_only(), Some(true));
451        assert_eq!(cookie.same_site(), Some(cookie::SameSite::Lax));
452    }
453
454    #[test]
455    fn cookie_arg_rejects_missing_equals() {
456        let err = parse_cookie_arg("sid").err().unwrap();
457        assert_eq!(err.error_code, ErrorCode::InvalidArgument);
458    }
459}
agent_first_http/cli/cmd/fetch.rs

agent_first_http/cli/cmd/
fetch.rs
