agent_first_http/cli/cmd/
fetch.rs

1//! `afhttp fetch` subcommand.
2
3use std::path::PathBuf;
4use std::time::Duration;
5
6use clap::Args as ClapArgs;
7use clap::ValueEnum;
8
9use crate::cli::output;
10use crate::host::bootstrap::BrowserChoice;
11use crate::sdk::fetch::{
12    FetchCookie, NetworkBodies, RenderMode, Wait, DEFAULT_NETWORK_BODY_MAX_BYTES,
13};
14use crate::sdk::{Client, InlineConfig};
15use crate::shared::artifacts::Artifact;
16use crate::shared::error::{Error, ErrorCode};
17use crate::shared::ids::TabId;
18
19#[derive(ValueEnum, Debug, Clone, Copy, Default)]
20pub enum NetworkBodiesArg {
21    #[default]
22    Off,
23    Xhr,
24    All,
25}
26
27impl From<NetworkBodiesArg> for NetworkBodies {
28    fn from(v: NetworkBodiesArg) -> Self {
29        match v {
30            NetworkBodiesArg::Off => NetworkBodies::Off,
31            NetworkBodiesArg::Xhr => NetworkBodies::Xhr,
32            NetworkBodiesArg::All => NetworkBodies::All,
33        }
34    }
35}
36
37impl std::fmt::Display for NetworkBodiesArg {
38    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
39        f.write_str(match self {
40            Self::Off => "off",
41            Self::Xhr => "xhr",
42            Self::All => "all",
43        })
44    }
45}
46
47#[derive(ValueEnum, Debug, Clone, Copy, Default)]
48pub enum NetworkRedactArg {
49    #[default]
50    On,
51    Off,
52}
53
54impl std::fmt::Display for NetworkRedactArg {
55    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
56        f.write_str(match self {
57            Self::On => "on",
58            Self::Off => "off",
59        })
60    }
61}
62
63#[derive(ClapArgs, Debug)]
64pub struct Args {
65    /// URL to fetch.
66    pub url: String,
67    /// CDP endpoint of a running host. Omit to spawn an inline ephemeral host
68    /// for this one fetch.
69    #[arg(long = "endpoint-url", help_heading = "Connection")]
70    pub endpoint: Option<String>,
71    /// Bearer token, if the host was started with `--token-secret`.
72    #[arg(long = "token-secret", help_heading = "Connection")]
73    pub token: Option<String>,
74    /// Browser backend for the inline host: auto, chromium, chrome,
75    /// chrome_shell, fingerprint-chromium, edge, brave, lightpanda, camoufox.
76    /// Ignored when --endpoint-url is set (the host owns its browser).
77    #[arg(long, default_value = "auto", help_heading = "Connection")]
78    pub browser: String,
79    /// Browser binary path for the inline host, for when auto-discovery can't
80    /// find one. Ignored when --endpoint-url is set.
81    #[arg(long = "browser-bin", value_name = "PATH", help_heading = "Connection")]
82    pub browser_bin: Option<PathBuf>,
83    /// Render strategy: none (HTTP fast path, no browser), auto (HTTP first,
84    /// escalate to the browser on failure), or always (browser only).
85    #[arg(long, default_value = "auto", help_heading = "Rendering")]
86    pub render: String,
87    /// Tab target to use. "new" allocates a temporary target and closes it
88    /// after fetch; an id reuses that target and leaves it open.
89    #[arg(
90        long,
91        default_value = "new",
92        value_name = "new|<id>",
93        help_heading = "Connection"
94    )]
95    pub tab: String,
96    /// Readiness signal before capture on the browser path:
97    /// auto | load | idle | selector:<css> | selector-visible:<css> | ms:<n>.
98    #[arg(long, default_value = "auto", help_heading = "Rendering")]
99    pub wait: String,
100    /// Add a request header (repeatable). Format: `Name: value`.
101    #[arg(long = "header", value_name = "K:V", help_heading = "Request")]
102    pub headers: Vec<String>,
103    /// Add a request cookie (repeatable). Format: `name=value`.
104    #[arg(long = "cookie", value_name = "name=value", help_heading = "Request")]
105    pub cookies: Vec<String>,
106    /// Override the User-Agent header for this fetch.
107    #[arg(long, help_heading = "Request")]
108    pub user_agent: Option<String>,
109    /// JavaScript to evaluate after the wait condition resolves (repeatable).
110    /// Runs in page context before artifacts are captured.
111    #[arg(long, value_name = "js", help_heading = "Rendering")]
112    pub evaluate_after_wait: Vec<String>,
113    /// Artifacts to capture, comma-separated. Omit for all of: body,
114    /// rendered_html, text, screenshot, network, console, observation
115    /// (storage is opt-in only).
116    #[arg(long, value_delimiter = ',', help_heading = "Rendering")]
117    pub want: Vec<String>,
118    /// HTTP method. Common values: POST, PUT, PATCH, DELETE.
119    #[arg(long, default_value = "GET", help_heading = "Request")]
120    pub method: String,
121    /// Request body as a string. Prefix with `@` to read from a file path
122    /// (e.g. `--data @payload.json`). Mutually exclusive with `--form`.
123    #[arg(long, help_heading = "Request")]
124    pub data: Option<String>,
125    /// Request body from a file path. Mutually exclusive with `--form`.
126    #[arg(long, help_heading = "Request")]
127    pub data_file: Option<PathBuf>,
128    /// Add a form field (repeatable). Sends body as
129    /// `application/x-www-form-urlencoded`. Mutually exclusive with `--data`.
130    /// Format: `key=value`.
131    #[arg(long = "form", value_name = "key=value", help_heading = "Request")]
132    pub form: Vec<String>,
133    /// Capture response bodies for network requests: off, xhr (XHR/fetch
134    /// only), or all.
135    #[arg(long, default_value_t = NetworkBodiesArg::Off, help_heading = "Network capture")]
136    pub network_bodies: NetworkBodiesArg,
137    /// Per-body cap for captured network bodies, in bytes.
138    #[arg(long, default_value_t = DEFAULT_NETWORK_BODY_MAX_BYTES, help_heading = "Network capture")]
139    pub network_body_max_bytes: u64,
140    /// Network quiet window used by --wait auto, in milliseconds.
141    #[arg(long, default_value_t = 800, help_heading = "Rendering")]
142    pub readiness_idle_ms: u64,
143    /// DOM/text unchanged window used by --wait auto, in milliseconds.
144    #[arg(long, default_value_t = 500, help_heading = "Rendering")]
145    pub readiness_stable_ms: u64,
146    /// Low visible-text byte threshold for --wait auto quality warnings only.
147    #[arg(long, default_value_t = 32, help_heading = "Rendering")]
148    pub readiness_min_text_bytes: u64,
149    /// Redact sensitive values in network.json: on or off. On by default;
150    /// off writes raw Authorization/Cookie headers and token-bearing query
151    /// params to the artifact — only disable for trusted local debugging.
152    #[arg(long, default_value_t = NetworkRedactArg::On, help_heading = "Network capture")]
153    pub network_redact: NetworkRedactArg,
154    /// Directory to write artifacts into. Defaults to an `afhttp-out`
155    /// subdirectory of the working directory.
156    #[arg(long, help_heading = "Output")]
157    pub out: Option<PathBuf>,
158    /// Override the cookie-jar path. The default — derived from the host's
159    /// `GET /profile` — places the jar at `<profile-dir>/cookies.jar.json`.
160    /// This override is rejected with `invalid_argument` if it does not
161    /// match the host's profile path; the flag exists for tests and
162    /// forensic tooling, not production sessions. Honors
163    /// `AFHTTP_COOKIE_JAR` when omitted (same validation applies).
164    #[arg(long, help_heading = "Cookies")]
165    pub cookie_jar: Option<PathBuf>,
166    /// Opt out of cookie-jar persistence for this fetch. No cookies are
167    /// replayed from the jar and no `Set-Cookie` responses are merged
168    /// back.
169    #[arg(long, help_heading = "Cookies")]
170    pub no_cookie_jar: bool,
171    /// Upper bound on the browser-path wait for the main document network
172    /// event, in milliseconds. Raise for slow networks or low-end machines.
173    #[arg(long, default_value_t = 500, help_heading = "Rendering")]
174    pub observe_main_wait_ms: u64,
175    /// Upper bound on the HTTP-path response body, in bytes. Default
176    /// 1 GiB (`1073741824`). `0` disables the cap entirely. When the
177    /// cap is hit, the fetch returns successfully with a
178    /// `network_body_truncated` warning and the prefix bytes that
179    /// were collected.
180    #[arg(long, default_value_t = 1_073_741_824, help_heading = "HTTP transport")]
181    pub max_response_bytes: u64,
182    /// Number of additional attempts after the first. Retries fire
183    /// only when the error has `retryable: true` (e.g.
184    /// `host_unreachable`, `cdp_timeout`); non-retryable failures
185    /// (`tls_error`, `wait_selector_unmatched`, etc.) short-circuit.
186    /// Default 0 = single attempt.
187    #[arg(long, default_value_t = 0, help_heading = "Retry")]
188    pub retry: u32,
189    /// Fixed delay between retries, in milliseconds.
190    #[arg(long, default_value_t = 250, help_heading = "Retry")]
191    pub backoff_ms: u64,
192    /// Per-fetch upstream HTTP/HTTPS proxy for the HTTP fast path.
193    /// The SDK never honors `HTTP_PROXY` from the environment; this
194    /// flag is the only way to route an HTTP-path fetch through one.
195    /// Format: `http://user:pass@host:port` or `socks5://host:port`.
196    #[arg(long = "proxy-url", help_heading = "HTTP transport")]
197    pub proxy: Option<String>,
198    /// Path to a PEM file containing extra root CAs to trust for
199    /// this fetch's HTTP path. Useful for self-signed staging or
200    /// corporate MITM CAs.
201    #[arg(long, help_heading = "HTTP transport")]
202    pub ca_cert: Option<PathBuf>,
203    /// Disable TLS certificate verification for this fetch's HTTP
204    /// path. Dangerous; leaves the connection open to MITM. Use only
205    /// against known-self-signed environments.
206    #[arg(long, help_heading = "HTTP transport")]
207    pub tls_insecure: bool,
208    /// Overall fetch timeout, in milliseconds.
209    #[arg(
210        long = "timeout-ms",
211        default_value_t = 30_000,
212        help_heading = "HTTP transport"
213    )]
214    pub timeout_ms: u64,
215    /// Capture WebSocket frame payloads to network-bodies/<id>.frames.jsonl.
216    /// Frames may carry bearer tokens, session IDs, and message content —
217    /// treat the artifact as sensitive.
218    #[arg(long, help_heading = "Network capture")]
219    pub capture_ws: bool,
220    /// Capture SSE event payloads to network-bodies/<id>.frames.jsonl. Events
221    /// may carry PII; treat the artifact as sensitive.
222    #[arg(long, help_heading = "Network capture")]
223    pub capture_sse: bool,
224}
225
226pub async fn run(args: Args) -> Result<(), Error> {
227    match run_inner(args).await {
228        Ok(()) => Ok(()),
229        Err(FetchRunError::Plain(err)) => {
230            let stdout = std::io::stdout();
231            let mut handle = stdout.lock();
232            let _ = crate::shared::envelope::emit_error(&mut handle, &err);
233            Err(err)
234        }
235        Err(FetchRunError::Emitted(err)) => Err(err),
236    }
237}
238
239async fn run_inner(args: Args) -> Result<(), FetchRunError> {
240    let render = RenderMode::parse(&args.render)?;
241    let wait = Wait::parse(&args.wait)?;
242    let timeout = Duration::from_millis(args.timeout_ms);
243    let network_bodies = NetworkBodies::from(args.network_bodies);
244    let network_redact = matches!(args.network_redact, NetworkRedactArg::On);
245
246    let body_bytes = resolve_body(&args).await?;
247    let want = resolve_want(&args.want)?;
248    let client = build_client(&args, render).await?;
249
250    let mut builder = client
251        .fetch(args.url.clone())
252        .render(render)
253        .wait(wait)
254        .timeout(timeout)
255        .want(want)
256        .network_bodies(network_bodies)
257        .network_body_max_bytes(args.network_body_max_bytes)
258        .readiness_idle_ms(args.readiness_idle_ms)
259        .readiness_stable_ms(args.readiness_stable_ms)
260        .readiness_min_text_bytes(args.readiness_min_text_bytes)
261        .network_redact(network_redact)
262        .method(args.method);
263    if let Some(bytes) = body_bytes {
264        builder = builder.body(bytes);
265    }
266    for raw in &args.form {
267        let (k, v) = raw.split_once('=').ok_or_else(|| {
268            Error::new(
269                ErrorCode::InvalidArgument,
270                format!("--form: expected key=value, got {raw:?}"),
271            )
272        })?;
273        builder = builder.form_field(k, v);
274    }
275    for raw in args.headers {
276        let (name, value) = parse_header_arg(&raw)?;
277        builder = builder.header(name, value);
278    }
279    for raw in args.cookies {
280        builder = builder.cookie_full(parse_cookie_arg(&raw)?);
281    }
282    if let Some(user_agent) = args.user_agent {
283        builder = builder.user_agent(user_agent);
284    }
285    for js in args.evaluate_after_wait {
286        builder = builder.evaluate_after_wait(js);
287    }
288    if args.tab != "new" {
289        builder = builder.tab(TabId::new(args.tab));
290    }
291    if let Some(out) = args.out {
292        builder = builder.out_dir(out);
293    }
294    builder = builder.observe_main_wait_ms(args.observe_main_wait_ms);
295    builder = builder.max_response_bytes(args.max_response_bytes);
296    builder = builder.retry(args.retry).backoff_ms(args.backoff_ms);
297    if let Some(url) = args.proxy {
298        builder = builder.proxy(url);
299    }
300    if let Some(path) = args.ca_cert {
301        builder = builder.ca_cert(path);
302    }
303    if args.tls_insecure {
304        builder = builder.tls_insecure(true);
305    }
306    if args.capture_ws {
307        builder = builder.capture_ws(true);
308    }
309    if args.capture_sse {
310        builder = builder.capture_sse(true);
311    }
312    if args.no_cookie_jar {
313        builder = builder.no_cookie_jar();
314    } else {
315        let cookie_jar = args.cookie_jar.or_else(|| {
316            std::env::var_os("AFHTTP_COOKIE_JAR")
317                .filter(|v| !v.is_empty())
318                .map(PathBuf::from)
319        });
320        if let Some(jar) = cookie_jar {
321            builder = builder.cookie_jar(jar);
322        }
323    }
324
325    match builder.send_detailed().await {
326        Ok(result) => Ok(output::emit("fetch", &result)?),
327        Err(err) => {
328            output::emit("error", &err)?;
329            Err(FetchRunError::Emitted(err.into_error()))
330        }
331    }
332}
333
334enum FetchRunError {
335    Plain(Error),
336    Emitted(Error),
337}
338
339impl From<Error> for FetchRunError {
340    fn from(err: Error) -> Self {
341        Self::Plain(err)
342    }
343}
344
345/// Resolve the request body from `--data` / `--data-file`, rejecting the
346/// mutually exclusive combinations. `--form` fields are wired separately by
347/// the caller; this only enforces that they don't coexist with a raw body.
348async fn resolve_body(args: &Args) -> Result<Option<Vec<u8>>, Error> {
349    if args.data.is_some() && args.data_file.is_some() {
350        return Err(Error::new(
351            ErrorCode::InvalidArgument,
352            "--data and --data-file are mutually exclusive",
353        ));
354    }
355    if (args.data.is_some() || args.data_file.is_some()) && !args.form.is_empty() {
356        return Err(Error::new(
357            ErrorCode::InvalidArgument,
358            "--data/--data-file and --form are mutually exclusive",
359        ));
360    }
361    if let Some(data) = &args.data {
362        if let Some(path) = data.strip_prefix('@') {
363            return Ok(Some(tokio::fs::read(path).await.map_err(|e| {
364                Error::new(ErrorCode::IoError, format!("--data @{path}: {e}"))
365            })?));
366        }
367        return Ok(Some(data.as_bytes().to_vec()));
368    }
369    if let Some(path) = &args.data_file {
370        return Ok(Some(tokio::fs::read(path).await.map_err(|e| {
371            Error::new(
372                ErrorCode::IoError,
373                format!("--data-file {}: {e}", path.display()),
374            )
375        })?));
376    }
377    Ok(None)
378}
379
380/// Parse the `--want` tokens into an artifact set; an empty list means all
381/// default artifacts.
382fn resolve_want(want: &[String]) -> Result<std::collections::BTreeSet<Artifact>, Error> {
383    if want.is_empty() {
384        return Ok(Artifact::ALL.iter().copied().collect());
385    }
386    want.iter().map(|t| parse_artifact(t)).collect()
387}
388
389/// Construct the SDK client for this fetch: a remote connection when
390/// `--endpoint-url` is set, the HTTP-only client for `--render none`, or an
391/// inline ephemeral host otherwise (lazy for `auto`, eager for `always`).
392async fn build_client(args: &Args, render: RenderMode) -> Result<Client, Error> {
393    match args.endpoint.as_deref() {
394        Some(ep) => {
395            let mut c = Client::connect(ep)?;
396            if let Some(t) = args.token.as_deref() {
397                c = c.with_token(t);
398            }
399            Ok(c)
400        }
401        None if matches!(render, RenderMode::None) => Client::http_only(),
402        None => {
403            let browser = args
404                .browser
405                .parse::<BrowserChoice>()
406                .map_err(|e| Error::new(ErrorCode::InvalidArgument, format!("--browser: {e}")))?;
407            let cfg = InlineConfig {
408                browser,
409                browser_bin: args.browser_bin.clone(),
410            };
411            if matches!(render, RenderMode::Auto) {
412                Client::inline_ephemeral_lazy(cfg).await
413            } else {
414                Client::inline_ephemeral_with(cfg).await
415            }
416        }
417    }
418}
419
420fn parse_artifact(token: &str) -> Result<Artifact, Error> {
421    Ok(match token {
422        "body" => Artifact::Body,
423        "rendered_html" => Artifact::RenderedHtml,
424        "text" => Artifact::Text,
425        "screenshot" => Artifact::Screenshot,
426        "network" => Artifact::Network,
427        "console" => Artifact::Console,
428        "observation" => Artifact::Observation,
429        "storage" => Artifact::Storage,
430        other => {
431            return Err(Error::new(
432                ErrorCode::InvalidArgument,
433                format!("--want: unknown artifact {other:?}"),
434            ));
435        }
436    })
437}
438
439fn parse_header_arg(raw: &str) -> Result<(String, String), Error> {
440    let (name, value) = raw.split_once(':').ok_or_else(|| {
441        Error::new(
442            ErrorCode::InvalidArgument,
443            format!("--header: expected K:V, got {raw:?}"),
444        )
445    })?;
446    let name = name.trim();
447    if name.is_empty() {
448        return Err(Error::new(
449            ErrorCode::InvalidArgument,
450            format!("--header: header name must not be empty in {raw:?}"),
451        ));
452    }
453    Ok((name.to_string(), value.trim_start().to_string()))
454}
455
456fn parse_cookie_arg(raw: &str) -> Result<FetchCookie, Error> {
457    if !raw.contains('=') {
458        return Err(Error::new(
459            ErrorCode::InvalidArgument,
460            format!("--cookie: expected Set-Cookie style name=value, got {raw:?}"),
461        ));
462    }
463    let cookie = FetchCookie::parse(raw.to_string())
464        .map_err(|e| Error::new(ErrorCode::InvalidArgument, format!("--cookie: {e}")))?
465        .into_owned();
466    if cookie.name().trim().is_empty() {
467        return Err(Error::new(
468            ErrorCode::InvalidArgument,
469            format!("--cookie: cookie name must not be empty in {raw:?}"),
470        ));
471    }
472    Ok(cookie)
473}
474
475#[cfg(test)]
476mod tests {
477    use super::*;
478
479    #[test]
480    fn header_arg_accepts_colon_separator() {
481        assert_eq!(
482            parse_header_arg("X-Test: yes").unwrap(),
483            ("X-Test".to_string(), "yes".to_string())
484        );
485    }
486
487    #[test]
488    fn header_arg_rejects_missing_colon() {
489        let err = parse_header_arg("X-Test").err().unwrap();
490        assert_eq!(err.error_code, ErrorCode::InvalidArgument);
491    }
492
493    #[test]
494    fn cookie_arg_accepts_equals_separator() {
495        let cookie = parse_cookie_arg("sid=abc=def").unwrap();
496        assert_eq!(cookie.name_value(), ("sid", "abc=def"));
497    }
498
499    #[test]
500    fn cookie_arg_accepts_full_set_cookie_attributes() {
501        let cookie = parse_cookie_arg("sid=abc; Path=/; Secure; HttpOnly; SameSite=Lax").unwrap();
502        assert_eq!(cookie.name_value(), ("sid", "abc"));
503        assert_eq!(cookie.path(), Some("/"));
504        assert_eq!(cookie.secure(), Some(true));
505        assert_eq!(cookie.http_only(), Some(true));
506        assert_eq!(cookie.same_site(), Some(cookie::SameSite::Lax));
507    }
508
509    #[test]
510    fn cookie_arg_rejects_missing_equals() {
511        let err = parse_cookie_arg("sid").err().unwrap();
512        assert_eq!(err.error_code, ErrorCode::InvalidArgument);
513    }
514}
agent_first_http/cli/cmd/fetch.rs

agent_first_http/cli/cmd/
fetch.rs
