agent_first_http/cli/cmd/
fetch.rs

1//! `afhttp fetch` subcommand.
2
3use std::path::PathBuf;
4use std::time::Duration;
5
6use clap::Args as ClapArgs;
7use clap::ValueEnum;
8
9use crate::cli::cmd::argenums::{BrowserArg, RenderArg};
10use crate::cli::output;
11use crate::sdk::fetch::{
12    DEFAULT_NETWORK_BODY_MAX_BYTES, FetchCookie, NetworkBodies, RenderMode, Wait,
13};
14use crate::sdk::{Client, InlineConfig};
15use crate::shared::artifacts::Artifact;
16use crate::shared::error::{Error, ErrorCode};
17use crate::shared::ids::TabId;
18
19#[derive(ValueEnum, Debug, Clone, Copy, Default)]
20pub enum NetworkBodiesArg {
21    #[default]
22    Off,
23    Xhr,
24    All,
25}
26
27impl From<NetworkBodiesArg> for NetworkBodies {
28    fn from(v: NetworkBodiesArg) -> Self {
29        match v {
30            NetworkBodiesArg::Off => NetworkBodies::Off,
31            NetworkBodiesArg::Xhr => NetworkBodies::Xhr,
32            NetworkBodiesArg::All => NetworkBodies::All,
33        }
34    }
35}
36
37impl std::fmt::Display for NetworkBodiesArg {
38    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
39        f.write_str(match self {
40            Self::Off => "off",
41            Self::Xhr => "xhr",
42            Self::All => "all",
43        })
44    }
45}
46
47#[derive(ClapArgs, Debug)]
48pub struct Args {
49    /// URL to fetch.
50    pub url: String,
51    /// CDP endpoint of a running host.
52    ///
53    /// Omit to spawn an inline ephemeral host for ordinary browser fetches;
54    /// with --takeover, omission discovers the standard local `afhttp-host`.
55    /// Falls back to `AFHTTP_ENDPOINT_URL`.
56    #[arg(
57        long = "endpoint-url",
58        env = "AFHTTP_ENDPOINT_URL",
59        help_heading = "Connection"
60    )]
61    pub endpoint: Option<String>,
62    /// Bearer token for a protected host.
63    ///
64    /// Use when the host was started with `--token-secret`.
65    /// Falls back to `AFHTTP_TOKEN_SECRET`.
66    #[arg(
67        long = "token-secret",
68        env = "AFHTTP_TOKEN_SECRET",
69        help_heading = "Connection"
70    )]
71    pub token: Option<String>,
72    /// Browser backend for the inline host.
73    ///
74    /// Ignored when --endpoint-url is set because the host owns its browser.
75    #[arg(long, default_value = "auto", help_heading = "Inline host")]
76    pub browser: BrowserArg,
77    /// Browser binary path for the inline host.
78    ///
79    /// Use when auto-discovery cannot find one. Ignored when --endpoint-url is
80    /// set.
81    #[arg(
82        long = "browser-bin",
83        value_name = "PATH",
84        help_heading = "Inline host"
85    )]
86    pub browser_bin: Option<PathBuf>,
87    /// Render strategy: none, auto, or always.
88    ///
89    /// `none` uses the HTTP fast path without a browser, `auto` starts with
90    /// HTTP and escalates on failure, and `always` uses the browser only.
91    #[arg(long, default_value = "auto", help_heading = "Rendering")]
92    pub render: RenderArg,
93    /// Browser tab target: new or an existing CDP target ID.
94    ///
95    /// `new` allocates a temporary target and closes it after fetch; an ID
96    /// reuses that target and leaves it open (the same ID accepted by
97    /// `afhttp cdp`, `upload`, and `tabs`).
98    #[arg(
99        long,
100        default_value = "new",
101        value_name = "new|<id>",
102        help_heading = "Session"
103    )]
104    pub tab: String,
105    /// Escalate captcha, login, or 2FA walls to human takeover.
106    ///
107    /// Keeps a persistent tab open and returns its short-lived takeover URL in
108    /// `next_action`, plus a re-fetch command for the same tab once the human
109    /// clears the wall. Uses `--endpoint-url` / `AFHTTP_ENDPOINT_URL` when set;
110    /// otherwise discovers the standard local `afhttp-host` container (build
111    /// one with `afhttp container install`).
112    #[arg(long, help_heading = "Session")]
113    pub takeover: bool,
114    /// Host profile to use for this fetch.
115    ///
116    /// Switches the host's active profile if it differs (per-domain
117    /// isolation), relaunching its browser. With `--takeover` and no
118    /// `--profile`, the profile defaults to the URL's registrable domain
119    /// (eTLD+1). Requires a host via `--endpoint-url`, or the standard local
120    /// takeover host discovered by `--takeover`.
121    #[arg(long, help_heading = "Session")]
122    pub profile: Option<String>,
123    /// Readiness signal before browser capture.
124    ///
125    /// Values: auto | load | idle | selector:<css> |
126    /// selector-visible:<css> | ms:<n>.
127    #[arg(long, default_value = "auto", help_heading = "Rendering")]
128    pub wait: String,
129    /// Add a request header (repeatable).
130    ///
131    /// Format: `Name:value` (a space after the colon is allowed).
132    #[arg(long = "header", value_name = "NAME:VALUE", help_heading = "Request")]
133    pub headers: Vec<String>,
134    /// Add a request cookie (repeatable). Format: `name=value`.
135    #[arg(long = "cookie", value_name = "NAME=VALUE", help_heading = "Request")]
136    pub cookies: Vec<String>,
137    /// Override the User-Agent header for this fetch.
138    #[arg(long, help_heading = "Request")]
139    pub user_agent: Option<String>,
140    /// Evaluate JavaScript after readiness (repeatable).
141    ///
142    /// Runs in page context before artifacts are captured.
143    #[arg(long, value_name = "JS", help_heading = "Rendering")]
144    pub evaluate_after_wait: Vec<String>,
145    /// Artifacts to capture, comma-separated.
146    ///
147    /// Defaults to body on the HTTP fast path and browser-default artifacts
148    /// when rendering. `content` is the agent-oriented composed page view
149    /// (content.md); `content_json` is its structured form with link/action
150    /// candidates. `storage` is opt-in and sensitive (localStorage/IndexedDB).
151    #[arg(long, value_delimiter = ',', help_heading = "Rendering")]
152    pub want: Vec<String>,
153    /// HTTP method. Common values: POST, PUT, PATCH, DELETE.
154    #[arg(long, default_value = "GET", help_heading = "Request")]
155    pub method: String,
156    /// Request body as a string or `@FILE`.
157    ///
158    /// Prefix with `@` to read from a file path (for example,
159    /// `--data @payload.json`). Mutually exclusive with `--form`.
160    #[arg(long, value_name = "STRING|@FILE", help_heading = "Request")]
161    pub data: Option<String>,
162    /// Add a URL-encoded form field (repeatable).
163    ///
164    /// Sends `application/x-www-form-urlencoded`, uses `name=value`, and is
165    /// mutually exclusive with `--data`.
166    #[arg(long = "form", value_name = "NAME=VALUE", help_heading = "Request")]
167    pub form: Vec<String>,
168    /// Capture network response bodies: off, xhr, or all.
169    ///
170    /// `xhr` captures XHR/fetch only.
171    #[arg(long, default_value_t = NetworkBodiesArg::Off, help_heading = "Network capture")]
172    pub network_bodies: NetworkBodiesArg,
173    /// Per-body byte cap for captured network sub-requests.
174    ///
175    /// See `--max-response-bytes` for the main HTTP-path response body.
176    #[arg(long, default_value_t = DEFAULT_NETWORK_BODY_MAX_BYTES, help_heading = "Network capture")]
177    pub network_body_max_bytes: u64,
178    /// Network quiet window used by --wait auto, in milliseconds.
179    #[arg(long, default_value_t = 800, help_heading = "Readiness tuning")]
180    pub readiness_idle_ms: u64,
181    /// DOM/text unchanged window used by --wait auto, in milliseconds.
182    #[arg(long, default_value_t = 500, help_heading = "Readiness tuning")]
183    pub readiness_stable_ms: u64,
184    /// Low visible-text byte threshold for --wait auto quality warnings only.
185    #[arg(long, default_value_t = 32, help_heading = "Readiness tuning")]
186    pub readiness_min_text_bytes: u64,
187    /// Disable sensitive-value redaction in network.json.
188    ///
189    /// Writes raw Authorization/Cookie headers and token-bearing query
190    /// parameters to the artifact. Use only for trusted local debugging.
191    #[arg(long, help_heading = "Network capture")]
192    pub no_network_redact: bool,
193    /// Directory in which to write artifacts.
194    ///
195    /// Defaults to `afhttp-out` under the system temporary directory. Files
196    /// persist there for inspection.
197    #[arg(long, help_heading = "Output")]
198    pub out: Option<PathBuf>,
199    /// Override the profile cookie-jar path.
200    ///
201    /// The default, derived from the host's `GET /profile`, is
202    /// `<profile-dir>/cookies.jar.json`. A mismatch with the host profile is
203    /// rejected with `invalid_argument`; this flag is for tests and forensic
204    /// tooling, not production sessions. Honors `AFHTTP_COOKIE_JAR` when
205    /// omitted (same validation applies).
206    #[arg(long, help_heading = "Cookies")]
207    pub cookie_jar: Option<PathBuf>,
208    /// Disable cookie-jar replay and persistence for this fetch.
209    ///
210    /// No cookies are replayed and no `Set-Cookie` responses are merged back.
211    #[arg(long, help_heading = "Cookies")]
212    pub no_cookie_jar: bool,
213    /// Maximum wait for the browser's main-document event, in milliseconds.
214    ///
215    /// Raise for slow networks or low-end machines.
216    #[arg(long, default_value_t = 500, help_heading = "Readiness tuning")]
217    pub observe_main_wait_ms: u64,
218    /// Maximum main HTTP-path response body size in bytes.
219    ///
220    /// See `--network-body-max-bytes` for captured sub-requests. The default is
221    /// 1 GiB (`1073741824`); `0` disables the cap. Hitting the cap returns the
222    /// collected prefix successfully with a `network_body_truncated` warning.
223    #[arg(long, default_value_t = 1_073_741_824, help_heading = "HTTP transport")]
224    pub max_response_bytes: u64,
225    /// Number of retry attempts after the first.
226    ///
227    /// Retries run only for `retryable: true` errors such as
228    /// `host_unreachable` or `cdp_timeout`; non-retryable failures such as
229    /// `tls_error` short-circuit. The default `0` means one total attempt.
230    #[arg(long, default_value_t = 0, help_heading = "Retry")]
231    pub retry: u32,
232    /// Fixed delay between retries, in milliseconds.
233    #[arg(long, default_value_t = 250, help_heading = "Retry")]
234    pub backoff_ms: u64,
235    /// Upstream proxy for this fetch's HTTP fast path.
236    ///
237    /// The SDK never honors ambient `HTTP_PROXY`; this flag is the only proxy
238    /// route. Use `http://user:pass@host:port` or `socks5://host:port`.
239    #[arg(long = "proxy-url", help_heading = "HTTP transport")]
240    pub proxy: Option<String>,
241    /// PEM file with extra root CAs for the HTTP path.
242    ///
243    /// Useful for self-signed staging or corporate MITM CAs.
244    #[arg(long, help_heading = "HTTP transport")]
245    pub ca_cert: Option<PathBuf>,
246    /// Disable TLS certificate verification on the HTTP path.
247    ///
248    /// Dangerous: leaves the connection open to MITM. Use only against a
249    /// known self-signed environment.
250    #[arg(long, help_heading = "HTTP transport")]
251    pub tls_insecure: bool,
252    /// Overall fetch timeout, in milliseconds. Applies to both the HTTP fast
253    /// path and the browser path.
254    #[arg(
255        long = "timeout-ms",
256        default_value_t = 30_000,
257        help_heading = "HTTP transport"
258    )]
259    pub timeout_ms: u64,
260    /// Capture WebSocket frame payloads.
261    ///
262    /// Writes network-bodies/<id>.frames.jsonl. Frames may carry bearer
263    /// tokens, session IDs, and message content; treat it as sensitive.
264    #[arg(long, help_heading = "Network capture")]
265    pub capture_ws: bool,
266    /// Capture SSE event payloads.
267    ///
268    /// Writes network-bodies/<id>.frames.jsonl. Events may carry PII; treat it
269    /// as sensitive.
270    #[arg(long, help_heading = "Network capture")]
271    pub capture_sse: bool,
272}
273
274pub async fn run(args: Args) -> Result<(), Error> {
275    match run_inner(args).await {
276        Ok(()) => Ok(()),
277        Err(FetchRunError::Plain(err)) => {
278            let _ = crate::shared::afdata::emit_process_error(&err);
279            Err(err)
280        }
281        Err(FetchRunError::Emitted(err)) => Err(err),
282    }
283}
284
285async fn run_inner(mut args: Args) -> Result<(), FetchRunError> {
286    let render: RenderMode = args.render.into();
287    let endpoint_was_preconfigured = args.endpoint.is_some();
288    prepare_takeover_connection(&mut args, render, |token| async move {
289        crate::cli::cmd::container::discover_default_takeover_host(token.as_deref()).await
290    })
291    .await?;
292    // Resolve the host profile: explicit --profile wins; otherwise --takeover
293    // derives the URL's registrable domain (eTLD+1) for per-domain isolation.
294    let explicit_profile = args.profile.clone();
295    let resolved_profile: Option<String> = if let Some(p) = explicit_profile.clone() {
296        Some(p)
297    } else if args.takeover {
298        Some(default_profile_for_url(&args.url)?)
299    } else {
300        None
301    };
302    if resolved_profile.is_some() && args.endpoint.is_none() {
303        return Err(Error::new(
304            ErrorCode::InvalidArgument,
305            "--profile (and --takeover profile derivation) switch the host's active profile and require a host; pass --endpoint-url or set AFHTTP_ENDPOINT_URL",
306        )
307        .into());
308    }
309    let takeover = args.takeover;
310    let takeover_endpoint = args.endpoint.clone();
311    let recommended_endpoint =
312        takeover_recommended_endpoint(takeover_endpoint.as_deref(), endpoint_was_preconfigured);
313    let takeover_token = args.token.clone();
314    let wait = Wait::parse(&args.wait)?;
315    let timeout = Duration::from_millis(args.timeout_ms);
316    let network_bodies = NetworkBodies::from(args.network_bodies);
317    let network_redact = !args.no_network_redact;
318
319    let body_bytes = resolve_body(&args).await?;
320    let want = resolve_want(&args.want)?;
321    let mut client = build_client(&args, render).await?;
322    if let Some(profile) = &resolved_profile {
323        client = client.with_profile(profile.clone());
324    }
325
326    let mut builder = client
327        .fetch(args.url.clone())
328        .render(render)
329        .wait(wait)
330        .timeout(timeout)
331        .network_bodies(network_bodies)
332        .network_body_max_bytes(args.network_body_max_bytes)
333        .readiness_idle_ms(args.readiness_idle_ms)
334        .readiness_stable_ms(args.readiness_stable_ms)
335        .readiness_min_text_bytes(args.readiness_min_text_bytes)
336        .network_redact(network_redact)
337        .method(args.method);
338    if let Some(want) = want {
339        builder = builder.want(want);
340    }
341    if let Some(bytes) = body_bytes {
342        builder = builder.body(bytes);
343    }
344    for raw in &args.form {
345        let (k, v) = raw.split_once('=').ok_or_else(|| {
346            Error::new(
347                ErrorCode::InvalidArgument,
348                format!("--form: expected key=value, got {raw:?}"),
349            )
350        })?;
351        builder = builder.form_field(k, v);
352    }
353    for raw in args.headers {
354        let (name, value) = parse_header_arg(&raw)?;
355        builder = builder.header(name, value);
356    }
357    for raw in args.cookies {
358        builder = builder.cookie_full(parse_cookie_arg(&raw)?);
359    }
360    if let Some(user_agent) = args.user_agent {
361        builder = builder.user_agent(user_agent);
362    }
363    for js in args.evaluate_after_wait {
364        builder = builder.evaluate_after_wait(js);
365    }
366    if args.tab != "new" {
367        builder = builder.tab(TabId::new(args.tab));
368    }
369    if takeover {
370        // Keep the prepared tab open so a human can take it over.
371        builder = builder.keep_tab_open(true);
372    }
373    if let Some(out) = args.out {
374        builder = builder.out_dir(out);
375    }
376    builder = builder.observe_main_wait_ms(args.observe_main_wait_ms);
377    builder = builder.max_response_bytes(args.max_response_bytes);
378    builder = builder.retry(args.retry).backoff_ms(args.backoff_ms);
379    if let Some(url) = args.proxy {
380        builder = builder.proxy(url);
381    }
382    if let Some(path) = args.ca_cert {
383        builder = builder.ca_cert(path);
384    }
385    if args.tls_insecure {
386        builder = builder.tls_insecure(true);
387    }
388    if args.capture_ws {
389        builder = builder.capture_ws(true);
390    }
391    if args.capture_sse {
392        builder = builder.capture_sse(true);
393    }
394    if args.no_cookie_jar {
395        builder = builder.no_cookie_jar();
396    } else {
397        let cookie_jar = args.cookie_jar.or_else(|| {
398            std::env::var_os("AFHTTP_COOKIE_JAR")
399                .filter(|v| !v.is_empty())
400                .map(PathBuf::from)
401        });
402        if let Some(jar) = cookie_jar {
403            builder = builder.cookie_jar(jar);
404        }
405    }
406
407    match builder.send_detailed().await {
408        Ok(mut result) => {
409            if takeover
410                && result.next_action.is_some()
411                && let Some(endpoint) = takeover_endpoint.as_deref()
412            {
413                let mut handoff_client = Client::connect(endpoint)?;
414                if let Some(token) = takeover_token.as_deref() {
415                    handoff_client = handoff_client.with_token(token);
416                }
417                let tab_id = result.tab_id.as_ref().map(|t| t.as_str().to_string());
418                let handoff = handoff_client
419                    .takeover_handoff(None, tab_id.as_deref())
420                    .await?;
421                result.attach_takeover_with_context(
422                    handoff.takeover_url_secret,
423                    Some(handoff.takeover_url_expires_at_rfc3339),
424                    Some(handoff.takeover_url_ttl_s),
425                    Some(handoff.takeover_url_scope),
426                    recommended_endpoint,
427                    explicit_profile.as_deref(),
428                );
429            }
430            if takeover {
431                Ok(output::emit_revealing_takeover("fetch", &result)?)
432            } else {
433                Ok(output::emit("fetch", &result)?)
434            }
435        }
436        Err(err) => {
437            let trace = serde_json::to_value(&err.trace).map_err(|e| {
438                Error::new(
439                    ErrorCode::InternalError,
440                    format!("serialize fetch error trace: {e}"),
441                )
442            })?;
443            let err = err.into_error();
444            crate::shared::afdata::emit_process_error_with(
445                err.error_code.as_str(),
446                &err.detail,
447                serde_json::json!({"retryable": err.retryable}),
448                trace,
449            )?;
450            Err(FetchRunError::Emitted(err))
451        }
452    }
453}
454
455async fn prepare_takeover_connection<D, Fut>(
456    args: &mut Args,
457    render: RenderMode,
458    discover: D,
459) -> Result<(), Error>
460where
461    D: FnOnce(Option<String>) -> Fut,
462    Fut: std::future::Future<Output = Result<crate::cli::cmd::container::LocalTakeoverHost, Error>>,
463{
464    if !args.takeover {
465        return Ok(());
466    }
467    if matches!(render, RenderMode::None) {
468        return Err(Error::new(
469            ErrorCode::InvalidArgument,
470            "fetch --takeover needs a browser render; use --render auto or always",
471        ));
472    }
473    if args.endpoint.is_none() {
474        let discovered = discover(args.token.clone()).await?;
475        args.endpoint = Some(discovered.endpoint);
476        if args.token.is_none() {
477            args.token = discovered.token_secret;
478        }
479    }
480    Ok(())
481}
482
483fn takeover_recommended_endpoint(
484    endpoint: Option<&str>,
485    endpoint_was_preconfigured: bool,
486) -> Option<&str> {
487    if endpoint_was_preconfigured {
488        endpoint
489    } else {
490        None
491    }
492}
493
494/// Derive a default host profile from a URL's registrable domain (eTLD+1), so
495/// `fetch --takeover` gives each site its own isolated browser profile when
496/// `--profile` is omitted. Falls back to the full host for public-suffix
497/// tenants (e.g. `foo.github.io`) and bare IPs.
498fn default_profile_for_url(raw_url: &str) -> Result<String, Error> {
499    let parsed = url::Url::parse(raw_url).map_err(|e| {
500        Error::new(
501            ErrorCode::InvalidArgument,
502            format!(
503                "--takeover default profile needs a valid URL with a host; \
504                 could not parse {raw_url:?}: {e}; pass --profile <name>"
505            ),
506        )
507    })?;
508    let host = parsed.host().ok_or_else(|| {
509        Error::new(
510            ErrorCode::InvalidArgument,
511            format!(
512                "--takeover default profile needs URL {raw_url:?} to include a host; \
513                 pass --profile <name>"
514            ),
515        )
516    })?;
517    let (normalized_host, dns_name) = match host {
518        url::Host::Domain(domain) => (normalize_profile_host(domain), true),
519        url::Host::Ipv4(addr) => (addr.to_string(), false),
520        url::Host::Ipv6(addr) => (addr.to_string(), false),
521    };
522    let profile = if dns_name {
523        psl::domain_str(&normalized_host)
524            .unwrap_or(&normalized_host)
525            .to_string()
526    } else {
527        normalized_host.clone()
528    };
529    crate::sdk::profile::paths::validate_name(&profile).map_err(|e| {
530        Error::new(
531            e.error_code,
532            format!(
533                "derived --takeover profile {profile:?} from URL host {normalized_host:?} \
534                 is invalid: {}; pass --profile <name>",
535                e.detail
536            ),
537        )
538    })?;
539    Ok(profile)
540}
541
542fn normalize_profile_host(host: &str) -> String {
543    host.trim_end_matches('.').to_ascii_lowercase()
544}
545
546enum FetchRunError {
547    Plain(Error),
548    Emitted(Error),
549}
550
551impl From<Error> for FetchRunError {
552    fn from(err: Error) -> Self {
553        Self::Plain(err)
554    }
555}
556
557/// Resolve the request body from `--data` (literal, or `@path` to read a file),
558/// rejecting coexistence with `--form`. `--form` fields are wired separately by
559/// the caller; this only enforces that they don't coexist with a raw body.
560async fn resolve_body(args: &Args) -> Result<Option<Vec<u8>>, Error> {
561    if args.data.is_some() && !args.form.is_empty() {
562        return Err(Error::new(
563            ErrorCode::InvalidArgument,
564            "--data and --form are mutually exclusive",
565        ));
566    }
567    if let Some(data) = &args.data {
568        if let Some(path) = data.strip_prefix('@') {
569            return Ok(Some(tokio::fs::read(path).await.map_err(|e| {
570                Error::new(ErrorCode::IoError, format!("--data @{path}: {e}"))
571            })?));
572        }
573        return Ok(Some(data.as_bytes().to_vec()));
574    }
575    Ok(None)
576}
577
578/// Parse the `--want` tokens into an artifact set; `None` means use the
579/// render-mode-aware default.
580fn resolve_want(want: &[String]) -> Result<Option<std::collections::BTreeSet<Artifact>>, Error> {
581    if want.is_empty() {
582        return Ok(None);
583    }
584    want.iter()
585        .map(|t| parse_artifact(t))
586        .collect::<Result<std::collections::BTreeSet<_>, _>>()
587        .map(Some)
588}
589
590/// Construct the SDK client for this fetch: a remote connection when
591/// `--endpoint-url` is set, the HTTP-only client for `--render none`, or an
592/// inline ephemeral host otherwise (lazy for `auto`, eager for `always`).
593async fn build_client(args: &Args, render: RenderMode) -> Result<Client, Error> {
594    match args.endpoint.as_deref() {
595        Some(ep) => {
596            let mut c = Client::connect(ep)?;
597            if let Some(t) = args.token.as_deref() {
598                c = c.with_token(t);
599            }
600            Ok(c)
601        }
602        None if matches!(render, RenderMode::None) => Client::http_only(),
603        None => {
604            let cfg = InlineConfig {
605                browser: args.browser.into(),
606                browser_bin: args.browser_bin.clone(),
607            };
608            if matches!(render, RenderMode::Auto) {
609                Client::inline_ephemeral_lazy(cfg).await
610            } else {
611                Client::inline_ephemeral_with(cfg).await
612            }
613        }
614    }
615}
616
617fn parse_artifact(token: &str) -> Result<Artifact, Error> {
618    Ok(match token {
619        "body" => Artifact::Body,
620        "rendered_html" => Artifact::RenderedHtml,
621        "text" => Artifact::Text,
622        "content" => Artifact::Content,
623        "content_json" => Artifact::ContentJson,
624        "screenshot" => Artifact::Screenshot,
625        "network" => Artifact::Network,
626        "console" => Artifact::Console,
627        "observation" => Artifact::Observation,
628        "storage" => Artifact::Storage,
629        other => {
630            return Err(Error::new(
631                ErrorCode::InvalidArgument,
632                format!("--want: unknown artifact {other:?}"),
633            ));
634        }
635    })
636}
637
638fn parse_header_arg(raw: &str) -> Result<(String, String), Error> {
639    let (name, value) = raw.split_once(':').ok_or_else(|| {
640        Error::new(
641            ErrorCode::InvalidArgument,
642            format!("--header: expected K:V, got {raw:?}"),
643        )
644    })?;
645    let name = name.trim();
646    if name.is_empty() {
647        return Err(Error::new(
648            ErrorCode::InvalidArgument,
649            format!("--header: header name must not be empty in {raw:?}"),
650        ));
651    }
652    Ok((name.to_string(), value.trim_start().to_string()))
653}
654
655fn parse_cookie_arg(raw: &str) -> Result<FetchCookie, Error> {
656    if !raw.contains('=') {
657        return Err(Error::new(
658            ErrorCode::InvalidArgument,
659            format!("--cookie: expected Set-Cookie style name=value, got {raw:?}"),
660        ));
661    }
662    let cookie = FetchCookie::parse(raw.to_string())
663        .map_err(|e| Error::new(ErrorCode::InvalidArgument, format!("--cookie: {e}")))?
664        .into_owned();
665    if cookie.name().trim().is_empty() {
666        return Err(Error::new(
667            ErrorCode::InvalidArgument,
668            format!("--cookie: cookie name must not be empty in {raw:?}"),
669        ));
670    }
671    Ok(cookie)
672}
673
674#[cfg(test)]
675mod tests {
676    use super::*;
677
678    fn base_args(url: &str) -> Args {
679        Args {
680            url: url.to_string(),
681            endpoint: None,
682            token: None,
683            browser: BrowserArg::Auto,
684            browser_bin: None,
685            render: RenderArg::Auto,
686            tab: "new".into(),
687            takeover: false,
688            profile: None,
689            wait: "auto".into(),
690            headers: Vec::new(),
691            cookies: Vec::new(),
692            user_agent: None,
693            evaluate_after_wait: Vec::new(),
694            want: Vec::new(),
695            method: "GET".into(),
696            data: None,
697            form: Vec::new(),
698            network_bodies: NetworkBodiesArg::Off,
699            network_body_max_bytes: DEFAULT_NETWORK_BODY_MAX_BYTES,
700            readiness_idle_ms: 800,
701            readiness_stable_ms: 500,
702            readiness_min_text_bytes: 32,
703            no_network_redact: false,
704            out: None,
705            cookie_jar: None,
706            no_cookie_jar: false,
707            observe_main_wait_ms: 500,
708            max_response_bytes: 1_073_741_824,
709            retry: 0,
710            backoff_ms: 250,
711            proxy: None,
712            ca_cert: None,
713            tls_insecure: false,
714            timeout_ms: 30_000,
715            capture_ws: false,
716            capture_sse: false,
717        }
718    }
719
720    #[tokio::test]
721    async fn takeover_autodiscovery_fills_missing_endpoint_and_token() {
722        let mut args = base_args("https://contabo.com");
723        args.takeover = true;
724        prepare_takeover_connection(&mut args, RenderMode::Auto, |token| async move {
725            assert_eq!(token, None);
726            Ok(crate::cli::cmd::container::LocalTakeoverHost {
727                endpoint: "ws://127.0.0.1:9222".into(),
728                token_secret: Some("secret".into()),
729            })
730        })
731        .await
732        .unwrap();
733
734        assert_eq!(args.endpoint.as_deref(), Some("ws://127.0.0.1:9222"));
735        assert_eq!(args.token.as_deref(), Some("secret"));
736    }
737
738    #[tokio::test]
739    async fn takeover_autodiscovery_preserves_existing_token() {
740        let mut args = base_args("https://contabo.com");
741        args.takeover = true;
742        args.token = Some("env-token".into());
743        prepare_takeover_connection(&mut args, RenderMode::Auto, |token| async move {
744            assert_eq!(token.as_deref(), Some("env-token"));
745            Ok(crate::cli::cmd::container::LocalTakeoverHost {
746                endpoint: "ws://127.0.0.1:9222".into(),
747                token_secret: Some("container-token".into()),
748            })
749        })
750        .await
751        .unwrap();
752
753        assert_eq!(args.endpoint.as_deref(), Some("ws://127.0.0.1:9222"));
754        assert_eq!(args.token.as_deref(), Some("env-token"));
755    }
756
757    #[tokio::test]
758    async fn takeover_autodiscovery_surfaces_failure() {
759        let mut args = base_args("https://contabo.com");
760        args.takeover = true;
761        let err = prepare_takeover_connection(&mut args, RenderMode::Auto, |_| async {
762            Err(Error::new(
763                ErrorCode::InvalidArgument,
764                "default local container `afhttp-host` is not running",
765            ))
766        })
767        .await
768        .err()
769        .unwrap();
770
771        assert_eq!(err.error_code, ErrorCode::InvalidArgument);
772        assert!(err.detail.contains("afhttp-host"));
773        assert!(args.endpoint.is_none());
774    }
775
776    #[tokio::test]
777    async fn takeover_autodiscovery_rejects_render_none() {
778        let mut args = base_args("https://contabo.com");
779        args.takeover = true;
780        let err = prepare_takeover_connection(&mut args, RenderMode::None, |_| async {
781            Ok(crate::cli::cmd::container::LocalTakeoverHost {
782                endpoint: "ws://127.0.0.1:9222".into(),
783                token_secret: None,
784            })
785        })
786        .await
787        .err()
788        .unwrap();
789
790        assert_eq!(err.error_code, ErrorCode::InvalidArgument);
791        assert!(err.detail.contains("browser render"));
792    }
793
794    #[test]
795    fn takeover_recommendation_omits_auto_discovered_endpoint() {
796        assert_eq!(
797            takeover_recommended_endpoint(Some("ws://127.0.0.1:9222"), false),
798            None
799        );
800    }
801
802    #[test]
803    fn takeover_recommendation_keeps_preconfigured_endpoint() {
804        assert_eq!(
805            takeover_recommended_endpoint(Some("ws://127.0.0.1:9222"), true),
806            Some("ws://127.0.0.1:9222")
807        );
808    }
809
810    #[test]
811    fn default_profile_uses_registrable_domain() {
812        assert_eq!(
813            default_profile_for_url("https://www.court.gov.cn/foo").unwrap(),
814            "court.gov.cn"
815        );
816        assert_eq!(
817            default_profile_for_url("https://accounts.google.com/foo").unwrap(),
818            "google.com"
819        );
820        assert_eq!(
821            default_profile_for_url("https://contabo.com").unwrap(),
822            "contabo.com"
823        );
824    }
825
826    #[test]
827    fn default_profile_keeps_public_suffix_tenants_isolated() {
828        assert_eq!(
829            default_profile_for_url("https://foo.github.io/x").unwrap(),
830            "foo.github.io"
831        );
832        assert_eq!(
833            default_profile_for_url("https://tenant.vercel.app/x").unwrap(),
834            "tenant.vercel.app"
835        );
836    }
837
838    #[test]
839    fn default_profile_normalizes_case_and_trailing_dot() {
840        assert_eq!(
841            default_profile_for_url("https://WWW.Example.COM./foo").unwrap(),
842            "example.com"
843        );
844    }
845
846    #[test]
847    fn default_profile_falls_back_to_full_host_for_psl_misses_and_ips() {
848        assert_eq!(
849            default_profile_for_url("http://localhost:8080/foo").unwrap(),
850            "localhost"
851        );
852        assert_eq!(
853            default_profile_for_url("http://127.0.0.1:8080/foo").unwrap(),
854            "127.0.0.1"
855        );
856    }
857
858    #[test]
859    fn default_profile_errors_when_host_is_missing() {
860        assert!(default_profile_for_url("file:///tmp/page.html").is_err());
861    }
862
863    #[test]
864    fn header_arg_accepts_colon_separator() {
865        assert_eq!(
866            parse_header_arg("X-Test: yes").unwrap(),
867            ("X-Test".to_string(), "yes".to_string())
868        );
869    }
870
871    #[test]
872    fn header_arg_rejects_missing_colon() {
873        let err = parse_header_arg("X-Test").err().unwrap();
874        assert_eq!(err.error_code, ErrorCode::InvalidArgument);
875    }
876
877    #[test]
878    fn cookie_arg_accepts_equals_separator() {
879        let cookie = parse_cookie_arg("sid=abc=def").unwrap();
880        assert_eq!(cookie.name_value(), ("sid", "abc=def"));
881    }
882
883    #[test]
884    fn cookie_arg_accepts_full_set_cookie_attributes() {
885        let cookie = parse_cookie_arg("sid=abc; Path=/; Secure; HttpOnly; SameSite=Lax").unwrap();
886        assert_eq!(cookie.name_value(), ("sid", "abc"));
887        assert_eq!(cookie.path(), Some("/"));
888        assert_eq!(cookie.secure(), Some(true));
889        assert_eq!(cookie.http_only(), Some(true));
890        assert_eq!(cookie.same_site(), Some(cookie::SameSite::Lax));
891    }
892
893    #[test]
894    fn cookie_arg_rejects_missing_equals() {
895        let err = parse_cookie_arg("sid").err().unwrap();
896        assert_eq!(err.error_code, ErrorCode::InvalidArgument);
897    }
898}
agent_first_http/cli/cmd/fetch.rs

agent_first_http/cli/cmd/
fetch.rs
