agent_first_http/cli/cmd/
fetch.rs

1//! `afhttp fetch` subcommand.
2
3use std::path::PathBuf;
4use std::time::Duration;
5
6use clap::Args as ClapArgs;
7use clap::ValueEnum;
8
9use crate::cli::cmd::argenums::{BrowserArg, RenderArg};
10use crate::cli::output;
11use crate::sdk::fetch::{
12    DEFAULT_NETWORK_BODY_MAX_BYTES, FetchCookie, NetworkBodies, RenderMode, Wait,
13};
14use crate::sdk::{Client, InlineConfig};
15use crate::shared::artifacts::Artifact;
16use crate::shared::error::{Error, ErrorCode};
17use crate::shared::ids::TabId;
18
19#[derive(ValueEnum, Debug, Clone, Copy, Default)]
20pub enum NetworkBodiesArg {
21    #[default]
22    Off,
23    Xhr,
24    All,
25}
26
27impl From<NetworkBodiesArg> for NetworkBodies {
28    fn from(v: NetworkBodiesArg) -> Self {
29        match v {
30            NetworkBodiesArg::Off => NetworkBodies::Off,
31            NetworkBodiesArg::Xhr => NetworkBodies::Xhr,
32            NetworkBodiesArg::All => NetworkBodies::All,
33        }
34    }
35}
36
37impl std::fmt::Display for NetworkBodiesArg {
38    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
39        f.write_str(match self {
40            Self::Off => "off",
41            Self::Xhr => "xhr",
42            Self::All => "all",
43        })
44    }
45}
46
47#[derive(ClapArgs, Debug)]
48pub struct Args {
49    /// URL to fetch.
50    pub url: String,
51    /// CDP endpoint of a running host. Omit to spawn an inline ephemeral host
52    /// for ordinary browser fetches; with --takeover, omission discovers the
53    /// standard local `afhttp-host`. Falls back to `AFHTTP_ENDPOINT_URL`.
54    #[arg(
55        long = "endpoint-url",
56        env = "AFHTTP_ENDPOINT_URL",
57        help_heading = "Connection"
58    )]
59    pub endpoint: Option<String>,
60    /// Bearer token, if the host was started with `--token-secret`.
61    /// Falls back to `AFHTTP_TOKEN_SECRET`.
62    #[arg(
63        long = "token-secret",
64        env = "AFHTTP_TOKEN_SECRET",
65        help_heading = "Connection"
66    )]
67    pub token: Option<String>,
68    /// Browser backend for the inline host. Ignored when --endpoint-url is set
69    /// (the host owns its browser).
70    #[arg(long, default_value = "auto", help_heading = "Inline host")]
71    pub browser: BrowserArg,
72    /// Browser binary path for the inline host, for when auto-discovery can't
73    /// find one. Ignored when --endpoint-url is set.
74    #[arg(
75        long = "browser-bin",
76        value_name = "PATH",
77        help_heading = "Inline host"
78    )]
79    pub browser_bin: Option<PathBuf>,
80    /// Render strategy: none (HTTP fast path, no browser), auto (HTTP first,
81    /// escalate to the browser on failure), or always (browser only).
82    #[arg(long, default_value = "auto", help_heading = "Rendering")]
83    pub render: RenderArg,
84    /// Tab target: "new" allocates a temporary target and closes it after
85    /// fetch; a CDP target id reuses that target and leaves it open (the same
86    /// id `afhttp cdp`/`upload`/`tabs` accept).
87    #[arg(
88        long,
89        default_value = "new",
90        value_name = "new|<id>",
91        help_heading = "Session"
92    )]
93    pub tab: String,
94    /// Escalate to human takeover when a wall (captcha/login/2FA) is hit: keep
95    /// a persistent tab open and return its short-lived takeover URL in
96    /// `next_action`, plus a re-fetch command for the same tab once the human
97    /// clears the wall. Uses `--endpoint-url` / `AFHTTP_ENDPOINT_URL` when set;
98    /// otherwise discovers the standard local `afhttp-host` container (build
99    /// one with `afhttp container install`).
100    #[arg(long, help_heading = "Session")]
101    pub takeover: bool,
102    /// Host profile to use for this fetch. Switches the host's active profile
103    /// if it differs (per-domain isolation), relaunching its browser. With
104    /// `--takeover` and no `--profile`, the profile defaults to the URL's
105    /// registrable domain (eTLD+1). Requires a host via `--endpoint-url`, or
106    /// the standard local takeover host discovered by `--takeover`.
107    #[arg(long, help_heading = "Session")]
108    pub profile: Option<String>,
109    /// Readiness signal before capture on the browser path:
110    /// auto | load | idle | selector:<css> | selector-visible:<css> | ms:<n>.
111    #[arg(long, default_value = "auto", help_heading = "Rendering")]
112    pub wait: String,
113    /// Add a request header (repeatable). Format: `Name:value` (a space after
114    /// the colon is allowed).
115    #[arg(long = "header", value_name = "NAME:VALUE", help_heading = "Request")]
116    pub headers: Vec<String>,
117    /// Add a request cookie (repeatable). Format: `name=value`.
118    #[arg(long = "cookie", value_name = "NAME=VALUE", help_heading = "Request")]
119    pub cookies: Vec<String>,
120    /// Override the User-Agent header for this fetch.
121    #[arg(long, help_heading = "Request")]
122    pub user_agent: Option<String>,
123    /// JavaScript to evaluate after the wait condition resolves (repeatable).
124    /// Runs in page context before artifacts are captured.
125    #[arg(long, value_name = "JS", help_heading = "Rendering")]
126    pub evaluate_after_wait: Vec<String>,
127    /// Artifacts to capture, comma-separated. Default: body on the HTTP fast
128    /// path; browser default artifacts when rendering is used. `content` is the
129    /// agent-oriented composed page view (content.md); `content_json` its
130    /// structured form with link/action candidates. `storage` is opt-in
131    /// (sensitive: localStorage/IndexedDB).
132    #[arg(long, value_delimiter = ',', help_heading = "Rendering")]
133    pub want: Vec<String>,
134    /// HTTP method. Common values: POST, PUT, PATCH, DELETE.
135    #[arg(long, default_value = "GET", help_heading = "Request")]
136    pub method: String,
137    /// Request body as a string. Prefix with `@` to read from a file path
138    /// (e.g. `--data @payload.json`). Mutually exclusive with `--form`.
139    #[arg(long, value_name = "STRING|@FILE", help_heading = "Request")]
140    pub data: Option<String>,
141    /// Add a form field (repeatable). Sends body as
142    /// `application/x-www-form-urlencoded`. Mutually exclusive with `--data`.
143    /// Format: `name=value`.
144    #[arg(long = "form", value_name = "NAME=VALUE", help_heading = "Request")]
145    pub form: Vec<String>,
146    /// Capture response bodies for network requests: off, xhr (XHR/fetch
147    /// only), or all.
148    #[arg(long, default_value_t = NetworkBodiesArg::Off, help_heading = "Network capture")]
149    pub network_bodies: NetworkBodiesArg,
150    /// Per-body cap for each captured network sub-request body, in bytes (see
151    /// `--max-response-bytes` for the main HTTP-path response body).
152    #[arg(long, default_value_t = DEFAULT_NETWORK_BODY_MAX_BYTES, help_heading = "Network capture")]
153    pub network_body_max_bytes: u64,
154    /// Network quiet window used by --wait auto, in milliseconds.
155    #[arg(long, default_value_t = 800, help_heading = "Readiness tuning")]
156    pub readiness_idle_ms: u64,
157    /// DOM/text unchanged window used by --wait auto, in milliseconds.
158    #[arg(long, default_value_t = 500, help_heading = "Readiness tuning")]
159    pub readiness_stable_ms: u64,
160    /// Low visible-text byte threshold for --wait auto quality warnings only.
161    #[arg(long, default_value_t = 32, help_heading = "Readiness tuning")]
162    pub readiness_min_text_bytes: u64,
163    /// Disable redaction of sensitive values in network.json (redacted by
164    /// default). Writes raw Authorization/Cookie headers and token-bearing
165    /// query params to the artifact — only for trusted local debugging.
166    #[arg(long, help_heading = "Network capture")]
167    pub no_network_redact: bool,
168    /// Directory to write artifacts into. Defaults to `afhttp-out` under the
169    /// system temporary directory. Files persist there for inspection.
170    #[arg(long, help_heading = "Output")]
171    pub out: Option<PathBuf>,
172    /// Override the cookie-jar path. The default — derived from the host's
173    /// `GET /profile` — places the jar at `<profile-dir>/cookies.jar.json`.
174    /// This override is rejected with `invalid_argument` if it does not
175    /// match the host's profile path; the flag exists for tests and
176    /// forensic tooling, not production sessions. Honors
177    /// `AFHTTP_COOKIE_JAR` when omitted (same validation applies).
178    #[arg(long, help_heading = "Cookies")]
179    pub cookie_jar: Option<PathBuf>,
180    /// Opt out of cookie-jar persistence for this fetch. No cookies are
181    /// replayed from the jar and no `Set-Cookie` responses are merged
182    /// back.
183    #[arg(long, help_heading = "Cookies")]
184    pub no_cookie_jar: bool,
185    /// Upper bound on the browser-path wait for the main document network
186    /// event, in milliseconds. Raise for slow networks or low-end machines.
187    #[arg(long, default_value_t = 500, help_heading = "Readiness tuning")]
188    pub observe_main_wait_ms: u64,
189    /// Upper bound on the main HTTP-path response body, in bytes (see
190    /// `--network-body-max-bytes` for captured network sub-request bodies).
191    /// Default 1 GiB (`1073741824`). `0` disables the cap entirely. When the
192    /// cap is hit, the fetch returns successfully with a
193    /// `network_body_truncated` warning and the prefix bytes that
194    /// were collected.
195    #[arg(long, default_value_t = 1_073_741_824, help_heading = "HTTP transport")]
196    pub max_response_bytes: u64,
197    /// Number of additional attempts after the first. Retries fire
198    /// only when the error has `retryable: true` (e.g.
199    /// `host_unreachable`, `cdp_timeout`); non-retryable failures
200    /// (`tls_error`, `wait_selector_unmatched`, etc.) short-circuit.
201    /// Default 0 = single attempt.
202    #[arg(long, default_value_t = 0, help_heading = "Retry")]
203    pub retry: u32,
204    /// Fixed delay between retries, in milliseconds.
205    #[arg(long, default_value_t = 250, help_heading = "Retry")]
206    pub backoff_ms: u64,
207    /// Per-fetch upstream HTTP/HTTPS proxy for the HTTP fast path.
208    /// The SDK never honors `HTTP_PROXY` from the environment; this
209    /// flag is the only way to route an HTTP-path fetch through one.
210    /// Format: `http://user:pass@host:port` or `socks5://host:port`.
211    #[arg(long = "proxy-url", help_heading = "HTTP transport")]
212    pub proxy: Option<String>,
213    /// Path to a PEM file containing extra root CAs to trust for
214    /// this fetch's HTTP path. Useful for self-signed staging or
215    /// corporate MITM CAs.
216    #[arg(long, help_heading = "HTTP transport")]
217    pub ca_cert: Option<PathBuf>,
218    /// Disable TLS certificate verification for this fetch's HTTP
219    /// path. Dangerous; leaves the connection open to MITM. Use only
220    /// against known-self-signed environments.
221    #[arg(long, help_heading = "HTTP transport")]
222    pub tls_insecure: bool,
223    /// Overall fetch timeout, in milliseconds. Applies to both the HTTP fast
224    /// path and the browser path.
225    #[arg(
226        long = "timeout-ms",
227        default_value_t = 30_000,
228        help_heading = "HTTP transport"
229    )]
230    pub timeout_ms: u64,
231    /// Capture WebSocket frame payloads to network-bodies/<id>.frames.jsonl.
232    /// Frames may carry bearer tokens, session IDs, and message content —
233    /// treat the artifact as sensitive.
234    #[arg(long, help_heading = "Network capture")]
235    pub capture_ws: bool,
236    /// Capture SSE event payloads to network-bodies/<id>.frames.jsonl. Events
237    /// may carry PII; treat the artifact as sensitive.
238    #[arg(long, help_heading = "Network capture")]
239    pub capture_sse: bool,
240}
241
242pub async fn run(args: Args) -> Result<(), Error> {
243    match run_inner(args).await {
244        Ok(()) => Ok(()),
245        Err(FetchRunError::Plain(err)) => {
246            let stdout = std::io::stdout();
247            let mut handle = stdout.lock();
248            let _ = crate::shared::afdata::emit_error(&mut handle, &err);
249            Err(err)
250        }
251        Err(FetchRunError::Emitted(err)) => Err(err),
252    }
253}
254
255async fn run_inner(mut args: Args) -> Result<(), FetchRunError> {
256    let render: RenderMode = args.render.into();
257    let endpoint_was_preconfigured = args.endpoint.is_some();
258    prepare_takeover_connection(&mut args, render, |token| async move {
259        crate::cli::cmd::container::discover_default_takeover_host(token.as_deref()).await
260    })
261    .await?;
262    // Resolve the host profile: explicit --profile wins; otherwise --takeover
263    // derives the URL's registrable domain (eTLD+1) for per-domain isolation.
264    let explicit_profile = args.profile.clone();
265    let resolved_profile: Option<String> = if let Some(p) = explicit_profile.clone() {
266        Some(p)
267    } else if args.takeover {
268        Some(default_profile_for_url(&args.url)?)
269    } else {
270        None
271    };
272    if resolved_profile.is_some() && args.endpoint.is_none() {
273        return Err(Error::new(
274            ErrorCode::InvalidArgument,
275            "--profile (and --takeover profile derivation) switch the host's active profile and require a host; pass --endpoint-url or set AFHTTP_ENDPOINT_URL",
276        )
277        .into());
278    }
279    let takeover = args.takeover;
280    let takeover_endpoint = args.endpoint.clone();
281    let recommended_endpoint =
282        takeover_recommended_endpoint(takeover_endpoint.as_deref(), endpoint_was_preconfigured);
283    let takeover_token = args.token.clone();
284    let wait = Wait::parse(&args.wait)?;
285    let timeout = Duration::from_millis(args.timeout_ms);
286    let network_bodies = NetworkBodies::from(args.network_bodies);
287    let network_redact = !args.no_network_redact;
288
289    let body_bytes = resolve_body(&args).await?;
290    let want = resolve_want(&args.want)?;
291    let mut client = build_client(&args, render).await?;
292    if let Some(profile) = &resolved_profile {
293        client = client.with_profile(profile.clone());
294    }
295
296    let mut builder = client
297        .fetch(args.url.clone())
298        .render(render)
299        .wait(wait)
300        .timeout(timeout)
301        .network_bodies(network_bodies)
302        .network_body_max_bytes(args.network_body_max_bytes)
303        .readiness_idle_ms(args.readiness_idle_ms)
304        .readiness_stable_ms(args.readiness_stable_ms)
305        .readiness_min_text_bytes(args.readiness_min_text_bytes)
306        .network_redact(network_redact)
307        .method(args.method);
308    if let Some(want) = want {
309        builder = builder.want(want);
310    }
311    if let Some(bytes) = body_bytes {
312        builder = builder.body(bytes);
313    }
314    for raw in &args.form {
315        let (k, v) = raw.split_once('=').ok_or_else(|| {
316            Error::new(
317                ErrorCode::InvalidArgument,
318                format!("--form: expected key=value, got {raw:?}"),
319            )
320        })?;
321        builder = builder.form_field(k, v);
322    }
323    for raw in args.headers {
324        let (name, value) = parse_header_arg(&raw)?;
325        builder = builder.header(name, value);
326    }
327    for raw in args.cookies {
328        builder = builder.cookie_full(parse_cookie_arg(&raw)?);
329    }
330    if let Some(user_agent) = args.user_agent {
331        builder = builder.user_agent(user_agent);
332    }
333    for js in args.evaluate_after_wait {
334        builder = builder.evaluate_after_wait(js);
335    }
336    if args.tab != "new" {
337        builder = builder.tab(TabId::new(args.tab));
338    }
339    if takeover {
340        // Keep the prepared tab open so a human can take it over.
341        builder = builder.keep_tab_open(true);
342    }
343    if let Some(out) = args.out {
344        builder = builder.out_dir(out);
345    }
346    builder = builder.observe_main_wait_ms(args.observe_main_wait_ms);
347    builder = builder.max_response_bytes(args.max_response_bytes);
348    builder = builder.retry(args.retry).backoff_ms(args.backoff_ms);
349    if let Some(url) = args.proxy {
350        builder = builder.proxy(url);
351    }
352    if let Some(path) = args.ca_cert {
353        builder = builder.ca_cert(path);
354    }
355    if args.tls_insecure {
356        builder = builder.tls_insecure(true);
357    }
358    if args.capture_ws {
359        builder = builder.capture_ws(true);
360    }
361    if args.capture_sse {
362        builder = builder.capture_sse(true);
363    }
364    if args.no_cookie_jar {
365        builder = builder.no_cookie_jar();
366    } else {
367        let cookie_jar = args.cookie_jar.or_else(|| {
368            std::env::var_os("AFHTTP_COOKIE_JAR")
369                .filter(|v| !v.is_empty())
370                .map(PathBuf::from)
371        });
372        if let Some(jar) = cookie_jar {
373            builder = builder.cookie_jar(jar);
374        }
375    }
376
377    match builder.send_detailed().await {
378        Ok(mut result) => {
379            if takeover
380                && result.next_action.is_some()
381                && let Some(endpoint) = takeover_endpoint.as_deref()
382            {
383                let mut handoff_client = Client::connect(endpoint)?;
384                if let Some(token) = takeover_token.as_deref() {
385                    handoff_client = handoff_client.with_token(token);
386                }
387                let tab_id = result.tab_id.as_ref().map(|t| t.as_str().to_string());
388                let handoff = handoff_client
389                    .takeover_handoff(None, tab_id.as_deref())
390                    .await?;
391                result.attach_takeover_with_context(
392                    handoff.takeover_url,
393                    Some(handoff.takeover_url_expires_at_rfc3339),
394                    Some(handoff.takeover_url_ttl_s),
395                    Some(handoff.takeover_url_scope),
396                    recommended_endpoint,
397                    explicit_profile.as_deref(),
398                );
399            }
400            Ok(output::emit("fetch", &result)?)
401        }
402        Err(err) => {
403            let trace = serde_json::to_value(&err.trace).map_err(|e| {
404                Error::new(
405                    ErrorCode::InternalError,
406                    format!("serialize fetch error trace: {e}"),
407                )
408            })?;
409            let err = err.into_error();
410            let stdout = std::io::stdout();
411            let mut handle = stdout.lock();
412            crate::shared::afdata::emit_error_with(
413                &mut handle,
414                err.error_code.as_str(),
415                &err.detail,
416                serde_json::json!({"retryable": err.retryable}),
417                trace,
418            )?;
419            Err(FetchRunError::Emitted(err))
420        }
421    }
422}
423
424async fn prepare_takeover_connection<D, Fut>(
425    args: &mut Args,
426    render: RenderMode,
427    discover: D,
428) -> Result<(), Error>
429where
430    D: FnOnce(Option<String>) -> Fut,
431    Fut: std::future::Future<Output = Result<crate::cli::cmd::container::LocalTakeoverHost, Error>>,
432{
433    if !args.takeover {
434        return Ok(());
435    }
436    if matches!(render, RenderMode::None) {
437        return Err(Error::new(
438            ErrorCode::InvalidArgument,
439            "fetch --takeover needs a browser render; use --render auto or always",
440        ));
441    }
442    if args.endpoint.is_none() {
443        let discovered = discover(args.token.clone()).await?;
444        args.endpoint = Some(discovered.endpoint);
445        if args.token.is_none() {
446            args.token = discovered.token_secret;
447        }
448    }
449    Ok(())
450}
451
452fn takeover_recommended_endpoint(
453    endpoint: Option<&str>,
454    endpoint_was_preconfigured: bool,
455) -> Option<&str> {
456    if endpoint_was_preconfigured {
457        endpoint
458    } else {
459        None
460    }
461}
462
463/// Derive a default host profile from a URL's registrable domain (eTLD+1), so
464/// `fetch --takeover` gives each site its own isolated browser profile when
465/// `--profile` is omitted. Falls back to the full host for public-suffix
466/// tenants (e.g. `foo.github.io`) and bare IPs.
467fn default_profile_for_url(raw_url: &str) -> Result<String, Error> {
468    let parsed = url::Url::parse(raw_url).map_err(|e| {
469        Error::new(
470            ErrorCode::InvalidArgument,
471            format!(
472                "--takeover default profile needs a valid URL with a host; \
473                 could not parse {raw_url:?}: {e}; pass --profile <name>"
474            ),
475        )
476    })?;
477    let host = parsed.host().ok_or_else(|| {
478        Error::new(
479            ErrorCode::InvalidArgument,
480            format!(
481                "--takeover default profile needs URL {raw_url:?} to include a host; \
482                 pass --profile <name>"
483            ),
484        )
485    })?;
486    let (normalized_host, dns_name) = match host {
487        url::Host::Domain(domain) => (normalize_profile_host(domain), true),
488        url::Host::Ipv4(addr) => (addr.to_string(), false),
489        url::Host::Ipv6(addr) => (addr.to_string(), false),
490    };
491    let profile = if dns_name {
492        psl::domain_str(&normalized_host)
493            .unwrap_or(&normalized_host)
494            .to_string()
495    } else {
496        normalized_host.clone()
497    };
498    crate::sdk::profile::paths::validate_name(&profile).map_err(|e| {
499        Error::new(
500            e.error_code,
501            format!(
502                "derived --takeover profile {profile:?} from URL host {normalized_host:?} \
503                 is invalid: {}; pass --profile <name>",
504                e.detail
505            ),
506        )
507    })?;
508    Ok(profile)
509}
510
511fn normalize_profile_host(host: &str) -> String {
512    host.trim_end_matches('.').to_ascii_lowercase()
513}
514
515enum FetchRunError {
516    Plain(Error),
517    Emitted(Error),
518}
519
520impl From<Error> for FetchRunError {
521    fn from(err: Error) -> Self {
522        Self::Plain(err)
523    }
524}
525
526/// Resolve the request body from `--data` (literal, or `@path` to read a file),
527/// rejecting coexistence with `--form`. `--form` fields are wired separately by
528/// the caller; this only enforces that they don't coexist with a raw body.
529async fn resolve_body(args: &Args) -> Result<Option<Vec<u8>>, Error> {
530    if args.data.is_some() && !args.form.is_empty() {
531        return Err(Error::new(
532            ErrorCode::InvalidArgument,
533            "--data and --form are mutually exclusive",
534        ));
535    }
536    if let Some(data) = &args.data {
537        if let Some(path) = data.strip_prefix('@') {
538            return Ok(Some(tokio::fs::read(path).await.map_err(|e| {
539                Error::new(ErrorCode::IoError, format!("--data @{path}: {e}"))
540            })?));
541        }
542        return Ok(Some(data.as_bytes().to_vec()));
543    }
544    Ok(None)
545}
546
547/// Parse the `--want` tokens into an artifact set; `None` means use the
548/// render-mode-aware default.
549fn resolve_want(want: &[String]) -> Result<Option<std::collections::BTreeSet<Artifact>>, Error> {
550    if want.is_empty() {
551        return Ok(None);
552    }
553    want.iter()
554        .map(|t| parse_artifact(t))
555        .collect::<Result<std::collections::BTreeSet<_>, _>>()
556        .map(Some)
557}
558
559/// Construct the SDK client for this fetch: a remote connection when
560/// `--endpoint-url` is set, the HTTP-only client for `--render none`, or an
561/// inline ephemeral host otherwise (lazy for `auto`, eager for `always`).
562async fn build_client(args: &Args, render: RenderMode) -> Result<Client, Error> {
563    match args.endpoint.as_deref() {
564        Some(ep) => {
565            let mut c = Client::connect(ep)?;
566            if let Some(t) = args.token.as_deref() {
567                c = c.with_token(t);
568            }
569            Ok(c)
570        }
571        None if matches!(render, RenderMode::None) => Client::http_only(),
572        None => {
573            let cfg = InlineConfig {
574                browser: args.browser.into(),
575                browser_bin: args.browser_bin.clone(),
576            };
577            if matches!(render, RenderMode::Auto) {
578                Client::inline_ephemeral_lazy(cfg).await
579            } else {
580                Client::inline_ephemeral_with(cfg).await
581            }
582        }
583    }
584}
585
586fn parse_artifact(token: &str) -> Result<Artifact, Error> {
587    Ok(match token {
588        "body" => Artifact::Body,
589        "rendered_html" => Artifact::RenderedHtml,
590        "text" => Artifact::Text,
591        "content" => Artifact::Content,
592        "content_json" => Artifact::ContentJson,
593        "screenshot" => Artifact::Screenshot,
594        "network" => Artifact::Network,
595        "console" => Artifact::Console,
596        "observation" => Artifact::Observation,
597        "storage" => Artifact::Storage,
598        other => {
599            return Err(Error::new(
600                ErrorCode::InvalidArgument,
601                format!("--want: unknown artifact {other:?}"),
602            ));
603        }
604    })
605}
606
607fn parse_header_arg(raw: &str) -> Result<(String, String), Error> {
608    let (name, value) = raw.split_once(':').ok_or_else(|| {
609        Error::new(
610            ErrorCode::InvalidArgument,
611            format!("--header: expected K:V, got {raw:?}"),
612        )
613    })?;
614    let name = name.trim();
615    if name.is_empty() {
616        return Err(Error::new(
617            ErrorCode::InvalidArgument,
618            format!("--header: header name must not be empty in {raw:?}"),
619        ));
620    }
621    Ok((name.to_string(), value.trim_start().to_string()))
622}
623
624fn parse_cookie_arg(raw: &str) -> Result<FetchCookie, Error> {
625    if !raw.contains('=') {
626        return Err(Error::new(
627            ErrorCode::InvalidArgument,
628            format!("--cookie: expected Set-Cookie style name=value, got {raw:?}"),
629        ));
630    }
631    let cookie = FetchCookie::parse(raw.to_string())
632        .map_err(|e| Error::new(ErrorCode::InvalidArgument, format!("--cookie: {e}")))?
633        .into_owned();
634    if cookie.name().trim().is_empty() {
635        return Err(Error::new(
636            ErrorCode::InvalidArgument,
637            format!("--cookie: cookie name must not be empty in {raw:?}"),
638        ));
639    }
640    Ok(cookie)
641}
642
643#[cfg(test)]
644mod tests {
645    use super::*;
646
647    fn base_args(url: &str) -> Args {
648        Args {
649            url: url.to_string(),
650            endpoint: None,
651            token: None,
652            browser: BrowserArg::Auto,
653            browser_bin: None,
654            render: RenderArg::Auto,
655            tab: "new".into(),
656            takeover: false,
657            profile: None,
658            wait: "auto".into(),
659            headers: Vec::new(),
660            cookies: Vec::new(),
661            user_agent: None,
662            evaluate_after_wait: Vec::new(),
663            want: Vec::new(),
664            method: "GET".into(),
665            data: None,
666            form: Vec::new(),
667            network_bodies: NetworkBodiesArg::Off,
668            network_body_max_bytes: DEFAULT_NETWORK_BODY_MAX_BYTES,
669            readiness_idle_ms: 800,
670            readiness_stable_ms: 500,
671            readiness_min_text_bytes: 32,
672            no_network_redact: false,
673            out: None,
674            cookie_jar: None,
675            no_cookie_jar: false,
676            observe_main_wait_ms: 500,
677            max_response_bytes: 1_073_741_824,
678            retry: 0,
679            backoff_ms: 250,
680            proxy: None,
681            ca_cert: None,
682            tls_insecure: false,
683            timeout_ms: 30_000,
684            capture_ws: false,
685            capture_sse: false,
686        }
687    }
688
689    #[tokio::test]
690    async fn takeover_autodiscovery_fills_missing_endpoint_and_token() {
691        let mut args = base_args("https://contabo.com");
692        args.takeover = true;
693        prepare_takeover_connection(&mut args, RenderMode::Auto, |token| async move {
694            assert_eq!(token, None);
695            Ok(crate::cli::cmd::container::LocalTakeoverHost {
696                endpoint: "ws://127.0.0.1:9222".into(),
697                token_secret: Some("secret".into()),
698            })
699        })
700        .await
701        .unwrap();
702
703        assert_eq!(args.endpoint.as_deref(), Some("ws://127.0.0.1:9222"));
704        assert_eq!(args.token.as_deref(), Some("secret"));
705    }
706
707    #[tokio::test]
708    async fn takeover_autodiscovery_preserves_existing_token() {
709        let mut args = base_args("https://contabo.com");
710        args.takeover = true;
711        args.token = Some("env-token".into());
712        prepare_takeover_connection(&mut args, RenderMode::Auto, |token| async move {
713            assert_eq!(token.as_deref(), Some("env-token"));
714            Ok(crate::cli::cmd::container::LocalTakeoverHost {
715                endpoint: "ws://127.0.0.1:9222".into(),
716                token_secret: Some("container-token".into()),
717            })
718        })
719        .await
720        .unwrap();
721
722        assert_eq!(args.endpoint.as_deref(), Some("ws://127.0.0.1:9222"));
723        assert_eq!(args.token.as_deref(), Some("env-token"));
724    }
725
726    #[tokio::test]
727    async fn takeover_autodiscovery_surfaces_failure() {
728        let mut args = base_args("https://contabo.com");
729        args.takeover = true;
730        let err = prepare_takeover_connection(&mut args, RenderMode::Auto, |_| async {
731            Err(Error::new(
732                ErrorCode::InvalidArgument,
733                "default local container `afhttp-host` is not running",
734            ))
735        })
736        .await
737        .err()
738        .unwrap();
739
740        assert_eq!(err.error_code, ErrorCode::InvalidArgument);
741        assert!(err.detail.contains("afhttp-host"));
742        assert!(args.endpoint.is_none());
743    }
744
745    #[tokio::test]
746    async fn takeover_autodiscovery_rejects_render_none() {
747        let mut args = base_args("https://contabo.com");
748        args.takeover = true;
749        let err = prepare_takeover_connection(&mut args, RenderMode::None, |_| async {
750            Ok(crate::cli::cmd::container::LocalTakeoverHost {
751                endpoint: "ws://127.0.0.1:9222".into(),
752                token_secret: None,
753            })
754        })
755        .await
756        .err()
757        .unwrap();
758
759        assert_eq!(err.error_code, ErrorCode::InvalidArgument);
760        assert!(err.detail.contains("browser render"));
761    }
762
763    #[test]
764    fn takeover_recommendation_omits_auto_discovered_endpoint() {
765        assert_eq!(
766            takeover_recommended_endpoint(Some("ws://127.0.0.1:9222"), false),
767            None
768        );
769    }
770
771    #[test]
772    fn takeover_recommendation_keeps_preconfigured_endpoint() {
773        assert_eq!(
774            takeover_recommended_endpoint(Some("ws://127.0.0.1:9222"), true),
775            Some("ws://127.0.0.1:9222")
776        );
777    }
778
779    #[test]
780    fn default_profile_uses_registrable_domain() {
781        assert_eq!(
782            default_profile_for_url("https://www.court.gov.cn/foo").unwrap(),
783            "court.gov.cn"
784        );
785        assert_eq!(
786            default_profile_for_url("https://accounts.google.com/foo").unwrap(),
787            "google.com"
788        );
789        assert_eq!(
790            default_profile_for_url("https://contabo.com").unwrap(),
791            "contabo.com"
792        );
793    }
794
795    #[test]
796    fn default_profile_keeps_public_suffix_tenants_isolated() {
797        assert_eq!(
798            default_profile_for_url("https://foo.github.io/x").unwrap(),
799            "foo.github.io"
800        );
801        assert_eq!(
802            default_profile_for_url("https://tenant.vercel.app/x").unwrap(),
803            "tenant.vercel.app"
804        );
805    }
806
807    #[test]
808    fn default_profile_normalizes_case_and_trailing_dot() {
809        assert_eq!(
810            default_profile_for_url("https://WWW.Example.COM./foo").unwrap(),
811            "example.com"
812        );
813    }
814
815    #[test]
816    fn default_profile_falls_back_to_full_host_for_psl_misses_and_ips() {
817        assert_eq!(
818            default_profile_for_url("http://localhost:8080/foo").unwrap(),
819            "localhost"
820        );
821        assert_eq!(
822            default_profile_for_url("http://127.0.0.1:8080/foo").unwrap(),
823            "127.0.0.1"
824        );
825    }
826
827    #[test]
828    fn default_profile_errors_when_host_is_missing() {
829        assert!(default_profile_for_url("file:///tmp/page.html").is_err());
830    }
831
832    #[test]
833    fn header_arg_accepts_colon_separator() {
834        assert_eq!(
835            parse_header_arg("X-Test: yes").unwrap(),
836            ("X-Test".to_string(), "yes".to_string())
837        );
838    }
839
840    #[test]
841    fn header_arg_rejects_missing_colon() {
842        let err = parse_header_arg("X-Test").err().unwrap();
843        assert_eq!(err.error_code, ErrorCode::InvalidArgument);
844    }
845
846    #[test]
847    fn cookie_arg_accepts_equals_separator() {
848        let cookie = parse_cookie_arg("sid=abc=def").unwrap();
849        assert_eq!(cookie.name_value(), ("sid", "abc=def"));
850    }
851
852    #[test]
853    fn cookie_arg_accepts_full_set_cookie_attributes() {
854        let cookie = parse_cookie_arg("sid=abc; Path=/; Secure; HttpOnly; SameSite=Lax").unwrap();
855        assert_eq!(cookie.name_value(), ("sid", "abc"));
856        assert_eq!(cookie.path(), Some("/"));
857        assert_eq!(cookie.secure(), Some(true));
858        assert_eq!(cookie.http_only(), Some(true));
859        assert_eq!(cookie.same_site(), Some(cookie::SameSite::Lax));
860    }
861
862    #[test]
863    fn cookie_arg_rejects_missing_equals() {
864        let err = parse_cookie_arg("sid").err().unwrap();
865        assert_eq!(err.error_code, ErrorCode::InvalidArgument);
866    }
867}
agent_first_http/cli/cmd/fetch.rs

agent_first_http/cli/cmd/
fetch.rs
