agent_first_http/cli/cmd/
fetch.rs

1//! `afhttp fetch` subcommand.
2
3use std::path::PathBuf;
4use std::time::Duration;
5
6use crate::cli::output;
7use crate::host::bootstrap::BrowserChoice;
8use crate::sdk::fetch::{FetchCookie, NetworkBodies, RenderMode, Wait};
9use crate::sdk::{Client, InlineConfig};
10use crate::shared::artifacts::Artifact;
11use crate::shared::error::{Error, ErrorCode};
12use crate::shared::ids::TabId;
13
14/// One fetch, already narrowed to a registered shape: `data` and `form` are
15/// never both set, and `takeover` never coexists with [`RenderMode::None`].
16#[derive(Debug)]
17pub struct Args {
18    pub url: String,
19    pub endpoint: Option<String>,
20    pub token: Option<String>,
21    pub browser: BrowserChoice,
22    pub browser_bin: Option<PathBuf>,
23    pub render: RenderMode,
24    pub tab: String,
25    pub takeover: bool,
26    pub profile: Option<String>,
27    pub wait: String,
28    pub headers: Vec<String>,
29    pub cookies: Vec<String>,
30    pub user_agent: Option<String>,
31    pub evaluate_after_wait: Vec<String>,
32    pub want: Vec<String>,
33    pub method: String,
34    pub data: Option<String>,
35    pub form: Vec<String>,
36    pub network_bodies: NetworkBodies,
37    pub network_body_max_bytes: u64,
38    pub readiness_idle_ms: u64,
39    pub readiness_stable_ms: u64,
40    pub readiness_min_text_bytes: u64,
41    pub no_network_redact: bool,
42    pub out: Option<PathBuf>,
43    pub cookie_jar: Option<PathBuf>,
44    pub no_cookie_jar: bool,
45    pub observe_main_wait_ms: u64,
46    pub max_response_bytes: u64,
47    pub retry: u32,
48    pub backoff_ms: u64,
49    pub proxy: Option<String>,
50    pub ca_cert: Option<PathBuf>,
51    pub tls_insecure: bool,
52    pub timeout_ms: u64,
53    pub capture_ws: bool,
54    pub capture_sse: bool,
55}
56
57pub async fn run(args: Args) -> Result<(), Error> {
58    match run_inner(args).await {
59        Ok(()) => Ok(()),
60        Err(FetchRunError::Plain(err)) => {
61            let _ = crate::shared::afdata::emit_process_error(&err);
62            Err(err)
63        }
64        Err(FetchRunError::Emitted(err)) => Err(err),
65    }
66}
67
68async fn run_inner(mut args: Args) -> Result<(), FetchRunError> {
69    let render = args.render;
70    let endpoint_was_preconfigured = args.endpoint.is_some();
71    prepare_takeover_connection(&mut args, |token| async move {
72        crate::cli::cmd::container::discover_default_takeover_host(token.as_deref()).await
73    })
74    .await?;
75    // Resolve the host profile: explicit --profile wins; otherwise --takeover
76    // derives the URL's registrable domain (eTLD+1) for per-domain isolation.
77    let explicit_profile = args.profile.clone();
78    let resolved_profile: Option<String> = if let Some(p) = explicit_profile.clone() {
79        Some(p)
80    } else if args.takeover {
81        Some(default_profile_for_url(&args.url)?)
82    } else {
83        None
84    };
85    if resolved_profile.is_some() && args.endpoint.is_none() {
86        return Err(Error::new(
87            ErrorCode::InvalidArgument,
88            "--profile (and --takeover profile derivation) switch the host's active profile and require a host; pass --endpoint-url or set AFHTTP_ENDPOINT_URL",
89        )
90        .into());
91    }
92    let takeover = args.takeover;
93    let takeover_endpoint = args.endpoint.clone();
94    let recommended_endpoint =
95        takeover_recommended_endpoint(takeover_endpoint.as_deref(), endpoint_was_preconfigured);
96    let takeover_token = args.token.clone();
97    let wait = Wait::parse(&args.wait)?;
98    let timeout = Duration::from_millis(args.timeout_ms);
99    let network_bodies = args.network_bodies;
100    let network_redact = !args.no_network_redact;
101
102    let body_bytes = resolve_body(&args).await?;
103    let want = resolve_want(&args.want);
104    let mut client = build_client(&args, render).await?;
105    if let Some(profile) = &resolved_profile {
106        client = client.with_profile(profile.clone());
107    }
108
109    let mut builder = client
110        .fetch(args.url.clone())
111        .render(render)
112        .wait(wait)
113        .timeout(timeout)
114        .network_bodies(network_bodies)
115        .network_body_max_bytes(args.network_body_max_bytes)
116        .readiness_idle_ms(args.readiness_idle_ms)
117        .readiness_stable_ms(args.readiness_stable_ms)
118        .readiness_min_text_bytes(args.readiness_min_text_bytes)
119        .network_redact(network_redact)
120        .method(args.method);
121    if let Some(want) = want {
122        builder = builder.want(want);
123    }
124    if let Some(bytes) = body_bytes {
125        builder = builder.body(bytes);
126    }
127    for raw in &args.form {
128        let (k, v) = raw.split_once('=').ok_or_else(|| {
129            Error::new(
130                ErrorCode::InvalidArgument,
131                format!("--form: expected key=value, got {raw:?}"),
132            )
133        })?;
134        builder = builder.form_field(k, v);
135    }
136    for raw in args.headers {
137        let (name, value) = parse_header_arg(&raw)?;
138        builder = builder.header(name, value);
139    }
140    for raw in args.cookies {
141        builder = builder.cookie_full(parse_cookie_arg(&raw)?);
142    }
143    if let Some(user_agent) = args.user_agent {
144        builder = builder.user_agent(user_agent);
145    }
146    for js in args.evaluate_after_wait {
147        builder = builder.evaluate_after_wait(js);
148    }
149    if args.tab != "new" {
150        builder = builder.tab(TabId::new(args.tab));
151    }
152    if takeover {
153        // Keep the prepared tab open so a human can take it over.
154        builder = builder.keep_tab_open(true);
155    }
156    if let Some(out) = args.out {
157        builder = builder.out_dir(out);
158    }
159    builder = builder.observe_main_wait_ms(args.observe_main_wait_ms);
160    builder = builder.max_response_bytes(args.max_response_bytes);
161    builder = builder.retry(args.retry).backoff_ms(args.backoff_ms);
162    if let Some(url) = args.proxy {
163        builder = builder.proxy(url);
164    }
165    if let Some(path) = args.ca_cert {
166        builder = builder.ca_cert(path);
167    }
168    if args.tls_insecure {
169        builder = builder.tls_insecure(true);
170    }
171    if args.capture_ws {
172        builder = builder.capture_ws(true);
173    }
174    if args.capture_sse {
175        builder = builder.capture_sse(true);
176    }
177    if args.no_cookie_jar {
178        builder = builder.no_cookie_jar();
179    } else {
180        let cookie_jar = args.cookie_jar.or_else(|| {
181            std::env::var_os("AFHTTP_COOKIE_JAR")
182                .filter(|v| !v.is_empty())
183                .map(PathBuf::from)
184        });
185        if let Some(jar) = cookie_jar {
186            builder = builder.cookie_jar(jar);
187        }
188    }
189
190    match builder.send_detailed().await {
191        Ok(mut result) => {
192            if takeover
193                && result.next_action.is_some()
194                && let Some(endpoint) = takeover_endpoint.as_deref()
195            {
196                let mut handoff_client = Client::connect(endpoint)?;
197                if let Some(token) = takeover_token.as_deref() {
198                    handoff_client = handoff_client.with_token(token);
199                }
200                let tab_id = result.tab_id.as_ref().map(|t| t.as_str().to_string());
201                let handoff = handoff_client
202                    .takeover_handoff(None, tab_id.as_deref())
203                    .await?;
204                result.attach_takeover_with_context(
205                    handoff.takeover_url_secret,
206                    Some(handoff.takeover_url_expires_at_rfc3339),
207                    Some(handoff.takeover_url_ttl_s),
208                    Some(handoff.takeover_url_scope),
209                    recommended_endpoint,
210                    explicit_profile.as_deref(),
211                );
212            }
213            if takeover {
214                Ok(output::emit_revealing_takeover("fetch", &result)?)
215            } else {
216                Ok(output::emit("fetch", &result)?)
217            }
218        }
219        Err(err) => {
220            let trace = serde_json::to_value(&err.trace).map_err(|e| {
221                Error::new(
222                    ErrorCode::InternalError,
223                    format!("serialize fetch error trace: {e}"),
224                )
225            })?;
226            let err = err.into_error();
227            crate::shared::afdata::emit_process_error_with(
228                err.error_code.as_str(),
229                &err.detail,
230                serde_json::json!({"retryable": err.retryable}),
231                trace,
232            )?;
233            Err(FetchRunError::Emitted(err))
234        }
235    }
236}
237
238/// Fill in the host `--takeover` needs when the caller did not name one.
239///
240/// That `--takeover` requires a browser render is not checked here: the
241/// registry's takeover shapes pin `--render` to the modes that reach one, so an
242/// unusable mix never becomes an invocation.
243async fn prepare_takeover_connection<D, Fut>(args: &mut Args, discover: D) -> Result<(), Error>
244where
245    D: FnOnce(Option<String>) -> Fut,
246    Fut: std::future::Future<Output = Result<crate::cli::cmd::container::LocalTakeoverHost, Error>>,
247{
248    if !args.takeover {
249        return Ok(());
250    }
251    if args.endpoint.is_none() {
252        let discovered = discover(args.token.clone()).await?;
253        args.endpoint = Some(discovered.endpoint);
254        if args.token.is_none() {
255            args.token = discovered.token_secret;
256        }
257    }
258    Ok(())
259}
260
261fn takeover_recommended_endpoint(
262    endpoint: Option<&str>,
263    endpoint_was_preconfigured: bool,
264) -> Option<&str> {
265    if endpoint_was_preconfigured {
266        endpoint
267    } else {
268        None
269    }
270}
271
272/// Derive a default host profile from a URL's registrable domain (eTLD+1), so
273/// `fetch --takeover` gives each site its own isolated browser profile when
274/// `--profile` is omitted. Falls back to the full host for public-suffix
275/// tenants (e.g. `foo.github.io`) and bare IPs.
276fn default_profile_for_url(raw_url: &str) -> Result<String, Error> {
277    let parsed = url::Url::parse(raw_url).map_err(|e| {
278        Error::new(
279            ErrorCode::InvalidArgument,
280            format!(
281                "--takeover default profile needs a valid URL with a host; \
282                 could not parse {raw_url:?}: {e}; pass --profile <name>"
283            ),
284        )
285    })?;
286    let host = parsed.host().ok_or_else(|| {
287        Error::new(
288            ErrorCode::InvalidArgument,
289            format!(
290                "--takeover default profile needs URL {raw_url:?} to include a host; \
291                 pass --profile <name>"
292            ),
293        )
294    })?;
295    let (normalized_host, dns_name) = match host {
296        url::Host::Domain(domain) => (normalize_profile_host(domain), true),
297        url::Host::Ipv4(addr) => (addr.to_string(), false),
298        url::Host::Ipv6(addr) => (addr.to_string(), false),
299    };
300    let profile = if dns_name {
301        psl::domain_str(&normalized_host)
302            .unwrap_or(&normalized_host)
303            .to_string()
304    } else {
305        normalized_host.clone()
306    };
307    crate::sdk::profile::paths::validate_name(&profile).map_err(|e| {
308        Error::new(
309            e.error_code,
310            format!(
311                "derived --takeover profile {profile:?} from URL host {normalized_host:?} \
312                 is invalid: {}; pass --profile <name>",
313                e.detail
314            ),
315        )
316    })?;
317    Ok(profile)
318}
319
320fn normalize_profile_host(host: &str) -> String {
321    host.trim_end_matches('.').to_ascii_lowercase()
322}
323
324enum FetchRunError {
325    Plain(Error),
326    Emitted(Error),
327}
328
329impl From<Error> for FetchRunError {
330    fn from(err: Error) -> Self {
331        Self::Plain(err)
332    }
333}
334
335/// Resolve the request body from `--data` (literal, or `@path` to read a file).
336///
337/// `--form` fields are wired separately by the caller, and the registry's three
338/// body shapes already guarantee the two never arrive together.
339async fn resolve_body(args: &Args) -> Result<Option<Vec<u8>>, Error> {
340    if let Some(data) = &args.data {
341        if let Some(path) = data.strip_prefix('@') {
342            return Ok(Some(tokio::fs::read(path).await.map_err(|e| {
343                Error::new(ErrorCode::IoError, format!("--data @{path}: {e}"))
344            })?));
345        }
346        return Ok(Some(data.as_bytes().to_vec()));
347    }
348    Ok(None)
349}
350
351/// Parse the `--want` tokens into an artifact set; `None` means use the
352/// render-mode-aware default. The registry declares `--want` as a closed value
353/// set, so every token that reaches here names an artifact.
354fn resolve_want(want: &[String]) -> Option<std::collections::BTreeSet<Artifact>> {
355    if want.is_empty() {
356        return None;
357    }
358    Some(
359        want.iter()
360            .filter_map(|token| parse_artifact(token))
361            .collect(),
362    )
363}
364
365/// Construct the SDK client for this fetch: a remote connection when
366/// `--endpoint-url` is set, the HTTP-only client for `--render none`, or an
367/// inline ephemeral host otherwise (lazy for `auto`, eager for `always`).
368async fn build_client(args: &Args, render: RenderMode) -> Result<Client, Error> {
369    match args.endpoint.as_deref() {
370        Some(ep) => {
371            let mut c = Client::connect(ep)?;
372            if let Some(t) = args.token.as_deref() {
373                c = c.with_token(t);
374            }
375            Ok(c)
376        }
377        None if matches!(render, RenderMode::None) => Client::http_only(),
378        None => {
379            let cfg = InlineConfig {
380                browser: args.browser.clone(),
381                browser_bin: args.browser_bin.clone(),
382            };
383            if matches!(render, RenderMode::Auto) {
384                Client::inline_ephemeral_lazy(cfg).await
385            } else {
386                Client::inline_ephemeral_with(cfg).await
387            }
388        }
389    }
390}
391
392fn parse_artifact(token: &str) -> Option<Artifact> {
393    Some(match token {
394        "body" => Artifact::Body,
395        "rendered_html" => Artifact::RenderedHtml,
396        "text" => Artifact::Text,
397        "content" => Artifact::Content,
398        "content_json" => Artifact::ContentJson,
399        "screenshot" => Artifact::Screenshot,
400        "network" => Artifact::Network,
401        "console" => Artifact::Console,
402        "observation" => Artifact::Observation,
403        "storage" => Artifact::Storage,
404        _ => return None,
405    })
406}
407
408fn parse_header_arg(raw: &str) -> Result<(String, String), Error> {
409    let (name, value) = raw.split_once(':').ok_or_else(|| {
410        Error::new(
411            ErrorCode::InvalidArgument,
412            format!("--header: expected K:V, got {raw:?}"),
413        )
414    })?;
415    let name = name.trim();
416    if name.is_empty() {
417        return Err(Error::new(
418            ErrorCode::InvalidArgument,
419            format!("--header: header name must not be empty in {raw:?}"),
420        ));
421    }
422    Ok((name.to_string(), value.trim_start().to_string()))
423}
424
425fn parse_cookie_arg(raw: &str) -> Result<FetchCookie, Error> {
426    if !raw.contains('=') {
427        return Err(Error::new(
428            ErrorCode::InvalidArgument,
429            format!("--cookie: expected Set-Cookie style name=value, got {raw:?}"),
430        ));
431    }
432    let cookie = FetchCookie::parse(raw.to_string())
433        .map_err(|e| Error::new(ErrorCode::InvalidArgument, format!("--cookie: {e}")))?
434        .into_owned();
435    if cookie.name().trim().is_empty() {
436        return Err(Error::new(
437            ErrorCode::InvalidArgument,
438            format!("--cookie: cookie name must not be empty in {raw:?}"),
439        ));
440    }
441    Ok(cookie)
442}
443
444#[cfg(test)]
445mod tests {
446    use super::*;
447
448    use crate::sdk::fetch::DEFAULT_NETWORK_BODY_MAX_BYTES;
449
450    fn base_args(url: &str) -> Args {
451        Args {
452            url: url.to_string(),
453            endpoint: None,
454            token: None,
455            browser: BrowserChoice::Auto,
456            browser_bin: None,
457            render: RenderMode::Auto,
458            tab: "new".into(),
459            takeover: false,
460            profile: None,
461            wait: "auto".into(),
462            headers: Vec::new(),
463            cookies: Vec::new(),
464            user_agent: None,
465            evaluate_after_wait: Vec::new(),
466            want: Vec::new(),
467            method: "GET".into(),
468            data: None,
469            form: Vec::new(),
470            network_bodies: NetworkBodies::Off,
471            network_body_max_bytes: DEFAULT_NETWORK_BODY_MAX_BYTES,
472            readiness_idle_ms: 800,
473            readiness_stable_ms: 500,
474            readiness_min_text_bytes: 32,
475            no_network_redact: false,
476            out: None,
477            cookie_jar: None,
478            no_cookie_jar: false,
479            observe_main_wait_ms: 500,
480            max_response_bytes: 1_073_741_824,
481            retry: 0,
482            backoff_ms: 250,
483            proxy: None,
484            ca_cert: None,
485            tls_insecure: false,
486            timeout_ms: 30_000,
487            capture_ws: false,
488            capture_sse: false,
489        }
490    }
491
492    #[tokio::test]
493    async fn takeover_autodiscovery_fills_missing_endpoint_and_token() {
494        let mut args = base_args("https://contabo.com");
495        args.takeover = true;
496        prepare_takeover_connection(&mut args, |token| async move {
497            assert_eq!(token, None);
498            Ok(crate::cli::cmd::container::LocalTakeoverHost {
499                endpoint: "ws://127.0.0.1:9222".into(),
500                token_secret: Some("secret".into()),
501            })
502        })
503        .await
504        .unwrap();
505
506        assert_eq!(args.endpoint.as_deref(), Some("ws://127.0.0.1:9222"));
507        assert_eq!(args.token.as_deref(), Some("secret"));
508    }
509
510    #[tokio::test]
511    async fn takeover_autodiscovery_preserves_existing_token() {
512        let mut args = base_args("https://contabo.com");
513        args.takeover = true;
514        args.token = Some("env-token".into());
515        prepare_takeover_connection(&mut args, |token| async move {
516            assert_eq!(token.as_deref(), Some("env-token"));
517            Ok(crate::cli::cmd::container::LocalTakeoverHost {
518                endpoint: "ws://127.0.0.1:9222".into(),
519                token_secret: Some("container-token".into()),
520            })
521        })
522        .await
523        .unwrap();
524
525        assert_eq!(args.endpoint.as_deref(), Some("ws://127.0.0.1:9222"));
526        assert_eq!(args.token.as_deref(), Some("env-token"));
527    }
528
529    #[tokio::test]
530    async fn takeover_autodiscovery_surfaces_failure() {
531        let mut args = base_args("https://contabo.com");
532        args.takeover = true;
533        let err = prepare_takeover_connection(&mut args, |_| async {
534            Err(Error::new(
535                ErrorCode::InvalidArgument,
536                "default local container `afhttp-host` is not running",
537            ))
538        })
539        .await
540        .err()
541        .unwrap();
542
543        assert_eq!(err.error_code, ErrorCode::InvalidArgument);
544        assert!(err.detail.contains("afhttp-host"));
545        assert!(args.endpoint.is_none());
546    }
547
548    /// Every value the registry lets `--want` carry names a real artifact, so
549    /// the projection can be total.
550    #[test]
551    fn every_registry_want_value_names_an_artifact() {
552        for token in crate::cli::spec::ARTIFACTS {
553            assert!(parse_artifact(token).is_some(), "{token}");
554        }
555        let wanted = resolve_want(&["body".to_string(), "network".to_string()])
556            .expect("an explicit want is a set");
557        assert!(wanted.contains(&Artifact::Body));
558        assert!(wanted.contains(&Artifact::Network));
559        assert!(resolve_want(&[]).is_none(), "no want means the default set");
560    }
561
562    #[test]
563    fn takeover_recommendation_omits_auto_discovered_endpoint() {
564        assert_eq!(
565            takeover_recommended_endpoint(Some("ws://127.0.0.1:9222"), false),
566            None
567        );
568    }
569
570    #[test]
571    fn takeover_recommendation_keeps_preconfigured_endpoint() {
572        assert_eq!(
573            takeover_recommended_endpoint(Some("ws://127.0.0.1:9222"), true),
574            Some("ws://127.0.0.1:9222")
575        );
576    }
577
578    #[test]
579    fn default_profile_uses_registrable_domain() {
580        assert_eq!(
581            default_profile_for_url("https://www.court.gov.cn/foo").unwrap(),
582            "court.gov.cn"
583        );
584        assert_eq!(
585            default_profile_for_url("https://accounts.google.com/foo").unwrap(),
586            "google.com"
587        );
588        assert_eq!(
589            default_profile_for_url("https://contabo.com").unwrap(),
590            "contabo.com"
591        );
592    }
593
594    #[test]
595    fn default_profile_keeps_public_suffix_tenants_isolated() {
596        assert_eq!(
597            default_profile_for_url("https://foo.github.io/x").unwrap(),
598            "foo.github.io"
599        );
600        assert_eq!(
601            default_profile_for_url("https://tenant.vercel.app/x").unwrap(),
602            "tenant.vercel.app"
603        );
604    }
605
606    #[test]
607    fn default_profile_normalizes_case_and_trailing_dot() {
608        assert_eq!(
609            default_profile_for_url("https://WWW.Example.COM./foo").unwrap(),
610            "example.com"
611        );
612    }
613
614    #[test]
615    fn default_profile_falls_back_to_full_host_for_psl_misses_and_ips() {
616        assert_eq!(
617            default_profile_for_url("http://localhost:8080/foo").unwrap(),
618            "localhost"
619        );
620        assert_eq!(
621            default_profile_for_url("http://127.0.0.1:8080/foo").unwrap(),
622            "127.0.0.1"
623        );
624    }
625
626    #[test]
627    fn default_profile_errors_when_host_is_missing() {
628        assert!(default_profile_for_url("file:///tmp/page.html").is_err());
629    }
630
631    #[test]
632    fn header_arg_accepts_colon_separator() {
633        assert_eq!(
634            parse_header_arg("X-Test: yes").unwrap(),
635            ("X-Test".to_string(), "yes".to_string())
636        );
637    }
638
639    #[test]
640    fn header_arg_rejects_missing_colon() {
641        let err = parse_header_arg("X-Test").err().unwrap();
642        assert_eq!(err.error_code, ErrorCode::InvalidArgument);
643    }
644
645    #[test]
646    fn cookie_arg_accepts_equals_separator() {
647        let cookie = parse_cookie_arg("sid=abc=def").unwrap();
648        assert_eq!(cookie.name_value(), ("sid", "abc=def"));
649    }
650
651    #[test]
652    fn cookie_arg_accepts_full_set_cookie_attributes() {
653        let cookie = parse_cookie_arg("sid=abc; Path=/; Secure; HttpOnly; SameSite=Lax").unwrap();
654        assert_eq!(cookie.name_value(), ("sid", "abc"));
655        assert_eq!(cookie.path(), Some("/"));
656        assert_eq!(cookie.secure(), Some(true));
657        assert_eq!(cookie.http_only(), Some(true));
658        assert_eq!(cookie.same_site(), Some(cookie::SameSite::Lax));
659    }
660
661    #[test]
662    fn cookie_arg_rejects_missing_equals() {
663        let err = parse_cookie_arg("sid").err().unwrap();
664        assert_eq!(err.error_code, ErrorCode::InvalidArgument);
665    }
666}
agent_first_http/cli/cmd/fetch.rs

agent_first_http/cli/cmd/
fetch.rs
