1use std::path::PathBuf;
4use std::time::Duration;
5
6use crate::cli::connect::{Connection, Resolved};
7use crate::cli::output;
8use crate::cli::token_source;
9use crate::host::bootstrap::BrowserChoice;
10use crate::sdk::fetch::{FetchCookie, NetworkBodies, RenderMode, Wait};
11use crate::sdk::{Client, InlineConfig};
12use crate::shared::artifacts::Artifact;
13use crate::shared::error::{Error, ErrorCode};
14use crate::shared::ids::TabId;
15
16#[derive(Debug)]
19pub struct Args {
20 pub url: String,
21 pub connection: Connection,
22 pub browser: BrowserChoice,
23 pub browser_bin: Option<PathBuf>,
24 pub render: RenderMode,
25 pub tab: String,
26 pub takeover: bool,
27 pub profile: Option<String>,
28 pub wait: String,
29 pub headers: Vec<String>,
30 pub cookies: Vec<String>,
31 pub user_agent: Option<String>,
32 pub evaluate_after_wait: Vec<String>,
33 pub want: Vec<String>,
34 pub method: String,
35 pub data: Option<String>,
36 pub form: Vec<String>,
37 pub network_bodies: NetworkBodies,
38 pub network_body_max_bytes: u64,
39 pub readiness_idle_ms: u64,
40 pub readiness_stable_ms: u64,
41 pub readiness_min_text_bytes: u64,
42 pub no_network_redact: bool,
43 pub out: Option<PathBuf>,
44 pub cookie_jar: Option<PathBuf>,
45 pub no_cookie_jar: bool,
46 pub observe_main_wait_ms: u64,
47 pub max_response_bytes: u64,
48 pub retry: u32,
49 pub backoff_ms: u64,
50 pub proxy: Option<String>,
51 pub ca_cert: Option<PathBuf>,
52 pub tls_insecure: bool,
53 pub timeout_ms: u64,
54 pub capture_ws: bool,
55 pub capture_sse: bool,
56}
57
58pub async fn run(args: Args) -> Result<(), Error> {
59 match run_inner(args).await {
60 Ok(()) => Ok(()),
61 Err(FetchRunError::Plain(err)) => {
62 let _ = crate::shared::afdata::emit_process_error(&err);
63 Err(err)
64 }
65 Err(FetchRunError::Emitted(err)) => Err(err),
66 }
67}
68
69async fn run_inner(args: Args) -> Result<(), FetchRunError> {
70 let render = args.render;
71 let endpoint_was_preconfigured = args.connection.is_explicit();
72 let explicit_profile = args.profile.clone();
75 let resolved_profile: Option<String> = if let Some(p) = explicit_profile.clone() {
76 Some(p)
77 } else if args.takeover {
78 Some(default_profile_for_url(&args.url)?)
79 } else {
80 None
81 };
82 let host = if args.takeover {
87 prepare_host(
88 &args.connection,
89 true,
90 resolved_profile.as_deref(),
91 |token| async move {
92 crate::cli::cmd::container::discover_default_takeover_host(token.as_deref()).await
93 },
94 )
95 .await?
96 } else {
97 prepare_host(
98 &args.connection,
99 false,
100 resolved_profile.as_deref(),
101 |token| async move {
102 crate::cli::cmd::container::discover_default_local_host(token.as_deref()).await
103 },
104 )
105 .await?
106 };
107 let takeover = args.takeover;
108 let takeover_endpoint = host.as_ref().map(|host| host.endpoint.clone());
109 let recommended_endpoint =
110 takeover_recommended_endpoint(takeover_endpoint.as_deref(), endpoint_was_preconfigured);
111 let takeover_token = host.as_ref().and_then(|host| host.token.clone());
112 let wait = Wait::parse(&args.wait)?;
113 let timeout = Duration::from_millis(args.timeout_ms);
114 let network_bodies = args.network_bodies;
115 let network_redact = !args.no_network_redact;
116
117 let body_bytes = resolve_body(&args).await?;
118 let want = resolve_want(&args.want);
119 let mut client = build_client(&args, host.as_ref(), render).await?;
120 if let Some(profile) = &resolved_profile {
121 client = client.with_profile(profile.clone());
122 }
123
124 let mut builder = client
125 .fetch(args.url.clone())
126 .render(render)
127 .wait(wait)
128 .timeout(timeout)
129 .network_bodies(network_bodies)
130 .network_body_max_bytes(args.network_body_max_bytes)
131 .readiness_idle_ms(args.readiness_idle_ms)
132 .readiness_stable_ms(args.readiness_stable_ms)
133 .readiness_min_text_bytes(args.readiness_min_text_bytes)
134 .network_redact(network_redact)
135 .method(args.method);
136 if let Some(want) = want {
137 builder = builder.want(want);
138 }
139 if let Some(bytes) = body_bytes {
140 builder = builder.body(bytes);
141 }
142 for raw in &args.form {
143 let (k, v) = raw.split_once('=').ok_or_else(|| {
146 Error::new(
147 ErrorCode::InvalidArgument,
148 "--form: expected KEY=VALUE, with an equals sign separating them".to_string(),
149 )
150 })?;
151 builder = builder.form_field(k, v);
152 }
153 for raw in args.headers {
154 let (name, value) = parse_header_arg(&raw)?;
155 builder = builder.header(name, value);
156 }
157 for raw in args.cookies {
158 builder = builder.cookie_full(parse_cookie_arg(&raw)?);
159 }
160 if let Some(user_agent) = args.user_agent {
161 builder = builder.user_agent(user_agent);
162 }
163 for js in args.evaluate_after_wait {
164 builder = builder.evaluate_after_wait(js);
165 }
166 if args.tab != "new" {
167 builder = builder.tab(TabId::new(args.tab));
168 }
169 if takeover {
170 builder = builder.keep_tab_open(true);
172 }
173 if let Some(out) = args.out {
174 builder = builder.out_dir(out);
175 }
176 builder = builder.observe_main_wait_ms(args.observe_main_wait_ms);
177 builder = builder.max_response_bytes(args.max_response_bytes);
178 builder = builder.retry(args.retry).backoff_ms(args.backoff_ms);
179 if let Some(url) = args.proxy {
180 builder = builder.proxy(url);
181 }
182 if let Some(path) = args.ca_cert {
183 builder = builder.ca_cert(path);
184 }
185 if args.tls_insecure {
186 builder = builder.tls_insecure(true);
187 }
188 if args.capture_ws {
189 builder = builder.capture_ws(true);
190 }
191 if args.capture_sse {
192 builder = builder.capture_sse(true);
193 }
194 if args.no_cookie_jar {
195 builder = builder.no_cookie_jar();
196 } else {
197 let cookie_jar = args.cookie_jar.or_else(|| {
198 std::env::var_os("AFHTTP_COOKIE_JAR")
199 .filter(|v| !v.is_empty())
200 .map(PathBuf::from)
201 });
202 if let Some(jar) = cookie_jar {
203 builder = builder.cookie_jar(jar);
204 }
205 }
206
207 match builder.send_detailed().await {
208 Ok(mut result) => {
209 if takeover
210 && result.next_action.is_some()
211 && let Some(endpoint) = takeover_endpoint.as_deref()
212 {
213 let mut handoff_client = Client::connect(endpoint)?;
214 if let Some(token) = &takeover_token {
215 handoff_client = handoff_client.with_token(token.expose_secret());
216 }
217 let tab_id = result.tab_id.as_ref().map(|t| t.as_str().to_string());
218 let handoff = handoff_client
219 .takeover_handoff(None, tab_id.as_deref())
220 .await?;
221 result.attach_takeover_with_context(
222 handoff.takeover_url_secret,
223 Some(handoff.takeover_url_expires_at_rfc3339),
224 Some(handoff.takeover_url_ttl_s),
225 Some(handoff.takeover_url_scope),
226 recommended_endpoint,
227 explicit_profile.as_deref(),
228 );
229 }
230 if takeover {
231 Ok(output::emit_revealing_takeover("fetch", &result)?)
232 } else {
233 Ok(output::emit("fetch", &result)?)
234 }
235 }
236 Err(err) => {
237 let trace = serde_json::to_value(&err.trace).map_err(|e| {
238 Error::new(
239 ErrorCode::InternalError,
240 format!("serialize fetch error trace: {e}"),
241 )
242 })?;
243 let err = err.into_error();
244 crate::shared::afdata::emit_process_error_with(
245 err.error_code.as_str(),
246 &err.detail,
247 serde_json::json!({"retryable": err.retryable}),
248 trace,
249 )?;
250 Err(FetchRunError::Emitted(err))
251 }
252 }
253}
254
255async fn prepare_host<D, Fut>(
267 connection: &Connection,
268 takeover: bool,
269 profile: Option<&str>,
270 discover: D,
271) -> Result<Option<Resolved>, Error>
272where
273 D: FnOnce(Option<String>) -> Fut,
274 Fut: std::future::Future<Output = Result<crate::cli::cmd::container::LocalHost, Error>>,
275{
276 let token = connection
277 .token
278 .as_ref()
279 .map(token_source::read)
280 .transpose()?;
281 if let Some(endpoint) = connection.endpoint.clone() {
282 return Ok(Some(Resolved { endpoint, token }));
283 }
284 if !takeover && profile.is_none() {
285 return Ok(None);
286 }
287 let discovered = discover(
288 token
289 .as_ref()
290 .map(|token| token.expose_secret().to_string()),
291 )
292 .await?;
293 Ok(Some(Resolved {
294 endpoint: discovered.endpoint,
295 token: token.or(discovered.token_secret),
296 }))
297}
298
299fn takeover_recommended_endpoint(
300 endpoint: Option<&str>,
301 endpoint_was_preconfigured: bool,
302) -> Option<&str> {
303 if endpoint_was_preconfigured {
304 endpoint
305 } else {
306 None
307 }
308}
309
310fn default_profile_for_url(raw_url: &str) -> Result<String, Error> {
315 let parsed = url::Url::parse(raw_url).map_err(|e| {
316 Error::new(
317 ErrorCode::InvalidArgument,
318 format!(
319 "--takeover default profile needs a valid URL with a host; \
320 could not parse {raw_url:?}: {e}; pass --profile <name>"
321 ),
322 )
323 })?;
324 let host = parsed.host().ok_or_else(|| {
325 Error::new(
326 ErrorCode::InvalidArgument,
327 format!(
328 "--takeover default profile needs URL {raw_url:?} to include a host; \
329 pass --profile <name>"
330 ),
331 )
332 })?;
333 let (normalized_host, dns_name) = match host {
334 url::Host::Domain(domain) => (normalize_profile_host(domain), true),
335 url::Host::Ipv4(addr) => (addr.to_string(), false),
336 url::Host::Ipv6(addr) => (addr.to_string(), false),
337 };
338 let profile = if dns_name {
339 psl::domain_str(&normalized_host)
340 .unwrap_or(&normalized_host)
341 .to_string()
342 } else {
343 normalized_host.clone()
344 };
345 crate::sdk::profile::paths::validate_name(&profile).map_err(|e| {
346 Error::new(
347 e.error_code,
348 format!(
349 "derived --takeover profile {profile:?} from URL host {normalized_host:?} \
350 is invalid: {}; pass --profile <name>",
351 e.detail
352 ),
353 )
354 })?;
355 Ok(profile)
356}
357
358fn normalize_profile_host(host: &str) -> String {
359 host.trim_end_matches('.').to_ascii_lowercase()
360}
361
362enum FetchRunError {
363 Plain(Error),
364 Emitted(Error),
365}
366
367impl From<Error> for FetchRunError {
368 fn from(err: Error) -> Self {
369 Self::Plain(err)
370 }
371}
372
373async fn resolve_body(args: &Args) -> Result<Option<Vec<u8>>, Error> {
378 if let Some(data) = &args.data {
379 if let Some(path) = data.strip_prefix('@') {
380 return Ok(Some(tokio::fs::read(path).await.map_err(|e| {
381 Error::new(ErrorCode::IoError, format!("--data @{path}: {e}"))
382 })?));
383 }
384 return Ok(Some(data.as_bytes().to_vec()));
385 }
386 Ok(None)
387}
388
389fn resolve_want(want: &[String]) -> Option<std::collections::BTreeSet<Artifact>> {
393 if want.is_empty() {
394 return None;
395 }
396 Some(
397 want.iter()
398 .filter_map(|token| parse_artifact(token))
399 .collect(),
400 )
401}
402
403async fn build_client(
407 args: &Args,
408 host: Option<&Resolved>,
409 render: RenderMode,
410) -> Result<Client, Error> {
411 match host {
412 Some(host) => host.client(),
413 None if matches!(render, RenderMode::None) => Client::http_only(),
414 None => {
415 let cfg = InlineConfig {
416 browser: args.browser.clone(),
417 browser_bin: args.browser_bin.clone(),
418 };
419 if matches!(render, RenderMode::Auto) {
420 Client::inline_ephemeral_lazy(cfg).await
421 } else {
422 Client::inline_ephemeral_with(cfg).await
423 }
424 }
425 }
426}
427
428fn parse_artifact(token: &str) -> Option<Artifact> {
429 Some(match token {
430 "body" => Artifact::Body,
431 "rendered_html" => Artifact::RenderedHtml,
432 "text" => Artifact::Text,
433 "content" => Artifact::Content,
434 "content_json" => Artifact::ContentJson,
435 "screenshot" => Artifact::Screenshot,
436 "network" => Artifact::Network,
437 "console" => Artifact::Console,
438 "observation" => Artifact::Observation,
439 "storage" => Artifact::Storage,
440 _ => return None,
441 })
442}
443
444fn parse_header_arg(raw: &str) -> Result<(String, String), Error> {
450 let (name, value) = raw.split_once(':').ok_or_else(|| {
451 Error::new(
452 ErrorCode::InvalidArgument,
453 "--header: expected NAME:VALUE, with a colon separating them".to_string(),
454 )
455 })?;
456 let name = name.trim();
457 if name.is_empty() {
458 return Err(Error::new(
459 ErrorCode::InvalidArgument,
460 "--header: the header name before the colon must not be empty".to_string(),
461 ));
462 }
463 Ok((name.to_string(), value.trim_start().to_string()))
464}
465
466fn parse_cookie_arg(raw: &str) -> Result<FetchCookie, Error> {
467 if !raw.contains('=') {
468 return Err(Error::new(
469 ErrorCode::InvalidArgument,
470 "--cookie: expected Set-Cookie style NAME=VALUE, with an equals sign separating them"
471 .to_string(),
472 ));
473 }
474 let cookie = FetchCookie::parse(raw.to_string())
475 .map_err(|_| {
476 Error::new(
477 ErrorCode::InvalidArgument,
478 "--cookie: not a parseable Set-Cookie value".to_string(),
479 )
480 })?
481 .into_owned();
482 if cookie.name().trim().is_empty() {
483 return Err(Error::new(
484 ErrorCode::InvalidArgument,
485 "--cookie: the cookie name before the equals sign must not be empty".to_string(),
486 ));
487 }
488 Ok(cookie)
489}
490
491#[cfg(test)]
492mod tests {
493 use super::*;
494
495 fn exposed(token: &Option<agent_first_data::value_source::SecretString>) -> Option<&str> {
498 token
499 .as_ref()
500 .map(agent_first_data::value_source::SecretString::expose_secret)
501 }
502
503 fn discovered(token_secret: Option<&str>) -> crate::cli::cmd::container::LocalHost {
504 crate::cli::cmd::container::LocalHost {
505 endpoint: "ws://127.0.0.1:9222".into(),
506 token_secret: token_secret.map(agent_first_data::value_source::SecretString::new),
507 }
508 }
509
510 #[test]
514 fn a_malformed_credential_argument_is_never_quoted_back() {
515 const CANARY: &str = "CANARY-SECRET-VALUE";
516
517 let header = parse_header_arg(&format!("Authorization Bearer {CANARY}"))
518 .expect_err("a header with no colon is invalid");
519 assert!(
520 !header.to_string().contains(CANARY),
521 "header error leaked its value: {header}"
522 );
523
524 let empty_name =
525 parse_header_arg(&format!(": {CANARY}")).expect_err("a header with no name is invalid");
526 assert!(!empty_name.to_string().contains(CANARY));
527
528 let engine_env = crate::cli::cmd::host::parse_engine_env(&format!("PROXY{CANARY}"))
529 .expect_err("an engine env with no equals sign is invalid");
530 assert!(
531 !engine_env.to_string().contains(CANARY),
532 "engine env error leaked its value: {engine_env}"
533 );
534
535 let cookie = parse_cookie_arg(&format!("session{CANARY}"))
536 .expect_err("a cookie with no equals sign is invalid");
537 assert!(
538 !cookie.to_string().contains(CANARY),
539 "cookie error leaked its value: {cookie}"
540 );
541
542 let empty_cookie =
543 parse_cookie_arg(&format!("={CANARY}")).expect_err("a cookie with no name is invalid");
544 assert!(!empty_cookie.to_string().contains(CANARY));
545 }
546
547 #[tokio::test]
550 async fn a_plain_fetch_needs_no_host_and_discovers_nothing() {
551 let host = prepare_host(&Connection::default(), false, None, |_| async {
552 panic!("a plain fetch must not discover a host");
553 })
554 .await
555 .expect("no host is not an error");
556 assert!(host.is_none());
557 }
558
559 #[tokio::test]
560 async fn autodiscovery_fills_missing_endpoint_and_token() {
561 let host = prepare_host(
562 &Connection::default(),
563 true,
564 Some("contabo.com"),
565 |token| async move {
566 assert_eq!(token, None);
567 Ok(discovered(Some("secret")))
568 },
569 )
570 .await
571 .expect("discovery succeeds")
572 .expect("takeover needs a host");
573
574 assert_eq!(host.endpoint, "ws://127.0.0.1:9222");
575 assert_eq!(exposed(&host.token), Some("secret"));
576 }
577
578 #[tokio::test]
582 async fn a_profile_alone_discovers_a_host() {
583 let host = prepare_host(&Connection::default(), false, Some("work"), |_| async {
584 Ok(discovered(Some("secret")))
585 })
586 .await
587 .expect("discovery succeeds")
588 .expect("a profile needs a host");
589 assert_eq!(host.endpoint, "ws://127.0.0.1:9222");
590 }
591
592 #[tokio::test]
593 async fn autodiscovery_preserves_an_explicit_token() {
594 let connection = Connection::new(
595 None,
596 Some(agent_first_data::ValueSource::Literal("argv-token".into())),
597 );
598 let host = prepare_host(&connection, true, Some("contabo.com"), |token| async move {
599 assert_eq!(token.as_deref(), Some("argv-token"));
600 Ok(discovered(Some("container-token")))
601 })
602 .await
603 .expect("discovery succeeds")
604 .expect("takeover needs a host");
605
606 assert_eq!(host.endpoint, "ws://127.0.0.1:9222");
607 assert_eq!(exposed(&host.token), Some("argv-token"));
608 }
609
610 #[tokio::test]
611 async fn autodiscovery_surfaces_failure() {
612 let err = prepare_host(
613 &Connection::default(),
614 true,
615 Some("contabo.com"),
616 |_| async {
617 Err(Error::new(
618 ErrorCode::InvalidArgument,
619 "default local container `afhttp-host` is not running",
620 ))
621 },
622 )
623 .await
624 .expect_err("a missing container is an error");
625
626 assert_eq!(err.error_code, ErrorCode::InvalidArgument);
627 assert!(err.detail.contains("afhttp-host"));
628 }
629
630 #[tokio::test]
632 async fn a_named_endpoint_is_never_second_guessed() {
633 let connection = Connection::new(Some("ws://box:9222".into()), None);
634 let host = prepare_host(&connection, true, Some("contabo.com"), |_| async {
635 panic!("an explicit endpoint must not discover");
636 })
637 .await
638 .expect("explicit host resolves")
639 .expect("takeover needs a host");
640 assert_eq!(host.endpoint, "ws://box:9222");
641 assert!(host.token.is_none());
642 }
643
644 #[test]
647 fn every_registry_want_value_names_an_artifact() {
648 for token in crate::cli::spec::ARTIFACTS {
649 assert!(parse_artifact(token).is_some(), "{token}");
650 }
651 let wanted = resolve_want(&["body".to_string(), "network".to_string()])
652 .expect("an explicit want is a set");
653 assert!(wanted.contains(&Artifact::Body));
654 assert!(wanted.contains(&Artifact::Network));
655 assert!(resolve_want(&[]).is_none(), "no want means the default set");
656 }
657
658 #[test]
659 fn takeover_recommendation_omits_auto_discovered_endpoint() {
660 assert_eq!(
661 takeover_recommended_endpoint(Some("ws://127.0.0.1:9222"), false),
662 None
663 );
664 }
665
666 #[test]
667 fn takeover_recommendation_keeps_preconfigured_endpoint() {
668 assert_eq!(
669 takeover_recommended_endpoint(Some("ws://127.0.0.1:9222"), true),
670 Some("ws://127.0.0.1:9222")
671 );
672 }
673
674 #[test]
675 fn default_profile_uses_registrable_domain() {
676 assert_eq!(
677 default_profile_for_url("https://www.court.gov.cn/foo").unwrap(),
678 "court.gov.cn"
679 );
680 assert_eq!(
681 default_profile_for_url("https://accounts.google.com/foo").unwrap(),
682 "google.com"
683 );
684 assert_eq!(
685 default_profile_for_url("https://contabo.com").unwrap(),
686 "contabo.com"
687 );
688 }
689
690 #[test]
691 fn default_profile_keeps_public_suffix_tenants_isolated() {
692 assert_eq!(
693 default_profile_for_url("https://foo.github.io/x").unwrap(),
694 "foo.github.io"
695 );
696 assert_eq!(
697 default_profile_for_url("https://tenant.vercel.app/x").unwrap(),
698 "tenant.vercel.app"
699 );
700 }
701
702 #[test]
703 fn default_profile_normalizes_case_and_trailing_dot() {
704 assert_eq!(
705 default_profile_for_url("https://WWW.Example.COM./foo").unwrap(),
706 "example.com"
707 );
708 }
709
710 #[test]
711 fn default_profile_falls_back_to_full_host_for_psl_misses_and_ips() {
712 assert_eq!(
713 default_profile_for_url("http://localhost:8080/foo").unwrap(),
714 "localhost"
715 );
716 assert_eq!(
717 default_profile_for_url("http://127.0.0.1:8080/foo").unwrap(),
718 "127.0.0.1"
719 );
720 }
721
722 #[test]
723 fn default_profile_errors_when_host_is_missing() {
724 assert!(default_profile_for_url("file:///tmp/page.html").is_err());
725 }
726
727 #[test]
728 fn header_arg_accepts_colon_separator() {
729 assert_eq!(
730 parse_header_arg("X-Test: yes").unwrap(),
731 ("X-Test".to_string(), "yes".to_string())
732 );
733 }
734
735 #[test]
736 fn header_arg_rejects_missing_colon() {
737 let err = parse_header_arg("X-Test").err().unwrap();
738 assert_eq!(err.error_code, ErrorCode::InvalidArgument);
739 }
740
741 #[test]
742 fn cookie_arg_accepts_equals_separator() {
743 let cookie = parse_cookie_arg("sid=abc=def").unwrap();
744 assert_eq!(cookie.name_value(), ("sid", "abc=def"));
745 }
746
747 #[test]
748 fn cookie_arg_accepts_full_set_cookie_attributes() {
749 let cookie = parse_cookie_arg("sid=abc; Path=/; Secure; HttpOnly; SameSite=Lax").unwrap();
750 assert_eq!(cookie.name_value(), ("sid", "abc"));
751 assert_eq!(cookie.path(), Some("/"));
752 assert_eq!(cookie.secure(), Some(true));
753 assert_eq!(cookie.http_only(), Some(true));
754 assert_eq!(cookie.same_site(), Some(cookie::SameSite::Lax));
755 }
756
757 #[test]
758 fn cookie_arg_rejects_missing_equals() {
759 let err = parse_cookie_arg("sid").err().unwrap();
760 assert_eq!(err.error_code, ErrorCode::InvalidArgument);
761 }
762}