agent_first_http/sdk/fetch/
mod.rs

1//! Fetch pipeline: HTTP-only fast path, browser-backed render path
2//! (`--render none|auto|always`), and the fetch artifacts
3//! (`architecture.md §8`).
4
5pub mod artifacts;
6pub(crate) mod deadline;
7pub(crate) mod page_classification;
8pub mod pipeline;
9pub mod result;
10pub mod wait;
11pub mod writer;
12
13pub type FetchCookie = cookie::Cookie<'static>;
14pub use cookie::SameSite as FetchCookieSameSite;
15pub use pipeline::{NetworkBodies, RenderMode};
16pub use result::{FetchError, FetchResult, PageKind};
17pub use wait::Wait;
18
19use std::collections::BTreeSet;
20use std::path::PathBuf;
21use std::time::Duration;
22
23use crate::sdk::client::Client;
24use crate::shared::artifacts::Artifact;
25use crate::shared::error::Error;
26use crate::shared::ids::TabId;
27
28/// Default per-network-response body capture cap: 10 MiB.
29pub const DEFAULT_NETWORK_BODY_MAX_BYTES: u64 = 10 * 1024 * 1024;
30
31/// Builder for `Client::fetch(...).send().await`.
32#[derive(Clone)]
33pub struct FetchBuilder {
34    pub(crate) client: Client,
35    pub(crate) url: String,
36    pub(crate) render: RenderMode,
37    pub(crate) wait: Wait,
38    pub(crate) timeout: Duration,
39    pub(crate) want: BTreeSet<Artifact>,
40    pub(crate) want_explicit: bool,
41    pub(crate) tab: Option<TabId>,
42    /// Keep a freshly-opened ("new") target open after the fetch instead of
43    /// closing it, so a human can take the tab over. No effect when an explicit
44    /// `tab` is reused (those are already left open).
45    pub(crate) keep_tab_open: bool,
46    pub(crate) request: RequestOptions,
47    pub(crate) out_dir: Option<PathBuf>,
48    pub(crate) readiness: ReadinessOptions,
49    pub(crate) network: NetworkCapture,
50    pub(crate) retry: RetryOptions,
51    pub(crate) http: HttpOptions,
52    pub(crate) cookie_jar: CookieJarOptions,
53}
54
55/// Browser-path readiness tuning: how long `--wait auto` waits for network
56/// quiet and DOM/text stability, plus the main-document observation cap.
57#[derive(Clone)]
58pub(crate) struct ReadinessOptions {
59    pub(crate) idle_ms: u64,
60    pub(crate) stable_ms: u64,
61    pub(crate) min_text_bytes: u64,
62    /// Upper bound, in milliseconds, on how long the browser path waits for
63    /// the main document network event before falling back to capturing
64    /// artifacts with `main_request_observed: false`. Default 500ms suits
65    /// well-behaved pages on fast networks; slow networks may need more.
66    pub(crate) observe_main_wait_ms: u64,
67}
68
69/// Network-capture knobs: response-body capture mode/cap, header redaction,
70/// and WebSocket/SSE frame capture.
71#[derive(Clone)]
72pub(crate) struct NetworkCapture {
73    pub(crate) bodies: NetworkBodies,
74    pub(crate) body_max_bytes: u64,
75    pub(crate) redact: bool,
76    /// Capture WebSocket frame payloads to `network-bodies/<id>.frames.jsonl`.
77    pub(crate) capture_ws: bool,
78    /// Capture SSE event payloads to `network-bodies/<id>.frames.jsonl`.
79    pub(crate) capture_sse: bool,
80}
81
82/// Retry policy. The fetch is retried only when the error carries
83/// `retryable: true`.
84#[derive(Clone)]
85pub(crate) struct RetryOptions {
86    /// Number of additional attempts after the first one. `0` (the default)
87    /// keeps the single-attempt behavior.
88    pub(crate) attempts: u32,
89    /// Fixed delay between retry attempts, in milliseconds. Retry
90    /// orchestration beyond a fixed interval is the agent's job; the tool
91    /// just gives it the primitive.
92    pub(crate) backoff_ms: u64,
93}
94
95/// HTTP fast-path transport options: upstream proxy, extra trust anchors,
96/// TLS verification, and the response-body size cap.
97#[derive(Clone)]
98pub(crate) struct HttpOptions {
99    /// Per-fetch upstream proxy for the HTTP fast path. The SDK builds a
100    /// dedicated reqwest client when this (or `ca_cert` / `tls_insecure`) is
101    /// set so the per-Client default reqwest is not contaminated. `None`
102    /// keeps the default direct connection.
103    pub(crate) proxy: Option<String>,
104    /// Path to a PEM file containing extra root certificates to trust for
105    /// this fetch's HTTP path. Useful for self-signed targets or corporate
106    /// MITM CAs without weakening the global trust store.
107    pub(crate) ca_cert: Option<PathBuf>,
108    /// Disable TLS certificate verification for the HTTP path. The agent
109    /// must opt in explicitly — this is dangerous and the CLI help says so.
110    pub(crate) tls_insecure: bool,
111    /// Upper bound, in bytes, on the HTTP fast path's response body before
112    /// the pipeline stops accumulating and emits a `network_body_truncated`
113    /// warning instead. Default 1 GiB — generous enough that normal pages
114    /// and downloads never trip it, low enough that a pathological multi-GB
115    /// download cannot OOM the host. `0` disables the cap entirely.
116    pub(crate) max_response_bytes: u64,
117}
118
119/// Cookie-jar selection for the fetch.
120#[derive(Clone)]
121pub(crate) struct CookieJarOptions {
122    /// Explicit cookie-jar path override. Normally the pipeline derives
123    /// `<profile>/cookies.jar.json` from the host's `GET /profile` — setting
124    /// this tells the pipeline to use the given path instead. The override
125    /// must canonicalize to the host's profile directory or the pipeline
126    /// rejects with `invalid_argument`.
127    pub(crate) path: Option<PathBuf>,
128    pub(crate) warning: Option<String>,
129    /// Opt out of the cookie jar entirely for this fetch. Useful for agents
130    /// that want a clean request even when the host has a persistent profile
131    /// (e.g. recon traffic that should not carry authenticated session
132    /// cookies).
133    pub(crate) disabled: bool,
134}
135
136#[derive(Clone, Debug, Default)]
137pub(crate) struct RequestOptions {
138    pub(crate) headers: Vec<(String, String)>,
139    pub(crate) user_agent: Option<String>,
140    pub(crate) cookies: Vec<FetchCookie>,
141    pub(crate) evaluate_after_wait: Vec<String>,
142    /// HTTP method. `None` = GET (default). Uppercase recommended; the
143    /// pipeline normalises before sending.
144    pub(crate) method: Option<String>,
145    /// Raw request body bytes (mutually exclusive with `form`).
146    pub(crate) body: Option<Vec<u8>>,
147    /// Form fields sent as `application/x-www-form-urlencoded` (mutually
148    /// exclusive with `body`).
149    pub(crate) form: Vec<(String, String)>,
150}
151
152impl FetchBuilder {
153    pub(crate) fn new(client: Client, url: String) -> Self {
154        Self {
155            client,
156            url,
157            render: RenderMode::Auto,
158            wait: Wait::Auto,
159            timeout: Duration::from_secs(30),
160            want: Artifact::HTTP_DEFAULT.iter().copied().collect(),
161            want_explicit: false,
162            tab: None,
163            keep_tab_open: false,
164            request: RequestOptions::default(),
165            out_dir: None,
166            readiness: ReadinessOptions {
167                idle_ms: 800,
168                stable_ms: 500,
169                min_text_bytes: 32,
170                observe_main_wait_ms: 500,
171            },
172            network: NetworkCapture {
173                bodies: NetworkBodies::Off,
174                body_max_bytes: DEFAULT_NETWORK_BODY_MAX_BYTES,
175                redact: true,
176                capture_ws: false,
177                capture_sse: false,
178            },
179            retry: RetryOptions {
180                attempts: 0,
181                backoff_ms: 250,
182            },
183            http: HttpOptions {
184                proxy: None,
185                ca_cert: None,
186                tls_insecure: false,
187                max_response_bytes: 1_073_741_824,
188            },
189            cookie_jar: CookieJarOptions {
190                path: None,
191                warning: None,
192                disabled: false,
193            },
194        }
195    }
196
197    #[must_use]
198    pub fn render(mut self, mode: RenderMode) -> Self {
199        self.render = mode;
200        self
201    }
202
203    #[must_use]
204    pub fn wait(mut self, w: Wait) -> Self {
205        self.wait = w;
206        self
207    }
208
209    #[must_use]
210    pub fn timeout(mut self, d: Duration) -> Self {
211        self.timeout = d;
212        self
213    }
214
215    #[must_use]
216    pub fn readiness_idle_ms(mut self, ms: u64) -> Self {
217        self.readiness.idle_ms = ms;
218        self
219    }
220
221    #[must_use]
222    pub fn readiness_stable_ms(mut self, ms: u64) -> Self {
223        self.readiness.stable_ms = ms;
224        self
225    }
226
227    #[must_use]
228    pub fn readiness_min_text_bytes(mut self, bytes: u64) -> Self {
229        self.readiness.min_text_bytes = bytes;
230        self
231    }
232
233    #[must_use]
234    pub fn want<I: IntoIterator<Item = Artifact>>(mut self, items: I) -> Self {
235        self.want = items.into_iter().collect();
236        self.want_explicit = true;
237        self
238    }
239
240    #[must_use]
241    pub fn tab(mut self, tab: TabId) -> Self {
242        self.tab = Some(tab);
243        self
244    }
245
246    /// Keep a freshly-opened target open after the fetch (for human takeover).
247    #[must_use]
248    pub fn keep_tab_open(mut self, keep: bool) -> Self {
249        self.keep_tab_open = keep;
250        self
251    }
252
253    #[must_use]
254    pub fn network_bodies(mut self, mode: NetworkBodies) -> Self {
255        self.network.bodies = mode;
256        self
257    }
258
259    #[must_use]
260    pub fn network_body_max_bytes(mut self, n: u64) -> Self {
261        self.network.body_max_bytes = n;
262        self
263    }
264
265    #[must_use]
266    pub fn network_redact(mut self, on: bool) -> Self {
267        self.network.redact = on;
268        self
269    }
270
271    /// Add a request header. `User-Agent` is normalized to
272    /// [`Self::user_agent`] at send time so browser fetches use the CDP UA
273    /// override instead of a plain extra header.
274    #[must_use]
275    pub fn header(mut self, name: impl Into<String>, value: impl Into<String>) -> Self {
276        self.request.headers.push((name.into(), value.into()));
277        self
278    }
279
280    /// Add multiple request headers.
281    #[must_use]
282    pub fn headers<I, K, V>(mut self, headers: I) -> Self
283    where
284        I: IntoIterator<Item = (K, V)>,
285        K: Into<String>,
286        V: Into<String>,
287    {
288        self.request
289            .headers
290            .extend(headers.into_iter().map(|(k, v)| (k.into(), v.into())));
291        self
292    }
293
294    /// Override the browser/client user agent.
295    #[must_use]
296    pub fn user_agent(mut self, value: impl Into<String>) -> Self {
297        self.request.user_agent = Some(value.into());
298        self
299    }
300
301    /// Add a request cookie as a `name=value` pair.
302    #[must_use]
303    pub fn cookie(mut self, name: impl Into<String>, value: impl Into<String>) -> Self {
304        self.request
305            .cookies
306            .push(cookie::Cookie::new(name.into(), value.into()));
307        self
308    }
309
310    /// Add a full cookie, including optional Domain/Path/Secure/HttpOnly/
311    /// SameSite/Max-Age/Expires attributes.
312    #[must_use]
313    pub fn cookie_full(mut self, cookie: FetchCookie) -> Self {
314        self.request.cookies.push(cookie);
315        self
316    }
317
318    /// Add multiple request cookies as `name=value` pairs.
319    #[must_use]
320    pub fn cookies<I, K, V>(mut self, cookies: I) -> Self
321    where
322        I: IntoIterator<Item = (K, V)>,
323        K: Into<String>,
324        V: Into<String>,
325    {
326        self.request.cookies.extend(
327            cookies
328                .into_iter()
329                .map(|(k, v)| cookie::Cookie::new(k.into(), v.into())),
330        );
331        self
332    }
333
334    /// Evaluate JavaScript after the configured wait condition and before
335    /// artifact capture. Only browser-backed fetches can execute scripts.
336    #[must_use]
337    pub fn evaluate_after_wait(mut self, js: impl Into<String>) -> Self {
338        self.request.evaluate_after_wait.push(js.into());
339        self
340    }
341
342    #[must_use]
343    pub fn out_dir(mut self, dir: impl Into<PathBuf>) -> Self {
344        self.out_dir = Some(dir.into());
345        self
346    }
347
348    /// Override the cookie-jar path. The default — derived from the host's
349    /// `GET /profile` — places the jar at `<profile-dir>/cookies.jar.json`,
350    /// which is the only path the isolation invariant permits. This
351    /// override exists for tests and forensic tooling; the pipeline
352    /// canonicalizes the given path and rejects it with `invalid_argument`
353    /// if it doesn't match the host's profile directory.
354    #[must_use]
355    pub fn cookie_jar(mut self, path: impl Into<PathBuf>) -> Self {
356        self.cookie_jar.path = Some(path.into());
357        self
358    }
359
360    /// Opt out of cookie-jar persistence for this fetch. The request goes
361    /// out without any session cookies the jar might hold, and the
362    /// response's `Set-Cookie` headers are not merged back.
363    #[must_use]
364    pub fn no_cookie_jar(mut self) -> Self {
365        self.cookie_jar.disabled = true;
366        self
367    }
368
369    /// Upper bound on the browser-path wait for the main document
370    /// network event, in milliseconds. Default 500ms is tuned for
371    /// well-behaved pages on fast networks; raise for slow networks
372    /// or low-end machines.
373    #[must_use]
374    pub fn observe_main_wait_ms(mut self, ms: u64) -> Self {
375        self.readiness.observe_main_wait_ms = ms;
376        self
377    }
378
379    /// Upper bound on the HTTP-path response body, in bytes. Default
380    /// 1 GiB. `0` disables the cap entirely. When the cap is hit, the
381    /// fetch returns successfully with a `network_body_truncated`
382    /// warning and the prefix bytes that were collected.
383    #[must_use]
384    pub fn max_response_bytes(mut self, bytes: u64) -> Self {
385        self.http.max_response_bytes = bytes;
386        self
387    }
388
389    /// Number of additional attempts after the first. `0` (default)
390    /// keeps the single-attempt behavior. Retries only fire when the
391    /// pipeline error has `retryable: true`.
392    #[must_use]
393    pub fn retry(mut self, n: u32) -> Self {
394        self.retry.attempts = n;
395        self
396    }
397
398    /// Fixed delay between retry attempts, in milliseconds.
399    #[must_use]
400    pub fn backoff_ms(mut self, ms: u64) -> Self {
401        self.retry.backoff_ms = ms;
402        self
403    }
404
405    /// Per-fetch upstream proxy URL for the HTTP path. Format:
406    /// `http://user:pass@host:port` or `socks5://host:port`. The SDK
407    /// never honors `HTTP_PROXY`/`HTTPS_PROXY` from the environment;
408    /// this method is the only way to route an HTTP-path fetch
409    /// through a proxy.
410    #[must_use]
411    pub fn proxy(mut self, url: impl Into<String>) -> Self {
412        self.http.proxy = Some(url.into());
413        self
414    }
415
416    /// Path to a PEM file containing extra root CAs to trust for
417    /// this fetch's HTTP path. Stacks on top of the platform trust
418    /// store; does not replace it.
419    #[must_use]
420    pub fn ca_cert(mut self, path: impl Into<PathBuf>) -> Self {
421        self.http.ca_cert = Some(path.into());
422        self
423    }
424
425    /// Disable TLS certificate verification for this fetch's HTTP
426    /// path. Dangerous — leaves the connection open to MITM. Use
427    /// only against known-self-signed staging environments.
428    #[must_use]
429    pub fn tls_insecure(mut self, on: bool) -> Self {
430        self.http.tls_insecure = on;
431        self
432    }
433
434    /// HTTP method. Defaults to `GET`. Pass `"POST"`, `"PUT"`, etc.
435    #[must_use]
436    pub fn method(mut self, m: impl Into<String>) -> Self {
437        self.request.method = Some(m.into());
438        self
439    }
440
441    /// Raw request body. Mutually exclusive with [`Self::form_field`].
442    /// Sets the body bytes as-is; add `Content-Type` via
443    /// [`Self::header`] when needed.
444    #[must_use]
445    pub fn body(mut self, data: impl Into<Vec<u8>>) -> Self {
446        self.request.body = Some(data.into());
447        self
448    }
449
450    /// Add a form field. Mutually exclusive with [`Self::body`]. The
451    /// pipeline sends the fields as `application/x-www-form-urlencoded`
452    /// and sets the content-type header automatically.
453    #[must_use]
454    pub fn form_field(mut self, key: impl Into<String>, value: impl Into<String>) -> Self {
455        self.request.form.push((key.into(), value.into()));
456        self
457    }
458
459    /// Capture WebSocket frame payloads to
460    /// `network-bodies/<request_id>.frames.jsonl` during the browser path.
461    #[must_use]
462    pub fn capture_ws(mut self, on: bool) -> Self {
463        self.network.capture_ws = on;
464        self
465    }
466
467    /// Capture SSE event payloads to
468    /// `network-bodies/<request_id>.frames.jsonl` during the browser path.
469    #[must_use]
470    pub fn capture_sse(mut self, on: bool) -> Self {
471        self.network.capture_sse = on;
472        self
473    }
474
475    /// Execute the fetch, with retries when configured. Retries only
476    /// fire for errors carrying `retryable: true`; any other error
477    /// short-circuits immediately.
478    pub async fn send(self) -> Result<FetchResult, Error> {
479        self.send_detailed().await.map_err(FetchError::into_error)
480    }
481
482    /// Execute the fetch and preserve the fetch trace on failure.
483    ///
484    /// This is what the CLI uses to emit `code: "error"` envelopes with a
485    /// fetch-local `trace` without adding trace fields to the global `Error`.
486    pub async fn send_detailed(self) -> Result<FetchResult, FetchError> {
487        if self.retry.attempts == 0 {
488            return execute_once_with_timeout(self).await;
489        }
490        let max_attempts = self.retry.attempts.saturating_add(1);
491        let delay = std::time::Duration::from_millis(self.retry.backoff_ms);
492        let mut attempt: u32 = 0;
493        loop {
494            match execute_once_with_timeout(self.clone()).await {
495                Ok(r) => return Ok(r),
496                Err(e) if e.retryable && attempt + 1 < max_attempts => {
497                    tokio::time::sleep(delay).await;
498                    attempt += 1;
499                }
500                Err(e) => return Err(e),
501            }
502        }
503    }
504}
505
506async fn execute_once_with_timeout(builder: FetchBuilder) -> Result<FetchResult, FetchError> {
507    let timeout = builder.timeout;
508    let render_mode = builder.render.as_trace();
509    let deadline = deadline::FetchDeadline::new(timeout, render_mode);
510    match tokio::time::timeout(timeout, pipeline::execute(builder, deadline.clone())).await {
511        Ok(Ok(result)) => Ok(result),
512        Ok(Err(error)) => Err(FetchError::new(error, deadline.snapshot())),
513        Err(_) => {
514            let error = deadline.timeout_error();
515            Err(FetchError::new(error, deadline.snapshot()))
516        }
517    }
518}
agent_first_http/sdk/fetch/mod.rs

agent_first_http/sdk/fetch/
mod.rs
