agent_first_http/sdk/fetch/
mod.rs

1//! Fetch pipeline: HTTP-only fast path, browser-backed render path
2//! (`--render none|auto|always`), and the fetch artifacts
3//! (`architecture.md §8`).
4
5pub mod artifacts;
6pub(crate) mod deadline;
7pub(crate) mod page_classification;
8pub mod pipeline;
9pub mod result;
10pub mod wait;
11pub mod writer;
12
13pub type FetchCookie = cookie::Cookie<'static>;
14pub use cookie::SameSite as FetchCookieSameSite;
15pub use pipeline::{NetworkBodies, RenderMode};
16pub use result::{FetchError, FetchResult, PageKind};
17pub use wait::Wait;
18
19use std::collections::BTreeSet;
20use std::path::PathBuf;
21use std::time::Duration;
22
23use crate::sdk::client::Client;
24use crate::shared::artifacts::Artifact;
25use crate::shared::error::Error;
26use crate::shared::ids::TabId;
27
28/// Default per-network-response body capture cap: 10 MiB.
29pub const DEFAULT_NETWORK_BODY_MAX_BYTES: u64 = 10 * 1024 * 1024;
30
31/// Builder for `Client::fetch(...).send().await`.
32#[derive(Clone)]
33pub struct FetchBuilder {
34    pub(crate) client: Client,
35    pub(crate) url: String,
36    pub(crate) render: RenderMode,
37    pub(crate) wait: Wait,
38    pub(crate) timeout: Duration,
39    pub(crate) want: BTreeSet<Artifact>,
40    pub(crate) tab: Option<TabId>,
41    /// Keep a freshly-opened ("new") target open after the fetch instead of
42    /// closing it, so a human can take the tab over. No effect when an explicit
43    /// `tab` is reused (those are already left open).
44    pub(crate) keep_tab_open: bool,
45    pub(crate) request: RequestOptions,
46    pub(crate) out_dir: Option<PathBuf>,
47    pub(crate) readiness: ReadinessOptions,
48    pub(crate) network: NetworkCapture,
49    pub(crate) retry: RetryOptions,
50    pub(crate) http: HttpOptions,
51    pub(crate) cookie_jar: CookieJarOptions,
52}
53
54/// Browser-path readiness tuning: how long `--wait auto` waits for network
55/// quiet and DOM/text stability, plus the main-document observation cap.
56#[derive(Clone)]
57pub(crate) struct ReadinessOptions {
58    pub(crate) idle_ms: u64,
59    pub(crate) stable_ms: u64,
60    pub(crate) min_text_bytes: u64,
61    /// Upper bound, in milliseconds, on how long the browser path waits for
62    /// the main document network event before falling back to capturing
63    /// artifacts with `main_request_observed: false`. Default 500ms suits
64    /// well-behaved pages on fast networks; slow networks may need more.
65    pub(crate) observe_main_wait_ms: u64,
66}
67
68/// Network-capture knobs: response-body capture mode/cap, header redaction,
69/// and WebSocket/SSE frame capture.
70#[derive(Clone)]
71pub(crate) struct NetworkCapture {
72    pub(crate) bodies: NetworkBodies,
73    pub(crate) body_max_bytes: u64,
74    pub(crate) redact: bool,
75    /// Capture WebSocket frame payloads to `network-bodies/<id>.frames.jsonl`.
76    pub(crate) capture_ws: bool,
77    /// Capture SSE event payloads to `network-bodies/<id>.frames.jsonl`.
78    pub(crate) capture_sse: bool,
79}
80
81/// Retry policy. The fetch is retried only when the error carries
82/// `retryable: true`.
83#[derive(Clone)]
84pub(crate) struct RetryOptions {
85    /// Number of additional attempts after the first one. `0` (the default)
86    /// keeps the single-attempt behavior.
87    pub(crate) attempts: u32,
88    /// Fixed delay between retry attempts, in milliseconds. Retry
89    /// orchestration beyond a fixed interval is the agent's job; the tool
90    /// just gives it the primitive.
91    pub(crate) backoff_ms: u64,
92}
93
94/// HTTP fast-path transport options: upstream proxy, extra trust anchors,
95/// TLS verification, and the response-body size cap.
96#[derive(Clone)]
97pub(crate) struct HttpOptions {
98    /// Per-fetch upstream proxy for the HTTP fast path. The SDK builds a
99    /// dedicated reqwest client when this (or `ca_cert` / `tls_insecure`) is
100    /// set so the per-Client default reqwest is not contaminated. `None`
101    /// keeps the default direct connection.
102    pub(crate) proxy: Option<String>,
103    /// Path to a PEM file containing extra root certificates to trust for
104    /// this fetch's HTTP path. Useful for self-signed targets or corporate
105    /// MITM CAs without weakening the global trust store.
106    pub(crate) ca_cert: Option<PathBuf>,
107    /// Disable TLS certificate verification for the HTTP path. The agent
108    /// must opt in explicitly — this is dangerous and the CLI help says so.
109    pub(crate) tls_insecure: bool,
110    /// Upper bound, in bytes, on the HTTP fast path's response body before
111    /// the pipeline stops accumulating and emits a `network_body_truncated`
112    /// warning instead. Default 1 GiB — generous enough that normal pages
113    /// and downloads never trip it, low enough that a pathological multi-GB
114    /// download cannot OOM the host. `0` disables the cap entirely.
115    pub(crate) max_response_bytes: u64,
116}
117
118/// Cookie-jar selection for the fetch.
119#[derive(Clone)]
120pub(crate) struct CookieJarOptions {
121    /// Explicit cookie-jar path override. Normally the pipeline derives
122    /// `<profile>/cookies.jar.json` from the host's `GET /profile` — setting
123    /// this tells the pipeline to use the given path instead. The override
124    /// must canonicalize to the host's profile directory or the pipeline
125    /// rejects with `invalid_argument`.
126    pub(crate) path: Option<PathBuf>,
127    pub(crate) warning: Option<String>,
128    /// Opt out of the cookie jar entirely for this fetch. Useful for agents
129    /// that want a clean request even when the host has a persistent profile
130    /// (e.g. recon traffic that should not carry authenticated session
131    /// cookies).
132    pub(crate) disabled: bool,
133}
134
135#[derive(Clone, Debug, Default)]
136pub(crate) struct RequestOptions {
137    pub(crate) headers: Vec<(String, String)>,
138    pub(crate) user_agent: Option<String>,
139    pub(crate) cookies: Vec<FetchCookie>,
140    pub(crate) evaluate_after_wait: Vec<String>,
141    /// HTTP method. `None` = GET (default). Uppercase recommended; the
142    /// pipeline normalises before sending.
143    pub(crate) method: Option<String>,
144    /// Raw request body bytes (mutually exclusive with `form`).
145    pub(crate) body: Option<Vec<u8>>,
146    /// Form fields sent as `application/x-www-form-urlencoded` (mutually
147    /// exclusive with `body`).
148    pub(crate) form: Vec<(String, String)>,
149}
150
151impl FetchBuilder {
152    pub(crate) fn new(client: Client, url: String) -> Self {
153        Self {
154            client,
155            url,
156            render: RenderMode::Auto,
157            wait: Wait::Auto,
158            timeout: Duration::from_secs(30),
159            want: Artifact::ALL.iter().copied().collect(),
160            tab: None,
161            keep_tab_open: false,
162            request: RequestOptions::default(),
163            out_dir: None,
164            readiness: ReadinessOptions {
165                idle_ms: 800,
166                stable_ms: 500,
167                min_text_bytes: 32,
168                observe_main_wait_ms: 500,
169            },
170            network: NetworkCapture {
171                bodies: NetworkBodies::Off,
172                body_max_bytes: DEFAULT_NETWORK_BODY_MAX_BYTES,
173                redact: true,
174                capture_ws: false,
175                capture_sse: false,
176            },
177            retry: RetryOptions {
178                attempts: 0,
179                backoff_ms: 250,
180            },
181            http: HttpOptions {
182                proxy: None,
183                ca_cert: None,
184                tls_insecure: false,
185                max_response_bytes: 1_073_741_824,
186            },
187            cookie_jar: CookieJarOptions {
188                path: None,
189                warning: None,
190                disabled: false,
191            },
192        }
193    }
194
195    #[must_use]
196    pub fn render(mut self, mode: RenderMode) -> Self {
197        self.render = mode;
198        self
199    }
200
201    #[must_use]
202    pub fn wait(mut self, w: Wait) -> Self {
203        self.wait = w;
204        self
205    }
206
207    #[must_use]
208    pub fn timeout(mut self, d: Duration) -> Self {
209        self.timeout = d;
210        self
211    }
212
213    #[must_use]
214    pub fn readiness_idle_ms(mut self, ms: u64) -> Self {
215        self.readiness.idle_ms = ms;
216        self
217    }
218
219    #[must_use]
220    pub fn readiness_stable_ms(mut self, ms: u64) -> Self {
221        self.readiness.stable_ms = ms;
222        self
223    }
224
225    #[must_use]
226    pub fn readiness_min_text_bytes(mut self, bytes: u64) -> Self {
227        self.readiness.min_text_bytes = bytes;
228        self
229    }
230
231    #[must_use]
232    pub fn want<I: IntoIterator<Item = Artifact>>(mut self, items: I) -> Self {
233        self.want = items.into_iter().collect();
234        self
235    }
236
237    #[must_use]
238    pub fn tab(mut self, tab: TabId) -> Self {
239        self.tab = Some(tab);
240        self
241    }
242
243    /// Keep a freshly-opened target open after the fetch (for human takeover).
244    #[must_use]
245    pub fn keep_tab_open(mut self, keep: bool) -> Self {
246        self.keep_tab_open = keep;
247        self
248    }
249
250    #[must_use]
251    pub fn network_bodies(mut self, mode: NetworkBodies) -> Self {
252        self.network.bodies = mode;
253        self
254    }
255
256    #[must_use]
257    pub fn network_body_max_bytes(mut self, n: u64) -> Self {
258        self.network.body_max_bytes = n;
259        self
260    }
261
262    #[must_use]
263    pub fn network_redact(mut self, on: bool) -> Self {
264        self.network.redact = on;
265        self
266    }
267
268    /// Add a request header. `User-Agent` is normalized to
269    /// [`Self::user_agent`] at send time so browser fetches use the CDP UA
270    /// override instead of a plain extra header.
271    #[must_use]
272    pub fn header(mut self, name: impl Into<String>, value: impl Into<String>) -> Self {
273        self.request.headers.push((name.into(), value.into()));
274        self
275    }
276
277    /// Add multiple request headers.
278    #[must_use]
279    pub fn headers<I, K, V>(mut self, headers: I) -> Self
280    where
281        I: IntoIterator<Item = (K, V)>,
282        K: Into<String>,
283        V: Into<String>,
284    {
285        self.request
286            .headers
287            .extend(headers.into_iter().map(|(k, v)| (k.into(), v.into())));
288        self
289    }
290
291    /// Override the browser/client user agent.
292    #[must_use]
293    pub fn user_agent(mut self, value: impl Into<String>) -> Self {
294        self.request.user_agent = Some(value.into());
295        self
296    }
297
298    /// Add a request cookie as a `name=value` pair.
299    #[must_use]
300    pub fn cookie(mut self, name: impl Into<String>, value: impl Into<String>) -> Self {
301        self.request
302            .cookies
303            .push(cookie::Cookie::new(name.into(), value.into()));
304        self
305    }
306
307    /// Add a full cookie, including optional Domain/Path/Secure/HttpOnly/
308    /// SameSite/Max-Age/Expires attributes.
309    #[must_use]
310    pub fn cookie_full(mut self, cookie: FetchCookie) -> Self {
311        self.request.cookies.push(cookie);
312        self
313    }
314
315    /// Add multiple request cookies as `name=value` pairs.
316    #[must_use]
317    pub fn cookies<I, K, V>(mut self, cookies: I) -> Self
318    where
319        I: IntoIterator<Item = (K, V)>,
320        K: Into<String>,
321        V: Into<String>,
322    {
323        self.request.cookies.extend(
324            cookies
325                .into_iter()
326                .map(|(k, v)| cookie::Cookie::new(k.into(), v.into())),
327        );
328        self
329    }
330
331    /// Evaluate JavaScript after the configured wait condition and before
332    /// artifact capture. Only browser-backed fetches can execute scripts.
333    #[must_use]
334    pub fn evaluate_after_wait(mut self, js: impl Into<String>) -> Self {
335        self.request.evaluate_after_wait.push(js.into());
336        self
337    }
338
339    #[must_use]
340    pub fn out_dir(mut self, dir: impl Into<PathBuf>) -> Self {
341        self.out_dir = Some(dir.into());
342        self
343    }
344
345    /// Override the cookie-jar path. The default — derived from the host's
346    /// `GET /profile` — places the jar at `<profile-dir>/cookies.jar.json`,
347    /// which is the only path the isolation invariant permits. This
348    /// override exists for tests and forensic tooling; the pipeline
349    /// canonicalizes the given path and rejects it with `invalid_argument`
350    /// if it doesn't match the host's profile directory.
351    #[must_use]
352    pub fn cookie_jar(mut self, path: impl Into<PathBuf>) -> Self {
353        self.cookie_jar.path = Some(path.into());
354        self
355    }
356
357    /// Opt out of cookie-jar persistence for this fetch. The request goes
358    /// out without any session cookies the jar might hold, and the
359    /// response's `Set-Cookie` headers are not merged back.
360    #[must_use]
361    pub fn no_cookie_jar(mut self) -> Self {
362        self.cookie_jar.disabled = true;
363        self
364    }
365
366    /// Upper bound on the browser-path wait for the main document
367    /// network event, in milliseconds. Default 500ms is tuned for
368    /// well-behaved pages on fast networks; raise for slow networks
369    /// or low-end machines.
370    #[must_use]
371    pub fn observe_main_wait_ms(mut self, ms: u64) -> Self {
372        self.readiness.observe_main_wait_ms = ms;
373        self
374    }
375
376    /// Upper bound on the HTTP-path response body, in bytes. Default
377    /// 1 GiB. `0` disables the cap entirely. When the cap is hit, the
378    /// fetch returns successfully with a `network_body_truncated`
379    /// warning and the prefix bytes that were collected.
380    #[must_use]
381    pub fn max_response_bytes(mut self, bytes: u64) -> Self {
382        self.http.max_response_bytes = bytes;
383        self
384    }
385
386    /// Number of additional attempts after the first. `0` (default)
387    /// keeps the single-attempt behavior. Retries only fire when the
388    /// pipeline error has `retryable: true`.
389    #[must_use]
390    pub fn retry(mut self, n: u32) -> Self {
391        self.retry.attempts = n;
392        self
393    }
394
395    /// Fixed delay between retry attempts, in milliseconds.
396    #[must_use]
397    pub fn backoff_ms(mut self, ms: u64) -> Self {
398        self.retry.backoff_ms = ms;
399        self
400    }
401
402    /// Per-fetch upstream proxy URL for the HTTP path. Format:
403    /// `http://user:pass@host:port` or `socks5://host:port`. The SDK
404    /// never honors `HTTP_PROXY`/`HTTPS_PROXY` from the environment;
405    /// this method is the only way to route an HTTP-path fetch
406    /// through a proxy.
407    #[must_use]
408    pub fn proxy(mut self, url: impl Into<String>) -> Self {
409        self.http.proxy = Some(url.into());
410        self
411    }
412
413    /// Path to a PEM file containing extra root CAs to trust for
414    /// this fetch's HTTP path. Stacks on top of the platform trust
415    /// store; does not replace it.
416    #[must_use]
417    pub fn ca_cert(mut self, path: impl Into<PathBuf>) -> Self {
418        self.http.ca_cert = Some(path.into());
419        self
420    }
421
422    /// Disable TLS certificate verification for this fetch's HTTP
423    /// path. Dangerous — leaves the connection open to MITM. Use
424    /// only against known-self-signed staging environments.
425    #[must_use]
426    pub fn tls_insecure(mut self, on: bool) -> Self {
427        self.http.tls_insecure = on;
428        self
429    }
430
431    /// HTTP method. Defaults to `GET`. Pass `"POST"`, `"PUT"`, etc.
432    #[must_use]
433    pub fn method(mut self, m: impl Into<String>) -> Self {
434        self.request.method = Some(m.into());
435        self
436    }
437
438    /// Raw request body. Mutually exclusive with [`Self::form_field`].
439    /// Sets the body bytes as-is; add `Content-Type` via
440    /// [`Self::header`] when needed.
441    #[must_use]
442    pub fn body(mut self, data: impl Into<Vec<u8>>) -> Self {
443        self.request.body = Some(data.into());
444        self
445    }
446
447    /// Add a form field. Mutually exclusive with [`Self::body`]. The
448    /// pipeline sends the fields as `application/x-www-form-urlencoded`
449    /// and sets the content-type header automatically.
450    #[must_use]
451    pub fn form_field(mut self, key: impl Into<String>, value: impl Into<String>) -> Self {
452        self.request.form.push((key.into(), value.into()));
453        self
454    }
455
456    /// Capture WebSocket frame payloads to
457    /// `network-bodies/<request_id>.frames.jsonl` during the browser path.
458    #[must_use]
459    pub fn capture_ws(mut self, on: bool) -> Self {
460        self.network.capture_ws = on;
461        self
462    }
463
464    /// Capture SSE event payloads to
465    /// `network-bodies/<request_id>.frames.jsonl` during the browser path.
466    #[must_use]
467    pub fn capture_sse(mut self, on: bool) -> Self {
468        self.network.capture_sse = on;
469        self
470    }
471
472    /// Execute the fetch, with retries when configured. Retries only
473    /// fire for errors carrying `retryable: true`; any other error
474    /// short-circuits immediately.
475    pub async fn send(self) -> Result<FetchResult, Error> {
476        self.send_detailed().await.map_err(FetchError::into_error)
477    }
478
479    /// Execute the fetch and preserve the fetch trace on failure.
480    ///
481    /// This is what the CLI uses to emit `code: "error"` envelopes with a
482    /// fetch-local `trace` without adding trace fields to the global `Error`.
483    pub async fn send_detailed(self) -> Result<FetchResult, FetchError> {
484        if self.retry.attempts == 0 {
485            return execute_once_with_timeout(self).await;
486        }
487        let max_attempts = self.retry.attempts.saturating_add(1);
488        let delay = std::time::Duration::from_millis(self.retry.backoff_ms);
489        let mut attempt: u32 = 0;
490        loop {
491            match execute_once_with_timeout(self.clone()).await {
492                Ok(r) => return Ok(r),
493                Err(e) if e.retryable && attempt + 1 < max_attempts => {
494                    tokio::time::sleep(delay).await;
495                    attempt += 1;
496                }
497                Err(e) => return Err(e),
498            }
499        }
500    }
501}
502
503async fn execute_once_with_timeout(builder: FetchBuilder) -> Result<FetchResult, FetchError> {
504    let timeout = builder.timeout;
505    let render_mode = builder.render.as_trace();
506    let deadline = deadline::FetchDeadline::new(timeout, render_mode);
507    match tokio::time::timeout(timeout, pipeline::execute(builder, deadline.clone())).await {
508        Ok(Ok(result)) => Ok(result),
509        Ok(Err(error)) => Err(FetchError::new(error, deadline.snapshot())),
510        Err(_) => {
511            let error = deadline.timeout_error();
512            Err(FetchError::new(error, deadline.snapshot()))
513        }
514    }
515}
agent_first_http/sdk/fetch/mod.rs

agent_first_http/sdk/fetch/
mod.rs
