agent_first_http/sdk/fetch/
mod.rs

1//! Fetch pipeline: HTTP-only fast path, browser-backed render path
2//! (`--render none|auto|always`), and the fetch artifacts
3//! (`architecture.md §8`).
4
5pub mod artifacts;
6pub(crate) mod deadline;
7pub(crate) mod page_classification;
8pub mod pipeline;
9pub mod result;
10pub mod wait;
11pub mod writer;
12
13pub type FetchCookie = cookie::Cookie<'static>;
14pub use cookie::SameSite as FetchCookieSameSite;
15pub use pipeline::{NetworkBodies, RenderMode};
16pub use result::{FetchError, FetchResult, PageKind};
17pub use wait::Wait;
18
19use std::collections::BTreeSet;
20use std::path::PathBuf;
21use std::time::Duration;
22
23use crate::sdk::client::Client;
24use crate::shared::artifacts::Artifact;
25use crate::shared::error::Error;
26use crate::shared::ids::TabId;
27
28/// Default per-network-response body capture cap: 10 MiB.
29pub const DEFAULT_NETWORK_BODY_MAX_BYTES: u64 = 10 * 1024 * 1024;
30
31/// Builder for `Client::fetch(...).send().await`.
32#[derive(Clone)]
33pub struct FetchBuilder {
34    pub(crate) client: Client,
35    pub(crate) url: String,
36    pub(crate) render: RenderMode,
37    pub(crate) wait: Wait,
38    pub(crate) timeout: Duration,
39    pub(crate) want: BTreeSet<Artifact>,
40    pub(crate) tab: Option<TabId>,
41    pub(crate) request: RequestOptions,
42    pub(crate) out_dir: Option<PathBuf>,
43    pub(crate) readiness: ReadinessOptions,
44    pub(crate) network: NetworkCapture,
45    pub(crate) retry: RetryOptions,
46    pub(crate) http: HttpOptions,
47    pub(crate) cookie_jar: CookieJarOptions,
48}
49
50/// Browser-path readiness tuning: how long `--wait auto` waits for network
51/// quiet and DOM/text stability, plus the main-document observation cap.
52#[derive(Clone)]
53pub(crate) struct ReadinessOptions {
54    pub(crate) idle_ms: u64,
55    pub(crate) stable_ms: u64,
56    pub(crate) min_text_bytes: u64,
57    /// Upper bound, in milliseconds, on how long the browser path waits for
58    /// the main document network event before falling back to capturing
59    /// artifacts with `main_request_observed: false`. Default 500ms suits
60    /// well-behaved pages on fast networks; slow networks may need more.
61    pub(crate) observe_main_wait_ms: u64,
62}
63
64/// Network-capture knobs: response-body capture mode/cap, header redaction,
65/// and WebSocket/SSE frame capture.
66#[derive(Clone)]
67pub(crate) struct NetworkCapture {
68    pub(crate) bodies: NetworkBodies,
69    pub(crate) body_max_bytes: u64,
70    pub(crate) redact: bool,
71    /// Capture WebSocket frame payloads to `network-bodies/<id>.frames.jsonl`.
72    pub(crate) capture_ws: bool,
73    /// Capture SSE event payloads to `network-bodies/<id>.frames.jsonl`.
74    pub(crate) capture_sse: bool,
75}
76
77/// Retry policy. The fetch is retried only when the error carries
78/// `retryable: true`.
79#[derive(Clone)]
80pub(crate) struct RetryOptions {
81    /// Number of additional attempts after the first one. `0` (the default)
82    /// keeps the single-attempt behavior.
83    pub(crate) attempts: u32,
84    /// Fixed delay between retry attempts, in milliseconds. Retry
85    /// orchestration beyond a fixed interval is the agent's job; the tool
86    /// just gives it the primitive.
87    pub(crate) backoff_ms: u64,
88}
89
90/// HTTP fast-path transport options: upstream proxy, extra trust anchors,
91/// TLS verification, and the response-body size cap.
92#[derive(Clone)]
93pub(crate) struct HttpOptions {
94    /// Per-fetch upstream proxy for the HTTP fast path. The SDK builds a
95    /// dedicated reqwest client when this (or `ca_cert` / `tls_insecure`) is
96    /// set so the per-Client default reqwest is not contaminated. `None`
97    /// keeps the default direct connection.
98    pub(crate) proxy: Option<String>,
99    /// Path to a PEM file containing extra root certificates to trust for
100    /// this fetch's HTTP path. Useful for self-signed targets or corporate
101    /// MITM CAs without weakening the global trust store.
102    pub(crate) ca_cert: Option<PathBuf>,
103    /// Disable TLS certificate verification for the HTTP path. The agent
104    /// must opt in explicitly — this is dangerous and the CLI help says so.
105    pub(crate) tls_insecure: bool,
106    /// Upper bound, in bytes, on the HTTP fast path's response body before
107    /// the pipeline stops accumulating and emits a `network_body_truncated`
108    /// warning instead. Default 1 GiB — generous enough that normal pages
109    /// and downloads never trip it, low enough that a pathological multi-GB
110    /// download cannot OOM the host. `0` disables the cap entirely.
111    pub(crate) max_response_bytes: u64,
112}
113
114/// Cookie-jar selection for the fetch.
115#[derive(Clone)]
116pub(crate) struct CookieJarOptions {
117    /// Explicit cookie-jar path override. Normally the pipeline derives
118    /// `<profile>/cookies.jar.json` from the host's `GET /profile` — setting
119    /// this tells the pipeline to use the given path instead. The override
120    /// must canonicalize to the host's profile directory or the pipeline
121    /// rejects with `invalid_argument`.
122    pub(crate) path: Option<PathBuf>,
123    pub(crate) warning: Option<String>,
124    /// Opt out of the cookie jar entirely for this fetch. Useful for agents
125    /// that want a clean request even when the host has a persistent profile
126    /// (e.g. recon traffic that should not carry authenticated session
127    /// cookies).
128    pub(crate) disabled: bool,
129}
130
131#[derive(Clone, Debug, Default)]
132pub(crate) struct RequestOptions {
133    pub(crate) headers: Vec<(String, String)>,
134    pub(crate) user_agent: Option<String>,
135    pub(crate) cookies: Vec<FetchCookie>,
136    pub(crate) evaluate_after_wait: Vec<String>,
137    /// HTTP method. `None` = GET (default). Uppercase recommended; the
138    /// pipeline normalises before sending.
139    pub(crate) method: Option<String>,
140    /// Raw request body bytes (mutually exclusive with `form`).
141    pub(crate) body: Option<Vec<u8>>,
142    /// Form fields sent as `application/x-www-form-urlencoded` (mutually
143    /// exclusive with `body`).
144    pub(crate) form: Vec<(String, String)>,
145}
146
147impl FetchBuilder {
148    pub(crate) fn new(client: Client, url: String) -> Self {
149        Self {
150            client,
151            url,
152            render: RenderMode::Auto,
153            wait: Wait::Auto,
154            timeout: Duration::from_secs(30),
155            want: Artifact::ALL.iter().copied().collect(),
156            tab: None,
157            request: RequestOptions::default(),
158            out_dir: None,
159            readiness: ReadinessOptions {
160                idle_ms: 800,
161                stable_ms: 500,
162                min_text_bytes: 32,
163                observe_main_wait_ms: 500,
164            },
165            network: NetworkCapture {
166                bodies: NetworkBodies::Off,
167                body_max_bytes: DEFAULT_NETWORK_BODY_MAX_BYTES,
168                redact: true,
169                capture_ws: false,
170                capture_sse: false,
171            },
172            retry: RetryOptions {
173                attempts: 0,
174                backoff_ms: 250,
175            },
176            http: HttpOptions {
177                proxy: None,
178                ca_cert: None,
179                tls_insecure: false,
180                max_response_bytes: 1_073_741_824,
181            },
182            cookie_jar: CookieJarOptions {
183                path: None,
184                warning: None,
185                disabled: false,
186            },
187        }
188    }
189
190    #[must_use]
191    pub fn render(mut self, mode: RenderMode) -> Self {
192        self.render = mode;
193        self
194    }
195
196    #[must_use]
197    pub fn wait(mut self, w: Wait) -> Self {
198        self.wait = w;
199        self
200    }
201
202    #[must_use]
203    pub fn timeout(mut self, d: Duration) -> Self {
204        self.timeout = d;
205        self
206    }
207
208    #[must_use]
209    pub fn readiness_idle_ms(mut self, ms: u64) -> Self {
210        self.readiness.idle_ms = ms;
211        self
212    }
213
214    #[must_use]
215    pub fn readiness_stable_ms(mut self, ms: u64) -> Self {
216        self.readiness.stable_ms = ms;
217        self
218    }
219
220    #[must_use]
221    pub fn readiness_min_text_bytes(mut self, bytes: u64) -> Self {
222        self.readiness.min_text_bytes = bytes;
223        self
224    }
225
226    #[must_use]
227    pub fn want<I: IntoIterator<Item = Artifact>>(mut self, items: I) -> Self {
228        self.want = items.into_iter().collect();
229        self
230    }
231
232    #[must_use]
233    pub fn tab(mut self, tab: TabId) -> Self {
234        self.tab = Some(tab);
235        self
236    }
237
238    #[must_use]
239    pub fn network_bodies(mut self, mode: NetworkBodies) -> Self {
240        self.network.bodies = mode;
241        self
242    }
243
244    #[must_use]
245    pub fn network_body_max_bytes(mut self, n: u64) -> Self {
246        self.network.body_max_bytes = n;
247        self
248    }
249
250    #[must_use]
251    pub fn network_redact(mut self, on: bool) -> Self {
252        self.network.redact = on;
253        self
254    }
255
256    /// Add a request header. `User-Agent` is normalized to
257    /// [`Self::user_agent`] at send time so browser fetches use the CDP UA
258    /// override instead of a plain extra header.
259    #[must_use]
260    pub fn header(mut self, name: impl Into<String>, value: impl Into<String>) -> Self {
261        self.request.headers.push((name.into(), value.into()));
262        self
263    }
264
265    /// Add multiple request headers.
266    #[must_use]
267    pub fn headers<I, K, V>(mut self, headers: I) -> Self
268    where
269        I: IntoIterator<Item = (K, V)>,
270        K: Into<String>,
271        V: Into<String>,
272    {
273        self.request
274            .headers
275            .extend(headers.into_iter().map(|(k, v)| (k.into(), v.into())));
276        self
277    }
278
279    /// Override the browser/client user agent.
280    #[must_use]
281    pub fn user_agent(mut self, value: impl Into<String>) -> Self {
282        self.request.user_agent = Some(value.into());
283        self
284    }
285
286    /// Add a request cookie as a `name=value` pair.
287    #[must_use]
288    pub fn cookie(mut self, name: impl Into<String>, value: impl Into<String>) -> Self {
289        self.request
290            .cookies
291            .push(cookie::Cookie::new(name.into(), value.into()));
292        self
293    }
294
295    /// Add a full cookie, including optional Domain/Path/Secure/HttpOnly/
296    /// SameSite/Max-Age/Expires attributes.
297    #[must_use]
298    pub fn cookie_full(mut self, cookie: FetchCookie) -> Self {
299        self.request.cookies.push(cookie);
300        self
301    }
302
303    /// Add multiple request cookies as `name=value` pairs.
304    #[must_use]
305    pub fn cookies<I, K, V>(mut self, cookies: I) -> Self
306    where
307        I: IntoIterator<Item = (K, V)>,
308        K: Into<String>,
309        V: Into<String>,
310    {
311        self.request.cookies.extend(
312            cookies
313                .into_iter()
314                .map(|(k, v)| cookie::Cookie::new(k.into(), v.into())),
315        );
316        self
317    }
318
319    /// Evaluate JavaScript after the configured wait condition and before
320    /// artifact capture. Only browser-backed fetches can execute scripts.
321    #[must_use]
322    pub fn evaluate_after_wait(mut self, js: impl Into<String>) -> Self {
323        self.request.evaluate_after_wait.push(js.into());
324        self
325    }
326
327    #[must_use]
328    pub fn out_dir(mut self, dir: impl Into<PathBuf>) -> Self {
329        self.out_dir = Some(dir.into());
330        self
331    }
332
333    /// Override the cookie-jar path. The default — derived from the host's
334    /// `GET /profile` — places the jar at `<profile-dir>/cookies.jar.json`,
335    /// which is the only path the isolation invariant permits. This
336    /// override exists for tests and forensic tooling; the pipeline
337    /// canonicalizes the given path and rejects it with `invalid_argument`
338    /// if it doesn't match the host's profile directory.
339    #[must_use]
340    pub fn cookie_jar(mut self, path: impl Into<PathBuf>) -> Self {
341        self.cookie_jar.path = Some(path.into());
342        self
343    }
344
345    /// Opt out of cookie-jar persistence for this fetch. The request goes
346    /// out without any session cookies the jar might hold, and the
347    /// response's `Set-Cookie` headers are not merged back.
348    #[must_use]
349    pub fn no_cookie_jar(mut self) -> Self {
350        self.cookie_jar.disabled = true;
351        self
352    }
353
354    /// Upper bound on the browser-path wait for the main document
355    /// network event, in milliseconds. Default 500ms is tuned for
356    /// well-behaved pages on fast networks; raise for slow networks
357    /// or low-end machines.
358    #[must_use]
359    pub fn observe_main_wait_ms(mut self, ms: u64) -> Self {
360        self.readiness.observe_main_wait_ms = ms;
361        self
362    }
363
364    /// Upper bound on the HTTP-path response body, in bytes. Default
365    /// 1 GiB. `0` disables the cap entirely. When the cap is hit, the
366    /// fetch returns successfully with a `network_body_truncated`
367    /// warning and the prefix bytes that were collected.
368    #[must_use]
369    pub fn max_response_bytes(mut self, bytes: u64) -> Self {
370        self.http.max_response_bytes = bytes;
371        self
372    }
373
374    /// Number of additional attempts after the first. `0` (default)
375    /// keeps the single-attempt behavior. Retries only fire when the
376    /// pipeline error has `retryable: true`.
377    #[must_use]
378    pub fn retry(mut self, n: u32) -> Self {
379        self.retry.attempts = n;
380        self
381    }
382
383    /// Fixed delay between retry attempts, in milliseconds.
384    #[must_use]
385    pub fn backoff_ms(mut self, ms: u64) -> Self {
386        self.retry.backoff_ms = ms;
387        self
388    }
389
390    /// Per-fetch upstream proxy URL for the HTTP path. Format:
391    /// `http://user:pass@host:port` or `socks5://host:port`. The SDK
392    /// never honors `HTTP_PROXY`/`HTTPS_PROXY` from the environment;
393    /// this method is the only way to route an HTTP-path fetch
394    /// through a proxy.
395    #[must_use]
396    pub fn proxy(mut self, url: impl Into<String>) -> Self {
397        self.http.proxy = Some(url.into());
398        self
399    }
400
401    /// Path to a PEM file containing extra root CAs to trust for
402    /// this fetch's HTTP path. Stacks on top of the platform trust
403    /// store; does not replace it.
404    #[must_use]
405    pub fn ca_cert(mut self, path: impl Into<PathBuf>) -> Self {
406        self.http.ca_cert = Some(path.into());
407        self
408    }
409
410    /// Disable TLS certificate verification for this fetch's HTTP
411    /// path. Dangerous — leaves the connection open to MITM. Use
412    /// only against known-self-signed staging environments.
413    #[must_use]
414    pub fn tls_insecure(mut self, on: bool) -> Self {
415        self.http.tls_insecure = on;
416        self
417    }
418
419    /// HTTP method. Defaults to `GET`. Pass `"POST"`, `"PUT"`, etc.
420    #[must_use]
421    pub fn method(mut self, m: impl Into<String>) -> Self {
422        self.request.method = Some(m.into());
423        self
424    }
425
426    /// Raw request body. Mutually exclusive with [`Self::form_field`].
427    /// Sets the body bytes as-is; add `Content-Type` via
428    /// [`Self::header`] when needed.
429    #[must_use]
430    pub fn body(mut self, data: impl Into<Vec<u8>>) -> Self {
431        self.request.body = Some(data.into());
432        self
433    }
434
435    /// Add a form field. Mutually exclusive with [`Self::body`]. The
436    /// pipeline sends the fields as `application/x-www-form-urlencoded`
437    /// and sets the content-type header automatically.
438    #[must_use]
439    pub fn form_field(mut self, key: impl Into<String>, value: impl Into<String>) -> Self {
440        self.request.form.push((key.into(), value.into()));
441        self
442    }
443
444    /// Capture WebSocket frame payloads to
445    /// `network-bodies/<request_id>.frames.jsonl` during the browser path.
446    #[must_use]
447    pub fn capture_ws(mut self, on: bool) -> Self {
448        self.network.capture_ws = on;
449        self
450    }
451
452    /// Capture SSE event payloads to
453    /// `network-bodies/<request_id>.frames.jsonl` during the browser path.
454    #[must_use]
455    pub fn capture_sse(mut self, on: bool) -> Self {
456        self.network.capture_sse = on;
457        self
458    }
459
460    /// Execute the fetch, with retries when configured. Retries only
461    /// fire for errors carrying `retryable: true`; any other error
462    /// short-circuits immediately.
463    pub async fn send(self) -> Result<FetchResult, Error> {
464        self.send_detailed().await.map_err(FetchError::into_error)
465    }
466
467    /// Execute the fetch and preserve the fetch trace on failure.
468    ///
469    /// This is what the CLI uses to emit `code: "error"` envelopes with a
470    /// fetch-local `trace` without adding trace fields to the global `Error`.
471    pub async fn send_detailed(self) -> Result<FetchResult, FetchError> {
472        if self.retry.attempts == 0 {
473            return execute_once_with_timeout(self).await;
474        }
475        let max_attempts = self.retry.attempts.saturating_add(1);
476        let delay = std::time::Duration::from_millis(self.retry.backoff_ms);
477        let mut attempt: u32 = 0;
478        loop {
479            match execute_once_with_timeout(self.clone()).await {
480                Ok(r) => return Ok(r),
481                Err(e) if e.retryable && attempt + 1 < max_attempts => {
482                    tokio::time::sleep(delay).await;
483                    attempt += 1;
484                }
485                Err(e) => return Err(e),
486            }
487        }
488    }
489}
490
491async fn execute_once_with_timeout(builder: FetchBuilder) -> Result<FetchResult, FetchError> {
492    let timeout = builder.timeout;
493    let render_mode = builder.render.as_trace();
494    let deadline = deadline::FetchDeadline::new(timeout, render_mode);
495    match tokio::time::timeout(timeout, pipeline::execute(builder, deadline.clone())).await {
496        Ok(Ok(result)) => Ok(result),
497        Ok(Err(error)) => Err(FetchError::new(error, deadline.snapshot())),
498        Err(_) => {
499            let error = deadline.timeout_error();
500            Err(FetchError::new(error, deadline.snapshot()))
501        }
502    }
503}
agent_first_http/sdk/fetch/mod.rs

agent_first_http/sdk/fetch/
mod.rs
