lean_ctx/core/web/
fetch.rs

1//! Bounded, SSRF-aware HTTP fetch built on `ureq`.
2//!
3//! Redirects are followed manually so every hop passes back through
4//! [`url_guard`], closing the redirect-to-internal SSRF hole that automatic
5//! redirect following would open. Response bodies are capped to a byte budget so
6//! a hostile server cannot exhaust memory.
7
8use std::io::Read;
9use std::time::Duration;
10
11use super::url_guard::{self, SafeUrl};
12
13/// Default response body cap (4 MiB) — generous for articles, safe for memory.
14pub const DEFAULT_MAX_BYTES: usize = 4 * 1024 * 1024;
15/// Default total request timeout in seconds.
16pub const DEFAULT_TIMEOUT_SECS: u64 = 20;
17
18const MAX_REDIRECTS: u32 = 5;
19const USER_AGENT: &str = "lean-ctx/3.7 (+https://leanctx.com; ctx_url_read)";
20const ACCEPT: &str = "text/html,application/xhtml+xml,text/plain;q=0.9,*/*;q=0.5";
21
22/// A fetched document with its raw body bytes and resolved metadata.
23///
24/// The body is kept as bytes so binary payloads (e.g. PDF) survive intact;
25/// textual callers use [`FetchedDoc::body_text`] for a lossy UTF-8 view.
26pub struct FetchedDoc {
27    pub final_url: String,
28    /// Lower-cased MIME type without parameters (e.g. `text/html`).
29    pub content_type: String,
30    pub bytes: Vec<u8>,
31    pub status: u16,
32    pub truncated: bool,
33}
34
35impl FetchedDoc {
36    /// Lossy UTF-8 view of the body, for textual content (HTML, JSON, …).
37    pub fn body_text(&self) -> String {
38        String::from_utf8_lossy(&self.bytes).into_owned()
39    }
40}
41
42/// Fetch `url`, following up to `MAX_REDIRECTS` re-validated redirects.
43pub fn fetch(url: &str, max_bytes: usize, timeout_secs: u64) -> Result<FetchedDoc, String> {
44    let mut current = url_guard::validate(url).map_err(|e| e.to_string())?;
45    current
46        .ensure_resolves_safely()
47        .map_err(|e| e.to_string())?;
48
49    let agent = build_agent(timeout_secs);
50    let mut hops = 0u32;
51
52    loop {
53        let resp = agent
54            .get(&current.normalized)
55            .header("user-agent", USER_AGENT)
56            .header("accept", ACCEPT)
57            .header("accept-language", "en,*;q=0.5")
58            .call()
59            .map_err(|e| format!("request failed: {e}"))?;
60
61        let status = resp.status().as_u16();
62
63        if (300..400).contains(&status)
64            && hops < MAX_REDIRECTS
65            && let Some(location) = header_value(&resp, "location")
66        {
67            let next = resolve_redirect(&current, &location);
68            let next_url = url_guard::validate(&next).map_err(|e| e.to_string())?;
69            next_url
70                .ensure_resolves_safely()
71                .map_err(|e| e.to_string())?;
72            current = next_url;
73            hops += 1;
74            continue;
75        }
76
77        let content_type = header_value(&resp, "content-type")
78            .and_then(|v| v.split(';').next().map(|m| m.trim().to_ascii_lowercase()))
79            .unwrap_or_default();
80        let (bytes, truncated) = read_bounded(resp, max_bytes)?;
81
82        return Ok(FetchedDoc {
83            final_url: current.normalized.clone(),
84            content_type,
85            bytes,
86            status,
87            truncated,
88        });
89    }
90}
91
92/// POST `body` to `url` (SSRF-guarded, bounded, redirects not followed).
93///
94/// Needed for JSON-RPC style endpoints — e.g. YouTube's InnerTube `player`
95/// API, whose caption URLs are server-fetchable (unlike the watch-page ones).
96/// `user_agent` is explicit because some APIs validate it against the declared
97/// client.
98pub fn post(
99    url: &str,
100    content_type: &str,
101    user_agent: &str,
102    body: &str,
103    max_bytes: usize,
104    timeout_secs: u64,
105) -> Result<FetchedDoc, String> {
106    let target = url_guard::validate(url).map_err(|e| e.to_string())?;
107    target.ensure_resolves_safely().map_err(|e| e.to_string())?;
108
109    let agent = build_agent(timeout_secs);
110    let resp = agent
111        .post(&target.normalized)
112        .header("user-agent", user_agent)
113        .header("content-type", content_type)
114        .header("accept", "application/json, text/xml;q=0.9, */*;q=0.5")
115        .send(body.as_bytes())
116        .map_err(|e| format!("request failed: {e}"))?;
117
118    let status = resp.status().as_u16();
119    let content_type = header_value(&resp, "content-type")
120        .and_then(|v| v.split(';').next().map(|m| m.trim().to_ascii_lowercase()))
121        .unwrap_or_default();
122    let (bytes, truncated) = read_bounded(resp, max_bytes)?;
123
124    Ok(FetchedDoc {
125        final_url: target.normalized,
126        content_type,
127        bytes,
128        status,
129        truncated,
130    })
131}
132
133fn build_agent(timeout_secs: u64) -> ureq::Agent {
134    // #ssrf-rebinding: use the SSRF-pinning resolver instead of the plain
135    // `http_client::ureq_agent` helper — see `url_guard::SsrfSafeResolver`.
136    let config = ureq::config::Config::builder()
137        .tls_config(crate::core::http_client::platform_tls_config())
138        .timeout_global(Some(Duration::from_secs(timeout_secs)))
139        .max_redirects(0)
140        .http_status_as_error(false)
141        .build();
142    ureq::Agent::with_parts(
143        config,
144        ureq::unversioned::transport::DefaultConnector::default(),
145        url_guard::SsrfSafeResolver::default(),
146    )
147}
148
149fn header_value<B>(resp: &ureq::http::Response<B>, name: &str) -> Option<String> {
150    resp.headers()
151        .get(name)
152        .and_then(|v| v.to_str().ok())
153        .map(str::to_string)
154}
155
156fn read_bounded(
157    resp: ureq::http::Response<ureq::Body>,
158    max_bytes: usize,
159) -> Result<(Vec<u8>, bool), String> {
160    let mut reader = resp.into_body().into_reader();
161    let mut buf: Vec<u8> = Vec::with_capacity(8192.min(max_bytes.max(1)));
162    let mut chunk = [0u8; 8192];
163    let mut truncated = false;
164
165    loop {
166        let n = reader
167            .read(&mut chunk)
168            .map_err(|e| format!("failed to read body: {e}"))?;
169        if n == 0 {
170            break;
171        }
172        let remaining = max_bytes.saturating_sub(buf.len());
173        if remaining == 0 {
174            truncated = true;
175            break;
176        }
177        let take = n.min(remaining);
178        buf.extend_from_slice(&chunk[..take]);
179        if take < n {
180            truncated = true;
181            break;
182        }
183    }
184
185    Ok((buf, truncated))
186}
187
188/// Resolve a (possibly relative) `location` (redirect target or link href)
189/// against a base URL.
190pub(crate) fn resolve_redirect(base: &SafeUrl, location: &str) -> String {
191    let loc = location.trim();
192
193    if loc.starts_with("http://") || loc.starts_with("https://") {
194        return loc.to_string();
195    }
196    if let Some(rest) = loc.strip_prefix("//") {
197        return format!("{}://{rest}", base.scheme);
198    }
199    if loc.starts_with('/') {
200        return format!("{}://{}{loc}", base.scheme, base.authority);
201    }
202
203    // Path-relative: join against the directory of the current path.
204    let base_path = base_path(base);
205    let dir = match base_path.rfind('/') {
206        Some(i) => &base_path[..=i],
207        None => "/",
208    };
209    format!("{}://{}{dir}{loc}", base.scheme, base.authority)
210}
211
212fn base_path(base: &SafeUrl) -> &str {
213    let prefix_len = base.scheme.len() + 3 + base.authority.len();
214    let path = base.normalized.get(prefix_len..).unwrap_or("");
215    let path = path.split(['?', '#']).next().unwrap_or("");
216    if path.is_empty() { "/" } else { path }
217}
218
219#[cfg(test)]
220mod tests {
221    use super::*;
222
223    fn safe(url: &str) -> SafeUrl {
224        url_guard::validate(url).unwrap()
225    }
226
227    #[test]
228    fn redirect_absolute_is_passthrough() {
229        let base = safe("https://a.com/x");
230        assert_eq!(
231            resolve_redirect(&base, "https://b.com/y"),
232            "https://b.com/y"
233        );
234    }
235
236    #[test]
237    fn redirect_scheme_relative() {
238        let base = safe("https://a.com/x");
239        assert_eq!(resolve_redirect(&base, "//c.com/z"), "https://c.com/z");
240    }
241
242    #[test]
243    fn redirect_root_relative() {
244        let base = safe("https://a.com/deep/path?q=1");
245        assert_eq!(resolve_redirect(&base, "/new"), "https://a.com/new");
246    }
247
248    #[test]
249    fn redirect_path_relative_joins_dir() {
250        let base = safe("https://a.com/dir/page.html");
251        assert_eq!(
252            resolve_redirect(&base, "other.html"),
253            "https://a.com/dir/other.html"
254        );
255    }
256
257    #[test]
258    fn redirect_path_relative_from_root() {
259        let base = safe("https://a.com");
260        assert_eq!(resolve_redirect(&base, "page"), "https://a.com/page");
261    }
262}
lean_ctx/core/web/fetch.rs

lean_ctx/core/web/
fetch.rs
