lean_ctx/core/web/
fetch.rs

1//! Bounded, SSRF-aware HTTP fetch built on `ureq`.
2//!
3//! Redirects are followed manually so every hop passes back through
4//! [`url_guard`], closing the redirect-to-internal SSRF hole that automatic
5//! redirect following would open. Response bodies are capped to a byte budget so
6//! a hostile server cannot exhaust memory.
7
8use std::io::Read;
9use std::time::Duration;
10
11use super::url_guard::{self, SafeUrl};
12
13/// Default response body cap (4 MiB) — generous for articles, safe for memory.
14pub const DEFAULT_MAX_BYTES: usize = 4 * 1024 * 1024;
15/// Default total request timeout in seconds.
16pub const DEFAULT_TIMEOUT_SECS: u64 = 20;
17
18const MAX_REDIRECTS: u32 = 5;
19const USER_AGENT: &str = "lean-ctx/3.7 (+https://leanctx.com; ctx_url_read)";
20const ACCEPT: &str = "text/html,application/xhtml+xml,text/plain;q=0.9,*/*;q=0.5";
21
22/// A fetched document with its raw body bytes and resolved metadata.
23///
24/// The body is kept as bytes so binary payloads (e.g. PDF) survive intact;
25/// textual callers use [`FetchedDoc::body_text`] for a lossy UTF-8 view.
26pub struct FetchedDoc {
27    pub final_url: String,
28    /// Lower-cased MIME type without parameters (e.g. `text/html`).
29    pub content_type: String,
30    pub bytes: Vec<u8>,
31    pub status: u16,
32    pub truncated: bool,
33}
34
35impl FetchedDoc {
36    /// Lossy UTF-8 view of the body, for textual content (HTML, JSON, …).
37    pub fn body_text(&self) -> String {
38        String::from_utf8_lossy(&self.bytes).into_owned()
39    }
40}
41
42/// Fetch `url`, following up to `MAX_REDIRECTS` re-validated redirects.
43pub fn fetch(url: &str, max_bytes: usize, timeout_secs: u64) -> Result<FetchedDoc, String> {
44    let mut current = url_guard::validate(url).map_err(|e| e.to_string())?;
45    current
46        .ensure_resolves_safely()
47        .map_err(|e| e.to_string())?;
48
49    let agent = build_agent(timeout_secs);
50    let mut hops = 0u32;
51
52    loop {
53        let resp = agent
54            .get(&current.normalized)
55            .header("user-agent", USER_AGENT)
56            .header("accept", ACCEPT)
57            .header("accept-language", "en,*;q=0.5")
58            .call()
59            .map_err(|e| format!("request failed: {e}"))?;
60
61        let status = resp.status().as_u16();
62
63        if (300..400).contains(&status)
64            && hops < MAX_REDIRECTS
65            && let Some(location) = header_value(&resp, "location")
66        {
67            let next = resolve_redirect(&current, &location);
68            let next_url = url_guard::validate(&next).map_err(|e| e.to_string())?;
69            next_url
70                .ensure_resolves_safely()
71                .map_err(|e| e.to_string())?;
72            current = next_url;
73            hops += 1;
74            continue;
75        }
76
77        let content_type = header_value(&resp, "content-type")
78            .and_then(|v| v.split(';').next().map(|m| m.trim().to_ascii_lowercase()))
79            .unwrap_or_default();
80        let (bytes, truncated) = read_bounded(resp, max_bytes)?;
81
82        return Ok(FetchedDoc {
83            final_url: current.normalized.clone(),
84            content_type,
85            bytes,
86            status,
87            truncated,
88        });
89    }
90}
91
92/// POST `body` to `url` (SSRF-guarded, bounded, redirects not followed).
93///
94/// Needed for JSON-RPC style endpoints — e.g. YouTube's InnerTube `player`
95/// API, whose caption URLs are server-fetchable (unlike the watch-page ones).
96/// `user_agent` is explicit because some APIs validate it against the declared
97/// client.
98pub fn post(
99    url: &str,
100    content_type: &str,
101    user_agent: &str,
102    body: &str,
103    max_bytes: usize,
104    timeout_secs: u64,
105) -> Result<FetchedDoc, String> {
106    let target = url_guard::validate(url).map_err(|e| e.to_string())?;
107    target.ensure_resolves_safely().map_err(|e| e.to_string())?;
108
109    let agent = build_agent(timeout_secs);
110    let resp = agent
111        .post(&target.normalized)
112        .header("user-agent", user_agent)
113        .header("content-type", content_type)
114        .header("accept", "application/json, text/xml;q=0.9, */*;q=0.5")
115        .send(body.as_bytes())
116        .map_err(|e| format!("request failed: {e}"))?;
117
118    let status = resp.status().as_u16();
119    let content_type = header_value(&resp, "content-type")
120        .and_then(|v| v.split(';').next().map(|m| m.trim().to_ascii_lowercase()))
121        .unwrap_or_default();
122    let (bytes, truncated) = read_bounded(resp, max_bytes)?;
123
124    Ok(FetchedDoc {
125        final_url: target.normalized,
126        content_type,
127        bytes,
128        status,
129        truncated,
130    })
131}
132
133fn build_agent(timeout_secs: u64) -> ureq::Agent {
134    ureq::Agent::new_with_config(
135        ureq::config::Config::builder()
136            .timeout_global(Some(Duration::from_secs(timeout_secs)))
137            .max_redirects(0)
138            .http_status_as_error(false)
139            .build(),
140    )
141}
142
143fn header_value<B>(resp: &ureq::http::Response<B>, name: &str) -> Option<String> {
144    resp.headers()
145        .get(name)
146        .and_then(|v| v.to_str().ok())
147        .map(str::to_string)
148}
149
150fn read_bounded(
151    resp: ureq::http::Response<ureq::Body>,
152    max_bytes: usize,
153) -> Result<(Vec<u8>, bool), String> {
154    let mut reader = resp.into_body().into_reader();
155    let mut buf: Vec<u8> = Vec::with_capacity(8192.min(max_bytes.max(1)));
156    let mut chunk = [0u8; 8192];
157    let mut truncated = false;
158
159    loop {
160        let n = reader
161            .read(&mut chunk)
162            .map_err(|e| format!("failed to read body: {e}"))?;
163        if n == 0 {
164            break;
165        }
166        let remaining = max_bytes.saturating_sub(buf.len());
167        if remaining == 0 {
168            truncated = true;
169            break;
170        }
171        let take = n.min(remaining);
172        buf.extend_from_slice(&chunk[..take]);
173        if take < n {
174            truncated = true;
175            break;
176        }
177    }
178
179    Ok((buf, truncated))
180}
181
182/// Resolve a (possibly relative) `location` (redirect target or link href)
183/// against a base URL.
184pub(crate) fn resolve_redirect(base: &SafeUrl, location: &str) -> String {
185    let loc = location.trim();
186
187    if loc.starts_with("http://") || loc.starts_with("https://") {
188        return loc.to_string();
189    }
190    if let Some(rest) = loc.strip_prefix("//") {
191        return format!("{}://{rest}", base.scheme);
192    }
193    if loc.starts_with('/') {
194        return format!("{}://{}{loc}", base.scheme, base.authority);
195    }
196
197    // Path-relative: join against the directory of the current path.
198    let base_path = base_path(base);
199    let dir = match base_path.rfind('/') {
200        Some(i) => &base_path[..=i],
201        None => "/",
202    };
203    format!("{}://{}{dir}{loc}", base.scheme, base.authority)
204}
205
206fn base_path(base: &SafeUrl) -> &str {
207    let prefix_len = base.scheme.len() + 3 + base.authority.len();
208    let path = base.normalized.get(prefix_len..).unwrap_or("");
209    let path = path.split(['?', '#']).next().unwrap_or("");
210    if path.is_empty() { "/" } else { path }
211}
212
213#[cfg(test)]
214mod tests {
215    use super::*;
216
217    fn safe(url: &str) -> SafeUrl {
218        url_guard::validate(url).unwrap()
219    }
220
221    #[test]
222    fn redirect_absolute_is_passthrough() {
223        let base = safe("https://a.com/x");
224        assert_eq!(
225            resolve_redirect(&base, "https://b.com/y"),
226            "https://b.com/y"
227        );
228    }
229
230    #[test]
231    fn redirect_scheme_relative() {
232        let base = safe("https://a.com/x");
233        assert_eq!(resolve_redirect(&base, "//c.com/z"), "https://c.com/z");
234    }
235
236    #[test]
237    fn redirect_root_relative() {
238        let base = safe("https://a.com/deep/path?q=1");
239        assert_eq!(resolve_redirect(&base, "/new"), "https://a.com/new");
240    }
241
242    #[test]
243    fn redirect_path_relative_joins_dir() {
244        let base = safe("https://a.com/dir/page.html");
245        assert_eq!(
246            resolve_redirect(&base, "other.html"),
247            "https://a.com/dir/other.html"
248        );
249    }
250
251    #[test]
252    fn redirect_path_relative_from_root() {
253        let base = safe("https://a.com");
254        assert_eq!(resolve_redirect(&base, "page"), "https://a.com/page");
255    }
256}
lean_ctx/core/web/fetch.rs

lean_ctx/core/web/
fetch.rs
