lean_ctx/core/web/
fetch.rs

1//! Bounded, SSRF-aware HTTP fetch built on `ureq`.
2//!
3//! Redirects are followed manually so every hop passes back through
4//! [`url_guard`], closing the redirect-to-internal SSRF hole that automatic
5//! redirect following would open. Response bodies are capped to a byte budget so
6//! a hostile server cannot exhaust memory.
7
8use std::io::Read;
9use std::time::Duration;
10
11use super::url_guard::{self, SafeUrl};
12
13/// Default response body cap (4 MiB) — generous for articles, safe for memory.
14pub const DEFAULT_MAX_BYTES: usize = 4 * 1024 * 1024;
15/// Default total request timeout in seconds.
16pub const DEFAULT_TIMEOUT_SECS: u64 = 20;
17
18const MAX_REDIRECTS: u32 = 5;
19const USER_AGENT: &str = "lean-ctx/3.7 (+https://leanctx.com; ctx_url_read)";
20const ACCEPT: &str = "text/html,application/xhtml+xml,text/plain;q=0.9,*/*;q=0.5";
21
22/// A fetched document with its raw body bytes and resolved metadata.
23///
24/// The body is kept as bytes so binary payloads (e.g. PDF) survive intact;
25/// textual callers use [`FetchedDoc::body_text`] for a lossy UTF-8 view.
26pub struct FetchedDoc {
27    pub final_url: String,
28    /// Lower-cased MIME type without parameters (e.g. `text/html`).
29    pub content_type: String,
30    pub bytes: Vec<u8>,
31    pub status: u16,
32    pub truncated: bool,
33}
34
35impl FetchedDoc {
36    /// Lossy UTF-8 view of the body, for textual content (HTML, JSON, …).
37    pub fn body_text(&self) -> String {
38        String::from_utf8_lossy(&self.bytes).into_owned()
39    }
40}
41
42/// Fetch `url`, following up to `MAX_REDIRECTS` re-validated redirects.
43pub fn fetch(url: &str, max_bytes: usize, timeout_secs: u64) -> Result<FetchedDoc, String> {
44    let mut current = url_guard::validate(url).map_err(|e| e.to_string())?;
45    current
46        .ensure_resolves_safely()
47        .map_err(|e| e.to_string())?;
48
49    let agent = build_agent(timeout_secs);
50    let mut hops = 0u32;
51
52    loop {
53        let resp = agent
54            .get(&current.normalized)
55            .header("user-agent", USER_AGENT)
56            .header("accept", ACCEPT)
57            .header("accept-language", "en,*;q=0.5")
58            .call()
59            .map_err(|e| format!("request failed: {e}"))?;
60
61        let status = resp.status().as_u16();
62
63        if (300..400).contains(&status)
64            && hops < MAX_REDIRECTS
65            && let Some(location) = header_value(&resp, "location")
66        {
67            let next = resolve_redirect(&current, &location);
68            let next_url = url_guard::validate(&next).map_err(|e| e.to_string())?;
69            next_url
70                .ensure_resolves_safely()
71                .map_err(|e| e.to_string())?;
72            current = next_url;
73            hops += 1;
74            continue;
75        }
76
77        let content_type = header_value(&resp, "content-type")
78            .and_then(|v| v.split(';').next().map(|m| m.trim().to_ascii_lowercase()))
79            .unwrap_or_default();
80        let (bytes, truncated) = read_bounded(resp, max_bytes)?;
81
82        return Ok(FetchedDoc {
83            final_url: current.normalized.clone(),
84            content_type,
85            bytes,
86            status,
87            truncated,
88        });
89    }
90}
91
92/// POST `body` to `url` (SSRF-guarded, bounded, redirects not followed).
93///
94/// Needed for JSON-RPC style endpoints — e.g. YouTube's InnerTube `player`
95/// API, whose caption URLs are server-fetchable (unlike the watch-page ones).
96/// `user_agent` is explicit because some APIs validate it against the declared
97/// client.
98pub fn post(
99    url: &str,
100    content_type: &str,
101    user_agent: &str,
102    body: &str,
103    max_bytes: usize,
104    timeout_secs: u64,
105) -> Result<FetchedDoc, String> {
106    let target = url_guard::validate(url).map_err(|e| e.to_string())?;
107    target.ensure_resolves_safely().map_err(|e| e.to_string())?;
108
109    let agent = build_agent(timeout_secs);
110    let resp = agent
111        .post(&target.normalized)
112        .header("user-agent", user_agent)
113        .header("content-type", content_type)
114        .header("accept", "application/json, text/xml;q=0.9, */*;q=0.5")
115        .send(body.as_bytes())
116        .map_err(|e| format!("request failed: {e}"))?;
117
118    let status = resp.status().as_u16();
119    let content_type = header_value(&resp, "content-type")
120        .and_then(|v| v.split(';').next().map(|m| m.trim().to_ascii_lowercase()))
121        .unwrap_or_default();
122    let (bytes, truncated) = read_bounded(resp, max_bytes)?;
123
124    Ok(FetchedDoc {
125        final_url: target.normalized,
126        content_type,
127        bytes,
128        status,
129        truncated,
130    })
131}
132
133fn build_agent(timeout_secs: u64) -> ureq::Agent {
134    crate::core::http_client::ureq_agent(
135        ureq::config::Config::builder()
136            .tls_config(crate::core::http_client::platform_tls_config())
137            .timeout_global(Some(Duration::from_secs(timeout_secs)))
138            .max_redirects(0)
139            .http_status_as_error(false)
140            .build(),
141    )
142}
143
144fn header_value<B>(resp: &ureq::http::Response<B>, name: &str) -> Option<String> {
145    resp.headers()
146        .get(name)
147        .and_then(|v| v.to_str().ok())
148        .map(str::to_string)
149}
150
151fn read_bounded(
152    resp: ureq::http::Response<ureq::Body>,
153    max_bytes: usize,
154) -> Result<(Vec<u8>, bool), String> {
155    let mut reader = resp.into_body().into_reader();
156    let mut buf: Vec<u8> = Vec::with_capacity(8192.min(max_bytes.max(1)));
157    let mut chunk = [0u8; 8192];
158    let mut truncated = false;
159
160    loop {
161        let n = reader
162            .read(&mut chunk)
163            .map_err(|e| format!("failed to read body: {e}"))?;
164        if n == 0 {
165            break;
166        }
167        let remaining = max_bytes.saturating_sub(buf.len());
168        if remaining == 0 {
169            truncated = true;
170            break;
171        }
172        let take = n.min(remaining);
173        buf.extend_from_slice(&chunk[..take]);
174        if take < n {
175            truncated = true;
176            break;
177        }
178    }
179
180    Ok((buf, truncated))
181}
182
183/// Resolve a (possibly relative) `location` (redirect target or link href)
184/// against a base URL.
185pub(crate) fn resolve_redirect(base: &SafeUrl, location: &str) -> String {
186    let loc = location.trim();
187
188    if loc.starts_with("http://") || loc.starts_with("https://") {
189        return loc.to_string();
190    }
191    if let Some(rest) = loc.strip_prefix("//") {
192        return format!("{}://{rest}", base.scheme);
193    }
194    if loc.starts_with('/') {
195        return format!("{}://{}{loc}", base.scheme, base.authority);
196    }
197
198    // Path-relative: join against the directory of the current path.
199    let base_path = base_path(base);
200    let dir = match base_path.rfind('/') {
201        Some(i) => &base_path[..=i],
202        None => "/",
203    };
204    format!("{}://{}{dir}{loc}", base.scheme, base.authority)
205}
206
207fn base_path(base: &SafeUrl) -> &str {
208    let prefix_len = base.scheme.len() + 3 + base.authority.len();
209    let path = base.normalized.get(prefix_len..).unwrap_or("");
210    let path = path.split(['?', '#']).next().unwrap_or("");
211    if path.is_empty() { "/" } else { path }
212}
213
214#[cfg(test)]
215mod tests {
216    use super::*;
217
218    fn safe(url: &str) -> SafeUrl {
219        url_guard::validate(url).unwrap()
220    }
221
222    #[test]
223    fn redirect_absolute_is_passthrough() {
224        let base = safe("https://a.com/x");
225        assert_eq!(
226            resolve_redirect(&base, "https://b.com/y"),
227            "https://b.com/y"
228        );
229    }
230
231    #[test]
232    fn redirect_scheme_relative() {
233        let base = safe("https://a.com/x");
234        assert_eq!(resolve_redirect(&base, "//c.com/z"), "https://c.com/z");
235    }
236
237    #[test]
238    fn redirect_root_relative() {
239        let base = safe("https://a.com/deep/path?q=1");
240        assert_eq!(resolve_redirect(&base, "/new"), "https://a.com/new");
241    }
242
243    #[test]
244    fn redirect_path_relative_joins_dir() {
245        let base = safe("https://a.com/dir/page.html");
246        assert_eq!(
247            resolve_redirect(&base, "other.html"),
248            "https://a.com/dir/other.html"
249        );
250    }
251
252    #[test]
253    fn redirect_path_relative_from_root() {
254        let base = safe("https://a.com");
255        assert_eq!(resolve_redirect(&base, "page"), "https://a.com/page");
256    }
257}
lean_ctx/core/web/fetch.rs

lean_ctx/core/web/
fetch.rs
