lean_ctx/core/web/
mod.rs

1//! Web & research context layer.
2//!
3//! Turns an arbitrary URL (web page or YouTube video) into compressed,
4//! citation-backed context for an agent. The flow is:
5//!
6//! 1. [`url_guard`] validates the URL and blocks SSRF targets.
7//! 2. [`fetch`] downloads it (bounded, manual-redirect, SSRF-revalidated) — or
8//!    [`youtube`] pulls a transcript for video URLs.
9//! 3. [`html_to_text`] renders HTML to clean Markdown.
10//! 4. [`distill`] applies the requested research-compression mode.
11//! 5. [`citation`] attaches source attribution.
12//!
13//! The single entry point is [`read_url`]; the [`crate::tools::registered::ctx_url_read`]
14//! MCP tool is a thin wrapper over it.
15
16pub mod citation;
17pub mod distill;
18pub mod feed;
19pub mod fetch;
20pub mod html_to_text;
21pub mod pdf;
22pub mod rewrite;
23pub mod url_guard;
24pub mod youtube;
25
26use crate::core::evidence::Claim;
27
28use citation::Citation;
29
30/// Default token budget for returned content.
31pub const DEFAULT_MAX_TOKENS: usize = 6000;
32/// Default number of items for `facts` / `quotes` modes.
33pub const DEFAULT_MAX_ITEMS: usize = 12;
34const MAX_LINKS: usize = 100;
35
36/// How fetched content should be distilled before returning.
37#[derive(Debug, Clone, Copy, PartialEq, Eq)]
38pub enum ReadMode {
39    /// Pick a sensible mode from the content type (Markdown for pages,
40    /// transcript summary for videos).
41    Auto,
42    /// Clean Markdown of the main content.
43    Markdown,
44    /// Plain text (Markdown decorations stripped).
45    Text,
46    /// Extracted hyperlinks.
47    Links,
48    /// Sentences carrying factual signals.
49    Facts,
50    /// Central / query-relevant sentences as evidence.
51    Quotes,
52    /// De-duplicated, filler-stripped transcript (best for videos).
53    Transcript,
54}
55
56impl ReadMode {
57    pub fn parse(s: &str) -> Option<Self> {
58        match s.trim().to_ascii_lowercase().as_str() {
59            "auto" => Some(Self::Auto),
60            "markdown" | "md" => Some(Self::Markdown),
61            "text" | "plain" => Some(Self::Text),
62            "links" => Some(Self::Links),
63            "facts" => Some(Self::Facts),
64            "quotes" => Some(Self::Quotes),
65            "transcript" | "summary" => Some(Self::Transcript),
66            _ => None,
67        }
68    }
69
70    pub fn label(self) -> &'static str {
71        match self {
72            Self::Auto => "auto",
73            Self::Markdown => "markdown",
74            Self::Text => "text",
75            Self::Links => "links",
76            Self::Facts => "facts",
77            Self::Quotes => "quotes",
78            Self::Transcript => "transcript",
79        }
80    }
81}
82
83/// Parameters for [`read_url`].
84pub struct ReadOptions<'a> {
85    pub url: &'a str,
86    pub mode: ReadMode,
87    pub query: Option<&'a str>,
88    pub max_tokens: usize,
89    pub max_items: usize,
90    pub timeout_secs: u64,
91}
92
93impl<'a> ReadOptions<'a> {
94    pub fn new(url: &'a str) -> Self {
95        Self {
96            url,
97            mode: ReadMode::Auto,
98            query: None,
99            max_tokens: DEFAULT_MAX_TOKENS,
100            max_items: DEFAULT_MAX_ITEMS,
101            timeout_secs: fetch::DEFAULT_TIMEOUT_SECS,
102        }
103    }
104}
105
106/// Result of a successful [`read_url`].
107pub struct ReadResult {
108    /// Distilled content with the citation footer already appended.
109    pub content: String,
110    /// Effective mode after `Auto` resolution.
111    pub mode: ReadMode,
112    /// Token count of the raw fetched payload (for savings accounting).
113    pub original_tokens: usize,
114    pub final_url: String,
115}
116
117/// Fetch and distill a URL into citation-backed context.
118pub fn read_url(opts: &ReadOptions) -> Result<ReadResult, String> {
119    // Rewrite known page URLs (e.g. GitHub blob → raw) to their clean-content
120    // equivalent before fetching, so the agent gets the file instead of chrome.
121    let rewritten = rewrite::rewrite_url(opts.url);
122    let url = rewritten.as_deref().unwrap_or(opts.url);
123
124    if let Some(id) = youtube::video_id(url) {
125        return read_youtube(&id, opts);
126    }
127
128    let effective = ReadOptions {
129        url,
130        mode: opts.mode,
131        query: opts.query,
132        max_tokens: opts.max_tokens,
133        max_items: opts.max_items,
134        timeout_secs: opts.timeout_secs,
135    };
136    read_web(&effective)
137}
138
139fn read_web(opts: &ReadOptions) -> Result<ReadResult, String> {
140    let doc = fetch::fetch(opts.url, fetch::DEFAULT_MAX_BYTES, opts.timeout_secs)?;
141    if doc.status >= 400 {
142        return Err(format!("HTTP {} from {}", doc.status, doc.final_url));
143    }
144
145    let is_pdf = doc.content_type.contains("pdf")
146        || (doc.content_type.is_empty() && pdf::looks_like_pdf(&doc.bytes));
147
148    let (title, markdown, links, original_tokens) = if is_pdf {
149        let text = pdf::extract_text(&doc.bytes)?;
150        let tokens = crate::core::tokens::count_tokens(&text);
151        (None, text, Vec::new(), tokens)
152    } else {
153        let body = doc.body_text();
154        let tokens = crate::core::tokens::count_tokens(&body);
155        let looks_html = body.trim_start().starts_with('<');
156        // RSS/Atom feeds are XML, so check them before the HTML branch (which
157        // would otherwise flatten a feed into unreadable text — GH #feedback).
158        if feed::looks_like_feed(&doc.content_type, &body) {
159            let parsed = feed::parse(&body, &doc.final_url);
160            (parsed.title, parsed.markdown, Vec::new(), tokens)
161        } else if is_html(&doc.content_type) || (doc.content_type.is_empty() && looks_html) {
162            let parsed = html_to_text::parse(&body);
163            (parsed.title, parsed.markdown, parsed.links, tokens)
164        } else if is_textual(&doc.content_type) {
165            (None, body, Vec::new(), tokens)
166        } else {
167            return Err(format!(
168                "unsupported content type '{}' for {} (extractable: HTML, PDF, plain text)",
169                doc.content_type, doc.final_url
170            ));
171        }
172    };
173
174    let effective = match opts.mode {
175        ReadMode::Auto => ReadMode::Markdown,
176        other => other,
177    };
178
179    let body = render_mode(effective, &markdown, &links, &doc.final_url, opts);
180    let trimmed = enforce_budget(&body, opts.max_tokens);
181    let citation = Citation::new(&doc.final_url, title);
182
183    Ok(ReadResult {
184        content: format!("{trimmed}{}", citation.footer()),
185        mode: effective,
186        original_tokens,
187        final_url: doc.final_url,
188    })
189}
190
191fn read_youtube(video_id: &str, opts: &ReadOptions) -> Result<ReadResult, String> {
192    let transcript = youtube::fetch_transcript(video_id, opts.timeout_secs)?;
193    let original_tokens = crate::core::tokens::count_tokens(&transcript.full_text);
194
195    let effective = match opts.mode {
196        ReadMode::Auto => ReadMode::Transcript,
197        other => other,
198    };
199
200    let body = match effective {
201        ReadMode::Facts => render_facts(&claims_from(
202            distill::facts_scored(&transcript.full_text, opts.query, opts.max_items),
203            &transcript.source_url,
204        )),
205        ReadMode::Quotes => render_quotes(&claims_from(
206            distill::quotes_scored(&transcript.full_text, opts.query, opts.max_items),
207            &transcript.source_url,
208        )),
209        ReadMode::Links => "Links are not available for video transcripts.".to_string(),
210        _ => distill::summarize_prose(
211            &transcript.full_text,
212            opts.max_tokens.saturating_mul(4),
213            opts.query,
214        ),
215    };
216
217    let trimmed = enforce_budget(&body, opts.max_tokens);
218    let citation = Citation::new(&transcript.source_url, transcript.title);
219
220    Ok(ReadResult {
221        content: format!("{trimmed}{}", citation.footer()),
222        mode: effective,
223        original_tokens,
224        final_url: transcript.source_url,
225    })
226}
227
228fn render_mode(
229    mode: ReadMode,
230    markdown: &str,
231    links: &[html_to_text::Link],
232    base_url: &str,
233    opts: &ReadOptions,
234) -> String {
235    match mode {
236        ReadMode::Markdown | ReadMode::Auto => markdown.to_string(),
237        ReadMode::Text => html_to_text::markdown_to_text(markdown),
238        ReadMode::Links => render_links(links, base_url),
239        ReadMode::Facts => {
240            let plain = html_to_text::markdown_to_text(markdown);
241            let claims = claims_from(
242                distill::facts_scored(&plain, opts.query, opts.max_items),
243                base_url,
244            );
245            render_facts(&claims)
246        }
247        ReadMode::Quotes => {
248            let plain = html_to_text::markdown_to_text(markdown);
249            let claims = claims_from(
250                distill::quotes_scored(&plain, opts.query, opts.max_items),
251                base_url,
252            );
253            render_quotes(&claims)
254        }
255        ReadMode::Transcript => {
256            let plain = html_to_text::markdown_to_text(markdown);
257            distill::summarize_prose(&plain, opts.max_tokens.saturating_mul(4), opts.query)
258        }
259    }
260}
261
262fn render_links(links: &[html_to_text::Link], base_url: &str) -> String {
263    if links.is_empty() {
264        return "No links found.".to_string();
265    }
266    let base = url_guard::validate(base_url).ok();
267    let mut seen = std::collections::HashSet::new();
268    let mut out = Vec::new();
269    for link in links {
270        let abs = absolutize(&link.href, base.as_ref());
271        if seen.insert(abs.clone()) {
272            out.push(format!("- [{}]({abs})", link.text));
273            if out.len() >= MAX_LINKS {
274                break;
275            }
276        }
277    }
278    out.join("\n")
279}
280
281fn absolutize(href: &str, base: Option<&url_guard::SafeUrl>) -> String {
282    if href.starts_with("http://") || href.starts_with("https://") {
283        return href.to_string();
284    }
285    match base {
286        Some(b) => fetch::resolve_redirect(b, href),
287        None => href.to_string(),
288    }
289}
290
291/// Build attributable claims from scored sentences, tagging each with `source`.
292fn claims_from(scored: Vec<(String, f32)>, source: &str) -> Vec<Claim> {
293    scored
294        .into_iter()
295        .map(|(text, conf)| Claim::new(text, conf).with_source(source))
296        .collect()
297}
298
299/// Render facts as a confidence-prefixed bullet list. The shared source lives in
300/// the citation footer, so it is not repeated per line (token-lean).
301fn render_facts(claims: &[Claim]) -> String {
302    if claims.is_empty() {
303        return "No matching content found.".to_string();
304    }
305    claims
306        .iter()
307        .map(|c| format!("- ({:.2}) {}", c.confidence, c.text))
308        .collect::<Vec<_>>()
309        .join("\n")
310}
311
312fn render_quotes(claims: &[Claim]) -> String {
313    if claims.is_empty() {
314        return "No quotable content found.".to_string();
315    }
316    claims
317        .iter()
318        .map(|c| format!("> ({:.2}) {}", c.confidence, c.text))
319        .collect::<Vec<_>>()
320        .join("\n\n")
321}
322
323fn enforce_budget(content: &str, max_tokens: usize) -> String {
324    let tokens = crate::core::tokens::count_tokens(content);
325    if tokens <= max_tokens {
326        return content.to_string();
327    }
328    let total_chars = content.chars().count();
329    let ratio = max_tokens as f64 / tokens as f64;
330    let keep = ((total_chars as f64 * ratio) as usize).max(1);
331    let truncated: String = content.chars().take(keep).collect();
332    format!("{truncated}\n\n…[truncated to fit ~{max_tokens} token budget]")
333}
334
335fn is_html(content_type: &str) -> bool {
336    content_type.contains("html") || content_type.contains("xml")
337}
338
339fn is_textual(content_type: &str) -> bool {
340    content_type.starts_with("text/")
341        || content_type.contains("json")
342        || content_type.contains("markdown")
343        || content_type.contains("plain")
344}
345
346#[cfg(test)]
347mod tests {
348    use super::*;
349
350    #[test]
351    fn read_mode_parsing_is_lenient() {
352        assert_eq!(ReadMode::parse("MD"), Some(ReadMode::Markdown));
353        assert_eq!(ReadMode::parse(" transcript "), Some(ReadMode::Transcript));
354        assert_eq!(ReadMode::parse("summary"), Some(ReadMode::Transcript));
355        assert_eq!(ReadMode::parse("bogus"), None);
356    }
357
358    #[test]
359    fn content_type_classification() {
360        assert!(is_html("text/html"));
361        assert!(is_html("application/xhtml+xml"));
362        assert!(is_textual("text/plain"));
363        assert!(is_textual("application/json"));
364        assert!(!is_html("application/pdf"));
365        assert!(!is_textual("application/pdf"));
366    }
367
368    #[test]
369    fn claim_renderers_handle_empty_and_confidence() {
370        assert_eq!(render_facts(&[]), "No matching content found.");
371        assert_eq!(render_quotes(&[]), "No quotable content found.");
372
373        let claims = claims_from(
374            vec![("Alpha".to_string(), 0.9), ("Beta".to_string(), 0.5)],
375            "https://src.example/page",
376        );
377        assert_eq!(render_facts(&claims), "- (0.90) Alpha\n- (0.50) Beta");
378        assert_eq!(
379            claims[0].source_url.as_deref(),
380            Some("https://src.example/page")
381        );
382    }
383
384    #[test]
385    fn render_links_absolutizes_and_dedupes() {
386        let links = vec![
387            html_to_text::Link {
388                text: "rel".into(),
389                href: "/about".into(),
390            },
391            html_to_text::Link {
392                text: "abs".into(),
393                href: "https://y.com/z".into(),
394            },
395            html_to_text::Link {
396                text: "dup".into(),
397                href: "https://y.com/z".into(),
398            },
399        ];
400        let out = render_links(&links, "https://x.com/dir/page");
401        assert!(out.contains("[rel](https://x.com/about)"));
402        assert!(out.contains("[abs](https://y.com/z)"));
403        assert_eq!(out.matches("https://y.com/z").count(), 1);
404    }
405
406    #[test]
407    fn enforce_budget_truncates_when_over() {
408        let big = "word ".repeat(5000);
409        let out = enforce_budget(&big, 50);
410        assert!(out.contains("[truncated"));
411        assert!(crate::core::tokens::count_tokens(&out) < crate::core::tokens::count_tokens(&big));
412    }
413
414    #[test]
415    fn enforce_budget_keeps_small_content() {
416        let small = "short content";
417        assert_eq!(enforce_budget(small, 1000), small);
418    }
419}
lean_ctx/core/web/mod.rs

lean_ctx/core/web/
mod.rs
