lean_ctx/core/web/
mod.rs

1//! Web & research context layer.
2//!
3//! Turns an arbitrary URL (web page or YouTube video) into compressed,
4//! citation-backed context for an agent. The flow is:
5//!
6//! 1. [`url_guard`] validates the URL and blocks SSRF targets.
7//! 2. [`fetch`] downloads it (bounded, manual-redirect, SSRF-revalidated) — or
8//!    [`youtube`] pulls a transcript for video URLs.
9//! 3. [`html_to_text`] renders HTML to clean Markdown.
10//! 4. [`distill`] applies the requested research-compression mode.
11//! 5. [`citation`] attaches source attribution.
12//!
13//! The single entry point is [`read_url`]; the [`crate::tools::registered::ctx_url_read`]
14//! MCP tool is a thin wrapper over it.
15
16pub mod citation;
17pub mod distill;
18pub mod feed;
19pub mod fetch;
20pub mod html_to_text;
21pub mod pdf;
22pub mod rewrite;
23pub mod url_guard;
24pub mod youtube;
25
26use crate::core::evidence::Claim;
27
28use citation::Citation;
29
30/// Default token budget for returned content.
31pub const DEFAULT_MAX_TOKENS: usize = 6000;
32/// Default number of items for `facts` / `quotes` modes.
33pub const DEFAULT_MAX_ITEMS: usize = 12;
34const MAX_LINKS: usize = 100;
35
36/// How fetched content should be distilled before returning.
37#[derive(Debug, Clone, Copy, PartialEq, Eq)]
38pub enum ReadMode {
39    /// Pick a sensible mode from the content type (Markdown for pages,
40    /// transcript summary for videos).
41    Auto,
42    /// Clean Markdown of the main content.
43    Markdown,
44    /// Plain text (Markdown decorations stripped).
45    Text,
46    /// Extracted hyperlinks.
47    Links,
48    /// Sentences carrying factual signals.
49    Facts,
50    /// Central / query-relevant sentences as evidence.
51    Quotes,
52    /// De-duplicated, filler-stripped transcript (best for videos).
53    Transcript,
54}
55
56impl ReadMode {
57    pub fn parse(s: &str) -> Option<Self> {
58        match s.trim().to_ascii_lowercase().as_str() {
59            "auto" => Some(Self::Auto),
60            "markdown" | "md" => Some(Self::Markdown),
61            "text" | "plain" => Some(Self::Text),
62            "links" => Some(Self::Links),
63            "facts" => Some(Self::Facts),
64            "quotes" => Some(Self::Quotes),
65            "transcript" | "summary" => Some(Self::Transcript),
66            _ => None,
67        }
68    }
69
70    pub fn label(self) -> &'static str {
71        match self {
72            Self::Auto => "auto",
73            Self::Markdown => "markdown",
74            Self::Text => "text",
75            Self::Links => "links",
76            Self::Facts => "facts",
77            Self::Quotes => "quotes",
78            Self::Transcript => "transcript",
79        }
80    }
81}
82
83/// Parameters for [`read_url`].
84pub struct ReadOptions<'a> {
85    pub url: &'a str,
86    pub mode: ReadMode,
87    pub query: Option<&'a str>,
88    pub max_tokens: usize,
89    pub max_items: usize,
90    pub timeout_secs: u64,
91}
92
93impl<'a> ReadOptions<'a> {
94    pub fn new(url: &'a str) -> Self {
95        Self {
96            url,
97            mode: ReadMode::Auto,
98            query: None,
99            max_tokens: DEFAULT_MAX_TOKENS,
100            max_items: DEFAULT_MAX_ITEMS,
101            timeout_secs: fetch::DEFAULT_TIMEOUT_SECS,
102        }
103    }
104}
105
106/// Result of a successful [`read_url`].
107pub struct ReadResult {
108    /// Distilled content with the citation footer already appended.
109    pub content: String,
110    /// Effective mode after `Auto` resolution.
111    pub mode: ReadMode,
112    /// Token count of the raw fetched payload (for savings accounting).
113    pub original_tokens: usize,
114    pub final_url: String,
115}
116
117/// Fetch and distill a URL into citation-backed context.
118pub fn read_url(opts: &ReadOptions) -> Result<ReadResult, String> {
119    // Rewrite known page URLs (e.g. GitHub blob → raw) to their clean-content
120    // equivalent before fetching, so the agent gets the file instead of chrome.
121    let rewritten = rewrite::rewrite_url(opts.url);
122    let url = rewritten.as_deref().unwrap_or(opts.url);
123
124    if let Some(id) = youtube::video_id(url) {
125        return read_youtube(&id, opts);
126    }
127
128    let effective = ReadOptions {
129        url,
130        mode: opts.mode,
131        query: opts.query,
132        max_tokens: opts.max_tokens,
133        max_items: opts.max_items,
134        timeout_secs: opts.timeout_secs,
135    };
136    read_web(&effective)
137}
138
139fn read_web(opts: &ReadOptions) -> Result<ReadResult, String> {
140    let doc = fetch::fetch(opts.url, fetch::DEFAULT_MAX_BYTES, opts.timeout_secs)?;
141    if doc.status >= 400 {
142        return Err(format!("HTTP {} from {}", doc.status, doc.final_url));
143    }
144
145    let is_pdf = doc.content_type.contains("pdf")
146        || (doc.content_type.is_empty() && pdf::looks_like_pdf(&doc.bytes));
147
148    let (title, markdown, links, original_tokens) = if is_pdf {
149        let text = pdf::extract_text(&doc.bytes)?;
150        let tokens = crate::core::tokens::count_tokens(&text);
151        (None, text, Vec::new(), tokens)
152    } else {
153        let body = doc.body_text();
154        let tokens = crate::core::tokens::count_tokens(&body);
155        let looks_html = body.trim_start().starts_with('<');
156        // RSS/Atom feeds are XML, so check them before the HTML branch (which
157        // would otherwise flatten a feed into unreadable text — GH #feedback).
158        if feed::looks_like_feed(&doc.content_type, &body) {
159            let parsed = feed::parse(&body, &doc.final_url);
160            (parsed.title, parsed.markdown, Vec::new(), tokens)
161        } else if is_html(&doc.content_type) || (doc.content_type.is_empty() && looks_html) {
162            let parsed = html_to_text::parse(&body);
163            (parsed.title, parsed.markdown, parsed.links, tokens)
164        } else if is_textual(&doc.content_type) {
165            (None, body, Vec::new(), tokens)
166        } else {
167            return Err(format!(
168                "unsupported content type '{}' for {} (extractable: HTML, PDF, plain text)",
169                doc.content_type, doc.final_url
170            ));
171        }
172    };
173
174    let effective = match opts.mode {
175        ReadMode::Auto => ReadMode::Markdown,
176        other => other,
177    };
178
179    let body = render_mode(effective, &markdown, &links, &doc.final_url, opts);
180    let trimmed = enforce_budget(&body, opts.max_tokens);
181    let citation = Citation::new(&doc.final_url, title);
182
183    Ok(ReadResult {
184        content: format!("{trimmed}{}", citation.footer()),
185        mode: effective,
186        original_tokens,
187        final_url: doc.final_url,
188    })
189}
190
191fn read_youtube(video_id: &str, opts: &ReadOptions) -> Result<ReadResult, String> {
192    let transcript = youtube::fetch_transcript(video_id, opts.timeout_secs)?;
193    let original_tokens = crate::core::tokens::count_tokens(&transcript.full_text);
194
195    let effective = match opts.mode {
196        ReadMode::Auto => ReadMode::Transcript,
197        other => other,
198    };
199
200    let body = match effective {
201        ReadMode::Facts => render_facts(&claims_from(
202            distill::facts_scored(&transcript.full_text, opts.query, opts.max_items),
203            &transcript.source_url,
204        )),
205        ReadMode::Quotes => render_quotes(&claims_from(
206            distill::quotes_scored(&transcript.full_text, opts.query, opts.max_items),
207            &transcript.source_url,
208        )),
209        ReadMode::Links => "Links are not available for video transcripts.".to_string(),
210        _ => distill::transcript_summary(&transcript.full_text, opts.max_tokens.saturating_mul(4)),
211    };
212
213    let trimmed = enforce_budget(&body, opts.max_tokens);
214    let citation = Citation::new(&transcript.source_url, transcript.title);
215
216    Ok(ReadResult {
217        content: format!("{trimmed}{}", citation.footer()),
218        mode: effective,
219        original_tokens,
220        final_url: transcript.source_url,
221    })
222}
223
224fn render_mode(
225    mode: ReadMode,
226    markdown: &str,
227    links: &[html_to_text::Link],
228    base_url: &str,
229    opts: &ReadOptions,
230) -> String {
231    match mode {
232        ReadMode::Markdown | ReadMode::Auto => markdown.to_string(),
233        ReadMode::Text => html_to_text::markdown_to_text(markdown),
234        ReadMode::Links => render_links(links, base_url),
235        ReadMode::Facts => {
236            let plain = html_to_text::markdown_to_text(markdown);
237            let claims = claims_from(
238                distill::facts_scored(&plain, opts.query, opts.max_items),
239                base_url,
240            );
241            render_facts(&claims)
242        }
243        ReadMode::Quotes => {
244            let plain = html_to_text::markdown_to_text(markdown);
245            let claims = claims_from(
246                distill::quotes_scored(&plain, opts.query, opts.max_items),
247                base_url,
248            );
249            render_quotes(&claims)
250        }
251        ReadMode::Transcript => {
252            let plain = html_to_text::markdown_to_text(markdown);
253            distill::transcript_summary(&plain, opts.max_tokens.saturating_mul(4))
254        }
255    }
256}
257
258fn render_links(links: &[html_to_text::Link], base_url: &str) -> String {
259    if links.is_empty() {
260        return "No links found.".to_string();
261    }
262    let base = url_guard::validate(base_url).ok();
263    let mut seen = std::collections::HashSet::new();
264    let mut out = Vec::new();
265    for link in links {
266        let abs = absolutize(&link.href, base.as_ref());
267        if seen.insert(abs.clone()) {
268            out.push(format!("- [{}]({abs})", link.text));
269            if out.len() >= MAX_LINKS {
270                break;
271            }
272        }
273    }
274    out.join("\n")
275}
276
277fn absolutize(href: &str, base: Option<&url_guard::SafeUrl>) -> String {
278    if href.starts_with("http://") || href.starts_with("https://") {
279        return href.to_string();
280    }
281    match base {
282        Some(b) => fetch::resolve_redirect(b, href),
283        None => href.to_string(),
284    }
285}
286
287/// Build attributable claims from scored sentences, tagging each with `source`.
288fn claims_from(scored: Vec<(String, f32)>, source: &str) -> Vec<Claim> {
289    scored
290        .into_iter()
291        .map(|(text, conf)| Claim::new(text, conf).with_source(source))
292        .collect()
293}
294
295/// Render facts as a confidence-prefixed bullet list. The shared source lives in
296/// the citation footer, so it is not repeated per line (token-lean).
297fn render_facts(claims: &[Claim]) -> String {
298    if claims.is_empty() {
299        return "No matching content found.".to_string();
300    }
301    claims
302        .iter()
303        .map(|c| format!("- ({:.2}) {}", c.confidence, c.text))
304        .collect::<Vec<_>>()
305        .join("\n")
306}
307
308fn render_quotes(claims: &[Claim]) -> String {
309    if claims.is_empty() {
310        return "No quotable content found.".to_string();
311    }
312    claims
313        .iter()
314        .map(|c| format!("> ({:.2}) {}", c.confidence, c.text))
315        .collect::<Vec<_>>()
316        .join("\n\n")
317}
318
319fn enforce_budget(content: &str, max_tokens: usize) -> String {
320    let tokens = crate::core::tokens::count_tokens(content);
321    if tokens <= max_tokens {
322        return content.to_string();
323    }
324    let total_chars = content.chars().count();
325    let ratio = max_tokens as f64 / tokens as f64;
326    let keep = ((total_chars as f64 * ratio) as usize).max(1);
327    let truncated: String = content.chars().take(keep).collect();
328    format!("{truncated}\n\n…[truncated to fit ~{max_tokens} token budget]")
329}
330
331fn is_html(content_type: &str) -> bool {
332    content_type.contains("html") || content_type.contains("xml")
333}
334
335fn is_textual(content_type: &str) -> bool {
336    content_type.starts_with("text/")
337        || content_type.contains("json")
338        || content_type.contains("markdown")
339        || content_type.contains("plain")
340}
341
342#[cfg(test)]
343mod tests {
344    use super::*;
345
346    #[test]
347    fn read_mode_parsing_is_lenient() {
348        assert_eq!(ReadMode::parse("MD"), Some(ReadMode::Markdown));
349        assert_eq!(ReadMode::parse(" transcript "), Some(ReadMode::Transcript));
350        assert_eq!(ReadMode::parse("summary"), Some(ReadMode::Transcript));
351        assert_eq!(ReadMode::parse("bogus"), None);
352    }
353
354    #[test]
355    fn content_type_classification() {
356        assert!(is_html("text/html"));
357        assert!(is_html("application/xhtml+xml"));
358        assert!(is_textual("text/plain"));
359        assert!(is_textual("application/json"));
360        assert!(!is_html("application/pdf"));
361        assert!(!is_textual("application/pdf"));
362    }
363
364    #[test]
365    fn claim_renderers_handle_empty_and_confidence() {
366        assert_eq!(render_facts(&[]), "No matching content found.");
367        assert_eq!(render_quotes(&[]), "No quotable content found.");
368
369        let claims = claims_from(
370            vec![("Alpha".to_string(), 0.9), ("Beta".to_string(), 0.5)],
371            "https://src.example/page",
372        );
373        assert_eq!(render_facts(&claims), "- (0.90) Alpha\n- (0.50) Beta");
374        assert_eq!(
375            claims[0].source_url.as_deref(),
376            Some("https://src.example/page")
377        );
378    }
379
380    #[test]
381    fn render_links_absolutizes_and_dedupes() {
382        let links = vec![
383            html_to_text::Link {
384                text: "rel".into(),
385                href: "/about".into(),
386            },
387            html_to_text::Link {
388                text: "abs".into(),
389                href: "https://y.com/z".into(),
390            },
391            html_to_text::Link {
392                text: "dup".into(),
393                href: "https://y.com/z".into(),
394            },
395        ];
396        let out = render_links(&links, "https://x.com/dir/page");
397        assert!(out.contains("[rel](https://x.com/about)"));
398        assert!(out.contains("[abs](https://y.com/z)"));
399        assert_eq!(out.matches("https://y.com/z").count(), 1);
400    }
401
402    #[test]
403    fn enforce_budget_truncates_when_over() {
404        let big = "word ".repeat(5000);
405        let out = enforce_budget(&big, 50);
406        assert!(out.contains("[truncated"));
407        assert!(crate::core::tokens::count_tokens(&out) < crate::core::tokens::count_tokens(&big));
408    }
409
410    #[test]
411    fn enforce_budget_keeps_small_content() {
412        let small = "short content";
413        assert_eq!(enforce_budget(small, 1000), small);
414    }
415}
lean_ctx/core/web/mod.rs

lean_ctx/core/web/
mod.rs
