lean_ctx/core/web/
mod.rs

1//! Web & research context layer.
2//!
3//! Turns an arbitrary URL (web page or YouTube video) into compressed,
4//! citation-backed context for an agent. The flow is:
5//!
6//! 1. [`url_guard`] validates the URL and blocks SSRF targets.
7//! 2. [`fetch`] downloads it (bounded, manual-redirect, SSRF-revalidated) — or
8//!    [`youtube`] pulls a transcript for video URLs.
9//! 3. [`html_to_text`] renders HTML to clean Markdown.
10//! 4. [`distill`] applies the requested research-compression mode.
11//! 5. [`citation`] attaches source attribution.
12//!
13//! The single entry point is [`read_url`]; the [`crate::tools::registered::ctx_url_read`]
14//! MCP tool is a thin wrapper over it.
15
16pub mod citation;
17pub mod distill;
18pub mod feed;
19pub mod fetch;
20pub mod html_to_text;
21pub mod pdf;
22pub mod rewrite;
23pub mod url_guard;
24pub mod youtube;
25
26use crate::core::evidence::Claim;
27
28use citation::Citation;
29
30/// Default token budget for returned content.
31pub const DEFAULT_MAX_TOKENS: usize = 6000;
32/// Default number of items for `facts` / `quotes` modes.
33pub const DEFAULT_MAX_ITEMS: usize = 12;
34const MAX_LINKS: usize = 100;
35
36/// How fetched content should be distilled before returning.
37#[derive(Debug, Clone, Copy, PartialEq, Eq)]
38pub enum ReadMode {
39    /// Pick a sensible mode from the content type (Markdown for pages,
40    /// transcript summary for videos).
41    Auto,
42    /// Clean Markdown of the main content.
43    Markdown,
44    /// Plain text (Markdown decorations stripped).
45    Text,
46    /// Extracted hyperlinks.
47    Links,
48    /// Sentences carrying factual signals.
49    Facts,
50    /// Central / query-relevant sentences as evidence.
51    Quotes,
52    /// De-duplicated, filler-stripped transcript (best for videos).
53    Transcript,
54}
55
56impl ReadMode {
57    pub fn parse(s: &str) -> Option<Self> {
58        match s.trim().to_ascii_lowercase().as_str() {
59            "auto" => Some(Self::Auto),
60            "markdown" | "md" => Some(Self::Markdown),
61            "text" | "plain" => Some(Self::Text),
62            "links" => Some(Self::Links),
63            "facts" => Some(Self::Facts),
64            "quotes" => Some(Self::Quotes),
65            "transcript" | "summary" => Some(Self::Transcript),
66            _ => None,
67        }
68    }
69
70    pub fn label(self) -> &'static str {
71        match self {
72            Self::Auto => "auto",
73            Self::Markdown => "markdown",
74            Self::Text => "text",
75            Self::Links => "links",
76            Self::Facts => "facts",
77            Self::Quotes => "quotes",
78            Self::Transcript => "transcript",
79        }
80    }
81}
82
83/// Parameters for [`read_url`].
84pub struct ReadOptions<'a> {
85    pub url: &'a str,
86    pub mode: ReadMode,
87    pub query: Option<&'a str>,
88    pub max_tokens: usize,
89    pub max_items: usize,
90    pub timeout_secs: u64,
91}
92
93impl<'a> ReadOptions<'a> {
94    pub fn new(url: &'a str) -> Self {
95        Self {
96            url,
97            mode: ReadMode::Auto,
98            query: None,
99            max_tokens: DEFAULT_MAX_TOKENS,
100            max_items: DEFAULT_MAX_ITEMS,
101            timeout_secs: fetch::DEFAULT_TIMEOUT_SECS,
102        }
103    }
104}
105
106/// Result of a successful [`read_url`].
107pub struct ReadResult {
108    /// Distilled content with the citation footer already appended.
109    pub content: String,
110    /// Effective mode after `Auto` resolution.
111    pub mode: ReadMode,
112    /// Token count of the raw fetched payload (for savings accounting).
113    pub original_tokens: usize,
114    pub final_url: String,
115}
116
117/// Fetch and distill a URL into citation-backed context.
118pub fn read_url(opts: &ReadOptions) -> Result<ReadResult, String> {
119    // Rewrite known page URLs (e.g. GitHub blob → raw) to their clean-content
120    // equivalent before fetching, so the agent gets the file instead of chrome.
121    let rewritten = rewrite::rewrite_url(opts.url);
122    let url = rewritten.as_deref().unwrap_or(opts.url);
123
124    if let Some(id) = youtube::video_id(url) {
125        return read_youtube(&id, opts);
126    }
127
128    let effective = ReadOptions {
129        url,
130        mode: opts.mode,
131        query: opts.query,
132        max_tokens: opts.max_tokens,
133        max_items: opts.max_items,
134        timeout_secs: opts.timeout_secs,
135    };
136    read_web(&effective)
137}
138
139fn read_web(opts: &ReadOptions) -> Result<ReadResult, String> {
140    let doc = fetch::fetch(opts.url, fetch::DEFAULT_MAX_BYTES, opts.timeout_secs)?;
141    if doc.status >= 400 {
142        return Err(format!("HTTP {} from {}", doc.status, doc.final_url));
143    }
144
145    let is_pdf = doc.content_type.contains("pdf")
146        || (doc.content_type.is_empty() && pdf::looks_like_pdf(&doc.bytes));
147
148    let (title, markdown, links, original_tokens) = if is_pdf {
149        let text = pdf::extract_text(&doc.bytes)?;
150        let tokens = crate::core::tokens::count_tokens(&text);
151        (None, text, Vec::new(), tokens)
152    } else {
153        let body = doc.body_text();
154        let tokens = crate::core::tokens::count_tokens(&body);
155        let looks_html = body.trim_start().starts_with('<');
156        // RSS/Atom feeds are XML, so check them before the HTML branch (which
157        // would otherwise flatten a feed into unreadable text — GH #feedback).
158        if feed::looks_like_feed(&doc.content_type, &body) {
159            let parsed = feed::parse(&body, &doc.final_url);
160            (parsed.title, parsed.markdown, Vec::new(), tokens)
161        } else if is_html(&doc.content_type) || (doc.content_type.is_empty() && looks_html) {
162            let parsed = html_to_text::parse(&body);
163            (parsed.title, parsed.markdown, parsed.links, tokens)
164        } else if is_textual(&doc.content_type) {
165            (None, body, Vec::new(), tokens)
166        } else {
167            return Err(format!(
168                "unsupported content type '{}' for {} (extractable: HTML, PDF, plain text)",
169                doc.content_type, doc.final_url
170            ));
171        }
172    };
173
174    let effective = match opts.mode {
175        ReadMode::Auto => ReadMode::Markdown,
176        other => other,
177    };
178
179    let body = render_mode(effective, &markdown, &links, &doc.final_url, opts);
180    let body = apply_persona_compressor(body, effective);
181    let trimmed = enforce_budget(&body, opts.max_tokens);
182    let citation = Citation::new(&doc.final_url, title);
183
184    Ok(ReadResult {
185        content: format!("{trimmed}{}", citation.footer()),
186        mode: effective,
187        original_tokens,
188        final_url: doc.final_url,
189    })
190}
191
192fn read_youtube(video_id: &str, opts: &ReadOptions) -> Result<ReadResult, String> {
193    let transcript = youtube::fetch_transcript(video_id, opts.timeout_secs)?;
194    let original_tokens = crate::core::tokens::count_tokens(&transcript.full_text);
195
196    let effective = match opts.mode {
197        ReadMode::Auto => ReadMode::Transcript,
198        other => other,
199    };
200
201    let body = match effective {
202        ReadMode::Facts => render_facts(&claims_from(
203            distill::facts_scored(&transcript.full_text, opts.query, opts.max_items),
204            &transcript.source_url,
205        )),
206        ReadMode::Quotes => render_quotes(&claims_from(
207            distill::quotes_scored(&transcript.full_text, opts.query, opts.max_items),
208            &transcript.source_url,
209        )),
210        ReadMode::Links => "Links are not available for video transcripts.".to_string(),
211        _ => distill::summarize_prose(
212            &transcript.full_text,
213            opts.max_tokens.saturating_mul(4),
214            opts.query,
215        ),
216    };
217
218    let body = apply_persona_compressor(body, effective);
219    let trimmed = enforce_budget(&body, opts.max_tokens);
220    let citation = Citation::new(&transcript.source_url, transcript.title);
221
222    Ok(ReadResult {
223        content: format!("{trimmed}{}", citation.footer()),
224        mode: effective,
225        original_tokens,
226        final_url: transcript.source_url,
227    })
228}
229
230fn render_mode(
231    mode: ReadMode,
232    markdown: &str,
233    links: &[html_to_text::Link],
234    base_url: &str,
235    opts: &ReadOptions,
236) -> String {
237    match mode {
238        ReadMode::Markdown | ReadMode::Auto => markdown.to_string(),
239        ReadMode::Text => html_to_text::markdown_to_text(markdown),
240        ReadMode::Links => render_links(links, base_url),
241        ReadMode::Facts => {
242            let plain = html_to_text::markdown_to_text(markdown);
243            let claims = claims_from(
244                distill::facts_scored(&plain, opts.query, opts.max_items),
245                base_url,
246            );
247            render_facts(&claims)
248        }
249        ReadMode::Quotes => {
250            let plain = html_to_text::markdown_to_text(markdown);
251            let claims = claims_from(
252                distill::quotes_scored(&plain, opts.query, opts.max_items),
253                base_url,
254            );
255            render_quotes(&claims)
256        }
257        ReadMode::Transcript => {
258            let plain = html_to_text::markdown_to_text(markdown);
259            distill::summarize_prose(&plain, opts.max_tokens.saturating_mul(4), opts.query)
260        }
261    }
262}
263
264fn render_links(links: &[html_to_text::Link], base_url: &str) -> String {
265    if links.is_empty() {
266        return "No links found.".to_string();
267    }
268    let base = url_guard::validate(base_url).ok();
269    let mut seen = std::collections::HashSet::new();
270    let mut out = Vec::new();
271    for link in links {
272        let abs = absolutize(&link.href, base.as_ref());
273        if seen.insert(abs.clone()) {
274            out.push(format!("- [{}]({abs})", link.text));
275            if out.len() >= MAX_LINKS {
276                break;
277            }
278        }
279    }
280    out.join("\n")
281}
282
283fn absolutize(href: &str, base: Option<&url_guard::SafeUrl>) -> String {
284    if href.starts_with("http://") || href.starts_with("https://") {
285        return href.to_string();
286    }
287    match base {
288        Some(b) => fetch::resolve_redirect(b, href),
289        None => href.to_string(),
290    }
291}
292
293/// Build attributable claims from scored sentences, tagging each with `source`.
294fn claims_from(scored: Vec<(String, f32)>, source: &str) -> Vec<Claim> {
295    scored
296        .into_iter()
297        .map(|(text, conf)| Claim::new(text, conf).with_source(source))
298        .collect()
299}
300
301/// Render facts as a confidence-prefixed bullet list. The shared source lives in
302/// the citation footer, so it is not repeated per line (token-lean).
303fn render_facts(claims: &[Claim]) -> String {
304    if claims.is_empty() {
305        return "No matching content found.".to_string();
306    }
307    claims
308        .iter()
309        .map(|c| format!("- ({:.2}) {}", c.confidence, c.text))
310        .collect::<Vec<_>>()
311        .join("\n")
312}
313
314fn render_quotes(claims: &[Claim]) -> String {
315    if claims.is_empty() {
316        return "No quotable content found.".to_string();
317    }
318    claims
319        .iter()
320        .map(|c| format!("> ({:.2}) {}", c.confidence, c.text))
321        .collect::<Vec<_>>()
322        .join("\n\n")
323}
324
325/// persona-spec-v1 — apply the active persona's registry compressor to
326/// flowing-text modes (`research` → `markdown`, `support`/`lead-gen` →
327/// `prose`). Extractive modes (facts/quotes/links) stay verbatim: their lines
328/// are claims and citations whose wording must not be rewritten. `identity`
329/// (the `coding` default) is a guaranteed no-op and skips the registry lookup.
330fn apply_persona_compressor(body: String, mode: ReadMode) -> String {
331    if !matches!(
332        mode,
333        ReadMode::Auto | ReadMode::Markdown | ReadMode::Text | ReadMode::Transcript
334    ) {
335        return body;
336    }
337    let name = crate::core::persona::active().compressor;
338    if name == "identity" {
339        return body;
340    }
341    let Some(compressor) = crate::core::extension_registry::global()
342        .read()
343        .ok()
344        .and_then(|reg| reg.compressor(&name))
345    else {
346        tracing::debug!("persona compressor '{name}' not registered; passing through");
347        return body;
348    };
349    compressor.compress(&body, None)
350}
351
352fn enforce_budget(content: &str, max_tokens: usize) -> String {
353    let tokens = crate::core::tokens::count_tokens(content);
354    if tokens <= max_tokens {
355        return content.to_string();
356    }
357    // persona-spec-v1 — cut at the persona chunker's boundaries (paragraphs
358    // for research/support/lead-gen, line windows for coding/data-analysis)
359    // so the truncation lands between units of meaning, not mid-sentence.
360    if let Some(trimmed) = trim_at_chunk_boundaries(content, max_tokens) {
361        return trimmed;
362    }
363    // Fallback: proportional character cut (chunker unavailable, content is a
364    // single chunk, or the first chunk alone exceeds the budget).
365    let total_chars = content.chars().count();
366    let ratio = max_tokens as f64 / tokens as f64;
367    let keep = ((total_chars as f64 * ratio) as usize).max(1);
368    let truncated: String = content.chars().take(keep).collect();
369    format!("{truncated}\n\n…[truncated to fit ~{max_tokens} token budget]")
370}
371
372/// Trim `content` to whole persona-chunker chunks within `max_tokens`.
373///
374/// Each kept chunk is located back in the original text (chunkers may trim
375/// separators), so the cut always lands on real source bytes. Returns `None`
376/// when the chunker yields fewer than two chunks, a chunk cannot be located
377/// verbatim (e.g. a transforming format chunker), or not even the first chunk
378/// fits — callers then fall back to the proportional character cut.
379fn trim_at_chunk_boundaries(content: &str, max_tokens: usize) -> Option<String> {
380    use crate::core::tokens::count_tokens;
381    let name = crate::core::persona::active().chunker;
382    let chunker = crate::core::extension_registry::global()
383        .read()
384        .ok()?
385        .chunker(&name)?;
386    let chunks = chunker.chunk(content);
387    if chunks.len() < 2 {
388        return None;
389    }
390    let mut cut = 0usize;
391    let mut used = 0usize;
392    let mut search_from = 0usize;
393    for chunk in &chunks {
394        let chunk_tokens = count_tokens(chunk);
395        if used + chunk_tokens > max_tokens {
396            break;
397        }
398        let rel = content.get(search_from..)?.find(chunk.as_str())?;
399        let end = search_from + rel + chunk.len();
400        used += chunk_tokens;
401        cut = end;
402        search_from = end;
403    }
404    if cut == 0 {
405        return None;
406    }
407    Some(format!(
408        "{}\n\n…[truncated to fit ~{max_tokens} token budget]",
409        content[..cut].trim_end()
410    ))
411}
412
413fn is_html(content_type: &str) -> bool {
414    content_type.contains("html") || content_type.contains("xml")
415}
416
417fn is_textual(content_type: &str) -> bool {
418    content_type.starts_with("text/")
419        || content_type.contains("json")
420        || content_type.contains("markdown")
421        || content_type.contains("plain")
422}
423
424#[cfg(test)]
425mod tests {
426    use super::*;
427
428    #[test]
429    fn read_mode_parsing_is_lenient() {
430        assert_eq!(ReadMode::parse("MD"), Some(ReadMode::Markdown));
431        assert_eq!(ReadMode::parse(" transcript "), Some(ReadMode::Transcript));
432        assert_eq!(ReadMode::parse("summary"), Some(ReadMode::Transcript));
433        assert_eq!(ReadMode::parse("bogus"), None);
434    }
435
436    #[test]
437    fn content_type_classification() {
438        assert!(is_html("text/html"));
439        assert!(is_html("application/xhtml+xml"));
440        assert!(is_textual("text/plain"));
441        assert!(is_textual("application/json"));
442        assert!(!is_html("application/pdf"));
443        assert!(!is_textual("application/pdf"));
444    }
445
446    #[test]
447    fn claim_renderers_handle_empty_and_confidence() {
448        assert_eq!(render_facts(&[]), "No matching content found.");
449        assert_eq!(render_quotes(&[]), "No quotable content found.");
450
451        let claims = claims_from(
452            vec![("Alpha".to_string(), 0.9), ("Beta".to_string(), 0.5)],
453            "https://src.example/page",
454        );
455        assert_eq!(render_facts(&claims), "- (0.90) Alpha\n- (0.50) Beta");
456        assert_eq!(
457            claims[0].source_url.as_deref(),
458            Some("https://src.example/page")
459        );
460    }
461
462    #[test]
463    fn render_links_absolutizes_and_dedupes() {
464        let links = vec![
465            html_to_text::Link {
466                text: "rel".into(),
467                href: "/about".into(),
468            },
469            html_to_text::Link {
470                text: "abs".into(),
471                href: "https://y.com/z".into(),
472            },
473            html_to_text::Link {
474                text: "dup".into(),
475                href: "https://y.com/z".into(),
476            },
477        ];
478        let out = render_links(&links, "https://x.com/dir/page");
479        assert!(out.contains("[rel](https://x.com/about)"));
480        assert!(out.contains("[abs](https://y.com/z)"));
481        assert_eq!(out.matches("https://y.com/z").count(), 1);
482    }
483
484    #[test]
485    fn enforce_budget_truncates_when_over() {
486        let big = "word ".repeat(5000);
487        let out = enforce_budget(&big, 50);
488        assert!(out.contains("[truncated"));
489        assert!(crate::core::tokens::count_tokens(&out) < crate::core::tokens::count_tokens(&big));
490    }
491
492    #[test]
493    fn enforce_budget_keeps_small_content() {
494        let small = "short content";
495        assert_eq!(enforce_budget(small, 1000), small);
496    }
497
498    #[test]
499    fn persona_chunker_trims_at_paragraph_boundaries() {
500        let _guard = crate::core::data_dir::test_env_lock();
501        crate::test_env::set_var("LEAN_CTX_PERSONA", "research");
502
503        let para = "alpha beta gamma delta epsilon zeta eta theta";
504        let content = format!("{para}\n\n{para}\n\n{para}\n\n{para}");
505        let budget = crate::core::tokens::count_tokens(para) * 2 + 1;
506        let out = enforce_budget(&content, budget);
507
508        crate::test_env::remove_var("LEAN_CTX_PERSONA");
509
510        assert!(out.contains("[truncated"), "{out}");
511        // The cut lands on a paragraph boundary: kept paragraphs verbatim,
512        // no mid-word fragment before the marker.
513        let body = out.split("\n\n…[truncated").next().unwrap();
514        assert_eq!(body, format!("{para}\n\n{para}"), "{out}");
515    }
516
517    #[test]
518    fn persona_compressor_strips_markdown_noise_for_research() {
519        let _guard = crate::core::data_dir::test_env_lock();
520        crate::test_env::set_var("LEAN_CTX_PERSONA", "research");
521
522        let body = "Intro ![badge](https://img.example/b.svg)\n\n\
523                    See [the docs](https://docs.example/page) for details."
524            .to_string();
525        let out = apply_persona_compressor(body, ReadMode::Markdown);
526
527        crate::test_env::remove_var("LEAN_CTX_PERSONA");
528
529        assert!(out.contains("the docs"), "{out}");
530        assert!(!out.contains("https://docs.example"), "{out}");
531        assert!(!out.contains("img.example"), "{out}");
532    }
533
534    #[test]
535    fn persona_compressor_leaves_extractive_modes_verbatim() {
536        let _guard = crate::core::data_dir::test_env_lock();
537        crate::test_env::set_var("LEAN_CTX_PERSONA", "research");
538
539        let body = "- (0.90) A [cited](https://src.example) claim.".to_string();
540        let out = apply_persona_compressor(body.clone(), ReadMode::Facts);
541
542        crate::test_env::remove_var("LEAN_CTX_PERSONA");
543
544        assert_eq!(out, body);
545    }
546
547    #[test]
548    fn persona_compressor_is_identity_for_coding() {
549        let _guard = crate::core::data_dir::test_env_lock();
550        crate::test_env::set_var("LEAN_CTX_PERSONA", "coding");
551
552        let body = "Anything ![badge](https://img.example/b.svg) stays.".to_string();
553        let out = apply_persona_compressor(body.clone(), ReadMode::Markdown);
554
555        crate::test_env::remove_var("LEAN_CTX_PERSONA");
556
557        assert_eq!(out, body);
558    }
559}
lean_ctx/core/web/mod.rs

lean_ctx/core/web/
mod.rs
