lean_ctx/core/web/
feed.rs

1//! Dependency-free RSS / Atom / RDF feed → Markdown rendering.
2//!
3//! Feeds are XML, so the generic [`super::html_to_text`] renderer flattens them
4//! into an unreadable mush (every `<title>`/`<link>`/`<description>` mashed
5//! together). This module instead understands feed structure and emits a clean,
6//! token-lean item list: a feed title heading followed by one linked entry per
7//! item with its date and a short, tag-stripped summary (GH feedback: "add RSS
8//! feed support to url_read").
9//!
10//! Supports RSS 2.0 (`<item>`), Atom (`<entry>`), and RSS 1.0 / RDF (`<item>`),
11//! including `<![CDATA[…]]>` payloads and namespaced date fields (`dc:date`).
12
13use super::html_to_text::decode_entities;
14
15/// Cap on rendered items so a 200-entry feed stays token-bounded; the overall
16/// token budget in `mod.rs` trims further if needed.
17const MAX_ITEMS: usize = 50;
18/// Per-item summary length cap (characters) — enough to triage, lean on tokens.
19const SUMMARY_CHARS: usize = 280;
20
21/// A parsed feed ready to hand back through the normal `read_web` pipeline.
22pub struct FeedDoc {
23    pub title: Option<String>,
24    pub markdown: String,
25}
26
27/// True when the response is an RSS/Atom/RDF feed, by MIME type or a sniff of
28/// the document root. Deliberately stricter than "is XML" so XHTML pages still
29/// go to the HTML renderer.
30pub fn looks_like_feed(content_type: &str, body: &str) -> bool {
31    let ct = content_type.to_ascii_lowercase();
32    if ct.contains("rss") || ct.contains("atom") {
33        return true;
34    }
35    // An explicit HTML type is a page, never a feed — don't let a stray "<rss"
36    // mention in the markup misroute it to the feed renderer.
37    if ct.contains("html") {
38        return false;
39    }
40    // Otherwise (generic xml / text / empty) sniff the root for a feed element.
41    let head: String = body
42        .chars()
43        .take(1024)
44        .collect::<String>()
45        .to_ascii_lowercase();
46    head.contains("<rss") || head.contains("<feed") || head.contains("<rdf:rdf")
47}
48
49/// Parse a feed document into a clean Markdown item list.
50pub fn parse(xml: &str, _source_url: &str) -> FeedDoc {
51    let doc = Xml::new(xml);
52    let title = feed_title(&doc);
53
54    let mut out = String::new();
55    if let Some(t) = &title {
56        out.push_str("# ");
57        out.push_str(t);
58        out.push_str("\n\n");
59    }
60
61    let blocks = doc.all_inner("item");
62    let blocks = if blocks.is_empty() {
63        doc.all_inner("entry")
64    } else {
65        blocks
66    };
67
68    let total = blocks.len();
69    let mut rendered = 0;
70    for block in blocks.into_iter().take(MAX_ITEMS) {
71        let item = Xml::new(block);
72        let it = parse_item(&item);
73        if it.title.is_empty() && it.link.is_none() {
74            continue;
75        }
76        out.push_str(&it.render());
77        out.push_str("\n\n");
78        rendered += 1;
79    }
80
81    if total > rendered {
82        out.push_str(&format!("_…and {} more item(s)._", total - rendered));
83    }
84    if rendered == 0 && title.is_none() {
85        out.push_str("No feed items found.");
86    }
87
88    FeedDoc {
89        title,
90        markdown: out.trim_end().to_string(),
91    }
92}
93
94struct Item {
95    title: String,
96    link: Option<String>,
97    date: Option<String>,
98    summary: Option<String>,
99}
100
101impl Item {
102    fn render(&self) -> String {
103        let heading = match (&self.link, self.title.is_empty()) {
104            (Some(link), false) => format!("## [{}]({link})", self.title),
105            (Some(link), true) => format!("## [{link}]({link})"),
106            (None, false) => format!("## {}", self.title),
107            (None, true) => "## (untitled)".to_string(),
108        };
109        let mut meta = Vec::new();
110        if let Some(d) = &self.date {
111            meta.push(d.clone());
112        }
113        if let Some(s) = &self.summary {
114            meta.push(s.clone());
115        }
116        if meta.is_empty() {
117            heading
118        } else {
119            format!("{heading}\n{}", meta.join(" · "))
120        }
121    }
122}
123
124fn parse_item(item: &Xml) -> Item {
125    let title = item
126        .inner_text("title")
127        .map(|t| clean_inline(&t))
128        .unwrap_or_default();
129
130    // RSS: <link>url</link>. Atom: <link href="url" rel="alternate"/>.
131    let link = item
132        .inner_text("link")
133        .map(|l| l.trim().to_string())
134        .filter(|l| !l.is_empty())
135        .or_else(|| atom_link(item));
136
137    let date = ["pubdate", "published", "updated", "dc:date", "date"]
138        .into_iter()
139        .find_map(|tag| item.inner_text(tag))
140        .map(|d| clean_inline(&d))
141        .filter(|d| !d.is_empty());
142
143    let summary = ["description", "summary", "content"]
144        .into_iter()
145        .find_map(|tag| item.inner_text(tag))
146        .map(|s| summarize(&s))
147        .filter(|s| !s.is_empty());
148
149    Item {
150        title,
151        link,
152        date,
153        summary,
154    }
155}
156
157/// Atom `<link>` carries the URL in an `href` attribute; prefer `rel="alternate"`
158/// (or no `rel`) over `self`/`edit` link relations.
159fn atom_link(item: &Xml) -> Option<String> {
160    let mut fallback = None;
161    let mut from = 0;
162    while let Some((open, content_or_end)) = item.open_tag("link", from) {
163        from = content_or_end;
164        let href = attr(open, "href")?;
165        if href.is_empty() {
166            continue;
167        }
168        match attr(open, "rel").as_deref() {
169            None | Some("alternate") => return Some(href),
170            Some(_) => {
171                if fallback.is_none() {
172                    fallback = Some(href);
173                }
174            }
175        }
176    }
177    fallback
178}
179
180fn feed_title(doc: &Xml) -> Option<String> {
181    // The channel/feed title is the first <title> before any item/entry.
182    let cut = [doc.find("<item"), doc.find("<entry")]
183        .into_iter()
184        .flatten()
185        .min()
186        .unwrap_or(doc.raw.len());
187    let head = Xml::new(&doc.raw[..cut]);
188    head.inner_text("title")
189        .map(|t| clean_inline(&t))
190        .filter(|s| !s.is_empty())
191}
192
193/// Strip HTML tags + entities from a feed summary and truncate to a lean length.
194///
195/// Tags are stripped twice around entity decoding: Atom `type="html"` payloads
196/// arrive entity-*encoded* (`&lt;p&gt;`), so a single pre-decode strip would
197/// leave visible `<p>` once decoded. RSS CDATA payloads carry literal tags, so
198/// the first strip catches those.
199fn summarize(raw: &str) -> String {
200    let unwrapped = strip_cdata(raw);
201    let once = strip_tags(&unwrapped);
202    let decoded = decode_entities(&once);
203    let twice = strip_tags(&decoded);
204    let text = collapse_ws(&twice);
205    let text = text.trim();
206    if text.chars().count() > SUMMARY_CHARS {
207        let truncated: String = text.chars().take(SUMMARY_CHARS).collect();
208        format!("{}…", truncated.trim_end())
209    } else {
210        text.to_string()
211    }
212}
213
214/// Decode entities + collapse whitespace for a short inline value (title/date).
215fn clean_inline(raw: &str) -> String {
216    collapse_ws(&decode_entities(&strip_cdata(raw)))
217        .trim()
218        .to_string()
219}
220
221fn strip_cdata(s: &str) -> String {
222    let mut out = String::with_capacity(s.len());
223    let mut rest = s;
224    while let Some(start) = rest.find("<![CDATA[") {
225        out.push_str(&rest[..start]);
226        let after = &rest[start + "<![CDATA[".len()..];
227        let Some(end) = after.find("]]>") else {
228            out.push_str(after);
229            return out;
230        };
231        out.push_str(&after[..end]);
232        rest = &after[end + 3..];
233    }
234    out.push_str(rest);
235    out
236}
237
238fn strip_tags(s: &str) -> String {
239    let mut out = String::with_capacity(s.len());
240    let mut in_tag = false;
241    for c in s.chars() {
242        match c {
243            '<' => in_tag = true,
244            '>' => {
245                in_tag = false;
246                out.push(' ');
247            }
248            _ if !in_tag => out.push(c),
249            _ => {}
250        }
251    }
252    out
253}
254
255fn collapse_ws(s: &str) -> String {
256    let mut out = String::with_capacity(s.len());
257    let mut prev_space = false;
258    for c in s.chars() {
259        if c.is_whitespace() {
260            if !prev_space {
261                out.push(' ');
262                prev_space = true;
263            }
264        } else {
265            out.push(c);
266            prev_space = false;
267        }
268    }
269    out
270}
271
272fn attr(open_tag: &str, key: &str) -> Option<String> {
273    let lower = open_tag.to_ascii_lowercase();
274    let mut from = 0;
275    while let Some(pos) = lower[from..].find(key) {
276        let idx = from + pos;
277        let boundary = idx == 0 || lower.as_bytes()[idx - 1].is_ascii_whitespace();
278        let after = idx + key.len();
279        let rest = open_tag[after..].trim_start();
280        if boundary && rest.starts_with('=') {
281            let val = rest[1..].trim_start();
282            let bytes = val.as_bytes();
283            if let Some(&q) = bytes.first() {
284                if q == b'"' || q == b'\'' {
285                    let quote = q as char;
286                    return val[1..]
287                        .find(quote)
288                        .map(|end| val[1..=end].to_string())
289                        .or_else(|| Some(val[1..].to_string()));
290                }
291            }
292            return val
293                .split_whitespace()
294                .next()
295                .map(|v| v.trim_end_matches("/>").to_string());
296        }
297        from = after;
298    }
299    None
300}
301
302/// A lower-cased index over an XML slice for case-insensitive element lookups.
303struct Xml<'a> {
304    raw: &'a str,
305    lower: String,
306}
307
308impl<'a> Xml<'a> {
309    fn new(raw: &'a str) -> Self {
310        Self {
311            raw,
312            lower: raw.to_ascii_lowercase(),
313        }
314    }
315
316    fn find(&self, needle: &str) -> Option<usize> {
317        self.lower.find(needle)
318    }
319
320    /// Locate `<tag …>` at/after `from`, returning `(open_tag_str, content_start)`
321    /// where `open_tag_str` is the full `<…>` (for attribute parsing) and
322    /// `content_start` is the byte index just past the `>`.
323    fn open_tag(&self, tag: &str, from: usize) -> Option<(&'a str, usize)> {
324        let needle = format!("<{}", tag.to_ascii_lowercase());
325        let mut search = from;
326        loop {
327            let rel = self.lower[search..].find(&needle)?;
328            let pos = search + rel;
329            let after = pos + needle.len();
330            let delim_ok = self.lower[after..]
331                .chars()
332                .next()
333                .is_some_and(|c| matches!(c, '>' | ' ' | '\t' | '\n' | '\r' | '/'));
334            if delim_ok {
335                let gt = self.lower[pos..].find('>')? + pos;
336                return Some((&self.raw[pos..=gt], gt + 1));
337            }
338            search = after;
339        }
340    }
341
342    /// Inner text of the first `<tag>…</tag>` at/after document start.
343    fn inner_text(&self, tag: &str) -> Option<String> {
344        let (_, content_start) = self.open_tag(tag, 0)?;
345        let close = format!("</{}", tag.to_ascii_lowercase());
346        let end = self.lower[content_start..].find(&close)? + content_start;
347        Some(self.raw[content_start..end].to_string())
348    }
349
350    /// Inner slices of every `<tag>…</tag>` block (non-nested).
351    fn all_inner(&self, tag: &str) -> Vec<&'a str> {
352        let mut out = Vec::new();
353        let close = format!("</{}", tag.to_ascii_lowercase());
354        let mut from = 0;
355        while let Some((_, content_start)) = self.open_tag(tag, from) {
356            let Some(rel) = self.lower[content_start..].find(&close) else {
357                break;
358            };
359            let end = content_start + rel;
360            out.push(&self.raw[content_start..end]);
361            from = end + close.len();
362        }
363        out
364    }
365}
366
367#[cfg(test)]
368mod tests {
369    use super::*;
370
371    #[test]
372    fn detects_feed_by_mime_and_sniff() {
373        assert!(looks_like_feed("application/rss+xml", ""));
374        assert!(looks_like_feed("application/atom+xml", ""));
375        assert!(looks_like_feed(
376            "text/xml",
377            "<?xml version='1.0'?><rss version='2.0'>"
378        ));
379        assert!(looks_like_feed(
380            "",
381            "<feed xmlns='http://www.w3.org/2005/Atom'>"
382        ));
383        assert!(!looks_like_feed("text/html", "<!doctype html><html><body>"));
384        // An HTML page that merely mentions "<rss" must not be misrouted.
385        assert!(!looks_like_feed(
386            "text/html",
387            "<html><body>How to read <rss> feeds</body></html>"
388        ));
389    }
390
391    #[test]
392    fn parses_rss_items_into_markdown() {
393        let xml = r#"<?xml version="1.0"?>
394        <rss version="2.0"><channel>
395          <title>My Feed</title>
396          <link>https://feed.example/</link>
397          <description>Channel desc</description>
398          <item>
399            <title>First post</title>
400            <link>https://feed.example/1</link>
401            <pubDate>Mon, 02 Jun 2026 10:00:00 GMT</pubDate>
402            <description><![CDATA[<p>Hello <b>world</b> with detail.</p>]]></description>
403          </item>
404          <item>
405            <title>Second &amp; last</title>
406            <link>https://feed.example/2</link>
407            <description>Plain summary</description>
408          </item>
409        </channel></rss>"#;
410        let doc = parse(xml, "https://feed.example/feed.xml");
411        assert_eq!(doc.title.as_deref(), Some("My Feed"));
412        assert!(doc.markdown.starts_with("# My Feed"));
413        assert!(
414            doc.markdown
415                .contains("## [First post](https://feed.example/1)"),
416            "item must be a linked heading: {}",
417            doc.markdown
418        );
419        assert!(doc.markdown.contains("Mon, 02 Jun 2026 10:00:00 GMT"));
420        assert!(
421            doc.markdown.contains("Hello world with detail."),
422            "CDATA HTML summary must be stripped to text: {}",
423            doc.markdown
424        );
425        assert!(
426            doc.markdown
427                .contains("## [Second & last](https://feed.example/2)"),
428            "entities in titles must decode: {}",
429            doc.markdown
430        );
431    }
432
433    #[test]
434    fn parses_atom_entries_with_href_links() {
435        let xml = r#"<feed xmlns="http://www.w3.org/2005/Atom">
436          <title>Atom Feed</title>
437          <entry>
438            <title>Atom entry</title>
439            <link href="https://a.example/self" rel="self"/>
440            <link href="https://a.example/post" rel="alternate"/>
441            <updated>2026-06-02T00:00:00Z</updated>
442            <summary>An atom summary.</summary>
443          </entry>
444        </feed>"#;
445        let doc = parse(xml, "https://a.example/atom");
446        assert_eq!(doc.title.as_deref(), Some("Atom Feed"));
447        assert!(
448            doc.markdown
449                .contains("## [Atom entry](https://a.example/post)"),
450            "must prefer rel=alternate link: {}",
451            doc.markdown
452        );
453        assert!(doc.markdown.contains("An atom summary."));
454    }
455
456    #[test]
457    fn strips_entity_encoded_html_in_atom_summary() {
458        // Atom type="html" content is entity-encoded; the rendered summary must
459        // not leak visible <p>/<a> tags (regression for the live Rust-blog feed).
460        let xml = r#"<feed xmlns="http://www.w3.org/2005/Atom">
461          <title>F</title>
462          <entry>
463            <title>E</title>
464            <link href="https://e/1"/>
465            <content type="html">&lt;p&gt;Hello &lt;a href="https://x"&gt;link&lt;/a&gt; there.&lt;/p&gt;</content>
466          </entry>
467        </feed>"#;
468        let doc = parse(xml, "https://e");
469        assert!(
470            doc.markdown.contains("Hello link there."),
471            "entity-encoded HTML must be stripped to text: {}",
472            doc.markdown
473        );
474        assert!(
475            !doc.markdown.contains("<p>") && !doc.markdown.contains("&lt;"),
476            "no raw/encoded tags may remain: {}",
477            doc.markdown
478        );
479    }
480
481    #[test]
482    fn truncates_long_summaries() {
483        let long = "x ".repeat(400);
484        let xml = format!(
485            "<rss><channel><title>F</title><item><title>T</title>\
486             <link>https://e/1</link><description>{long}</description></item></channel></rss>"
487        );
488        let doc = parse(&xml, "https://e/feed");
489        assert!(
490            doc.markdown.contains('…'),
491            "long summary should be truncated"
492        );
493    }
494
495    #[test]
496    fn handles_empty_feed_gracefully() {
497        let doc = parse(
498            "<rss><channel><title>Empty</title></channel></rss>",
499            "https://e",
500        );
501        assert_eq!(doc.title.as_deref(), Some("Empty"));
502        assert!(doc.markdown.contains("# Empty"));
503    }
504}
lean_ctx/core/web/feed.rs

lean_ctx/core/web/
feed.rs
