rss_cli/
discover.rs

1//! Feed autodiscovery from a website homepage. **Owner: `cli` agent.**
2//!
3//! ## Requirements
4//! - GET the `site_url` HTML via [`HttpClient::get_bytes`].
5//! - Parse the `<head>` for `<link rel="alternate" type="application/rss+xml">` and
6//!   `type="application/atom+xml"` (also accept `application/json` / `feed+json`). Use the
7//!   lightweight `tl` HTML parser — do not pull a heavyweight DOM stack.
8//! - Resolve each `href` to an absolute URL against `site_url` (the `url` crate).
9//! - Map the `type` to `feed_type` (`"rss" | "atom" | "json"`), carry the `title` attr.
10//! - If the page *is itself* a feed (content sniffing optional), or if no `<link>` tags are
11//!   found, return an empty list rather than erroring (the CLI decides exit behavior).
12
13use std::collections::HashSet;
14
15use url::Url;
16
17use crate::error::RssError;
18use crate::fetch::HttpClient;
19use crate::model::{DiscoverOutput, DiscoveredFeed};
20
21/// Discover feeds advertised on `site_url`. **Owner: `cli` agent.**
22pub async fn discover(site_url: &str, http: &HttpClient) -> Result<DiscoverOutput, RssError> {
23    let (bytes, final_url) = http.get_bytes(site_url).await?;
24    let html = String::from_utf8_lossy(&bytes);
25    // Resolve relative hrefs against the post-redirect URL when available.
26    let feeds = extract_feeds(&html, &final_url);
27    Ok(DiscoverOutput::new(site_url, feeds))
28}
29
30/// Extract `<link rel="alternate">` feed references from an HTML document, resolving each
31/// `href` to an absolute URL against `base_url`. Factored out so it can be unit-tested on
32/// inline HTML without the network. De-dup is order-preserving on the resolved URL.
33fn extract_feeds(html: &str, base_url: &str) -> Vec<DiscoveredFeed> {
34    let dom = match tl::parse(html, tl::ParserOptions::default()) {
35        Ok(dom) => dom,
36        // A malformed homepage should not error the whole command; treat it as no feeds.
37        Err(_) => return Vec::new(),
38    };
39    let base = Url::parse(base_url).ok();
40
41    let mut feeds: Vec<DiscoveredFeed> = Vec::new();
42    let mut seen: HashSet<String> = HashSet::new();
43
44    for node in dom.nodes() {
45        let Some(tag) = node.as_tag() else { continue };
46        if !tag.name().as_bytes().eq_ignore_ascii_case(b"link") {
47            continue;
48        }
49        let attrs = tag.attributes();
50        let rel = attr(attrs, "rel");
51        // Only consider `<link rel="alternate">` (rel may carry multiple space-separated tokens).
52        if !rel
53            .as_deref()
54            .is_some_and(|r| r.to_ascii_lowercase().contains("alternate"))
55        {
56            continue;
57        }
58
59        let href = match attr(attrs, "href") {
60            Some(h) if !h.trim().is_empty() => h,
61            _ => continue,
62        };
63        let ty = attr(attrs, "type");
64        let feed_type = match classify(ty.as_deref(), &href) {
65            Some(t) => t,
66            None => continue,
67        };
68
69        // Resolve to an absolute URL; skip entries we cannot resolve deterministically.
70        let resolved = match &base {
71            Some(b) => match b.join(href.trim()) {
72                Ok(u) => u.to_string(),
73                Err(_) => continue,
74            },
75            None => href.trim().to_string(),
76        };
77
78        if !seen.insert(resolved.clone()) {
79            continue;
80        }
81        feeds.push(DiscoveredFeed {
82            url: resolved,
83            feed_type: feed_type.to_string(),
84            title: attr(attrs, "title"),
85        });
86    }
87
88    feeds
89}
90
91/// Read an attribute as an owned `String` (HTML entities decoded by `tl`).
92fn attr(attrs: &tl::Attributes<'_>, key: &str) -> Option<String> {
93    attrs
94        .get(key)
95        .flatten()
96        .map(|b| b.as_utf8_str().into_owned())
97}
98
99/// Map a `<link>`'s `type` (and `href` as a fallback) to a feed type, or `None` to reject.
100fn classify(ty: Option<&str>, href: &str) -> Option<&'static str> {
101    if let Some(ty) = ty {
102        // Strip any `; charset=…` parameter and normalize case.
103        let mime = ty
104            .split(';')
105            .next()
106            .unwrap_or(ty)
107            .trim()
108            .to_ascii_lowercase();
109        return match mime.as_str() {
110            "application/rss+xml" => Some("rss"),
111            "application/atom+xml" => Some("atom"),
112            "application/json" | "application/feed+json" => Some("json"),
113            _ => None,
114        };
115    }
116    // No `type`: fall back to a coarse extension heuristic on the href.
117    let lower = href
118        .split(['?', '#'])
119        .next()
120        .unwrap_or(href)
121        .to_ascii_lowercase();
122    if lower.ends_with(".rss") || lower.ends_with(".atom") || lower.ends_with(".xml") {
123        Some("unknown")
124    } else {
125        None
126    }
127}
128
129#[cfg(test)]
130mod tests {
131    use super::*;
132
133    #[test]
134    fn extracts_rss_and_atom_links() {
135        let html = r#"
136            <html><head>
137              <link rel="alternate" type="application/rss+xml" title="RSS Feed" href="/feed.rss">
138              <link rel="alternate" type="application/atom+xml" title="Atom Feed" href="https://other.example/atom.xml">
139            </head><body></body></html>
140        "#;
141        let feeds = extract_feeds(html, "https://example.com/blog/");
142
143        assert_eq!(feeds.len(), 2);
144
145        assert_eq!(feeds[0].feed_type, "rss");
146        assert_eq!(feeds[0].title.as_deref(), Some("RSS Feed"));
147        // Relative href resolved against the base URL.
148        assert_eq!(feeds[0].url, "https://example.com/feed.rss");
149
150        assert_eq!(feeds[1].feed_type, "atom");
151        // Absolute href left intact.
152        assert_eq!(feeds[1].url, "https://other.example/atom.xml");
153    }
154
155    #[test]
156    fn accepts_json_and_strips_charset() {
157        let html = r#"<link rel="alternate" type="application/feed+json; charset=utf-8" href="/feed.json">"#;
158        let feeds = extract_feeds(html, "https://example.com/");
159        assert_eq!(feeds.len(), 1);
160        assert_eq!(feeds[0].feed_type, "json");
161        assert_eq!(feeds[0].url, "https://example.com/feed.json");
162    }
163
164    #[test]
165    fn rejects_non_feed_links() {
166        // stylesheet, html alternate, and a bare anchor must all be ignored.
167        let html = r#"
168            <link rel="stylesheet" href="/style.css">
169            <link rel="alternate" type="text/html" href="/index.html">
170            <link rel="alternate" hreflang="fr" href="/fr/">
171        "#;
172        let feeds = extract_feeds(html, "https://example.com/");
173        assert!(feeds.is_empty(), "expected no feeds, got {feeds:?}");
174    }
175
176    #[test]
177    fn extension_heuristic_when_type_missing() {
178        let html = r#"<link rel="alternate" href="/posts.atom">"#;
179        let feeds = extract_feeds(html, "https://example.com/");
180        assert_eq!(feeds.len(), 1);
181        assert_eq!(feeds[0].feed_type, "unknown");
182    }
183
184    #[test]
185    fn dedup_is_order_preserving() {
186        let html = r#"
187            <link rel="alternate" type="application/rss+xml" href="/feed.rss">
188            <link rel="alternate" type="application/rss+xml" href="/feed.rss">
189            <link rel="alternate" type="application/atom+xml" href="/feed.atom">
190        "#;
191        let feeds = extract_feeds(html, "https://example.com/");
192        assert_eq!(feeds.len(), 2);
193        assert_eq!(feeds[0].url, "https://example.com/feed.rss");
194        assert_eq!(feeds[1].url, "https://example.com/feed.atom");
195    }
196}
rss_cli/discover.rs

rss_cli/
discover.rs
