feed_rs/parser/
mod.rs

1use std::error::Error;
2use std::fmt;
3use std::fmt::Debug;
4use std::hash::Hasher;
5use std::io::{BufRead, BufReader, Read};
6
7use crate::model;
8use crate::parser::util::{IdGenerator, TimestampParser};
9use crate::xml;
10use crate::xml::NS;
11use chrono::{DateTime, Utc};
12use siphasher::sip128::{Hasher128, SipHasher};
13
14mod atom;
15mod common;
16mod itunes;
17mod json;
18mod mediarss;
19mod rss0;
20mod rss1;
21mod rss2;
22pub(crate) mod util;
23
24pub type ParseFeedResult<T> = Result<T, ParseFeedError>;
25
26/// An error returned when parsing a feed from a source fails
27#[derive(Debug)]
28pub enum ParseFeedError {
29    ParseError(ParseErrorKind),
30    // IO error
31    IoError(std::io::Error),
32    // Underlying issue with JSON (poorly formatted etc.)
33    JsonSerde(serde_json::error::Error),
34    // Unsupported version of the JSON feed
35    JsonUnsupportedVersion(String),
36    // Underlying issue with XML (poorly formatted etc.)
37    XmlReader(xml::XmlError),
38}
39
40impl From<serde_json::error::Error> for ParseFeedError {
41    fn from(err: serde_json::error::Error) -> Self {
42        ParseFeedError::JsonSerde(err)
43    }
44}
45
46impl From<std::io::Error> for ParseFeedError {
47    fn from(err: std::io::Error) -> Self {
48        ParseFeedError::IoError(err)
49    }
50}
51
52impl From<xml::XmlError> for ParseFeedError {
53    fn from(err: xml::XmlError) -> Self {
54        ParseFeedError::XmlReader(err)
55    }
56}
57
58impl fmt::Display for ParseFeedError {
59    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
60        match self {
61            ParseFeedError::ParseError(pe) => write!(f, "unable to parse feed: {}", pe),
62            ParseFeedError::IoError(ie) => write!(f, "unable to read feed: {}", ie),
63            ParseFeedError::JsonSerde(je) => write!(f, "unable to parse JSON: {}", je),
64            ParseFeedError::JsonUnsupportedVersion(version) => write!(f, "unsupported version: {}", version),
65            ParseFeedError::XmlReader(xe) => write!(f, "unable to parse XML: {}", xe),
66        }
67    }
68}
69
70impl Error for ParseFeedError {
71    fn source(&self) -> Option<&(dyn Error + 'static)> {
72        match self {
73            ParseFeedError::IoError(ie) => Some(ie),
74            ParseFeedError::JsonSerde(je) => Some(je),
75            ParseFeedError::XmlReader(xe) => Some(xe),
76            _ => None,
77        }
78    }
79}
80
81/// Underlying cause of the parse failure
82#[derive(Debug)]
83pub enum ParseErrorKind {
84    /// Could not find the expected root element (e.g. "channel" for RSS 2, a JSON node etc.)
85    NoFeedRoot,
86    /// The content type is unsupported, and we cannot parse the value into a known representation
87    UnknownMimeType { position: u64, mime: String },
88    /// Required content within the source was not found e.g. the XML child text element for a "content" element
89    MissingContent { position: u64, reference: &'static str },
90    /// Unknown enum variant
91    UnknownEnumVariant { position: u64, reference: String },
92    /// Unexpected state
93    IllegalState { position: u64, reference: String },
94}
95
96impl fmt::Display for ParseErrorKind {
97    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
98        match self {
99            ParseErrorKind::NoFeedRoot => f.write_str("no root element"),
100            ParseErrorKind::UnknownMimeType { position, mime } => write!(f, "unsupported content type {} at position {}", mime, position),
101            ParseErrorKind::MissingContent { position, reference } => write!(f, "missing {} at position {}", reference, position),
102            ParseErrorKind::UnknownEnumVariant { position, reference } => write!(f, "unknown enum variant {} at position {}", reference, position),
103            ParseErrorKind::IllegalState { position, reference } => write!(f, "illegal parser state {} at position {}", reference, position),
104        }
105    }
106}
107
108/// Parser for various feed formats
109pub struct Parser {
110    base_uri: Option<String>,
111    id_generator: Box<IdGenerator>,
112    sanitize_content: bool,
113    timestamp_parser: Box<TimestampParser>,
114
115    // state for MediaRSS handling
116    mediarss: mediarss::MediaRssState,
117}
118
119impl Parser {
120    /// Parse the input (Atom, a flavour of RSS or JSON Feed) into our model
121    ///
122    /// # Arguments
123    ///
124    /// * `input` - A source of content such as a string, file etc.
125    ///
126    /// NOTE: feed-rs uses the encoding attribute in the XML prolog to decode content.
127    /// HTTP libraries (such as reqwest) provide a `text()` method which applies the content-encoding header and decodes the source into UTF-8.
128    /// This then causes feed-rs to fail when it attempts to interpret the UTF-8 stream as a different character set.
129    /// Instead, pass the raw, encoded source to feed-rs e.g. the `.bytes()` method if using reqwest.
130    ///
131    /// # Examples
132    ///
133    /// ```
134    /// use feed_rs::parser;
135    /// let xml = r#"
136    /// <feed>
137    ///    <title type="text">sample feed</title>
138    ///    <updated>2005-07-31T12:29:29Z</updated>
139    ///    <id>feed1</id>
140    ///    <entry>
141    ///        <title>sample entry</title>
142    ///        <id>entry1</id>
143    ///    </entry>
144    /// </feed>
145    /// "#;
146    /// let feed_from_xml = parser::parse(xml.as_bytes()).unwrap();
147    ///
148    ///
149    /// ```
150    pub fn parse<R: Read>(&self, source: R) -> ParseFeedResult<model::Feed> {
151        // Buffer the reader for performance (e.g. when streaming from a network) and so we can peek to determine the type of content
152        let mut input = BufReader::new(source);
153
154        // Determine whether this is XML or JSON and call the appropriate parser
155        input.fill_buf()?;
156        let first_char = input.buffer().iter().find(|b| **b == b'<' || **b == b'{').map(|b| *b as char);
157        let result = match first_char {
158            Some('<') => self.parse_xml(input),
159
160            Some('{') => self.parse_json(input),
161
162            _ => Err(ParseFeedError::ParseError(ParseErrorKind::NoFeedRoot)),
163        };
164
165        // Post-processing as required
166        if let Ok(mut feed) = result {
167            // Some feeds do not provide IDs, so we need to generate a consistent identifier
168            assign_missing_ids(&self.id_generator, &mut feed, self.base_uri.as_deref());
169
170            // Sanitise content as required
171            if self.sanitize_content {
172                sanitise(&mut feed);
173            }
174
175            Ok(feed)
176        } else {
177            result
178        }
179    }
180
181    // Handles JSON content
182    fn parse_json<R: BufRead>(&self, source: R) -> ParseFeedResult<model::Feed> {
183        json::parse(self, source)
184    }
185
186    // Parses timestamps with the configured parser (internal, or supplied via the builder)
187    fn parse_timestamp(&self, text: &str) -> Option<DateTime<Utc>> {
188        (self.timestamp_parser)(text)
189    }
190
191    // Handles XML content
192    fn parse_xml<R: BufRead>(&self, source: R) -> ParseFeedResult<model::Feed> {
193        // Set up the source of XML elements from the input
194        let element_source = xml::ElementSource::new(source, self.base_uri.as_deref())?;
195        if let Ok(Some(root)) = element_source.root() {
196            // Dispatch to the correct parser
197            let version = root.attr_value("version");
198            match (root.name.as_str(), version.as_deref()) {
199                ("feed", _) => {
200                    element_source.set_default_default_namespace(NS::Atom);
201                    return atom::parse_feed(self, root);
202                }
203                ("entry", _) => {
204                    element_source.set_default_default_namespace(NS::Atom);
205                    return atom::parse_entry(self, root);
206                }
207                ("rss", Some("2.0")) | ("rss", Some("")) => {
208                    element_source.set_default_default_namespace(NS::RSS);
209                    return rss2::parse(self, root);
210                }
211                ("rss", Some("0.91")) | ("rss", Some("0.92")) => {
212                    element_source.set_default_default_namespace(NS::RSS);
213                    return rss0::parse(self, root);
214                }
215                ("RDF", _) => {
216                    element_source.set_default_default_namespace(NS::RSS);
217                    return rss1::parse(self, root);
218                }
219                _ => {}
220            };
221        }
222
223        // Couldn't find a recognised feed within the provided XML stream
224        Err(ParseFeedError::ParseError(ParseErrorKind::NoFeedRoot))
225    }
226}
227
228/// Parses the provided source with the defaults
229///
230/// Customisation of the parser (e.g. base URI, custom timestamp parsers etc. can be configured through the builder.
231pub fn parse<R: Read>(source: R) -> ParseFeedResult<model::Feed> {
232    Builder::new().build().parse(source)
233}
234
235/// Builder to create instances of `FeedParser`
236pub struct Builder {
237    base_uri: Option<String>,
238    id_generator: Box<IdGenerator>,
239    sanitize_content: bool,
240    timestamp_parser: Box<TimestampParser>,
241}
242
243impl Builder {
244    /// Create a new instance of the builder
245    pub fn new() -> Builder {
246        Builder::default()
247    }
248
249    /// Source of the content, used to resolve relative URLs in XML based feeds
250    pub fn base_uri<S: AsRef<str>>(mut self, uri: Option<S>) -> Self {
251        self.base_uri = uri.map(|s| s.as_ref().to_string());
252        self
253    }
254
255    /// Create a new instance of the parser
256    pub fn build(self) -> Parser {
257        Parser {
258            base_uri: self.base_uri,
259            id_generator: self.id_generator,
260            sanitize_content: self.sanitize_content,
261            timestamp_parser: self.timestamp_parser,
262            mediarss: mediarss::MediaRssState::new(),
263        }
264    }
265
266    /// Registers an ID generator
267    pub fn id_generator<F>(mut self, generator: F) -> Self
268    where
269        F: Fn(&[model::Link], &Option<model::Text>, Option<&str>) -> String + 'static + Send + Sync,
270    {
271        self.id_generator = Box::new(generator);
272        self
273    }
274
275    /// Registers an ID generator compatible with v0.2 of feed-rs
276    pub fn id_generator_v0_2(self) -> Self {
277        self.id_generator(|links, title, _uri| {
278            // If we have a link without relative components, use that
279            if let Some(link) = links.iter().find(|l| l.rel.is_none()) {
280                // Trim the trailing slash if it exists
281                let mut link = model::Link::new(link.href.clone(), None);
282                if link.href.ends_with('/') {
283                    link.href.pop();
284                }
285
286                generate_id_from_link_and_title(&link, title)
287            } else {
288                util::uuid_gen()
289            }
290        })
291    }
292
293    /// Registers the flag for sanitizing content when the "sanitize" feature
294    /// is available
295    pub fn sanitize_content(mut self, flag: bool) -> Self {
296        self.sanitize_content = flag;
297        self
298    }
299
300    /// Registers a custom timestamp parser
301    pub fn timestamp_parser<F>(mut self, ts_parser: F) -> Self
302    where
303        F: Fn(&str) -> Option<DateTime<Utc>> + 'static + Send + Sync,
304    {
305        self.timestamp_parser = Box::new(ts_parser);
306        self
307    }
308}
309
310/// Creates a parser instance with sensible defaults
311impl Default for Builder {
312    fn default() -> Self {
313        Builder {
314            base_uri: None,
315            id_generator: Box::new(generate_id),
316            sanitize_content: true,
317            timestamp_parser: Box::new(util::parse_timestamp_lenient),
318        }
319    }
320}
321
322// Assigns IDs to missing feed + entries as required
323fn assign_missing_ids(id_generator: &IdGenerator, feed: &mut model::Feed, uri: Option<&str>) {
324    if feed.id.is_empty() {
325        feed.id = id_generator(&feed.links, &feed.title, uri);
326    }
327
328    for entry in feed.entries.iter_mut() {
329        if entry.id.is_empty() {
330            entry.id = id_generator(&entry.links, &entry.title, uri);
331        }
332    }
333}
334
335// Sanitises the various text fields in the feed + entries
336fn sanitise(feed: &mut model::Feed) {
337    if let Some(t) = feed.description.as_mut() {
338        t.sanitize()
339    }
340    if let Some(t) = feed.rights.as_mut() {
341        t.sanitize()
342    }
343    if let Some(t) = feed.title.as_mut() {
344        t.sanitize()
345    }
346
347    for entry in feed.entries.iter_mut() {
348        if let Some(c) = entry.content.as_mut() {
349            c.sanitize()
350        }
351        if let Some(t) = entry.rights.as_mut() {
352            t.sanitize()
353        }
354        if let Some(t) = entry.summary.as_mut() {
355            t.sanitize()
356        }
357        if let Some(t) = entry.title.as_mut() {
358            t.sanitize()
359        }
360    }
361}
362
363const LINK_HASH_KEY1: u64 = 0x5d78_4074_2887_2d60;
364const LINK_HASH_KEY2: u64 = 0x90ee_ca4c_90a5_e228;
365
366// Creates a unique ID by trying the following in order:
367// 1) the first link + optional title
368// 2) the uri + title provided
369// 3) a UUID
370pub fn generate_id(links: &[model::Link], title: &Option<model::Text>, uri: Option<&str>) -> String {
371    if let Some(link) = links.first() {
372        generate_id_from_link_and_title(link, title)
373    } else if let (Some(uri), Some(title)) = (uri, title) {
374        generate_id_from_uri_and_title(uri, title)
375    } else {
376        // Generate a UUID as last resort
377        util::uuid_gen()
378    }
379}
380
381// Generate an ID from the link + title
382pub fn generate_id_from_link_and_title(link: &model::Link, title: &Option<model::Text>) -> String {
383    let mut hasher = SipHasher::new_with_keys(LINK_HASH_KEY1, LINK_HASH_KEY2);
384    hasher.write(link.href.as_bytes());
385    if let Some(title) = title {
386        hasher.write(title.content.as_bytes());
387    }
388    let hash = hasher.finish128();
389    format!("{:x}{:x}", hash.h1, hash.h2)
390}
391
392// Generate an ID from the URI and title
393pub fn generate_id_from_uri_and_title(uri: &str, title: &model::Text) -> String {
394    let mut hasher = SipHasher::new_with_keys(LINK_HASH_KEY1, LINK_HASH_KEY2);
395    hasher.write(uri.as_bytes());
396    hasher.write(title.content.as_bytes());
397    let hash = hasher.finish128();
398    format!("{:x}{:x}", hash.h1, hash.h2)
399}
400
401mod podcast;
402#[cfg(test)]
403mod tests;
feed_rs/parser/mod.rs

feed_rs/parser/
mod.rs
