1use std::pin::Pin;
8use std::task::{Context, Poll};
9
10use jiff::Timestamp;
11use quick_xml::NsReader;
12use quick_xml::events::Event;
13use rama_core::futures::Stream;
14use rama_core::futures::StreamExt as _;
15use rama_core::futures::async_stream::stream_fn;
16use rama_core::futures::stream::BoxStream;
17use rama_core::telemetry::tracing;
18use rama_net::uri::Uri;
19use tokio::io::AsyncBufRead;
20
21use super::names::elem;
22use super::{Rss2Category, Rss2Feed, Rss2Guid, Rss2Image, Rss2Item, Rss2Source};
23use crate::protocols::rss::atom::AtomLink;
24use crate::protocols::rss::atom::names::elem as atom_elem;
25use crate::protocols::rss::error::{CollectError, FeedParseError, Rss2CollectError};
26use crate::protocols::rss::feed_ext::FeedExtensions;
27use crate::protocols::rss::feed_ext::names::attr;
28use crate::protocols::rss::feed_ext::parse::{FeedExtAcc, ItemExtAcc, Ns, classify_ns};
29use crate::protocols::rss::parse_util::{
30 attr_uri, attr_value, enclosure_from_attrs, end_event_parts, parse_rss2_date, parse_uri,
31 push_general_ref, push_text,
32};
33
34#[derive(Debug, Clone, PartialEq)]
38pub struct Rss2Channel {
39 pub title: String,
40 pub link: Uri,
41 pub description: String,
42 pub language: Option<String>,
43 pub copyright: Option<String>,
44 pub managing_editor: Option<String>,
45 pub web_master: Option<String>,
46 pub pub_date: Option<Timestamp>,
47 pub last_build_date: Option<Timestamp>,
48 pub categories: Vec<Rss2Category>,
49 pub generator: Option<String>,
50 pub docs: Option<String>,
51 pub ttl: Option<u32>,
52 pub image: Option<Rss2Image>,
53 pub atom_links: Vec<AtomLink>,
54 pub extensions: FeedExtensions,
55}
56
57impl Default for Rss2Channel {
58 fn default() -> Self {
59 Self {
60 title: String::new(),
61 link: Uri::from_static("/"),
62 description: String::new(),
63 language: None,
64 copyright: None,
65 managing_editor: None,
66 web_master: None,
67 pub_date: None,
68 last_build_date: None,
69 categories: Vec::new(),
70 generator: None,
71 docs: None,
72 ttl: None,
73 image: None,
74 atom_links: Vec::new(),
75 extensions: FeedExtensions::default(),
76 }
77 }
78}
79
80impl Rss2Channel {
81 #[must_use]
83 pub fn into_feed_with_items<I>(self, items: I) -> Rss2Feed
84 where
85 I: IntoIterator<Item = Rss2Item>,
86 {
87 Rss2Feed {
88 title: self.title,
89 link: self.link,
90 description: self.description,
91 language: self.language,
92 copyright: self.copyright,
93 managing_editor: self.managing_editor,
94 web_master: self.web_master,
95 pub_date: self.pub_date,
96 last_build_date: self.last_build_date,
97 categories: self.categories,
98 generator: self.generator,
99 docs: self.docs,
100 ttl: self.ttl,
101 image: self.image,
102 atom_links: self.atom_links,
103 items: items.into_iter().collect(),
104 extensions: self.extensions,
105 }
106 }
107}
108
109pub struct Rss2FeedStream {
121 channel: Rss2Channel,
122 items: BoxStream<'static, Result<Rss2Item, FeedParseError>>,
123}
124
125impl Rss2FeedStream {
126 pub async fn new<R>(reader: R) -> Result<Self, FeedParseError>
129 where
130 R: AsyncBufRead + Unpin + Send + 'static,
131 {
132 Self::new_with_mode(reader, false).await
133 }
134
135 pub async fn new_strict<R>(reader: R) -> Result<Self, FeedParseError>
137 where
138 R: AsyncBufRead + Unpin + Send + 'static,
139 {
140 Self::new_with_mode(reader, true).await
141 }
142
143 pub(in crate::protocols::rss) async fn new_with_mode<R>(
144 reader: R,
145 strict: bool,
146 ) -> Result<Self, FeedParseError>
147 where
148 R: AsyncBufRead + Unpin + Send + 'static,
149 {
150 let mut state = Rss2Reader::new(reader, strict);
151 let channel = state.read_channel().await?;
152 let items: BoxStream<'static, Result<Rss2Item, FeedParseError>> =
153 Box::pin(stream_fn(move |mut yielder| async move {
154 let mut state = state;
155 loop {
156 match state.read_next_item().await {
157 Ok(Some(item)) => yielder.yield_item(Ok(item)).await,
158 Ok(None) => return,
159 Err(e) => {
160 yielder.yield_item(Err(e)).await;
161 return;
162 }
163 }
164 }
165 }));
166 Ok(Self { channel, items })
167 }
168
169 #[must_use]
171 pub fn channel(&self) -> &Rss2Channel {
172 &self.channel
173 }
174
175 #[must_use]
178 pub fn drain(
179 self,
180 ) -> (
181 Rss2Channel,
182 BoxStream<'static, Result<Rss2Item, FeedParseError>>,
183 ) {
184 (self.channel, self.items)
185 }
186
187 pub async fn collect(mut self) -> Result<Rss2Feed, Rss2CollectError> {
191 let mut items = Vec::new();
192 while let Some(item) = self.items.next().await {
193 match item {
194 Ok(it) => items.push(it),
195 Err(error) => {
196 return Err(CollectError {
197 error,
198 partial: self.channel.into_feed_with_items(items),
199 });
200 }
201 }
202 }
203 Ok(self.channel.into_feed_with_items(items))
204 }
205
206 pub async fn collect_lossy(mut self) -> Rss2Feed {
211 let mut items = Vec::new();
212 while let Some(item) = self.items.next().await {
213 match item {
214 Ok(it) => items.push(it),
215 Err(err) => tracing::debug!(error = %err, "rss item dropped by collect_lossy"),
216 }
217 }
218 self.channel.into_feed_with_items(items)
219 }
220
221 pub async fn collect_filtered<F>(
225 mut self,
226 mut predicate: F,
227 ) -> Result<Rss2Feed, Rss2CollectError>
228 where
229 F: FnMut(&Rss2Item) -> bool + Send,
230 {
231 let mut items = Vec::new();
232 while let Some(item) = self.items.next().await {
233 match item {
234 Ok(it) => {
235 if predicate(&it) {
236 items.push(it);
237 }
238 }
239 Err(error) => {
240 return Err(CollectError {
241 error,
242 partial: self.channel.into_feed_with_items(items),
243 });
244 }
245 }
246 }
247 Ok(self.channel.into_feed_with_items(items))
248 }
249}
250
251impl Stream for Rss2FeedStream {
252 type Item = Result<Rss2Item, FeedParseError>;
253 fn poll_next(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Option<Self::Item>> {
254 let this = Pin::into_inner(self);
255 this.items.poll_next_unpin(cx)
256 }
257}
258
259enum Action {
265 Continue,
267 FirstItemStarted,
270 ItemFinished(Box<Rss2Item>),
273 Eof,
275}
276
277struct Rss2Reader<R: AsyncBufRead + Unpin + Send> {
278 nsr: NsReader<R>,
279 buf: Vec<u8>,
280 strict: bool,
281
282 text_buf: String,
284 depth: i32,
285 saw_root: bool,
286 channel_link_set: bool,
287
288 channel: Rss2Channel,
290 feed_acc: FeedExtAcc,
291
292 in_image_block: bool,
294 image_url: Option<Uri>,
295 image_title: String,
296 image_link: Option<Uri>,
297 image_width: Option<u32>,
298 image_height: Option<u32>,
299 image_description: Option<String>,
300
301 in_item: bool,
303 current_item: Rss2Item,
304 item_acc: ItemExtAcc,
305
306 pending_category_domain: Option<String>,
308 pending_source_url: Option<Uri>,
309}
310
311impl<R: AsyncBufRead + Unpin + Send> Rss2Reader<R> {
312 fn new(reader: R, strict: bool) -> Self {
313 let mut nsr = NsReader::from_reader(reader);
314 nsr.config_mut().trim_text(false);
320 Self {
321 nsr,
322 buf: Vec::with_capacity(4096),
323 strict,
324 text_buf: String::new(),
325 depth: 0,
326 saw_root: false,
327 channel_link_set: false,
328 channel: Rss2Channel::default(),
329 feed_acc: FeedExtAcc::default(),
330 in_image_block: false,
331 image_url: None,
332 image_title: String::new(),
333 image_link: None,
334 image_width: None,
335 image_height: None,
336 image_description: None,
337 in_item: false,
338 current_item: Rss2Item::default(),
339 item_acc: ItemExtAcc::default(),
340 pending_category_domain: None,
341 pending_source_url: None,
342 }
343 }
344
345 async fn read_channel(&mut self) -> Result<Rss2Channel, FeedParseError> {
349 loop {
350 match self.step().await? {
351 Action::Continue => {}
352 Action::FirstItemStarted | Action::Eof => {
353 return self.take_channel();
354 }
355 Action::ItemFinished(_) => {
356 return Err(FeedParseError::new(
358 "internal: item finished during channel phase",
359 ));
360 }
361 }
362 }
363 }
364
365 async fn read_next_item(&mut self) -> Result<Option<Rss2Item>, FeedParseError> {
367 loop {
368 match self.step().await? {
369 Action::Continue | Action::FirstItemStarted => {}
375 Action::ItemFinished(item) => return Ok(Some(*item)),
376 Action::Eof => return Ok(None),
377 }
378 }
379 }
380
381 fn take_channel(&mut self) -> Result<Rss2Channel, FeedParseError> {
384 if !self.saw_root {
385 return Err(FeedParseError::new("no <rss>/<channel> root encountered"));
386 }
387 let mut channel = std::mem::take(&mut self.channel);
388 channel.extensions = std::mem::take(&mut self.feed_acc).finish();
389 if self.strict {
390 if channel.title.is_empty() {
391 return Err(FeedParseError::new(
392 "RSS 2.0 channel missing required <title>",
393 ));
394 }
395 if !self.channel_link_set {
396 return Err(FeedParseError::new(
397 "RSS 2.0 channel missing required <link>",
398 ));
399 }
400 if channel.description.is_empty() {
401 return Err(FeedParseError::new(
402 "RSS 2.0 channel missing required <description>",
403 ));
404 }
405 }
406 Ok(channel)
407 }
408
409 async fn step(&mut self) -> Result<Action, FeedParseError> {
411 self.buf.clear();
412 let (rr, ev) = match self.nsr.read_resolved_event_into_async(&mut self.buf).await {
413 Ok(p) => p,
414 Err(e) => {
415 if self.strict {
416 return Err(FeedParseError::new(format!("xml error: {e}")));
417 }
418 tracing::debug!("rss2 stream xml error (lenient): {e}");
419 return Ok(Action::Eof);
420 }
421 };
422
423 match ev {
424 Event::Start(e) => {
425 self.depth += 1;
426 let ns = classify_ns(&rr);
427 let local_name = e.local_name();
428 let local = std::str::from_utf8(local_name.as_ref()).unwrap_or("");
429 self.text_buf.clear();
430
431 let consumed = if self.in_item {
432 self.item_acc.on_start(ns, local, &e)
433 } else {
434 self.feed_acc.on_start(ns, local, &e)
435 };
436 if !consumed
437 && !self.in_item
438 && ns == Ns::Atom
439 && local == atom_elem::LINK
440 && let Some(link) = crate::protocols::rss::parse_util::atom_link_from_attrs(&e)
441 {
442 self.channel.atom_links.push(link);
443 return Ok(Action::Continue);
444 }
445 if consumed || ns != Ns::None {
446 return Ok(Action::Continue);
447 }
448
449 match local {
450 elem::RSS | elem::CHANNEL => {
451 self.saw_root = true;
452 Ok(Action::Continue)
453 }
454 elem::ITEM => {
455 let first_item = !self.in_item;
458 if !first_item {
459 if self.strict {
466 return Err(FeedParseError::new(format!(
467 "RSS 2.0: nested or re-opened <item> at depth {}",
468 self.depth,
469 )));
470 }
471 tracing::debug!(
472 "rss2: nested or re-opened <item> at depth {} — \
473 partial outer item discarded",
474 self.depth,
475 );
476 }
477 self.in_item = true;
478 self.current_item = Rss2Item::default();
479 self.item_acc = ItemExtAcc::default();
480 if first_item {
481 Ok(Action::FirstItemStarted)
482 } else {
483 Ok(Action::Continue)
484 }
485 }
486 elem::IMAGE if !self.in_item => {
487 self.in_image_block = true;
488 Ok(Action::Continue)
489 }
490 elem::ENCLOSURE if self.in_item => {
491 if let Some(enclosure) = enclosure_from_attrs(&e) {
492 self.current_item.enclosures.push(enclosure);
493 }
494 Ok(Action::Continue)
495 }
496 elem::GUID if self.in_item => {
497 let permalink = attr_value(&e, attr::IS_PERMALINK)
498 .map(|v| v != "false")
499 .unwrap_or(true);
500 self.current_item.guid = Some(Rss2Guid {
501 value: String::new(),
502 permalink,
503 });
504 Ok(Action::Continue)
505 }
506 elem::SOURCE if self.in_item => {
507 self.pending_source_url = attr_uri(&e, attr::URL);
508 Ok(Action::Continue)
509 }
510 elem::CATEGORY => {
511 self.pending_category_domain = attr_value(&e, attr::DOMAIN);
512 Ok(Action::Continue)
513 }
514 _ => Ok(Action::Continue),
515 }
516 }
517 Event::Empty(e) => {
518 let ns = classify_ns(&rr);
519 let local_name = e.local_name();
520 let local = std::str::from_utf8(local_name.as_ref()).unwrap_or("");
521
522 let consumed = if self.in_item {
523 self.item_acc.on_empty(ns, local, &e)
524 } else {
525 self.feed_acc.on_empty(ns, local, &e)
526 };
527 if consumed {
528 return Ok(Action::Continue);
529 }
530 if !self.in_item
531 && ns == Ns::Atom
532 && local == atom_elem::LINK
533 && let Some(link) = crate::protocols::rss::parse_util::atom_link_from_attrs(&e)
534 {
535 self.channel.atom_links.push(link);
536 return Ok(Action::Continue);
537 }
538 if ns == Ns::None
539 && self.in_item
540 && local == elem::ENCLOSURE
541 && let Some(enclosure) = enclosure_from_attrs(&e)
542 {
543 self.current_item.enclosures.push(enclosure);
544 }
545 Ok(Action::Continue)
546 }
547 Event::Text(e) => {
548 push_text(&mut self.text_buf, &e, self.strict)?;
549 Ok(Action::Continue)
550 }
551 Event::GeneralRef(e) => {
555 push_general_ref(&mut self.text_buf, &e, self.strict)?;
556 Ok(Action::Continue)
557 }
558 Event::CData(e) => {
559 match std::str::from_utf8(e.as_ref()) {
560 Ok(t) => self.text_buf.push_str(t),
561 Err(err) => {
562 if self.strict {
563 return Err(FeedParseError::new(format!("invalid CDATA: {err}")));
564 }
565 tracing::debug!("rss2 stream CDATA utf8 error (lenient): {err}");
566 self.text_buf.push_str(&String::from_utf8_lossy(e.as_ref()));
567 }
568 }
569 Ok(Action::Continue)
570 }
571 Event::End(e) => {
572 self.depth -= 1;
573 let ns = classify_ns(&rr);
574 let mut name = [0u8; 64];
575 let (local, text) = end_event_parts(e, &mut name, &mut self.text_buf);
576 self.handle_end(ns, local, text)
577 }
578 Event::Eof => {
579 if self.strict && self.depth > 0 {
580 return Err(FeedParseError::new(format!(
581 "truncated RSS 2.0 document ({} unclosed elements at EOF)",
582 self.depth
583 )));
584 }
585 Ok(Action::Eof)
586 }
587 _ => Ok(Action::Continue),
588 }
589 }
590
591 fn handle_end(&mut self, ns: Ns, local: &str, text: String) -> Result<Action, FeedParseError> {
592 if self.in_item {
593 let Some(text) = self.item_acc.on_end(ns, local, text) else {
594 return Ok(Action::Continue);
595 };
596 if ns != Ns::None {
597 return Ok(Action::Continue);
598 }
599 match local {
600 elem::TITLE => self.current_item.title = Some(text),
601 elem::LINK => self.current_item.link = parse_uri(&text),
602 elem::DESCRIPTION => self.current_item.description = Some(text),
603 elem::AUTHOR => self.current_item.author = Some(text),
604 elem::COMMENTS => self.current_item.comments = Some(text),
605 elem::PUB_DATE => self.current_item.pub_date = parse_rss2_date(&text),
606 elem::GUID => {
607 if let Some(guid) = &mut self.current_item.guid {
608 guid.value = text;
609 }
610 }
611 elem::CATEGORY => self.current_item.categories.push(Rss2Category {
612 name: text,
613 domain: self.pending_category_domain.take(),
614 }),
615 elem::SOURCE => {
616 if let Some(url) = self.pending_source_url.take() {
617 self.current_item.source = Some(Rss2Source { title: text, url });
618 }
619 }
620 elem::ITEM => {
621 self.current_item.extensions = std::mem::take(&mut self.item_acc).finish();
622 let item = std::mem::take(&mut self.current_item);
623 self.in_item = false;
624 if self.strict
628 && item.title.as_deref().is_none_or(str::is_empty)
629 && item.description.as_deref().is_none_or(str::is_empty)
630 {
631 return Err(FeedParseError::new(
632 "RSS 2.0 item must carry at least one of <title> or <description>",
633 ));
634 }
635 return Ok(Action::ItemFinished(Box::new(item)));
636 }
637 _ => {}
638 }
639 } else if self.in_image_block {
640 match local {
641 elem::URL => self.image_url = parse_uri(&text),
642 elem::TITLE => self.image_title = text,
643 elem::LINK => self.image_link = parse_uri(&text),
644 elem::WIDTH => self.image_width = text.parse().ok(),
645 elem::HEIGHT => self.image_height = text.parse().ok(),
646 elem::DESCRIPTION => self.image_description = Some(text),
647 elem::IMAGE => {
648 self.in_image_block = false;
649 if let (Some(url), Some(link)) = (self.image_url.take(), self.image_link.take())
650 {
651 self.channel.image = Some(Rss2Image {
652 url,
653 title: std::mem::take(&mut self.image_title),
654 link,
655 width: self.image_width.take(),
656 height: self.image_height.take(),
657 description: self.image_description.take(),
658 });
659 }
660 }
661 _ => {}
662 }
663 } else {
664 let Some(text) = self.feed_acc.on_end(ns, local, text) else {
665 return Ok(Action::Continue);
666 };
667 if ns != Ns::None {
668 return Ok(Action::Continue);
669 }
670 match local {
671 elem::TITLE => self.channel.title = text,
672 elem::LINK => {
673 if let Some(link) = parse_uri(&text) {
674 self.channel.link = link;
675 self.channel_link_set = true;
676 } else if self.strict {
677 return Err(FeedParseError::new(format!(
678 "RSS 2.0 channel <link> could not be parsed as URI: {text:?}"
679 )));
680 }
681 }
682 elem::DESCRIPTION => self.channel.description = text,
683 elem::LANGUAGE => self.channel.language = Some(text),
684 elem::COPYRIGHT => self.channel.copyright = Some(text),
685 elem::MANAGING_EDITOR => self.channel.managing_editor = Some(text),
686 elem::WEB_MASTER => self.channel.web_master = Some(text),
687 elem::PUB_DATE => self.channel.pub_date = parse_rss2_date(&text),
688 elem::LAST_BUILD_DATE => self.channel.last_build_date = parse_rss2_date(&text),
689 elem::GENERATOR => self.channel.generator = Some(text),
690 elem::TTL => self.channel.ttl = text.parse().ok(),
691 elem::DOCS => self.channel.docs = Some(text),
692 elem::CATEGORY => self.channel.categories.push(Rss2Category {
693 name: text,
694 domain: self.pending_category_domain.take(),
695 }),
696 _ => {}
697 }
698 }
699 Ok(Action::Continue)
700 }
701}