finance_query/feeds/
mod.rs

1//! RSS/Atom news feed aggregation.
2//!
3//! Fetches and parses RSS/Atom feeds from named financial sources or arbitrary URLs.
4//! Multiple feeds can be fetched and merged in one call with automatic deduplication.
5//!
6//! # Quick Start
7//!
8//! ```no_run
9//! use finance_query::feeds::{self, FeedSource};
10//!
11//! # async fn example() -> Result<(), Box<dyn std::error::Error>> {
12//! // Fetch Federal Reserve press releases
13//! let fed_news = feeds::fetch(FeedSource::FederalReserve).await?;
14//! for entry in fed_news.iter().take(5) {
15//!     println!("{}: {}", entry.published.as_deref().unwrap_or("?"), entry.title);
16//! }
17//!
18//! // Aggregate multiple sources
19//! let news = feeds::fetch_all([
20//!     FeedSource::FederalReserve,
21//!     FeedSource::SecPressReleases,
22//!     FeedSource::MarketWatch,
23//! ]).await?;
24//! println!("Total entries: {}", news.len());
25//! # Ok(())
26//! # }
27//! ```
28
29use futures::future::join_all;
30use serde::{Deserialize, Serialize};
31use std::collections::HashSet;
32use std::sync::OnceLock;
33use std::time::Duration;
34
35use crate::error::Result;
36
37mod parser;
38
39/// Cached User-Agent string, computed once from the environment.
40///
41/// Only the configuration (UA string) is stored as a singleton — not the
42/// `reqwest::Client` itself. `reqwest::Client` internally spawns hyper
43/// connection-pool tasks on whichever tokio runtime first uses it; when that
44/// runtime is dropped (e.g. after a `#[tokio::test]`), those tasks die and
45/// subsequent calls from a different runtime receive `DispatchGone`. Caching
46/// only the UA avoids this while still computing the environment lookup once.
47static FEED_UA: OnceLock<String> = OnceLock::new();
48
49const FEED_TIMEOUT_SECONDS: u64 = 30;
50
51fn feed_user_agent() -> &'static str {
52    FEED_UA.get_or_init(|| {
53        // SEC EDGAR requires "app/version (email)" — nothing else in the UA.
54        // Other sites accept any reasonable UA. We use the email format when
55        // EDGAR_EMAIL is set (same env var as the edgar module), falling back
56        // to a github URL for environments without EDGAR configured.
57        match std::env::var("EDGAR_EMAIL") {
58            Ok(email) if !email.trim().is_empty() => {
59                format!(
60                    "finance-query/{} ({})",
61                    env!("CARGO_PKG_VERSION"),
62                    email.trim()
63                )
64            }
65            _ => concat!(
66                "finance-query/",
67                env!("CARGO_PKG_VERSION"),
68                " (+https://github.com/Verdenroz/finance-query)"
69            )
70            .to_string(),
71        }
72    })
73}
74
75fn build_feed_client() -> Result<reqwest::Client> {
76    reqwest::Client::builder()
77        .user_agent(feed_user_agent())
78        .timeout(Duration::from_secs(FEED_TIMEOUT_SECONDS))
79        .build()
80        .map_err(crate::error::FinanceError::HttpError)
81}
82
83/// A named or custom RSS/Atom feed source.
84#[derive(Debug, Clone)]
85#[non_exhaustive]
86pub enum FeedSource {
87    /// Federal Reserve press releases and speeches
88    FederalReserve,
89    /// SEC press releases (enforcement actions, rule changes)
90    SecPressReleases,
91    /// SEC EDGAR filing feed — specify form type (e.g., `"10-K"`, `"8-K"`)
92    SecFilings(String),
93    /// MarketWatch top stories
94    MarketWatch,
95    /// CNBC Markets
96    Cnbc,
97    /// Bloomberg Markets news
98    Bloomberg,
99    /// Financial Times Markets section
100    FinancialTimes,
101    /// The New York Times Business section
102    NytBusiness,
103    /// The Guardian Business section
104    GuardianBusiness,
105    /// Investing.com all news
106    Investing,
107    /// U.S. Bureau of Economic Analysis data releases
108    Bea,
109    /// European Central Bank press releases and speeches
110    Ecb,
111    /// Consumer Financial Protection Bureau newsroom
112    Cfpb,
113    /// Wall Street Journal Markets top stories
114    WsjMarkets,
115    /// Fortune — business and finance news
116    Fortune,
117    /// Business Wire — official corporate press releases (earnings, dividends, M&A)
118    BusinessWire,
119    /// CoinDesk — cryptocurrency and blockchain news
120    CoinDesk,
121    /// CoinTelegraph — cryptocurrency news and analysis
122    CoinTelegraph,
123    /// TechCrunch — startup, VC, and tech industry news
124    TechCrunch,
125    /// Hacker News — community-curated tech posts with 100+ points
126    HackerNews,
127    /// OilPrice.com — crude oil, natural gas, and energy geopolitics
128    OilPrice,
129    /// Calculated Risk — housing starts, mortgage rates, and macro data
130    CalculatedRisk,
131    /// South China Morning Post — China business, regulation, and trade
132    Scmp,
133    /// Nikkei Asia — Japanese and Asian business news
134    NikkeiAsia,
135    /// Bank of England — UK monetary policy, rate decisions, and regulatory notices
136    BankOfEngland,
137    /// VentureBeat — AI funding rounds and enterprise technology
138    VentureBeat,
139    /// Y Combinator Blog — startup ecosystem announcements (low-frequency)
140    YCombinator,
141    /// The Economist — global economics and market analysis
142    TheEconomist,
143    /// Financial Post — Canadian market and business news
144    FinancialPost,
145    /// Financial Times Lex — short daily market commentary column
146    FtLex,
147    /// The Big Picture (Ritholtz) — macro finance analysis and commentary
148    RitholtzBigPicture,
149    /// Custom feed URL
150    Custom(String),
151}
152
153impl FeedSource {
154    /// Return the URL for this feed source.
155    pub fn url(&self) -> String {
156        match self {
157            Self::FederalReserve => {
158                "https://www.federalreserve.gov/feeds/press_all.xml".to_string()
159            }
160            Self::SecPressReleases => "https://www.sec.gov/news/pressreleases.rss".to_string(),
161            Self::SecFilings(form_type) => format!(
162                "https://www.sec.gov/cgi-bin/browse-edgar?action=getcurrent&type={form_type}&output=atom"
163            ),
164            Self::MarketWatch => {
165                "https://feeds.content.dowjones.io/public/rss/mw_topstories".to_string()
166            }
167            Self::Cnbc => "https://www.cnbc.com/id/100003114/device/rss/rss.html".to_string(),
168            Self::Bloomberg => "https://feeds.bloomberg.com/markets/news.rss".to_string(),
169            Self::FinancialTimes => "https://www.ft.com/markets?format=rss".to_string(),
170            Self::NytBusiness => {
171                "https://rss.nytimes.com/services/xml/rss/nyt/Business.xml".to_string()
172            }
173            Self::GuardianBusiness => "https://www.theguardian.com/business/rss".to_string(),
174            Self::Investing => "https://www.investing.com/rss/news.rss".to_string(),
175            Self::Bea => "https://apps.bea.gov/rss/rss.xml".to_string(),
176            Self::Ecb => "https://www.ecb.europa.eu/rss/press.html".to_string(),
177            Self::Cfpb => "https://www.consumerfinance.gov/about-us/newsroom/feed/".to_string(),
178            Self::WsjMarkets => "https://feeds.a.dj.com/rss/RSSMarketsMain.xml".to_string(),
179            Self::Fortune => "https://fortune.com/feed".to_string(),
180            Self::BusinessWire => {
181                "https://feed.businesswire.com/rss/home/?rss=G1QFDERJXkJeGVtQXw==".to_string()
182            }
183            Self::CoinDesk => "https://www.coindesk.com/arc/outboundfeeds/rss/".to_string(),
184            Self::CoinTelegraph => "https://cointelegraph.com/rss".to_string(),
185            Self::TechCrunch => "https://techcrunch.com/feed/".to_string(),
186            Self::HackerNews => "https://hnrss.org/newest?points=100".to_string(),
187            Self::OilPrice => "https://oilprice.com/rss/main".to_string(),
188            Self::CalculatedRisk => "https://calculatedrisk.substack.com/feed".to_string(),
189            Self::Scmp => "https://www.scmp.com/rss/91/feed".to_string(),
190            Self::NikkeiAsia => "https://asia.nikkei.com/rss/feed/nar".to_string(),
191            Self::BankOfEngland => "https://www.bankofengland.co.uk/rss/news".to_string(),
192            Self::VentureBeat => "https://venturebeat.com/feed/".to_string(),
193            Self::YCombinator => "https://blog.ycombinator.com/feed/".to_string(),
194            Self::TheEconomist => {
195                "https://www.economist.com/sections/economics/rss.xml".to_string()
196            }
197            Self::FinancialPost => "https://financialpost.com/feed".to_string(),
198            Self::FtLex => "https://www.ft.com/lex?format=rss".to_string(),
199            Self::RitholtzBigPicture => "https://ritholtz.com/feed/".to_string(),
200            Self::Custom(url) => url.clone(),
201        }
202    }
203
204    /// Human-readable source name, used in [`FeedEntry::source`].
205    pub fn name(&self) -> String {
206        match self {
207            Self::FederalReserve => "Federal Reserve".to_string(),
208            Self::SecPressReleases => "SEC".to_string(),
209            Self::SecFilings(form) => format!("SEC EDGAR ({form})"),
210            Self::MarketWatch => "MarketWatch".to_string(),
211            Self::Cnbc => "CNBC".to_string(),
212            Self::Bloomberg => "Bloomberg".to_string(),
213            Self::FinancialTimes => "Financial Times".to_string(),
214            Self::NytBusiness => "New York Times".to_string(),
215            Self::GuardianBusiness => "The Guardian".to_string(),
216            Self::Investing => "Investing.com".to_string(),
217            Self::Bea => "Bureau of Economic Analysis".to_string(),
218            Self::Ecb => "European Central Bank".to_string(),
219            Self::Cfpb => "CFPB".to_string(),
220            Self::WsjMarkets => "Wall Street Journal".to_string(),
221            Self::Fortune => "Fortune".to_string(),
222            Self::BusinessWire => "Business Wire".to_string(),
223            Self::CoinDesk => "CoinDesk".to_string(),
224            Self::CoinTelegraph => "CoinTelegraph".to_string(),
225            Self::TechCrunch => "TechCrunch".to_string(),
226            Self::HackerNews => "Hacker News".to_string(),
227            Self::OilPrice => "OilPrice.com".to_string(),
228            Self::CalculatedRisk => "Calculated Risk".to_string(),
229            Self::Scmp => "South China Morning Post".to_string(),
230            Self::NikkeiAsia => "Nikkei Asia".to_string(),
231            Self::BankOfEngland => "Bank of England".to_string(),
232            Self::VentureBeat => "VentureBeat".to_string(),
233            Self::YCombinator => "Y Combinator".to_string(),
234            Self::TheEconomist => "The Economist".to_string(),
235            Self::FinancialPost => "Financial Post".to_string(),
236            Self::FtLex => "Financial Times Lex".to_string(),
237            Self::RitholtzBigPicture => "The Big Picture".to_string(),
238            Self::Custom(url) => url.clone(),
239        }
240    }
241}
242
243/// A single entry from an RSS/Atom feed.
244#[derive(Debug, Clone, Serialize, Deserialize)]
245#[non_exhaustive]
246pub struct FeedEntry {
247    /// Article or item title
248    pub title: String,
249    /// Canonical link to the article
250    pub url: String,
251    /// Publication date/time as an RFC 3339 string (if available)
252    pub published: Option<String>,
253    /// Short summary or description
254    pub summary: Option<String>,
255    /// Name of the feed source
256    pub source: String,
257}
258
259/// Fetch and parse a single feed source.
260///
261/// Returns an empty `Vec` (not an error) when the feed is reachable but empty.
262pub async fn fetch(source: FeedSource) -> Result<Vec<FeedEntry>> {
263    let client = build_feed_client()?;
264    fetch_with_client(&client, &source.url(), &source.name()).await
265}
266
267/// Fetch multiple feed sources concurrently and merge the results.
268///
269/// Results are deduplicated by URL and sorted newest-first when dates are available.
270/// Feeds that fail individually are skipped (not propagated as errors).
271///
272/// A single `reqwest::Client` is shared across all concurrent fetches within
273/// this call, reusing connection pools and TLS state.
274pub async fn fetch_all(sources: impl IntoIterator<Item = FeedSource>) -> Result<Vec<FeedEntry>> {
275    let client = build_feed_client()?;
276    let pairs: Vec<(String, String)> = sources.into_iter().map(|s| (s.url(), s.name())).collect();
277    let futures: Vec<_> = pairs
278        .iter()
279        .map(|(url, name)| fetch_with_client(&client, url, name))
280        .collect();
281
282    let results = join_all(futures).await;
283
284    let mut seen_urls: HashSet<String> = HashSet::new();
285    let mut entries: Vec<FeedEntry> = results
286        .into_iter()
287        .flat_map(|r| r.unwrap_or_default())
288        .filter(|e| seen_urls.insert(e.url.clone()))
289        .collect();
290
291    // Sort newest-first where dates are present
292    entries.sort_by(|a, b| b.published.cmp(&a.published));
293
294    Ok(entries)
295}
296
297async fn fetch_with_client(
298    client: &reqwest::Client,
299    url: &str,
300    source_name: &str,
301) -> Result<Vec<FeedEntry>> {
302    let text = client.get(url).send().await?.text().await?;
303    parser::parse(text.as_bytes(), source_name)
304}
305
306/// Parse already-fetched RSS/Atom bytes into entries, without a network round-trip.
307///
308/// Used internally by [`fetch`]/[`fetch_all`]; also useful for callers that
309/// fetch feed bytes through their own HTTP client/cache/proxy, and for
310/// offline parsing benchmarks/tests.
311pub fn parse_bytes(bytes: &[u8], source_name: &str) -> Result<Vec<FeedEntry>> {
312    parser::parse(bytes, source_name)
313}
314
315#[cfg(test)]
316mod tests {
317    use super::*;
318
319    #[test]
320    fn test_feed_source_urls() {
321        assert!(FeedSource::FederalReserve.url().starts_with("https://"));
322        assert!(FeedSource::SecPressReleases.url().starts_with("https://"));
323        assert!(
324            FeedSource::SecFilings("10-K".to_string())
325                .url()
326                .contains("10-K")
327        );
328        assert_eq!(
329            FeedSource::Custom("https://example.com/feed.rss".to_string()).url(),
330            "https://example.com/feed.rss"
331        );
332        assert!(FeedSource::Bloomberg.url().starts_with("https://"));
333        assert!(FeedSource::FinancialTimes.url().starts_with("https://"));
334        assert!(FeedSource::NytBusiness.url().starts_with("https://"));
335        assert!(FeedSource::GuardianBusiness.url().starts_with("https://"));
336        assert!(FeedSource::Investing.url().starts_with("https://"));
337        assert!(FeedSource::Bea.url().starts_with("https://"));
338        assert!(FeedSource::Ecb.url().starts_with("https://"));
339        assert!(FeedSource::Cfpb.url().starts_with("https://"));
340        // New sources
341        assert!(FeedSource::WsjMarkets.url().contains("dj.com"));
342        assert!(FeedSource::Fortune.url().contains("fortune.com"));
343        assert!(FeedSource::BusinessWire.url().contains("businesswire.com"));
344        assert!(FeedSource::CoinDesk.url().contains("coindesk.com"));
345        assert!(
346            FeedSource::CoinTelegraph
347                .url()
348                .contains("cointelegraph.com")
349        );
350        assert!(FeedSource::TechCrunch.url().contains("techcrunch.com"));
351        assert!(FeedSource::HackerNews.url().contains("hnrss.org"));
352    }
353
354    #[test]
355    fn test_feed_source_names() {
356        assert_eq!(FeedSource::FederalReserve.name(), "Federal Reserve");
357        assert_eq!(FeedSource::MarketWatch.name(), "MarketWatch");
358        assert_eq!(FeedSource::Bloomberg.name(), "Bloomberg");
359        assert_eq!(FeedSource::FinancialTimes.name(), "Financial Times");
360        assert_eq!(FeedSource::NytBusiness.name(), "New York Times");
361        assert_eq!(FeedSource::GuardianBusiness.name(), "The Guardian");
362        assert_eq!(FeedSource::Investing.name(), "Investing.com");
363        assert_eq!(FeedSource::Bea.name(), "Bureau of Economic Analysis");
364        assert_eq!(FeedSource::Ecb.name(), "European Central Bank");
365        assert_eq!(FeedSource::Cfpb.name(), "CFPB");
366        // New sources
367        assert_eq!(FeedSource::WsjMarkets.name(), "Wall Street Journal");
368        assert_eq!(FeedSource::Fortune.name(), "Fortune");
369        assert_eq!(FeedSource::BusinessWire.name(), "Business Wire");
370        assert_eq!(FeedSource::CoinDesk.name(), "CoinDesk");
371        assert_eq!(FeedSource::CoinTelegraph.name(), "CoinTelegraph");
372        assert_eq!(FeedSource::TechCrunch.name(), "TechCrunch");
373        assert_eq!(FeedSource::HackerNews.name(), "Hacker News");
374    }
375
376    #[tokio::test]
377    #[ignore = "requires network access"]
378    async fn test_fetch_fed_reserve() {
379        let entries = fetch(FeedSource::FederalReserve).await;
380        assert!(entries.is_ok(), "Expected ok, got: {:?}", entries.err());
381        let entries = entries.unwrap();
382        assert!(!entries.is_empty());
383        for e in entries.iter().take(3) {
384            assert!(!e.title.is_empty());
385            assert!(!e.url.is_empty());
386        }
387    }
388
389    #[tokio::test]
390    #[ignore = "requires network access"]
391    async fn test_fetch_bloomberg() {
392        let entries = fetch(FeedSource::Bloomberg).await;
393        assert!(entries.is_ok(), "Expected ok, got: {:?}", entries.err());
394        let entries = entries.unwrap();
395        assert!(!entries.is_empty());
396        for e in entries.iter().take(3) {
397            assert!(!e.title.is_empty());
398            assert!(!e.url.is_empty());
399            assert_eq!(e.source, "Bloomberg");
400        }
401    }
402
403    #[tokio::test]
404    #[ignore = "requires network access"]
405    async fn test_fetch_financial_times() {
406        let entries = fetch(FeedSource::FinancialTimes).await;
407        assert!(entries.is_ok(), "Expected ok, got: {:?}", entries.err());
408        let entries = entries.unwrap();
409        assert!(!entries.is_empty());
410    }
411
412    #[tokio::test]
413    #[ignore = "requires network access"]
414    async fn test_fetch_bea() {
415        let entries = fetch(FeedSource::Bea).await;
416        assert!(entries.is_ok(), "Expected ok, got: {:?}", entries.err());
417        let entries = entries.unwrap();
418        assert!(!entries.is_empty());
419        assert_eq!(entries[0].source, "Bureau of Economic Analysis");
420    }
421
422    #[tokio::test]
423    #[ignore = "requires network access"]
424    async fn test_fetch_ecb() {
425        let entries = fetch(FeedSource::Ecb).await;
426        assert!(entries.is_ok(), "Expected ok, got: {:?}", entries.err());
427        let entries = entries.unwrap();
428        assert!(!entries.is_empty());
429    }
430}
finance_query/feeds/mod.rs

finance_query/feeds/
mod.rs
