feed/article_store/
fetch.rs

1use futures::future::join_all;
2
3use crate::article::Article;
4use crate::cache::CacheStore;
5use crate::config::ExtractorMethod;
6use crate::feed_source::{self, FetchResult};
7
8use super::ArticleStore;
9
10fn load_from_cache(
11    cache: &CacheStore,
12    feed_url: &str,
13    feed_name: &str,
14    extractor_method: &ExtractorMethod,
15) -> Vec<Article> {
16    cache
17        .load_feed(feed_url)
18        .map(|c| {
19            c.articles
20                .into_iter()
21                .map(|info| Article {
22                    title: info.title,
23                    url: info.url,
24                    published: info.published,
25                    feed_url: feed_url.to_string(),
26                    feed_name: feed_name.to_string(),
27                    extractor: extractor_method.clone(),
28                    read: info.read,
29                    rss_content: info.rss_content,
30                })
31                .collect()
32        })
33        .unwrap_or_default()
34}
35
36impl ArticleStore {
37    /// Fetch all feeds and update internal article list.
38    /// If cached_only=true, only loads from cache (no network).
39    pub async fn fetch(&mut self, cached_only: bool) {
40        let cache_enabled = self.config().cache.retention_days >= 0;
41        let default_extractor = self.config().content.extractor.clone();
42
43        if cached_only {
44            let futures: Vec<_> = self
45                .feeds()
46                .iter()
47                .map(|feed_entry| {
48                    let cache = self.cache().clone();
49                    let feed_url = feed_entry.url.clone();
50                    let feed_name = feed_entry.name.clone();
51                    let extractor_method = feed_entry
52                        .extractor
53                        .as_ref()
54                        .unwrap_or(&default_extractor)
55                        .clone();
56                    tokio::task::spawn_blocking(move || {
57                        load_from_cache(&cache, &feed_url, &feed_name, &extractor_method)
58                    })
59                })
60                .collect();
61
62            let mut all_articles: Vec<Article> = Vec::new();
63            for articles in join_all(futures).await.into_iter().flatten() {
64                all_articles.extend(articles);
65            }
66            all_articles.sort_by(|a, b| b.published.cmp(&a.published));
67            self.set_articles(all_articles);
68            return;
69        }
70
71        let futures: Vec<_> = self
72            .feeds()
73            .iter()
74            .map(|feed_entry| {
75                let client = self.client().clone();
76                let feed_url = feed_entry.url.clone();
77                let cache = self.cache().clone();
78                async move {
79                    let metadata = cache.load_http_metadata(&feed_url);
80                    feed_source::fetch(&client, &feed_url, &metadata).await
81                }
82            })
83            .collect();
84
85        let results = join_all(futures).await;
86
87        let mut all_articles: Vec<Article> = Vec::new();
88
89        for (feed_entry, result) in self.feeds().iter().zip(results) {
90            let extractor_method = feed_entry
91                .extractor
92                .as_ref()
93                .unwrap_or(&default_extractor)
94                .clone();
95            let feed_url = &feed_entry.url;
96            let feed_name = &feed_entry.name;
97
98            match result {
99                Ok(FetchResult::Fetched(feed)) => {
100                    if cache_enabled {
101                        let save_cache = self.cache().clone();
102                        let save_url = feed_url.to_string();
103                        let save_feed = feed.clone();
104                        let _ = tokio::task::spawn_blocking(move || {
105                            save_cache.save_feed(
106                                &save_url,
107                                &save_feed,
108                                save_feed.etag.as_deref(),
109                                save_feed.last_modified.as_deref(),
110                            )
111                        })
112                        .await;
113                        all_articles.extend(load_from_cache(
114                            self.cache(),
115                            feed_url,
116                            feed_name,
117                            &extractor_method,
118                        ));
119                    } else {
120                        all_articles.extend(feed.entries.into_iter().map(|e| Article {
121                            title: e.title,
122                            url: e.url,
123                            published: e.published,
124                            feed_url: feed_url.to_string(),
125                            feed_name: feed_name.to_string(),
126                            extractor: extractor_method.clone(),
127                            read: false,
128                            rss_content: e.rss_content,
129                        }));
130                    }
131                }
132                Ok(FetchResult::NotModified) => {
133                    all_articles.extend(load_from_cache(
134                        self.cache(),
135                        feed_url,
136                        feed_name,
137                        &extractor_method,
138                    ));
139                }
140                Err(e) => {
141                    eprintln!("Error fetching {}: {}", feed_name, e);
142                    all_articles.extend(load_from_cache(
143                        self.cache(),
144                        feed_url,
145                        feed_name,
146                        &extractor_method,
147                    ));
148                }
149            }
150        }
151
152        all_articles.sort_by(|a, b| b.published.cmp(&a.published));
153        self.set_articles(all_articles);
154    }
155}
feed/article_store/fetch.rs

feed/article_store/
fetch.rs
