rssume 0.2.0

RSS middleware with AI-powered translation and summarization
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110

use chrono::Utc;
use feed_rs::parser;

#[derive(Debug, Clone)]
pub struct RawArticle {
    pub title: String,
    pub link: String,
    pub content: String,
    pub published_at: String,
    pub author: Option<String>,
    pub categories: Vec<String>,
    pub guid: Option<String>,
    pub media_urls: Vec<MediaItem>,
}

#[derive(Debug, Clone)]
pub struct MediaItem {
    pub url: String,
    pub content_type: Option<String>,
    pub length: Option<u64>,
}

pub async fn fetch_feed(url: &str) -> Result<Vec<RawArticle>, crate::error::AppError> {
    let client = reqwest::Client::builder()
        .user_agent("rssume/0.1 (RSS middleware; +https://github.com/rssume/rssume)")
        .timeout(std::time::Duration::from_secs(30))
        .build()
        .map_err(|e| crate::error::AppError::Fetch(format!("client: {}", e)))?;

    let resp = client
        .get(url)
        .send()
        .await
        .map_err(|e| crate::error::AppError::Fetch(format!("fetch {}: {}", url, e)))?;
    if !resp.status().is_success() {
        return Err(crate::error::AppError::Fetch(format!(
            "HTTP {} for {}",
            resp.status(),
            url
        )));
    }

    let body = resp
        .bytes()
        .await
        .map_err(|e| crate::error::AppError::Fetch(format!("body: {}", e)))?;
    let feed = parser::parse(&body[..])
        .map_err(|e| crate::error::AppError::Parse(format!("parse: {}", e)))?;

    let articles: Vec<_> = feed
        .entries
        .into_iter()
        .map(|entry| {
            let content = entry
                .content
                .as_ref()
                .and_then(|c| c.body.as_ref())
                .cloned()
                .unwrap_or_default();
            let summary = entry
                .summary
                .as_ref()
                .map(|s| s.content.clone())
                .unwrap_or_default();
            let body = if !content.is_empty() {
                content
            } else {
                summary
            };
            let media_urls = entry
                .media
                .iter()
                .flat_map(|m| &m.content)
                .filter_map(|mc| {
                    mc.url.as_ref().map(|u| MediaItem {
                        url: u.to_string(),
                        content_type: mc.content_type.as_ref().map(|t| t.to_string()),
                        length: mc.size,
                    })
                })
                .collect();

            RawArticle {
                title: entry
                    .title
                    .as_ref()
                    .map(|t| t.content.clone())
                    .unwrap_or_default(),
                link: entry
                    .links
                    .first()
                    .map(|l| l.href.clone())
                    .unwrap_or_default(),
                content: body,
                published_at: entry
                    .published
                    .or(entry.updated)
                    .map(|d| d.to_rfc2822())
                    .unwrap_or_else(|| Utc::now().to_rfc2822()),
                author: entry.authors.first().map(|p| p.name.clone()),
                categories: entry.categories.into_iter().map(|c| c.term).collect(),
                guid: Some(entry.id),
                media_urls,
            }
        })
        .filter(|a| !a.link.is_empty() && !a.title.is_empty())
        .collect();

    Ok(articles)
}