Skip to main content

fetchium_core/search/
reddit.rs

1//! Reddit search backend — JSON API (`search.json`).
2//!
3//! Reddit's public JSON API appends `.json` to the search URL. No auth is
4//! required, but a descriptive `User-Agent` must be provided to avoid 429s.
5
6use crate::error::{FetchiumError, FetchiumResult};
7use crate::http::HttpClient;
8use crate::search::SearchBackend;
9use crate::types::{BackendId, ResultItem};
10use async_trait::async_trait;
11use serde::Deserialize;
12use std::sync::atomic::{AtomicU64, Ordering};
13use std::time::{SystemTime, UNIX_EPOCH};
14use tracing::debug;
15
16/// Reddit search JSON endpoint.
17const REDDIT_SEARCH: &str = "https://www.reddit.com/search.json";
18/// Backoff window for Reddit rate-limit / anti-bot bursts.
19const REDDIT_COOLDOWN_SECS: u64 = 180;
20static REDDIT_COOLDOWN_UNTIL_MS: AtomicU64 = AtomicU64::new(0);
21
22#[derive(Debug, Deserialize)]
23struct RedditResponse {
24    data: RedditData,
25}
26
27#[derive(Debug, Deserialize)]
28struct RedditData {
29    children: Vec<RedditChild>,
30}
31
32#[derive(Debug, Deserialize)]
33struct RedditChild {
34    data: RedditPost,
35}
36
37#[derive(Debug, Deserialize)]
38struct RedditPost {
39    title: String,
40    url: String,
41    selftext: Option<String>,
42    subreddit: String,
43    score: Option<i64>,
44    created_utc: Option<f64>,
45    permalink: Option<String>,
46}
47
48/// Reddit search backend using the public JSON API.
49///
50/// Reddit requires a descriptive `User-Agent` to avoid rate limiting.
51/// The client is built once at construction and reused across requests
52/// to take advantage of connection pooling and TLS session resumption.
53pub struct RedditBackend {
54    /// Pre-built client with Reddit-specific User-Agent and connection pooling.
55    client: reqwest::Client,
56}
57
58impl RedditBackend {
59    /// Create a new Reddit backend with the given HTTP client.
60    pub fn new(_http: HttpClient) -> Self {
61        let client = reqwest::Client::builder()
62            // Keep Reddit fail-fast to avoid dragging overall search tail latency.
63            .timeout(std::time::Duration::from_secs(3))
64            .connect_timeout(std::time::Duration::from_secs(2))
65            .user_agent("Fetchium:search-bot:v0.1 (research tool)")
66            .pool_max_idle_per_host(4)
67            .pool_idle_timeout(std::time::Duration::from_secs(90))
68            .build()
69            .unwrap_or_default();
70        Self { client }
71    }
72}
73
74#[async_trait]
75impl SearchBackend for RedditBackend {
76    fn id(&self) -> BackendId {
77        BackendId::Reddit
78    }
79
80    async fn search(&self, query: &str, max_results: u32) -> FetchiumResult<Vec<ResultItem>> {
81        let now = now_ms();
82        let cooldown_until = REDDIT_COOLDOWN_UNTIL_MS.load(Ordering::Relaxed);
83        if now < cooldown_until {
84            debug!("Reddit in cooldown window, skipping request");
85            return Ok(vec![]);
86        }
87
88        let limit = max_results.min(25);
89        let url = format!(
90            "{REDDIT_SEARCH}?q={}&limit={limit}&sort=relevance&type=link",
91            urlencoding_encode(query)
92        );
93
94        let resp = match self.client.get(&url).send().await {
95            Ok(r) => r,
96            Err(e) => {
97                let until = now_ms() + 30_000;
98                REDDIT_COOLDOWN_UNTIL_MS.store(until, Ordering::Relaxed);
99                return Err(FetchiumError::Search(format!(
100                    "Reddit request failed: {e} (cooldown 30s)"
101                )));
102            }
103        };
104
105        if !resp.status().is_success() {
106            let status = resp.status();
107            if status.as_u16() == 429 || status.as_u16() == 403 {
108                let until = now_ms() + REDDIT_COOLDOWN_SECS * 1000;
109                REDDIT_COOLDOWN_UNTIL_MS.store(until, Ordering::Relaxed);
110                return Err(FetchiumError::Search(format!(
111                    "Reddit HTTP {status} — cooling down for {}s",
112                    REDDIT_COOLDOWN_SECS
113                )));
114            }
115            debug!("Reddit non-success HTTP {status}, skipping");
116            return Ok(vec![]);
117        }
118
119        let body = match resp.text().await {
120            Ok(b) => b,
121            Err(e) => {
122                tracing::warn!("Reddit body read failed: {e}");
123                return Ok(vec![]);
124            }
125        };
126
127        let parsed: RedditResponse = match serde_json::from_str(&body) {
128            Ok(r) => r,
129            Err(e) => {
130                tracing::warn!("Reddit JSON parse failed: {e}");
131                return Ok(vec![]);
132            }
133        };
134
135        let results = parsed
136            .data
137            .children
138            .into_iter()
139            .enumerate()
140            .map(|(i, child)| {
141                let post = child.data;
142                let snippet = build_snippet(&post);
143
144                // For self-posts (Reddit-internal links), use the permalink
145                let final_url = if post.url.starts_with("https://www.reddit.com")
146                    || post.url.starts_with("http://www.reddit.com")
147                    || post.url.starts_with('/')
148                {
149                    post.permalink
150                        .as_deref()
151                        .map(|p| format!("https://www.reddit.com{p}"))
152                        .unwrap_or(post.url.clone())
153                } else {
154                    post.url.clone()
155                };
156
157                // Convert UTC epoch to ISO-8601 string
158                let published = post.created_utc.and_then(|ts| {
159                    chrono::DateTime::from_timestamp(ts as i64, 0)
160                        .map(|dt| dt.format("%Y-%m-%dT%H:%M:%SZ").to_string())
161                });
162
163                ResultItem {
164                    title: post.title,
165                    url: final_url,
166                    snippet,
167                    rank: (i + 1) as u32,
168                    backend: BackendId::Reddit,
169                    score: None,
170                    published_date: published,
171                }
172            })
173            .collect::<Vec<_>>();
174
175        debug!("Reddit: {} results for {:?}", results.len(), query);
176        Ok(results)
177    }
178}
179
180fn now_ms() -> u64 {
181    SystemTime::now()
182        .duration_since(UNIX_EPOCH)
183        .map(|d| d.as_millis() as u64)
184        .unwrap_or(0)
185}
186
187/// Build a human-readable snippet from post metadata.
188fn build_snippet(post: &RedditPost) -> String {
189    let mut parts = Vec::new();
190
191    // Include selftext if present and not deleted
192    if let Some(ref text) = post.selftext {
193        let text = text.trim();
194        if !text.is_empty() && text != "[deleted]" && text != "[removed]" {
195            let char_count = text.chars().count();
196            let truncated: String = text.chars().take(200).collect();
197            parts.push(if char_count > 200 {
198                format!("{truncated}...")
199            } else {
200                truncated
201            });
202        }
203    }
204
205    let meta = format!("r/{} | {} points", post.subreddit, post.score.unwrap_or(0));
206    parts.push(meta);
207
208    parts.join(" — ")
209}
210
211/// Percent-encode a query string for use in URLs.
212fn urlencoding_encode(s: &str) -> String {
213    url::form_urlencoded::Serializer::new(String::new())
214        .append_key_only(s)
215        .finish()
216}
217
218#[cfg(test)]
219mod tests {
220    use super::*;
221
222    #[test]
223    fn parse_reddit_response() {
224        let json = r#"{"data":{"children":[{"data":{
225            "title":"Rust 2024",
226            "url":"https://reddit.com/r/rust/comments/123",
227            "selftext":"Great year for Rust!",
228            "subreddit":"rust",
229            "score":500,
230            "created_utc":1700000000.0,
231            "permalink":"/r/rust/comments/123"
232        }}]}}"#;
233        let resp: RedditResponse = serde_json::from_str(json).unwrap();
234        assert_eq!(resp.data.children.len(), 1);
235        let post = &resp.data.children[0].data;
236        assert_eq!(post.title, "Rust 2024");
237        assert_eq!(post.subreddit, "rust");
238        assert_eq!(post.score, Some(500));
239    }
240
241    #[test]
242    fn snippet_includes_subreddit_and_score() {
243        let post = RedditPost {
244            title: "test".to_string(),
245            url: "https://example.com".to_string(),
246            selftext: None,
247            subreddit: "rust".to_string(),
248            score: Some(42),
249            created_utc: None,
250            permalink: None,
251        };
252        let s = build_snippet(&post);
253        assert!(s.contains("r/rust"), "Expected 'r/rust' in: {s}");
254        assert!(s.contains("42"), "Expected '42' in: {s}");
255    }
256
257    #[test]
258    fn snippet_includes_selftext_when_present() {
259        let post = RedditPost {
260            title: "test".to_string(),
261            url: "https://example.com".to_string(),
262            selftext: Some("This is the body of the post.".to_string()),
263            subreddit: "programming".to_string(),
264            score: Some(10),
265            created_utc: None,
266            permalink: None,
267        };
268        let s = build_snippet(&post);
269        assert!(s.contains("This is the body"), "Expected selftext in: {s}");
270    }
271
272    #[test]
273    fn snippet_skips_deleted_selftext() {
274        for deleted in &["[deleted]", "[removed]"] {
275            let post = RedditPost {
276                title: "test".to_string(),
277                url: "https://example.com".to_string(),
278                selftext: Some(deleted.to_string()),
279                subreddit: "test".to_string(),
280                score: Some(1),
281                created_utc: None,
282                permalink: None,
283            };
284            let s = build_snippet(&post);
285            assert!(
286                !s.contains("[deleted]") && !s.contains("[removed]"),
287                "Should skip deleted text, got: {s}"
288            );
289        }
290    }
291
292    #[test]
293    fn self_post_uses_permalink() {
294        let post = RedditPost {
295            title: "Ask Reddit: something".to_string(),
296            url: "https://www.reddit.com/r/AskReddit/comments/abc".to_string(),
297            selftext: Some("My question".to_string()),
298            subreddit: "AskReddit".to_string(),
299            score: Some(100),
300            created_utc: None,
301            permalink: Some("/r/AskReddit/comments/abc".to_string()),
302        };
303        let final_url = if post.url.starts_with("https://www.reddit.com") {
304            post.permalink
305                .as_deref()
306                .map(|p| format!("https://www.reddit.com{p}"))
307                .unwrap_or(post.url.clone())
308        } else {
309            post.url.clone()
310        };
311        assert_eq!(final_url, "https://www.reddit.com/r/AskReddit/comments/abc");
312    }
313
314    #[test]
315    fn external_link_post_uses_url() {
316        let post = RedditPost {
317            title: "Interesting article".to_string(),
318            url: "https://example.com/article".to_string(),
319            selftext: None,
320            subreddit: "programming".to_string(),
321            score: Some(200),
322            created_utc: None,
323            permalink: Some("/r/programming/comments/xyz".to_string()),
324        };
325        let final_url = if post.url.starts_with("https://www.reddit.com") {
326            post.permalink
327                .as_deref()
328                .map(|p| format!("https://www.reddit.com{p}"))
329                .unwrap_or(post.url.clone())
330        } else {
331            post.url.clone()
332        };
333        assert_eq!(final_url, "https://example.com/article");
334    }
335
336    #[test]
337    fn created_utc_converts_to_iso8601() {
338        // 1700000000 UTC = 2023-11-14T22:13:20Z
339        let ts = 1_700_000_000.0f64;
340        let published = chrono::DateTime::from_timestamp(ts as i64, 0)
341            .map(|dt| dt.format("%Y-%m-%dT%H:%M:%SZ").to_string());
342        assert!(published.is_some());
343        let s = published.unwrap();
344        assert!(s.starts_with("2023-11-"), "Got: {s}");
345    }
346}