Skip to main content

fetchium_core/search/
stackoverflow.rs

1//! StackOverflow search backend — StackExchange API v2.3.
2//!
3//! Uses the `/questions/search` endpoint. No API key is required for basic usage
4//! (300 requests/day). Set `STACKEXCHANGE_KEY` for a higher quota (10,000 req/day).
5//!
6//! StackExchange returns gzip-compressed responses by default; reqwest handles
7//! decompression automatically when built with the `gzip` feature.
8
9use crate::error::FetchiumError;
10use crate::error::FetchiumResult;
11use crate::http::HttpClient;
12use crate::search::SearchBackend;
13use crate::types::{BackendId, ResultItem};
14use async_trait::async_trait;
15use serde::Deserialize;
16use std::sync::atomic::{AtomicU64, Ordering};
17use std::time::{SystemTime, UNIX_EPOCH};
18use tracing::debug;
19
20/// StackExchange API v2.3 search endpoint.
21const SO_API: &str = "https://api.stackexchange.com/2.3/search";
22/// Backoff window when StackExchange starts rate-limiting aggressively.
23const SO_COOLDOWN_SECS: u64 = 300;
24static SO_COOLDOWN_UNTIL_MS: AtomicU64 = AtomicU64::new(0);
25
26#[derive(Debug, Deserialize)]
27struct SoResponse {
28    items: Vec<SoQuestion>,
29}
30
31#[derive(Debug, Deserialize)]
32struct SoQuestion {
33    title: String,
34    link: String,
35    score: i32,
36    answer_count: u32,
37    is_answered: bool,
38    #[serde(default)]
39    tags: Vec<String>,
40    creation_date: Option<u64>,
41    body_markdown: Option<String>,
42}
43
44/// StackOverflow search backend using the StackExchange API.
45///
46/// Searches for questions whose titles match the query. The `withbody` filter
47/// includes `body_markdown` for richer snippets (counts toward quota).
48/// The client is built once and reused to enable connection pooling and
49/// avoid repeated TLS handshakes (StackExchange always returns gzip).
50pub struct StackOverflowBackend {
51    /// Pre-built client with gzip support and connection pooling.
52    client: reqwest::Client,
53    api_key: Option<String>,
54}
55
56impl StackOverflowBackend {
57    /// Create a new StackOverflow backend. Reads `STACKEXCHANGE_KEY` from the environment.
58    pub fn new(_http: HttpClient) -> Self {
59        let api_key = std::env::var("STACKEXCHANGE_KEY").ok();
60        let client = reqwest::Client::builder()
61            // Keep StackOverflow fail-fast to avoid long-tail latency in aggregate search.
62            .timeout(std::time::Duration::from_secs(4))
63            .connect_timeout(std::time::Duration::from_secs(2))
64            .user_agent("Fetchium/0.1")
65            .gzip(true)
66            .pool_max_idle_per_host(4)
67            .pool_idle_timeout(std::time::Duration::from_secs(90))
68            .build()
69            .unwrap_or_default();
70        Self { client, api_key }
71    }
72
73    /// Create a StackOverflow backend with an explicit API key.
74    pub fn with_key(_http: HttpClient, api_key: impl Into<String>) -> Self {
75        let client = reqwest::Client::builder()
76            .timeout(std::time::Duration::from_secs(4))
77            .connect_timeout(std::time::Duration::from_secs(2))
78            .user_agent("Fetchium/0.1")
79            .gzip(true)
80            .pool_max_idle_per_host(4)
81            .pool_idle_timeout(std::time::Duration::from_secs(90))
82            .build()
83            .unwrap_or_default();
84        Self {
85            client,
86            api_key: Some(api_key.into()),
87        }
88    }
89}
90
91#[async_trait]
92impl SearchBackend for StackOverflowBackend {
93    fn id(&self) -> BackendId {
94        BackendId::StackOverflow
95    }
96
97    async fn search(&self, query: &str, max_results: u32) -> FetchiumResult<Vec<ResultItem>> {
98        let now = now_ms();
99        let cooldown_until = SO_COOLDOWN_UNTIL_MS.load(Ordering::Relaxed);
100        if now < cooldown_until {
101            debug!("StackOverflow in cooldown window, skipping request");
102            return Ok(vec![]);
103        }
104
105        let page_size = max_results.min(30);
106        let mut url = format!(
107            "{SO_API}?intitle={}&site=stackoverflow&order=desc&sort=relevance\
108             &pagesize={page_size}&filter=withbody",
109            urlencoding_encode(query)
110        );
111        if let Some(ref key) = self.api_key {
112            url.push_str(&format!("&key={key}"));
113        }
114
115        let resp = match self
116            .client
117            .get(&url)
118            .header("Accept-Encoding", "gzip")
119            .send()
120            .await
121        {
122            Ok(r) => r,
123            Err(e) => {
124                tracing::warn!("StackOverflow request failed: {e}");
125                return Ok(vec![]);
126            }
127        };
128
129        if !resp.status().is_success() {
130            let status = resp.status();
131            let body = resp.text().await.unwrap_or_default();
132            if status.as_u16() == 429 || body.contains("throttle_violation") {
133                let until = now_ms() + SO_COOLDOWN_SECS * 1000;
134                SO_COOLDOWN_UNTIL_MS.store(until, Ordering::Relaxed);
135                return Err(FetchiumError::Search(format!(
136                    "StackOverflow rate-limited (HTTP {status}) — cooling down for {}s",
137                    SO_COOLDOWN_SECS
138                )));
139            }
140            if status.is_server_error() {
141                return Err(FetchiumError::Search(format!(
142                    "StackOverflow upstream server error: HTTP {status}"
143                )));
144            }
145            debug!("StackOverflow non-success HTTP {status}, skipping");
146            return Ok(vec![]);
147        }
148
149        let body = match resp.text().await {
150            Ok(b) => b,
151            Err(e) => {
152                tracing::warn!("StackOverflow body read failed: {e}");
153                return Ok(vec![]);
154            }
155        };
156
157        let parsed: SoResponse = match serde_json::from_str(&body) {
158            Ok(r) => r,
159            Err(e) => {
160                tracing::warn!("StackOverflow JSON parse failed: {e}");
161                return Ok(vec![]);
162            }
163        };
164
165        let results = parsed
166            .items
167            .into_iter()
168            .enumerate()
169            .map(|(i, q)| {
170                let snippet = build_snippet(&q);
171                let published = q.creation_date.and_then(|ts| {
172                    chrono::DateTime::from_timestamp(ts as i64, 0)
173                        .map(|dt| dt.format("%Y-%m-%dT%H:%M:%SZ").to_string())
174                });
175                ResultItem {
176                    title: decode_html_entities(&q.title),
177                    url: q.link,
178                    snippet,
179                    rank: (i + 1) as u32,
180                    backend: BackendId::StackOverflow,
181                    score: None,
182                    published_date: published,
183                }
184            })
185            .collect::<Vec<_>>();
186
187        debug!("StackOverflow: {} results for {:?}", results.len(), query);
188        Ok(results)
189    }
190}
191
192fn now_ms() -> u64 {
193    SystemTime::now()
194        .duration_since(UNIX_EPOCH)
195        .map(|d| d.as_millis() as u64)
196        .unwrap_or(0)
197}
198
199/// Build a human-readable snippet from question metadata.
200fn build_snippet(q: &SoQuestion) -> String {
201    let mut parts = Vec::new();
202
203    // Include body if available
204    if let Some(ref body) = q.body_markdown {
205        let stripped = strip_html(body);
206        let stripped = stripped.trim();
207        if !stripped.is_empty() {
208            let char_count = stripped.chars().count();
209            let truncated: String = stripped.chars().take(200).collect();
210            parts.push(if char_count > 200 {
211                format!("{truncated}...")
212            } else {
213                truncated
214            });
215        }
216    }
217
218    let status = if q.is_answered {
219        "Answered"
220    } else {
221        "Unanswered"
222    };
223    parts.push(format!(
224        "{status} | {} answers | score {}",
225        q.answer_count, q.score
226    ));
227
228    if !q.tags.is_empty() {
229        parts.push(format!("Tags: {}", q.tags.join(", ")));
230    }
231
232    parts.join(" — ")
233}
234
235/// Strip HTML tags from a string (for StackExchange body_markdown fields).
236fn strip_html(s: &str) -> String {
237    let mut result = String::with_capacity(s.len());
238    let mut in_tag = false;
239    for c in s.chars() {
240        match c {
241            '<' => in_tag = true,
242            '>' => in_tag = false,
243            _ if !in_tag => result.push(c),
244            _ => {}
245        }
246    }
247    result.trim().to_string()
248}
249
250/// Decode the five predefined HTML entities in question titles.
251fn decode_html_entities(s: &str) -> String {
252    s.replace("&amp;", "&")
253        .replace("&lt;", "<")
254        .replace("&gt;", ">")
255        .replace("&quot;", "\"")
256        .replace("&#39;", "'")
257}
258
259/// Percent-encode a query string for use in URLs.
260fn urlencoding_encode(s: &str) -> String {
261    url::form_urlencoded::Serializer::new(String::new())
262        .append_key_only(s)
263        .finish()
264}
265
266#[cfg(test)]
267mod tests {
268    use super::*;
269
270    #[test]
271    fn parse_so_response() {
272        let json = r#"{"items":[{
273            "title":"How to use Rust?",
274            "link":"https://stackoverflow.com/q/123",
275            "score":10,
276            "answer_count":3,
277            "is_answered":true,
278            "tags":["rust","programming"],
279            "creation_date":1700000000
280        }]}"#;
281        let resp: SoResponse = serde_json::from_str(json).unwrap();
282        assert_eq!(resp.items.len(), 1);
283        let q = &resp.items[0];
284        assert_eq!(q.title, "How to use Rust?");
285        assert!(q.is_answered);
286        assert_eq!(q.answer_count, 3);
287        assert_eq!(q.score, 10);
288        assert_eq!(q.tags, ["rust", "programming"]);
289    }
290
291    #[test]
292    fn parse_so_response_empty_tags() {
293        let json = r#"{"items":[{
294            "title":"Test question",
295            "link":"https://stackoverflow.com/q/999",
296            "score":0,
297            "answer_count":0,
298            "is_answered":false,
299            "tags":[],
300            "creation_date":null
301        }]}"#;
302        let resp: SoResponse = serde_json::from_str(json).unwrap();
303        assert!(resp.items[0].tags.is_empty());
304        assert!(resp.items[0].creation_date.is_none());
305    }
306
307    #[test]
308    fn decode_html_entities_works() {
309        assert_eq!(decode_html_entities("A &amp; B"), "A & B");
310        assert_eq!(decode_html_entities("&lt;code&gt;"), "<code>");
311        assert_eq!(
312            decode_html_entities("it&#39;s &quot;quoted&quot;"),
313            "it's \"quoted\""
314        );
315    }
316
317    #[test]
318    fn strip_html_removes_tags() {
319        assert_eq!(strip_html("<p>Hello <b>world</b></p>"), "Hello world");
320        assert_eq!(strip_html("plain text"), "plain text");
321    }
322
323    #[test]
324    fn snippet_includes_answer_status_answered() {
325        let q = SoQuestion {
326            title: "test".to_string(),
327            link: "https://so.com/q/1".to_string(),
328            score: 5,
329            answer_count: 2,
330            is_answered: true,
331            tags: vec!["rust".to_string()],
332            creation_date: None,
333            body_markdown: None,
334        };
335        let s = build_snippet(&q);
336        assert!(s.contains("Answered"), "Expected 'Answered' in: {s}");
337        assert!(s.contains("2 answers"), "Expected '2 answers' in: {s}");
338        assert!(s.contains("score 5"), "Expected 'score 5' in: {s}");
339        assert!(s.contains("Tags: rust"), "Expected 'Tags: rust' in: {s}");
340    }
341
342    #[test]
343    fn snippet_includes_answer_status_unanswered() {
344        let q = SoQuestion {
345            title: "test".to_string(),
346            link: "https://so.com/q/2".to_string(),
347            score: -1,
348            answer_count: 0,
349            is_answered: false,
350            tags: vec![],
351            creation_date: None,
352            body_markdown: None,
353        };
354        let s = build_snippet(&q);
355        assert!(s.contains("Unanswered"), "Expected 'Unanswered' in: {s}");
356    }
357
358    #[test]
359    fn snippet_includes_body_when_present() {
360        let q = SoQuestion {
361            title: "test".to_string(),
362            link: "https://so.com/q/3".to_string(),
363            score: 3,
364            answer_count: 1,
365            is_answered: true,
366            tags: vec![],
367            creation_date: None,
368            body_markdown: Some("<p>How do I do this specific thing in Rust?</p>".to_string()),
369        };
370        let s = build_snippet(&q);
371        assert!(
372            s.contains("How do I do this specific thing"),
373            "Expected body text in: {s}"
374        );
375    }
376
377    #[test]
378    fn creation_date_converts_to_iso8601() {
379        let ts: u64 = 1_700_000_000;
380        let published = chrono::DateTime::from_timestamp(ts as i64, 0)
381            .map(|dt| dt.format("%Y-%m-%dT%H:%M:%SZ").to_string());
382        assert!(published.is_some());
383        let s = published.unwrap();
384        assert!(s.starts_with("2023-11-"), "Got: {s}");
385    }
386}