1use crate::error::FetchiumError;
10use crate::error::FetchiumResult;
11use crate::http::HttpClient;
12use crate::search::SearchBackend;
13use crate::types::{BackendId, ResultItem};
14use async_trait::async_trait;
15use serde::Deserialize;
16use std::sync::atomic::{AtomicU64, Ordering};
17use std::time::{SystemTime, UNIX_EPOCH};
18use tracing::debug;
19
20const SO_API: &str = "https://api.stackexchange.com/2.3/search";
22const SO_COOLDOWN_SECS: u64 = 300;
24static SO_COOLDOWN_UNTIL_MS: AtomicU64 = AtomicU64::new(0);
25
26#[derive(Debug, Deserialize)]
27struct SoResponse {
28 items: Vec<SoQuestion>,
29}
30
31#[derive(Debug, Deserialize)]
32struct SoQuestion {
33 title: String,
34 link: String,
35 score: i32,
36 answer_count: u32,
37 is_answered: bool,
38 #[serde(default)]
39 tags: Vec<String>,
40 creation_date: Option<u64>,
41 body_markdown: Option<String>,
42}
43
44pub struct StackOverflowBackend {
51 client: reqwest::Client,
53 api_key: Option<String>,
54}
55
56impl StackOverflowBackend {
57 pub fn new(_http: HttpClient) -> Self {
59 let api_key = std::env::var("STACKEXCHANGE_KEY").ok();
60 let client = reqwest::Client::builder()
61 .timeout(std::time::Duration::from_secs(4))
63 .connect_timeout(std::time::Duration::from_secs(2))
64 .user_agent("Fetchium/0.1")
65 .gzip(true)
66 .pool_max_idle_per_host(4)
67 .pool_idle_timeout(std::time::Duration::from_secs(90))
68 .build()
69 .unwrap_or_default();
70 Self { client, api_key }
71 }
72
73 pub fn with_key(_http: HttpClient, api_key: impl Into<String>) -> Self {
75 let client = reqwest::Client::builder()
76 .timeout(std::time::Duration::from_secs(4))
77 .connect_timeout(std::time::Duration::from_secs(2))
78 .user_agent("Fetchium/0.1")
79 .gzip(true)
80 .pool_max_idle_per_host(4)
81 .pool_idle_timeout(std::time::Duration::from_secs(90))
82 .build()
83 .unwrap_or_default();
84 Self {
85 client,
86 api_key: Some(api_key.into()),
87 }
88 }
89}
90
91#[async_trait]
92impl SearchBackend for StackOverflowBackend {
93 fn id(&self) -> BackendId {
94 BackendId::StackOverflow
95 }
96
97 async fn search(&self, query: &str, max_results: u32) -> FetchiumResult<Vec<ResultItem>> {
98 let now = now_ms();
99 let cooldown_until = SO_COOLDOWN_UNTIL_MS.load(Ordering::Relaxed);
100 if now < cooldown_until {
101 debug!("StackOverflow in cooldown window, skipping request");
102 return Ok(vec![]);
103 }
104
105 let page_size = max_results.min(30);
106 let mut url = format!(
107 "{SO_API}?intitle={}&site=stackoverflow&order=desc&sort=relevance\
108 &pagesize={page_size}&filter=withbody",
109 urlencoding_encode(query)
110 );
111 if let Some(ref key) = self.api_key {
112 url.push_str(&format!("&key={key}"));
113 }
114
115 let resp = match self
116 .client
117 .get(&url)
118 .header("Accept-Encoding", "gzip")
119 .send()
120 .await
121 {
122 Ok(r) => r,
123 Err(e) => {
124 tracing::warn!("StackOverflow request failed: {e}");
125 return Ok(vec![]);
126 }
127 };
128
129 if !resp.status().is_success() {
130 let status = resp.status();
131 let body = resp.text().await.unwrap_or_default();
132 if status.as_u16() == 429 || body.contains("throttle_violation") {
133 let until = now_ms() + SO_COOLDOWN_SECS * 1000;
134 SO_COOLDOWN_UNTIL_MS.store(until, Ordering::Relaxed);
135 return Err(FetchiumError::Search(format!(
136 "StackOverflow rate-limited (HTTP {status}) — cooling down for {}s",
137 SO_COOLDOWN_SECS
138 )));
139 }
140 if status.is_server_error() {
141 return Err(FetchiumError::Search(format!(
142 "StackOverflow upstream server error: HTTP {status}"
143 )));
144 }
145 debug!("StackOverflow non-success HTTP {status}, skipping");
146 return Ok(vec![]);
147 }
148
149 let body = match resp.text().await {
150 Ok(b) => b,
151 Err(e) => {
152 tracing::warn!("StackOverflow body read failed: {e}");
153 return Ok(vec![]);
154 }
155 };
156
157 let parsed: SoResponse = match serde_json::from_str(&body) {
158 Ok(r) => r,
159 Err(e) => {
160 tracing::warn!("StackOverflow JSON parse failed: {e}");
161 return Ok(vec![]);
162 }
163 };
164
165 let results = parsed
166 .items
167 .into_iter()
168 .enumerate()
169 .map(|(i, q)| {
170 let snippet = build_snippet(&q);
171 let published = q.creation_date.and_then(|ts| {
172 chrono::DateTime::from_timestamp(ts as i64, 0)
173 .map(|dt| dt.format("%Y-%m-%dT%H:%M:%SZ").to_string())
174 });
175 ResultItem {
176 title: decode_html_entities(&q.title),
177 url: q.link,
178 snippet,
179 rank: (i + 1) as u32,
180 backend: BackendId::StackOverflow,
181 score: None,
182 published_date: published,
183 }
184 })
185 .collect::<Vec<_>>();
186
187 debug!("StackOverflow: {} results for {:?}", results.len(), query);
188 Ok(results)
189 }
190}
191
192fn now_ms() -> u64 {
193 SystemTime::now()
194 .duration_since(UNIX_EPOCH)
195 .map(|d| d.as_millis() as u64)
196 .unwrap_or(0)
197}
198
199fn build_snippet(q: &SoQuestion) -> String {
201 let mut parts = Vec::new();
202
203 if let Some(ref body) = q.body_markdown {
205 let stripped = strip_html(body);
206 let stripped = stripped.trim();
207 if !stripped.is_empty() {
208 let char_count = stripped.chars().count();
209 let truncated: String = stripped.chars().take(200).collect();
210 parts.push(if char_count > 200 {
211 format!("{truncated}...")
212 } else {
213 truncated
214 });
215 }
216 }
217
218 let status = if q.is_answered {
219 "Answered"
220 } else {
221 "Unanswered"
222 };
223 parts.push(format!(
224 "{status} | {} answers | score {}",
225 q.answer_count, q.score
226 ));
227
228 if !q.tags.is_empty() {
229 parts.push(format!("Tags: {}", q.tags.join(", ")));
230 }
231
232 parts.join(" — ")
233}
234
235fn strip_html(s: &str) -> String {
237 let mut result = String::with_capacity(s.len());
238 let mut in_tag = false;
239 for c in s.chars() {
240 match c {
241 '<' => in_tag = true,
242 '>' => in_tag = false,
243 _ if !in_tag => result.push(c),
244 _ => {}
245 }
246 }
247 result.trim().to_string()
248}
249
250fn decode_html_entities(s: &str) -> String {
252 s.replace("&", "&")
253 .replace("<", "<")
254 .replace(">", ">")
255 .replace(""", "\"")
256 .replace("'", "'")
257}
258
259fn urlencoding_encode(s: &str) -> String {
261 url::form_urlencoded::Serializer::new(String::new())
262 .append_key_only(s)
263 .finish()
264}
265
266#[cfg(test)]
267mod tests {
268 use super::*;
269
270 #[test]
271 fn parse_so_response() {
272 let json = r#"{"items":[{
273 "title":"How to use Rust?",
274 "link":"https://stackoverflow.com/q/123",
275 "score":10,
276 "answer_count":3,
277 "is_answered":true,
278 "tags":["rust","programming"],
279 "creation_date":1700000000
280 }]}"#;
281 let resp: SoResponse = serde_json::from_str(json).unwrap();
282 assert_eq!(resp.items.len(), 1);
283 let q = &resp.items[0];
284 assert_eq!(q.title, "How to use Rust?");
285 assert!(q.is_answered);
286 assert_eq!(q.answer_count, 3);
287 assert_eq!(q.score, 10);
288 assert_eq!(q.tags, ["rust", "programming"]);
289 }
290
291 #[test]
292 fn parse_so_response_empty_tags() {
293 let json = r#"{"items":[{
294 "title":"Test question",
295 "link":"https://stackoverflow.com/q/999",
296 "score":0,
297 "answer_count":0,
298 "is_answered":false,
299 "tags":[],
300 "creation_date":null
301 }]}"#;
302 let resp: SoResponse = serde_json::from_str(json).unwrap();
303 assert!(resp.items[0].tags.is_empty());
304 assert!(resp.items[0].creation_date.is_none());
305 }
306
307 #[test]
308 fn decode_html_entities_works() {
309 assert_eq!(decode_html_entities("A & B"), "A & B");
310 assert_eq!(decode_html_entities("<code>"), "<code>");
311 assert_eq!(
312 decode_html_entities("it's "quoted""),
313 "it's \"quoted\""
314 );
315 }
316
317 #[test]
318 fn strip_html_removes_tags() {
319 assert_eq!(strip_html("<p>Hello <b>world</b></p>"), "Hello world");
320 assert_eq!(strip_html("plain text"), "plain text");
321 }
322
323 #[test]
324 fn snippet_includes_answer_status_answered() {
325 let q = SoQuestion {
326 title: "test".to_string(),
327 link: "https://so.com/q/1".to_string(),
328 score: 5,
329 answer_count: 2,
330 is_answered: true,
331 tags: vec!["rust".to_string()],
332 creation_date: None,
333 body_markdown: None,
334 };
335 let s = build_snippet(&q);
336 assert!(s.contains("Answered"), "Expected 'Answered' in: {s}");
337 assert!(s.contains("2 answers"), "Expected '2 answers' in: {s}");
338 assert!(s.contains("score 5"), "Expected 'score 5' in: {s}");
339 assert!(s.contains("Tags: rust"), "Expected 'Tags: rust' in: {s}");
340 }
341
342 #[test]
343 fn snippet_includes_answer_status_unanswered() {
344 let q = SoQuestion {
345 title: "test".to_string(),
346 link: "https://so.com/q/2".to_string(),
347 score: -1,
348 answer_count: 0,
349 is_answered: false,
350 tags: vec![],
351 creation_date: None,
352 body_markdown: None,
353 };
354 let s = build_snippet(&q);
355 assert!(s.contains("Unanswered"), "Expected 'Unanswered' in: {s}");
356 }
357
358 #[test]
359 fn snippet_includes_body_when_present() {
360 let q = SoQuestion {
361 title: "test".to_string(),
362 link: "https://so.com/q/3".to_string(),
363 score: 3,
364 answer_count: 1,
365 is_answered: true,
366 tags: vec![],
367 creation_date: None,
368 body_markdown: Some("<p>How do I do this specific thing in Rust?</p>".to_string()),
369 };
370 let s = build_snippet(&q);
371 assert!(
372 s.contains("How do I do this specific thing"),
373 "Expected body text in: {s}"
374 );
375 }
376
377 #[test]
378 fn creation_date_converts_to_iso8601() {
379 let ts: u64 = 1_700_000_000;
380 let published = chrono::DateTime::from_timestamp(ts as i64, 0)
381 .map(|dt| dt.format("%Y-%m-%dT%H:%M:%SZ").to_string());
382 assert!(published.is_some());
383 let s = published.unwrap();
384 assert!(s.starts_with("2023-11-"), "Got: {s}");
385 }
386}