use crate::error::FetchiumError;
use crate::error::FetchiumResult;
use crate::http::HttpClient;
use crate::search::SearchBackend;
use crate::types::{BackendId, ResultItem};
use async_trait::async_trait;
use serde::Deserialize;
use std::sync::atomic::{AtomicU64, Ordering};
use std::time::{SystemTime, UNIX_EPOCH};
use tracing::debug;
const SO_API: &str = "https://api.stackexchange.com/2.3/search";
const SO_COOLDOWN_SECS: u64 = 300;
static SO_COOLDOWN_UNTIL_MS: AtomicU64 = AtomicU64::new(0);
#[derive(Debug, Deserialize)]
struct SoResponse {
items: Vec<SoQuestion>,
}
#[derive(Debug, Deserialize)]
struct SoQuestion {
title: String,
link: String,
score: i32,
answer_count: u32,
is_answered: bool,
#[serde(default)]
tags: Vec<String>,
creation_date: Option<u64>,
body_markdown: Option<String>,
}
pub struct StackOverflowBackend {
client: reqwest::Client,
api_key: Option<String>,
}
impl StackOverflowBackend {
pub fn new(_http: HttpClient) -> Self {
let api_key = std::env::var("STACKEXCHANGE_KEY").ok();
let client = reqwest::Client::builder()
.timeout(std::time::Duration::from_secs(4))
.connect_timeout(std::time::Duration::from_secs(2))
.user_agent("Fetchium/0.1")
.gzip(true)
.pool_max_idle_per_host(4)
.pool_idle_timeout(std::time::Duration::from_secs(90))
.build()
.unwrap_or_default();
Self { client, api_key }
}
pub fn with_key(_http: HttpClient, api_key: impl Into<String>) -> Self {
let client = reqwest::Client::builder()
.timeout(std::time::Duration::from_secs(4))
.connect_timeout(std::time::Duration::from_secs(2))
.user_agent("Fetchium/0.1")
.gzip(true)
.pool_max_idle_per_host(4)
.pool_idle_timeout(std::time::Duration::from_secs(90))
.build()
.unwrap_or_default();
Self {
client,
api_key: Some(api_key.into()),
}
}
}
#[async_trait]
impl SearchBackend for StackOverflowBackend {
fn id(&self) -> BackendId {
BackendId::StackOverflow
}
async fn search(&self, query: &str, max_results: u32) -> FetchiumResult<Vec<ResultItem>> {
let now = now_ms();
let cooldown_until = SO_COOLDOWN_UNTIL_MS.load(Ordering::Relaxed);
if now < cooldown_until {
debug!("StackOverflow in cooldown window, skipping request");
return Ok(vec![]);
}
let page_size = max_results.min(30);
let mut url = format!(
"{SO_API}?intitle={}&site=stackoverflow&order=desc&sort=relevance\
&pagesize={page_size}&filter=withbody",
urlencoding_encode(query)
);
if let Some(ref key) = self.api_key {
url.push_str(&format!("&key={key}"));
}
let resp = match self
.client
.get(&url)
.header("Accept-Encoding", "gzip")
.send()
.await
{
Ok(r) => r,
Err(e) => {
tracing::warn!("StackOverflow request failed: {e}");
return Ok(vec![]);
}
};
if !resp.status().is_success() {
let status = resp.status();
let body = resp.text().await.unwrap_or_default();
if status.as_u16() == 429 || body.contains("throttle_violation") {
let until = now_ms() + SO_COOLDOWN_SECS * 1000;
SO_COOLDOWN_UNTIL_MS.store(until, Ordering::Relaxed);
return Err(FetchiumError::Search(format!(
"StackOverflow rate-limited (HTTP {status}) — cooling down for {}s",
SO_COOLDOWN_SECS
)));
}
if status.is_server_error() {
return Err(FetchiumError::Search(format!(
"StackOverflow upstream server error: HTTP {status}"
)));
}
debug!("StackOverflow non-success HTTP {status}, skipping");
return Ok(vec![]);
}
let body = match resp.text().await {
Ok(b) => b,
Err(e) => {
tracing::warn!("StackOverflow body read failed: {e}");
return Ok(vec![]);
}
};
let parsed: SoResponse = match serde_json::from_str(&body) {
Ok(r) => r,
Err(e) => {
tracing::warn!("StackOverflow JSON parse failed: {e}");
return Ok(vec![]);
}
};
let results = parsed
.items
.into_iter()
.enumerate()
.map(|(i, q)| {
let snippet = build_snippet(&q);
let published = q.creation_date.and_then(|ts| {
chrono::DateTime::from_timestamp(ts as i64, 0)
.map(|dt| dt.format("%Y-%m-%dT%H:%M:%SZ").to_string())
});
ResultItem {
title: decode_html_entities(&q.title),
url: q.link,
snippet,
rank: (i + 1) as u32,
backend: BackendId::StackOverflow,
score: None,
published_date: published,
}
})
.collect::<Vec<_>>();
debug!("StackOverflow: {} results for {:?}", results.len(), query);
Ok(results)
}
}
fn now_ms() -> u64 {
SystemTime::now()
.duration_since(UNIX_EPOCH)
.map(|d| d.as_millis() as u64)
.unwrap_or(0)
}
fn build_snippet(q: &SoQuestion) -> String {
let mut parts = Vec::new();
if let Some(ref body) = q.body_markdown {
let stripped = strip_html(body);
let stripped = stripped.trim();
if !stripped.is_empty() {
let char_count = stripped.chars().count();
let truncated: String = stripped.chars().take(200).collect();
parts.push(if char_count > 200 {
format!("{truncated}...")
} else {
truncated
});
}
}
let status = if q.is_answered {
"Answered"
} else {
"Unanswered"
};
parts.push(format!(
"{status} | {} answers | score {}",
q.answer_count, q.score
));
if !q.tags.is_empty() {
parts.push(format!("Tags: {}", q.tags.join(", ")));
}
parts.join(" — ")
}
fn strip_html(s: &str) -> String {
let mut result = String::with_capacity(s.len());
let mut in_tag = false;
for c in s.chars() {
match c {
'<' => in_tag = true,
'>' => in_tag = false,
_ if !in_tag => result.push(c),
_ => {}
}
}
result.trim().to_string()
}
fn decode_html_entities(s: &str) -> String {
s.replace("&", "&")
.replace("<", "<")
.replace(">", ">")
.replace(""", "\"")
.replace("'", "'")
}
fn urlencoding_encode(s: &str) -> String {
url::form_urlencoded::Serializer::new(String::new())
.append_key_only(s)
.finish()
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn parse_so_response() {
let json = r#"{"items":[{
"title":"How to use Rust?",
"link":"https://stackoverflow.com/q/123",
"score":10,
"answer_count":3,
"is_answered":true,
"tags":["rust","programming"],
"creation_date":1700000000
}]}"#;
let resp: SoResponse = serde_json::from_str(json).unwrap();
assert_eq!(resp.items.len(), 1);
let q = &resp.items[0];
assert_eq!(q.title, "How to use Rust?");
assert!(q.is_answered);
assert_eq!(q.answer_count, 3);
assert_eq!(q.score, 10);
assert_eq!(q.tags, ["rust", "programming"]);
}
#[test]
fn parse_so_response_empty_tags() {
let json = r#"{"items":[{
"title":"Test question",
"link":"https://stackoverflow.com/q/999",
"score":0,
"answer_count":0,
"is_answered":false,
"tags":[],
"creation_date":null
}]}"#;
let resp: SoResponse = serde_json::from_str(json).unwrap();
assert!(resp.items[0].tags.is_empty());
assert!(resp.items[0].creation_date.is_none());
}
#[test]
fn decode_html_entities_works() {
assert_eq!(decode_html_entities("A & B"), "A & B");
assert_eq!(decode_html_entities("<code>"), "<code>");
assert_eq!(
decode_html_entities("it's "quoted""),
"it's \"quoted\""
);
}
#[test]
fn strip_html_removes_tags() {
assert_eq!(strip_html("<p>Hello <b>world</b></p>"), "Hello world");
assert_eq!(strip_html("plain text"), "plain text");
}
#[test]
fn snippet_includes_answer_status_answered() {
let q = SoQuestion {
title: "test".to_string(),
link: "https://so.com/q/1".to_string(),
score: 5,
answer_count: 2,
is_answered: true,
tags: vec!["rust".to_string()],
creation_date: None,
body_markdown: None,
};
let s = build_snippet(&q);
assert!(s.contains("Answered"), "Expected 'Answered' in: {s}");
assert!(s.contains("2 answers"), "Expected '2 answers' in: {s}");
assert!(s.contains("score 5"), "Expected 'score 5' in: {s}");
assert!(s.contains("Tags: rust"), "Expected 'Tags: rust' in: {s}");
}
#[test]
fn snippet_includes_answer_status_unanswered() {
let q = SoQuestion {
title: "test".to_string(),
link: "https://so.com/q/2".to_string(),
score: -1,
answer_count: 0,
is_answered: false,
tags: vec![],
creation_date: None,
body_markdown: None,
};
let s = build_snippet(&q);
assert!(s.contains("Unanswered"), "Expected 'Unanswered' in: {s}");
}
#[test]
fn snippet_includes_body_when_present() {
let q = SoQuestion {
title: "test".to_string(),
link: "https://so.com/q/3".to_string(),
score: 3,
answer_count: 1,
is_answered: true,
tags: vec![],
creation_date: None,
body_markdown: Some("<p>How do I do this specific thing in Rust?</p>".to_string()),
};
let s = build_snippet(&q);
assert!(
s.contains("How do I do this specific thing"),
"Expected body text in: {s}"
);
}
#[test]
fn creation_date_converts_to_iso8601() {
let ts: u64 = 1_700_000_000;
let published = chrono::DateTime::from_timestamp(ts as i64, 0)
.map(|dt| dt.format("%Y-%m-%dT%H:%M:%SZ").to_string());
assert!(published.is_some());
let s = published.unwrap();
assert!(s.starts_with("2023-11-"), "Got: {s}");
}
}