Skip to main content

fetchium_core/search/
github.rs

1//! GitHub search backend — REST API v3 repository search.
2//!
3//! Searches GitHub repositories sorted by star count. Unauthenticated requests
4//! are limited to 10 req/min; set `GITHUB_TOKEN` for 30 req/min.
5//!
6//! GitHub's API requires a `User-Agent` header and recommends an `Accept` header.
7
8use crate::error::FetchiumResult;
9use crate::http::HttpClient;
10use crate::search::SearchBackend;
11use crate::types::{BackendId, ResultItem};
12use async_trait::async_trait;
13use serde::Deserialize;
14use tracing::debug;
15
16/// GitHub REST API search repositories endpoint.
17const GITHUB_API: &str = "https://api.github.com/search/repositories";
18
19#[derive(Debug, Deserialize)]
20struct GhSearchResponse {
21    items: Vec<GhRepo>,
22}
23
24#[derive(Debug, Deserialize)]
25struct GhRepo {
26    full_name: String,
27    html_url: String,
28    description: Option<String>,
29    stargazers_count: u32,
30    language: Option<String>,
31    updated_at: Option<String>,
32    topics: Option<Vec<String>>,
33}
34
35/// GitHub repository search backend.
36///
37/// Uses the GitHub REST API v3. Set `GITHUB_TOKEN` for higher rate limits
38/// (30 req/min authenticated vs 10 req/min unauthenticated).
39/// The client is built once and reused for connection pooling.
40pub struct GithubBackend {
41    /// Pre-built client with GitHub-required headers and connection pooling.
42    client: reqwest::Client,
43    token: Option<String>,
44}
45
46impl GithubBackend {
47    /// Create a new GitHub backend. Reads `GITHUB_TOKEN` from the environment.
48    pub fn new(_http: HttpClient) -> Self {
49        let token = std::env::var("GITHUB_TOKEN").ok();
50        let client = reqwest::Client::builder()
51            .timeout(std::time::Duration::from_secs(15))
52            .user_agent("Fetchium/0.1")
53            .pool_max_idle_per_host(4)
54            .pool_idle_timeout(std::time::Duration::from_secs(90))
55            .build()
56            .unwrap_or_default();
57        Self { client, token }
58    }
59
60    /// Create a GitHub backend with an explicit token (for testing or config injection).
61    pub fn with_token(_http: HttpClient, token: impl Into<String>) -> Self {
62        let client = reqwest::Client::builder()
63            .timeout(std::time::Duration::from_secs(15))
64            .user_agent("Fetchium/0.1")
65            .pool_max_idle_per_host(4)
66            .pool_idle_timeout(std::time::Duration::from_secs(90))
67            .build()
68            .unwrap_or_default();
69        Self {
70            client,
71            token: Some(token.into()),
72        }
73    }
74}
75
76#[async_trait]
77impl SearchBackend for GithubBackend {
78    fn id(&self) -> BackendId {
79        BackendId::Github
80    }
81
82    async fn search(&self, query: &str, max_results: u32) -> FetchiumResult<Vec<ResultItem>> {
83        let per_page = max_results.min(30);
84        let url = format!(
85            "{GITHUB_API}?q={}&sort=stars&order=desc&per_page={per_page}",
86            urlencoding_encode(query)
87        );
88
89        let mut req = self
90            .client
91            .get(&url)
92            .header("Accept", "application/vnd.github+json")
93            .header("X-GitHub-Api-Version", "2022-11-28");
94
95        if let Some(ref token) = self.token {
96            req = req.header("Authorization", format!("Bearer {token}"));
97        }
98
99        let resp = match req.send().await {
100            Ok(r) => r,
101            Err(e) => {
102                tracing::warn!("GitHub request failed: {e}");
103                return Ok(vec![]);
104            }
105        };
106
107        // Handle rate limiting gracefully
108        let status = resp.status();
109        if status == reqwest::StatusCode::FORBIDDEN
110            || status == reqwest::StatusCode::TOO_MANY_REQUESTS
111        {
112            tracing::warn!("GitHub rate limited (HTTP {status})");
113            return Ok(vec![]);
114        }
115
116        if !status.is_success() {
117            tracing::warn!("GitHub HTTP {status}");
118            return Ok(vec![]);
119        }
120
121        let body = match resp.text().await {
122            Ok(b) => b,
123            Err(e) => {
124                tracing::warn!("GitHub body read failed: {e}");
125                return Ok(vec![]);
126            }
127        };
128
129        let parsed: GhSearchResponse = match serde_json::from_str(&body) {
130            Ok(r) => r,
131            Err(e) => {
132                tracing::warn!("GitHub JSON parse failed: {e}");
133                return Ok(vec![]);
134            }
135        };
136
137        let results = parsed
138            .items
139            .into_iter()
140            .enumerate()
141            .map(|(i, repo)| {
142                let snippet = build_snippet(&repo);
143                ResultItem {
144                    title: repo.full_name,
145                    url: repo.html_url,
146                    snippet,
147                    rank: (i + 1) as u32,
148                    backend: BackendId::Github,
149                    score: None,
150                    published_date: repo.updated_at,
151                }
152            })
153            .collect::<Vec<_>>();
154
155        debug!("GitHub: {} results for {:?}", results.len(), query);
156        Ok(results)
157    }
158}
159
160/// Build a human-readable snippet from repository metadata.
161fn build_snippet(repo: &GhRepo) -> String {
162    let mut parts = Vec::new();
163
164    if let Some(ref desc) = repo.description {
165        if !desc.is_empty() {
166            parts.push(desc.clone());
167        }
168    }
169
170    let star_info = format!("{} stars", repo.stargazers_count);
171    let lang_info = if let Some(ref lang) = repo.language {
172        format!("{star_info} | {lang}")
173    } else {
174        star_info
175    };
176    parts.push(lang_info);
177
178    if let Some(ref topics) = repo.topics {
179        if !topics.is_empty() {
180            parts.push(format!("Topics: {}", topics.join(", ")));
181        }
182    }
183
184    parts.join(" — ")
185}
186
187/// Percent-encode a query string for use in URLs.
188fn urlencoding_encode(s: &str) -> String {
189    url::form_urlencoded::Serializer::new(String::new())
190        .append_key_only(s)
191        .finish()
192}
193
194#[cfg(test)]
195mod tests {
196    use super::*;
197
198    #[test]
199    fn parse_github_response() {
200        let json = r#"{"items":[{
201            "full_name":"rust-lang/rust",
202            "html_url":"https://github.com/rust-lang/rust",
203            "description":"Rust language",
204            "stargazers_count":100000,
205            "language":"Rust",
206            "updated_at":"2024-01-01",
207            "topics":["rust","systems"]
208        }]}"#;
209        let resp: GhSearchResponse = serde_json::from_str(json).unwrap();
210        assert_eq!(resp.items.len(), 1);
211        assert_eq!(resp.items[0].full_name, "rust-lang/rust");
212        assert_eq!(resp.items[0].stargazers_count, 100_000);
213        assert_eq!(resp.items[0].language.as_deref(), Some("Rust"));
214    }
215
216    #[test]
217    fn parse_github_response_no_description() {
218        let json = r#"{"items":[{
219            "full_name":"foo/bar",
220            "html_url":"https://github.com/foo/bar",
221            "description":null,
222            "stargazers_count":10,
223            "language":null,
224            "updated_at":null,
225            "topics":[]
226        }]}"#;
227        let resp: GhSearchResponse = serde_json::from_str(json).unwrap();
228        assert!(resp.items[0].description.is_none());
229        assert!(resp.items[0].language.is_none());
230    }
231
232    #[test]
233    fn build_snippet_includes_stars_and_lang() {
234        let repo = GhRepo {
235            full_name: "foo/bar".to_string(),
236            html_url: "https://github.com/foo/bar".to_string(),
237            description: Some("A great project".to_string()),
238            stargazers_count: 500,
239            language: Some("Rust".to_string()),
240            updated_at: None,
241            topics: Some(vec!["rust".to_string(), "cli".to_string()]),
242        };
243        let s = build_snippet(&repo);
244        assert!(s.contains("500 stars"), "Expected stars in: {s}");
245        assert!(s.contains("Rust"), "Expected language in: {s}");
246        assert!(s.contains("A great project"), "Expected desc in: {s}");
247        assert!(s.contains("rust, cli"), "Expected topics in: {s}");
248    }
249
250    #[test]
251    fn build_snippet_no_description_no_topics() {
252        let repo = GhRepo {
253            full_name: "foo/bar".to_string(),
254            html_url: "https://github.com/foo/bar".to_string(),
255            description: None,
256            stargazers_count: 42,
257            language: None,
258            updated_at: None,
259            topics: None,
260        };
261        let s = build_snippet(&repo);
262        assert!(s.contains("42 stars"), "Expected stars in: {s}");
263        // Without language or topics, snippet is just the star count
264        assert!(!s.contains("Topics:"));
265    }
266
267    #[test]
268    fn result_item_mapping() {
269        let repo = GhRepo {
270            full_name: "torvalds/linux".to_string(),
271            html_url: "https://github.com/torvalds/linux".to_string(),
272            description: Some("Linux kernel".to_string()),
273            stargazers_count: 200_000,
274            language: Some("C".to_string()),
275            updated_at: Some("2024-02-01".to_string()),
276            topics: Some(vec!["kernel".to_string()]),
277        };
278        let snippet = build_snippet(&repo);
279        let item = ResultItem {
280            title: repo.full_name.clone(),
281            url: repo.html_url.clone(),
282            snippet,
283            rank: 1,
284            backend: BackendId::Github,
285            score: None,
286            published_date: repo.updated_at.clone(),
287        };
288        assert_eq!(item.title, "torvalds/linux");
289        assert_eq!(item.published_date.as_deref(), Some("2024-02-01"));
290        assert!(item.snippet.contains("200000 stars"));
291    }
292}