fetchkit/fetchers/
github_code.rs

1//! GitHub source file fetcher
2//!
3//! Handles GitHub blob URLs, returning raw source file content with language
4//! metadata via the GitHub API, optimized for LLM consumption.
5
6use crate::client::FetchOptions;
7use crate::error::FetchError;
8use crate::fetchers::default::{read_full_body, transport_request};
9use crate::fetchers::Fetcher;
10use crate::types::{FetchRequest, FetchResponse};
11use crate::DEFAULT_USER_AGENT;
12use async_trait::async_trait;
13use reqwest::header::{HeaderMap, HeaderValue, ACCEPT, USER_AGENT};
14use serde::Deserialize;
15use std::time::Duration;
16use url::Url;
17
18const API_TIMEOUT: Duration = Duration::from_secs(10);
19
20/// GitHub API host and port (DNS-pinned per request).
21const GITHUB_API_HOST: &str = "api.github.com";
22const GITHUB_API_PORT: u16 = 443;
23
24/// Max file size we'll return inline (1 MB, matching GitHub contents API limit)
25const MAX_INLINE_SIZE: u64 = 1_048_576;
26
27/// GitHub source file fetcher
28///
29/// Matches `https://github.com/{owner}/{repo}/blob/{ref}/{path}` and returns
30/// raw file content with language metadata.
31pub struct GitHubCodeFetcher;
32
33impl GitHubCodeFetcher {
34    pub fn new() -> Self {
35        Self
36    }
37
38    /// Extract owner, repo, ref, and path from a GitHub blob URL
39    fn parse_url(url: &Url) -> Option<ParsedBlobUrl> {
40        if url.host_str() != Some("github.com") {
41            return None;
42        }
43
44        let segments: Vec<&str> = url.path_segments().map(|s| s.collect()).unwrap_or_default();
45
46        // Minimum: /{owner}/{repo}/blob/{ref}/{path} = 5+ segments
47        if segments.len() < 5 {
48            return None;
49        }
50
51        let owner = segments[0];
52        let repo = segments[1];
53        let kind = segments[2];
54        let git_ref = segments[3];
55
56        if owner.is_empty() || repo.is_empty() || git_ref.is_empty() {
57            return None;
58        }
59
60        if kind != "blob" {
61            return None;
62        }
63
64        // Exclude reserved owner paths
65        let reserved = [
66            "settings",
67            "explore",
68            "trending",
69            "collections",
70            "events",
71            "sponsors",
72            "notifications",
73            "marketplace",
74            "pulls",
75            "issues",
76            "codespaces",
77            "features",
78            "enterprise",
79            "organizations",
80            "pricing",
81            "about",
82            "team",
83            "security",
84            "login",
85            "join",
86        ];
87        if reserved.contains(&owner) {
88            return None;
89        }
90
91        // Path is everything after the ref
92        let file_path = segments[4..].join("/");
93        if file_path.is_empty() {
94            return None;
95        }
96
97        Some(ParsedBlobUrl {
98            owner: owner.to_string(),
99            repo: repo.to_string(),
100            git_ref: git_ref.to_string(),
101            path: file_path,
102        })
103    }
104}
105
106impl Default for GitHubCodeFetcher {
107    fn default() -> Self {
108        Self::new()
109    }
110}
111
112struct ParsedBlobUrl {
113    owner: String,
114    repo: String,
115    git_ref: String,
116    path: String,
117}
118
119#[derive(Debug, Deserialize)]
120struct GitHubContents {
121    name: String,
122    path: String,
123    size: u64,
124    #[serde(rename = "type")]
125    content_type: String,
126    content: Option<String>,
127    html_url: Option<String>,
128}
129
130#[async_trait]
131impl Fetcher for GitHubCodeFetcher {
132    fn name(&self) -> &'static str {
133        "github_code"
134    }
135
136    fn matches(&self, url: &Url) -> bool {
137        Self::parse_url(url).is_some()
138    }
139
140    async fn fetch(
141        &self,
142        request: &FetchRequest,
143        options: &FetchOptions,
144    ) -> Result<FetchResponse, FetchError> {
145        let request = request.normalized_for_fetch()?;
146        let url = Url::parse(&request.url).map_err(|_| FetchError::InvalidUrlScheme)?;
147
148        let parsed = Self::parse_url(&url)
149            .ok_or_else(|| FetchError::FetcherError("Not a valid GitHub blob URL".to_string()))?;
150
151        let user_agent = options.user_agent.as_deref().unwrap_or(DEFAULT_USER_AGENT);
152        let ua_header = HeaderValue::from_str(user_agent)
153            .unwrap_or_else(|_| HeaderValue::from_static(DEFAULT_USER_AGENT));
154        let accept_header = HeaderValue::from_static("application/vnd.github+json");
155
156        // Fetch file contents via GitHub API
157        let api_url = format!(
158            "https://api.github.com/repos/{}/{}/contents/{}?ref={}",
159            parsed.owner, parsed.repo, parsed.path, parsed.git_ref
160        );
161        let parsed_api_url = Url::parse(&api_url).map_err(|_| FetchError::InvalidUrlScheme)?;
162        // THREAT[TM-INPUT]: enforce host/port policy on the GitHub API subrequest (PR #131).
163        options.validate_url(&parsed_api_url)?;
164
165        let mut headers = HeaderMap::new();
166        headers.insert(USER_AGENT, ua_header);
167        headers.insert(ACCEPT, accept_header);
168
169        // THREAT[TM-SSRF-010]: single-hop request, redirects not followed.
170        let response = transport_request(
171            parsed_api_url,
172            reqwest::Method::GET,
173            headers,
174            options,
175            API_TIMEOUT,
176            GITHUB_API_HOST,
177            GITHUB_API_PORT,
178        )
179        .await?;
180
181        let status_code = response.status;
182        if !(200..300).contains(&status_code) {
183            let error_msg = if status_code == 404 {
184                format!(
185                    "{}/{}:{} {} not found",
186                    parsed.owner, parsed.repo, parsed.git_ref, parsed.path
187                )
188            } else if status_code == 403 {
189                "GitHub API rate limit exceeded".to_string()
190            } else {
191                format!("GitHub API error: HTTP {}", status_code)
192            };
193            return Ok(FetchResponse {
194                url: request.url.clone(),
195                status_code,
196                error: Some(error_msg),
197                ..Default::default()
198            });
199        }
200
201        let body = read_full_body(response, options).await?;
202        let contents: GitHubContents = serde_json::from_slice(&body)
203            .map_err(|e| FetchError::FetcherError(format!("Failed to parse contents: {}", e)))?;
204
205        // Handle directories (content_type == "dir")
206        if contents.content_type != "file" {
207            return Ok(FetchResponse {
208                url: request.url.clone(),
209                status_code: 200,
210                format: Some("github_file".to_string()),
211                error: Some(format!("Path is a {} (not a file)", contents.content_type)),
212                ..Default::default()
213            });
214        }
215
216        // Handle binary/large files — return metadata only
217        if contents.size > MAX_INLINE_SIZE || contents.content.is_none() {
218            let content = format_metadata_only(&parsed, &contents);
219            return Ok(FetchResponse {
220                url: request.url.clone(),
221                status_code: 200,
222                content_type: Some("text/markdown".to_string()),
223                format: Some("github_file".to_string()),
224                content: Some(content),
225                size: Some(contents.size),
226                ..Default::default()
227            });
228        }
229
230        // Decode base64 content
231        let raw_content = contents.content.as_deref().and_then(decode_base64_content);
232
233        let (file_content, is_binary) = match raw_content {
234            Some(bytes) => match String::from_utf8(bytes) {
235                Ok(text) => (Some(text), false),
236                Err(_) => (None, true),
237            },
238            None => (None, true),
239        };
240
241        if is_binary {
242            let content = format_metadata_only(&parsed, &contents);
243            return Ok(FetchResponse {
244                url: request.url.clone(),
245                status_code: 200,
246                content_type: Some("text/markdown".to_string()),
247                format: Some("github_file".to_string()),
248                content: Some(content),
249                size: Some(contents.size),
250                error: Some("Binary file — metadata only".to_string()),
251                ..Default::default()
252            });
253        }
254
255        let lang = detect_language(&contents.name);
256        let content = format_file_response(&parsed, &contents, file_content.as_deref(), lang);
257
258        Ok(FetchResponse {
259            url: request.url.clone(),
260            status_code: 200,
261            content_type: Some("text/markdown".to_string()),
262            format: Some("github_file".to_string()),
263            content: Some(content),
264            size: Some(contents.size),
265            ..Default::default()
266        })
267    }
268}
269
270/// Decode base64 with whitespace (GitHub API includes newlines in base64)
271fn decode_base64_content(encoded: &str) -> Option<Vec<u8>> {
272    let cleaned: String = encoded.chars().filter(|c| !c.is_whitespace()).collect();
273    base64_decode(&cleaned)
274}
275
276/// Basic base64 decoder (same approach as github_repo.rs)
277fn base64_decode(input: &str) -> Option<Vec<u8>> {
278    const ALPHABET: &[u8] = b"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/";
279
280    fn decode_char(c: u8) -> Option<u8> {
281        if c == b'=' {
282            return Some(0);
283        }
284        ALPHABET.iter().position(|&x| x == c).map(|p| p as u8)
285    }
286
287    let bytes: Vec<u8> = input.bytes().collect();
288    if !bytes.is_empty() && !bytes.len().is_multiple_of(4) {
289        return None;
290    }
291
292    let mut result = Vec::with_capacity(bytes.len() * 3 / 4);
293
294    for chunk in bytes.chunks(4) {
295        if chunk.len() != 4 {
296            return None;
297        }
298        let a = decode_char(chunk[0])?;
299        let b = decode_char(chunk[1])?;
300        let c = decode_char(chunk[2])?;
301        let d = decode_char(chunk[3])?;
302
303        result.push((a << 2) | (b >> 4));
304        if chunk[2] != b'=' {
305            result.push((b << 4) | (c >> 2));
306        }
307        if chunk[3] != b'=' {
308            result.push((c << 6) | d);
309        }
310    }
311
312    Some(result)
313}
314
315/// Simple language detection from file extension
316fn detect_language(filename: &str) -> Option<&'static str> {
317    let ext = filename.rsplit('.').next()?;
318    match ext.to_ascii_lowercase().as_str() {
319        "rs" => Some("rust"),
320        "py" => Some("python"),
321        "js" => Some("javascript"),
322        "ts" => Some("typescript"),
323        "tsx" => Some("tsx"),
324        "jsx" => Some("jsx"),
325        "rb" => Some("ruby"),
326        "go" => Some("go"),
327        "java" => Some("java"),
328        "kt" | "kts" => Some("kotlin"),
329        "swift" => Some("swift"),
330        "c" => Some("c"),
331        "cpp" | "cc" | "cxx" => Some("cpp"),
332        "h" | "hpp" => Some("cpp"),
333        "cs" => Some("csharp"),
334        "php" => Some("php"),
335        "sh" | "bash" => Some("bash"),
336        "zsh" => Some("zsh"),
337        "fish" => Some("fish"),
338        "yml" | "yaml" => Some("yaml"),
339        "json" => Some("json"),
340        "toml" => Some("toml"),
341        "xml" => Some("xml"),
342        "html" | "htm" => Some("html"),
343        "css" => Some("css"),
344        "scss" | "sass" => Some("scss"),
345        "sql" => Some("sql"),
346        "md" | "markdown" => Some("markdown"),
347        "dockerfile" => Some("dockerfile"),
348        "tf" => Some("terraform"),
349        "ex" | "exs" => Some("elixir"),
350        "erl" => Some("erlang"),
351        "hs" => Some("haskell"),
352        "ml" | "mli" => Some("ocaml"),
353        "r" => Some("r"),
354        "scala" => Some("scala"),
355        "lua" => Some("lua"),
356        "zig" => Some("zig"),
357        "nim" => Some("nim"),
358        "v" => Some("v"),
359        "dart" => Some("dart"),
360        "proto" => Some("protobuf"),
361        "graphql" | "gql" => Some("graphql"),
362        _ => None,
363    }
364}
365
366fn format_metadata_only(parsed: &ParsedBlobUrl, contents: &GitHubContents) -> String {
367    let lang = detect_language(&contents.name);
368    let mut out = String::new();
369    out.push_str(&format!("# {}\n\n", contents.path));
370    out.push_str("## File Info\n\n");
371    out.push_str(&format!(
372        "- **Repository:** {}/{}\n",
373        parsed.owner, parsed.repo
374    ));
375    out.push_str(&format!("- **Ref:** {}\n", parsed.git_ref));
376    out.push_str(&format!("- **Size:** {} bytes\n", contents.size));
377    if let Some(lang) = lang {
378        out.push_str(&format!("- **Language:** {}\n", lang));
379    }
380    if let Some(url) = &contents.html_url {
381        out.push_str(&format!("- **URL:** {}\n", url));
382    }
383    out
384}
385
386fn format_file_response(
387    parsed: &ParsedBlobUrl,
388    contents: &GitHubContents,
389    file_content: Option<&str>,
390    lang: Option<&str>,
391) -> String {
392    let mut out = String::new();
393
394    out.push_str(&format!("# {}\n\n", contents.path));
395    out.push_str("## File Info\n\n");
396    out.push_str(&format!(
397        "- **Repository:** {}/{}\n",
398        parsed.owner, parsed.repo
399    ));
400    out.push_str(&format!("- **Ref:** {}\n", parsed.git_ref));
401    out.push_str(&format!("- **Size:** {} bytes\n", contents.size));
402    if let Some(lang) = lang {
403        out.push_str(&format!("- **Language:** {}\n", lang));
404    }
405    if let Some(url) = &contents.html_url {
406        out.push_str(&format!("- **URL:** {}\n", url));
407    }
408
409    if let Some(content) = file_content {
410        let lang_hint = lang.unwrap_or("");
411        out.push_str(&format!(
412            "\n## Content\n\n```{}\n{}\n```\n",
413            lang_hint, content
414        ));
415    }
416
417    out
418}
419
420#[cfg(test)]
421mod tests {
422    use super::*;
423
424    #[test]
425    fn test_parse_blob_url() {
426        let url = Url::parse("https://github.com/owner/repo/blob/main/src/lib.rs").unwrap();
427        let parsed = GitHubCodeFetcher::parse_url(&url).unwrap();
428        assert_eq!(parsed.owner, "owner");
429        assert_eq!(parsed.repo, "repo");
430        assert_eq!(parsed.git_ref, "main");
431        assert_eq!(parsed.path, "src/lib.rs");
432    }
433
434    #[test]
435    fn test_parse_blob_url_nested_path() {
436        let url = Url::parse("https://github.com/owner/repo/blob/v1.0.0/crates/core/src/main.rs")
437            .unwrap();
438        let parsed = GitHubCodeFetcher::parse_url(&url).unwrap();
439        assert_eq!(parsed.git_ref, "v1.0.0");
440        assert_eq!(parsed.path, "crates/core/src/main.rs");
441    }
442
443    #[test]
444    fn test_rejects_non_blob() {
445        let url = Url::parse("https://github.com/owner/repo/tree/main/src").unwrap();
446        assert!(GitHubCodeFetcher::parse_url(&url).is_none());
447    }
448
449    #[test]
450    fn test_rejects_too_few_segments() {
451        let url = Url::parse("https://github.com/owner/repo/blob/main").unwrap();
452        assert!(GitHubCodeFetcher::parse_url(&url).is_none());
453    }
454
455    #[test]
456    fn test_rejects_non_github() {
457        let url = Url::parse("https://gitlab.com/owner/repo/blob/main/file.rs").unwrap();
458        assert!(GitHubCodeFetcher::parse_url(&url).is_none());
459    }
460
461    #[test]
462    fn test_rejects_reserved_owner() {
463        let url = Url::parse("https://github.com/settings/repo/blob/main/file.rs").unwrap();
464        assert!(GitHubCodeFetcher::parse_url(&url).is_none());
465    }
466
467    #[test]
468    fn test_fetcher_matches() {
469        let fetcher = GitHubCodeFetcher::new();
470
471        let url = Url::parse("https://github.com/rust-lang/rust/blob/master/Cargo.toml").unwrap();
472        assert!(fetcher.matches(&url));
473
474        let url = Url::parse("https://github.com/rust-lang/rust").unwrap();
475        assert!(!fetcher.matches(&url));
476
477        let url = Url::parse("https://github.com/rust-lang/rust/issues/1").unwrap();
478        assert!(!fetcher.matches(&url));
479    }
480
481    #[test]
482    fn test_detect_language() {
483        assert_eq!(detect_language("main.rs"), Some("rust"));
484        assert_eq!(detect_language("app.py"), Some("python"));
485        assert_eq!(detect_language("index.tsx"), Some("tsx"));
486        assert_eq!(detect_language("Cargo.toml"), Some("toml"));
487        assert_eq!(detect_language("unknown.xyz"), None);
488        assert_eq!(detect_language("Dockerfile"), Some("dockerfile"));
489    }
490
491    #[test]
492    fn test_format_file_response() {
493        let parsed = ParsedBlobUrl {
494            owner: "owner".to_string(),
495            repo: "repo".to_string(),
496            git_ref: "main".to_string(),
497            path: "src/lib.rs".to_string(),
498        };
499        let contents = GitHubContents {
500            name: "lib.rs".to_string(),
501            path: "src/lib.rs".to_string(),
502            size: 42,
503            content_type: "file".to_string(),
504            content: None,
505            html_url: Some("https://github.com/owner/repo/blob/main/src/lib.rs".to_string()),
506        };
507
508        let output = format_file_response(&parsed, &contents, Some("fn main() {}"), Some("rust"));
509
510        assert!(output.contains("# src/lib.rs"));
511        assert!(output.contains("**Repository:** owner/repo"));
512        assert!(output.contains("**Language:** rust"));
513        assert!(output.contains("```rust\nfn main() {}\n```"));
514    }
515
516    #[test]
517    fn test_base64_decode() {
518        // "Hello" in base64
519        assert_eq!(base64_decode("SGVsbG8="), Some(b"Hello".to_vec()));
520        assert_eq!(base64_decode(""), Some(vec![]));
521        assert_eq!(base64_decode("abc"), None);
522    }
523}
fetchkit/fetchers/github_code.rs

fetchkit/fetchers/
github_code.rs
