kowalski_core/tools/internal/
github.rs

1//! **Internal GitHub / HTTP fetch** — README API + raw blob URLs, optional `GITHUB_TOKEN`.
2//!
3//! This is the **default, dependency-light** path. Replace with a **GitHub MCP** (in-repo server,
4//! stdio MCP, or a [Docker MCP Toolkit](https://docs.docker.com/ai/mcp-catalog-and-toolkit/toolkit/)
5//! profile) when you need issues, search, or OAuth — wire-through is via `McpHub` + config, not
6//! by growing this module without bound.
7
8use reqwest::blocking::{Client, Response};
9use std::time::Duration;
10
11const DEFAULT_TIMEOUT_SECS: u64 = 90;
12const MAX_BODY_CHARS: usize = 240_000;
13
14/// How the URL was resolved for diagnostics (metadata table + section headers).
15#[derive(Debug, Clone, PartialEq, Eq)]
16pub enum GithubFetchKind {
17    ReadmeApi,
18    RawUserContent,
19    PlainHttp,
20}
21
22#[derive(Debug, Clone)]
23pub struct FetchedUrlBody {
24    pub text: String,
25    pub kind: GithubFetchKind,
26    pub resolved_url: String,
27}
28
29fn http_client() -> Result<Client, String> {
30    Client::builder()
31        .timeout(Duration::from_secs(DEFAULT_TIMEOUT_SECS))
32        .user_agent(concat!("Kowalski/", env!("CARGO_PKG_VERSION")))
33        .build()
34        .map_err(|e| e.to_string())
35}
36
37fn github_token() -> Option<String> {
38    std::env::var("GITHUB_TOKEN")
39        .ok()
40        .filter(|s| !s.trim().is_empty())
41}
42
43fn apply_github_auth(req: reqwest::blocking::RequestBuilder) -> reqwest::blocking::RequestBuilder {
44    if let Some(token) = github_token() {
45        req.header("Authorization", format!("Bearer {}", token.trim()))
46    } else {
47        req
48    }
49}
50
51/// Strip known `github.com` host prefixes; returns path after `owner/repo`.
52fn github_tail(url: &str) -> Option<String> {
53    let u = url.trim().trim_end_matches('/');
54    for prefix in [
55        "https://github.com/",
56        "http://github.com/",
57        "https://www.github.com/",
58        "http://www.github.com/",
59    ] {
60        if let Some(t) = u.strip_prefix(prefix) {
61            return Some(t.to_string());
62        }
63    }
64    None
65}
66
67/// `owner/repo` or `owner/repo/...` → README API URL.
68fn readme_api_url(owner: &str, repo: &str) -> String {
69    format!("https://api.github.com/repos/{owner}/{repo}/readme")
70}
71
72/// Convert `https://github.com/o/r/blob/ref/path/to/file` → raw.githubusercontent.com URL.
73fn blob_url_to_raw(owner: &str, repo: &str, git_ref: &str, path: &str) -> String {
74    format!("https://raw.githubusercontent.com/{owner}/{repo}/{git_ref}/{path}")
75}
76
77/// Parse GitHub browser URLs into a fetch strategy.
78pub fn resolve_github_fetch(url: &str) -> Option<ResolvedGithub> {
79    let tail = github_tail(url)?;
80    let segments: Vec<&str> = tail.split('/').filter(|s| !s.is_empty()).collect();
81    if segments.len() < 2 {
82        return None;
83    }
84    let owner = segments[0];
85    let repo = segments[1];
86
87    if segments.len() == 2 {
88        return Some(ResolvedGithub::Readme {
89            owner: owner.to_string(),
90            repo: repo.to_string(),
91        });
92    }
93
94    match segments.get(2).copied() {
95        Some("blob") if segments.len() >= 5 => {
96            let git_ref = segments[3];
97            let path = segments[4..].join("/");
98            Some(ResolvedGithub::RawFile {
99                owner: owner.to_string(),
100                repo: repo.to_string(),
101                git_ref: git_ref.to_string(),
102                path,
103            })
104        }
105        Some("raw") if segments.len() >= 5 => {
106            let git_ref = segments[3];
107            let path = segments[4..].join("/");
108            Some(ResolvedGithub::RawFile {
109                owner: owner.to_string(),
110                repo: repo.to_string(),
111                git_ref: git_ref.to_string(),
112                path,
113            })
114        }
115        _ => None,
116    }
117}
118
119#[derive(Debug, Clone)]
120pub enum ResolvedGithub {
121    Readme { owner: String, repo: String },
122    RawFile {
123        owner: String,
124        repo: String,
125        git_ref: String,
126        path: String,
127    },
128}
129
130impl ResolvedGithub {
131    fn fetch(&self, client: &Client) -> Result<Response, String> {
132        match self {
133            ResolvedGithub::Readme { owner, repo } => {
134                let u = readme_api_url(owner, repo);
135                let req = apply_github_auth(
136                    client
137                        .get(&u)
138                        .header("Accept", "application/vnd.github.raw+json"),
139                );
140                req.send().map_err(|e| e.to_string())
141            }
142            ResolvedGithub::RawFile {
143                owner,
144                repo,
145                git_ref,
146                path,
147            } => {
148                let u = blob_url_to_raw(owner, repo, git_ref, path);
149                let req = apply_github_auth(client.get(&u));
150                req.send().map_err(|e| e.to_string())
151            }
152        }
153    }
154
155    fn resolved_url_display(&self) -> String {
156        match self {
157            ResolvedGithub::Readme { owner, repo } => readme_api_url(owner, repo),
158            ResolvedGithub::RawFile {
159                owner,
160                repo,
161                git_ref,
162                path,
163            } => blob_url_to_raw(owner, repo, git_ref, path),
164        }
165    }
166
167    fn fetch_kind(&self) -> GithubFetchKind {
168        match self {
169            ResolvedGithub::Readme { .. } => GithubFetchKind::ReadmeApi,
170            ResolvedGithub::RawFile { .. } => GithubFetchKind::RawUserContent,
171        }
172    }
173}
174
175fn read_response_text(resp: Response) -> Result<String, String> {
176    if !resp.status().is_success() {
177        return Err(format!(
178            "HTTP {} {}",
179            resp.status().as_u16(),
180            resp.status().canonical_reason().unwrap_or("")
181        ));
182    }
183    let text = resp
184        .text()
185        .map_err(|e| e.to_string())?
186        .chars()
187        .take(MAX_BODY_CHARS)
188        .collect::<String>();
189    Ok(text)
190}
191
192/// Fetch URL body with GitHub-specific resolution when applicable.
193/// Falls back to a plain HTTP GET on the original URL if GitHub-specific fetching fails.
194pub fn fetch_url_for_ingest(original_url: &str) -> Result<FetchedUrlBody, String> {
195    let client = http_client()?;
196
197    if let Some(resolved) = resolve_github_fetch(original_url) {
198        let kind = resolved.fetch_kind();
199        let resolved_url = resolved.resolved_url_display();
200        if let Ok(resp) = resolved.fetch(&client)
201            && let Ok(text) = read_response_text(resp)
202        {
203            return Ok(FetchedUrlBody {
204                text,
205                kind,
206                resolved_url,
207            });
208        }
209        // GitHub resolution failed; fall through to plain GET of the browser URL.
210    }
211
212    let resp = client.get(original_url).send().map_err(|e| e.to_string())?;
213    let text = read_response_text(resp)?;
214    Ok(FetchedUrlBody {
215        text,
216        kind: GithubFetchKind::PlainHttp,
217        resolved_url: original_url.to_string(),
218    })
219}
220
221#[cfg(test)]
222mod tests {
223    use super::*;
224
225    #[test]
226    fn readme_repo_only() {
227        let r = resolve_github_fetch("https://github.com/octocat/Hello-World").unwrap();
228        match r {
229            ResolvedGithub::Readme { owner, repo } => {
230                assert_eq!(owner, "octocat");
231                assert_eq!(repo, "Hello-World");
232            }
233            _ => panic!("expected Readme"),
234        }
235    }
236
237    #[test]
238    fn readme_trailing_slash() {
239        assert!(resolve_github_fetch("https://github.com/foo/bar/").is_some());
240    }
241
242    #[test]
243    fn blob_to_raw_resolution() {
244        let r = resolve_github_fetch(
245            "https://github.com/rust-lang/rust/blob/master/README.md",
246        )
247        .unwrap();
248        match r {
249            ResolvedGithub::RawFile {
250                owner,
251                repo,
252                git_ref,
253                path,
254            } => {
255                assert_eq!(owner, "rust-lang");
256                assert_eq!(repo, "rust");
257                assert_eq!(git_ref, "master");
258                assert_eq!(path, "README.md");
259            }
260            _ => panic!("expected RawFile"),
261        }
262    }
263
264    #[test]
265    fn nested_blob_path() {
266        let r = resolve_github_fetch(
267            "https://github.com/o/r/blob/main/docs/guide.md",
268        )
269        .unwrap();
270        match r {
271            ResolvedGithub::RawFile { path, .. } => assert_eq!(path, "docs/guide.md"),
272            _ => panic!("expected RawFile"),
273        }
274    }
275
276    #[test]
277    fn non_github_returns_none() {
278        assert!(resolve_github_fetch("https://example.com/page").is_none());
279    }
280
281    #[test]
282    fn raw_display_url() {
283        let r = resolve_github_fetch(
284            "https://github.com/a/b/blob/v1.0/Cargo.toml",
285        )
286        .unwrap();
287        assert_eq!(
288            r.resolved_url_display(),
289            "https://raw.githubusercontent.com/a/b/v1.0/Cargo.toml"
290        );
291    }
292}
kowalski_core/tools/internal/github.rs

kowalski_core/tools/internal/
github.rs
