upstream_ontologist/
github.rs

1//! Helpers for accessing the GitHub API and raw repository files.
2//!
3//! Several providers need to talk to GitHub, either to query the REST API
4//! (`api.github.com`) or to download individual files. These helpers
5//! centralise that access, including authentication via the `GITHUB_TOKEN`
6//! environment variable, so call sites do not each reimplement it.
7
8use crate::{HTTPJSONError, UpstreamDatum};
9use reqwest::header::{HeaderMap, HeaderValue, ACCEPT, AUTHORIZATION};
10
11/// Base URL of the GitHub REST API.
12const API_BASE: &str = "https://api.github.com";
13
14/// Base URL for raw file access.
15const RAW_BASE: &str = "https://raw.githubusercontent.com";
16
17/// Builds the header map for a GitHub API request, adding an `Authorization`
18/// header when `GITHUB_TOKEN` is set in the environment.
19fn auth_headers(accept: &'static str) -> HeaderMap {
20    let mut headers = HeaderMap::new();
21    headers.insert(ACCEPT, HeaderValue::from_static(accept));
22    if let Ok(token) = std::env::var("GITHUB_TOKEN") {
23        if let Ok(value) = HeaderValue::from_str(&format!("Bearer {}", token)) {
24            headers.insert(AUTHORIZATION, value);
25        }
26    }
27    headers
28}
29
30async fn fetch(url: &str, accept: &'static str) -> Result<reqwest::Response, HTTPJSONError> {
31    let client = crate::http::build_client()
32        .default_headers(auth_headers(accept))
33        .build()
34        .map_err(HTTPJSONError::HTTPError)?;
35
36    let response = client
37        .get(url)
38        .send()
39        .await
40        .map_err(HTTPJSONError::HTTPError)?;
41
42    if !response.status().is_success() {
43        return Err(HTTPJSONError::Error {
44            url: response.url().clone(),
45            status: response.status().as_u16(),
46            response: Box::new(response),
47        });
48    }
49
50    Ok(response)
51}
52
53/// Fetches a GitHub REST API endpoint and parses the response as JSON.
54///
55/// `path` is the API path without the host, e.g. `repos/serde-rs/serde` or
56/// `repos/serde-rs/serde/tags`. The `GITHUB_TOKEN` environment variable, if
57/// set, is sent as a bearer token to raise the rate limit.
58pub async fn load_github_json(path: &str) -> Result<serde_json::Value, HTTPJSONError> {
59    let url = format!("{}/{}", API_BASE, path.trim_start_matches('/'));
60    let response = fetch(&url, "application/vnd.github+json").await?;
61    response.json().await.map_err(HTTPJSONError::HTTPError)
62}
63
64/// Downloads a raw file from a repository via `raw.githubusercontent.com`.
65///
66/// This does not consume the API rate limit, but the caller must know the
67/// branch or tag (`reference`) and exact `path`; a missing file or wrong
68/// reference yields a 404.
69pub async fn download_raw_file(
70    owner: &str,
71    repo: &str,
72    reference: &str,
73    path: &str,
74) -> Result<String, HTTPJSONError> {
75    let url = format!(
76        "{}/{}/{}/{}/{}",
77        RAW_BASE,
78        owner,
79        repo,
80        reference,
81        path.trim_start_matches('/')
82    );
83    let response = fetch(&url, "text/plain").await?;
84    response.text().await.map_err(HTTPJSONError::HTTPError)
85}
86
87/// Downloads a file via the GitHub contents API.
88///
89/// Unlike [`download_raw_file`] this resolves the repository's default branch
90/// automatically when `reference` is `None`, at the cost of consuming the API
91/// rate limit. The base64-encoded content returned by the API is decoded.
92pub async fn download_contents(
93    owner: &str,
94    repo: &str,
95    path: &str,
96    reference: Option<&str>,
97) -> Result<Vec<u8>, crate::ProviderError> {
98    let mut url = format!(
99        "{}/repos/{}/{}/contents/{}",
100        API_BASE,
101        owner,
102        repo,
103        path.trim_start_matches('/')
104    );
105    if let Some(reference) = reference {
106        url.push_str(&format!("?ref={}", reference));
107    }
108
109    let response = fetch(&url, "application/vnd.github+json").await?;
110    let data: serde_json::Value = response.json().await.map_err(HTTPJSONError::HTTPError)?;
111
112    let encoding = data["encoding"].as_str();
113    let content = data["content"].as_str().ok_or_else(|| {
114        crate::ProviderError::ParseError("contents API response missing content".to_string())
115    })?;
116
117    match encoding {
118        Some("base64") => {
119            use base64::Engine;
120            // The API wraps the base64 payload at column 60 with newlines.
121            let stripped: String = content.chars().filter(|c| !c.is_whitespace()).collect();
122            base64::engine::general_purpose::STANDARD
123                .decode(stripped.as_bytes())
124                .map_err(|e| {
125                    crate::ProviderError::ParseError(format!("invalid base64 content: {}", e))
126                })
127        }
128        other => Err(crate::ProviderError::ParseError(format!(
129            "unexpected contents encoding: {:?}",
130            other
131        ))),
132    }
133}
134
135/// Repository metadata returned by the GitHub repos API.
136///
137/// This covers the fields useful as upstream metadata; the API returns many
138/// more that are ignored here.
139#[derive(serde::Deserialize, Debug, Clone)]
140pub struct RepoMetadata {
141    /// Repository description.
142    pub description: Option<String>,
143    /// Homepage URL declared on the repository.
144    pub homepage: Option<String>,
145    /// Canonical repository URL.
146    pub html_url: Option<String>,
147    /// License information, if GitHub detected one.
148    pub license: Option<License>,
149    /// Whether the repository is archived.
150    #[serde(default)]
151    pub archived: bool,
152}
153
154/// License information from the GitHub repos API.
155#[derive(serde::Deserialize, Debug, Clone)]
156pub struct License {
157    /// SPDX identifier, e.g. `Apache-2.0`. May be `NOASSERTION` for licenses
158    /// GitHub could not map to SPDX.
159    pub spdx_id: Option<String>,
160}
161
162/// Fetches repository metadata from the GitHub repos API.
163///
164/// Returns `None` if the repository does not exist.
165pub async fn repo_metadata(
166    owner: &str,
167    repo: &str,
168) -> Result<Option<RepoMetadata>, crate::ProviderError> {
169    let path = format!("repos/{}/{}", owner, repo);
170    match load_github_json(&path).await {
171        Ok(value) => serde_json::from_value(value).map(Some).map_err(|e| {
172            crate::ProviderError::ParseError(format!("Failed to parse repo metadata: {}", e))
173        }),
174        Err(HTTPJSONError::Error { status: 404, .. }) => Ok(None),
175        Err(e) => Err(e.into()),
176    }
177}
178
179impl RepoMetadata {
180    /// Converts the repository metadata into upstream data items.
181    pub fn to_upstream_data(&self) -> Vec<UpstreamDatum> {
182        let mut results = Vec::new();
183        if let Some(description) = self.description.as_deref() {
184            if !description.is_empty() {
185                results.push(UpstreamDatum::Summary(description.to_string()));
186            }
187        }
188        if let Some(homepage) = self.homepage.as_deref() {
189            if !homepage.is_empty() {
190                results.push(UpstreamDatum::Homepage(homepage.to_string()));
191            }
192        }
193        if let Some(html_url) = self.html_url.as_deref() {
194            results.push(UpstreamDatum::Repository(html_url.to_string()));
195        }
196        if let Some(spdx) = self.license.as_ref().and_then(|l| l.spdx_id.as_deref()) {
197            if !spdx.is_empty() && spdx != "NOASSERTION" {
198                results.push(UpstreamDatum::License(spdx.to_string()));
199            }
200        }
201        results
202    }
203}
204
205#[cfg(test)]
206mod tests {
207    use super::*;
208
209    #[test]
210    fn test_parse_repo_metadata() {
211        let data = r#"{
212            "description": "Serialization framework for Rust",
213            "homepage": "https://serde.rs/",
214            "html_url": "https://github.com/serde-rs/serde",
215            "license": {"spdx_id": "Apache-2.0"},
216            "archived": false
217        }"#;
218        let meta: RepoMetadata = serde_json::from_str(data).unwrap();
219        assert_eq!(
220            meta.description.as_deref(),
221            Some("Serialization framework for Rust")
222        );
223        assert_eq!(
224            meta.license.as_ref().unwrap().spdx_id.as_deref(),
225            Some("Apache-2.0")
226        );
227
228        let data = meta.to_upstream_data();
229        assert_eq!(
230            data,
231            vec![
232                UpstreamDatum::Summary("Serialization framework for Rust".to_string()),
233                UpstreamDatum::Homepage("https://serde.rs/".to_string()),
234                UpstreamDatum::Repository("https://github.com/serde-rs/serde".to_string()),
235                UpstreamDatum::License("Apache-2.0".to_string()),
236            ]
237        );
238    }
239
240    #[test]
241    fn test_noassertion_license_dropped() {
242        let data = r#"{"license": {"spdx_id": "NOASSERTION"}}"#;
243        let meta: RepoMetadata = serde_json::from_str(data).unwrap();
244        assert_eq!(meta.to_upstream_data(), vec![]);
245    }
246}
upstream_ontologist/github.rs

upstream_ontologist/
github.rs
