use anyhow::{Context, Result};
use reqwest::Client;
use serde::Deserialize;
use std::collections::HashMap;
use crate::config::MAX_RULESET_DOWNLOAD_SIZE;
use crate::fingerprint::models::Technology;
use crate::security::validate_url_safe;
#[allow(clippy::too_many_lines)] pub(crate) async fn fetch_from_github_directory(
dir_url: &str,
client: &Client,
) -> Result<HashMap<String, Technology>> {
let (api_url, branch) = if dir_url.contains("raw.githubusercontent.com") {
let parts: Vec<&str> = dir_url.split('/').collect();
if parts.len() >= 7 {
let owner = parts[3];
let repo = parts[4];
let branch = parts.get(5).copied().unwrap_or("main"); let path = parts[6..].join("/");
let constructed_url =
format!("https://api.github.com/repos/{owner}/{repo}/contents/{path}");
validate_url_safe(&constructed_url)
.with_context(|| format!("Unsafe GitHub API URL constructed: {constructed_url}"))?;
(constructed_url, branch)
} else {
return Err(anyhow::anyhow!(
"Invalid GitHub URL format. Expected: \
'https://raw.githubusercontent.com/owner/repo/branch/path', got: '{dir_url}'"
));
}
} else {
validate_url_safe(dir_url).with_context(|| format!("Unsafe GitHub API URL: {dir_url}"))?;
(dir_url.to_string(), "main") };
log::info!("Fetching technology files from GitHub directory: {api_url}");
let api_url_with_ref = format!("{api_url}?ref={branch}");
log::debug!("Fetching directory listing from: {api_url_with_ref} (branch: {branch})");
let response = super::github_api_get(client, &api_url_with_ref).await?;
if !response.status().is_success() {
let status = response.status();
let error_text = response
.text()
.await
.unwrap_or_else(|_| "Unknown error".to_string());
if status.as_u16() == 403 && error_text.contains("rate limit") {
return Err(anyhow::anyhow!(
"GitHub API rate limit exceeded. \
Unauthenticated requests are limited to 60 requests/hour. \
To increase the limit to 5000 requests/hour, set the GITHUB_TOKEN environment variable. \
\
Get a token at: https://github.com/settings/tokens \
(no special permissions needed for public repositories). \
\
Alternatively, use a cached ruleset or wait before retrying. \
\
Error: {status} - {error_text}"
));
}
return Err(anyhow::anyhow!(
"Failed to fetch directory listing: {status} - {error_text}"
));
}
#[derive(Deserialize)]
struct FileEntry {
name: String,
download_url: Option<String>,
#[serde(rename = "type")]
file_type: String,
}
let json_text = response.text().await?;
let entries: Vec<FileEntry> =
serde_json::from_str(&json_text).context("Failed to parse GitHub API response")?;
let json_files: Vec<_> = entries
.into_iter()
.filter(|f| {
f.file_type == "file"
&& std::path::Path::new(&f.name)
.extension()
.is_some_and(|ext| ext.eq_ignore_ascii_case("json"))
})
.collect();
log::info!("Found {} JSON files to fetch", json_files.len());
let mut tasks = Vec::new();
for file in json_files {
if let Some(download_url) = file.download_url {
if let Err(e) = validate_url_safe(&download_url) {
log::warn!("Skipping unsafe download URL from GitHub API: {download_url} - {e}");
continue;
}
if !download_url.starts_with("https://raw.githubusercontent.com/") {
log::warn!(
"Suspicious download URL from GitHub API (not raw.githubusercontent.com): {download_url}"
);
continue;
}
let client = client.clone();
tasks.push(tokio::spawn(async move {
match client.get(&download_url).send().await {
Ok(resp) => {
if resp.status().is_success() {
if let Some(content_length) = resp.content_length() {
if content_length > MAX_RULESET_DOWNLOAD_SIZE as u64 {
log::warn!(
"File {download_url} too large ({content_length} bytes, max: {MAX_RULESET_DOWNLOAD_SIZE}), skipping"
);
return None;
}
}
match resp.text().await {
Ok(text) => {
if text.len() > MAX_RULESET_DOWNLOAD_SIZE {
log::warn!(
"File {} too large after download ({} bytes, max: {}), skipping",
download_url,
text.len(),
MAX_RULESET_DOWNLOAD_SIZE
);
return None;
}
match serde_json::from_str::<HashMap<String, Technology>>(&text)
{
Ok(techs) => Some(techs),
Err(e) => {
log::warn!("Failed to parse {download_url}: {e}");
None
}
}
}
Err(e) => {
log::warn!("Failed to read {download_url}: {e}");
None
}
}
} else {
log::warn!("Failed to fetch {}: {}", download_url, resp.status());
None
}
}
Err(e) => {
log::warn!("Failed to request {download_url}: {e}");
None
}
}
}));
}
}
let mut all_technologies = HashMap::new();
for task in tasks {
if let Ok(Some(techs)) = task.await {
all_technologies.extend(techs);
}
}
log::info!(
"Successfully loaded {} technologies",
all_technologies.len()
);
Ok(all_technologies)
}
#[cfg(test)]
mod tests {
use super::*;
use reqwest::Client;
fn create_test_client() -> Client {
Client::builder()
.timeout(std::time::Duration::from_secs(5))
.build()
.expect("Failed to create test client")
}
#[tokio::test]
async fn test_fetch_from_github_directory_invalid_url() {
let client = create_test_client();
let result = fetch_from_github_directory("not-a-valid-url", &client).await;
assert!(result.is_err());
}
#[tokio::test]
async fn test_fetch_from_github_directory_short_url() {
let client = create_test_client();
let result =
fetch_from_github_directory("https://raw.githubusercontent.com/owner", &client).await;
assert!(result.is_err());
let error_msg = result.unwrap_err().to_string();
assert!(
error_msg.contains("Invalid GitHub URL format"),
"Expected invalid URL format error, got: {}",
error_msg
);
}
#[tokio::test]
async fn test_fetch_from_github_directory_ssrf_protection() {
let client = create_test_client();
let result = fetch_from_github_directory("http://192.168.1.1/path", &client).await;
assert!(result.is_err());
let error_msg = result.unwrap_err().to_string();
assert!(
error_msg.contains("Unsafe") || error_msg.contains("private"),
"Expected SSRF protection error, got: {}",
error_msg
);
}
#[tokio::test]
async fn test_fetch_from_github_directory_nonexistent_repo() {
let client = create_test_client();
let url = "https://raw.githubusercontent.com/nonexistent/repo/main/path";
let result = fetch_from_github_directory(url, &client).await;
assert!(result.is_err());
}
}