use anyhow::{Context, Result};
use std::collections::HashMap;
use std::time::Duration;
use crate::config::MAX_RULESET_DOWNLOAD_SIZE;
use crate::fingerprint::models::Technology;
use crate::security::validate_url_safe;
use crate::utils::retry::with_network_download_retry;
use super::github::fetch_from_github_directory;
pub(crate) async fn fetch_from_url(url: &str) -> Result<HashMap<String, Technology>> {
validate_url_safe(url).with_context(|| format!("Unsafe ruleset URL rejected: {url}"))?;
let client = crate::initialization::build_download_client(Duration::from_secs(60))?;
if url.contains("raw.githubusercontent.com")
&& !std::path::Path::new(url)
.extension()
.is_some_and(|ext| ext.eq_ignore_ascii_case("json"))
{
log::debug!("Detected GitHub directory URL, fetching via API");
return fetch_from_github_directory(url, &client).await;
}
log::debug!("Fetching single file from: {url}");
with_network_download_retry(
&format!("fetch ruleset from {url}"),
1, || fetch_single_file_with_size_limit(&client, url),
)
.await
}
async fn fetch_single_file_with_size_limit(
client: &reqwest::Client,
url: &str,
) -> Result<HashMap<String, Technology>> {
use crate::fetch::stream::reject_if_content_length_exceeds;
let response = client.get(url).send().await?;
if !response.status().is_success() {
return Err(anyhow::anyhow!(
"Failed to fetch {}: {}",
url,
response.status()
));
}
reject_if_content_length_exceeds(&response, MAX_RULESET_DOWNLOAD_SIZE, "Ruleset file")?;
let bytes = response.bytes().await?;
if bytes.len() > MAX_RULESET_DOWNLOAD_SIZE {
return Err(anyhow::anyhow!(
"Ruleset file too large: {} bytes (max: {} bytes)",
bytes.len(),
MAX_RULESET_DOWNLOAD_SIZE
));
}
let json_text =
String::from_utf8(bytes.to_vec()).context("Ruleset file contains invalid UTF-8")?;
let technologies: HashMap<String, Technology> =
serde_json::from_str(&json_text).context("Failed to parse technologies JSON")?;
Ok(technologies)
}
#[cfg(test)]
mod tests {
use super::*;
use httptest::{matchers::*, responders::*, Expectation, Server};
#[tokio::test]
async fn test_fetch_from_url_blocks_private_ip() {
let result = fetch_from_url("http://127.0.0.1/technologies.json").await;
assert!(result.is_err());
assert!(result
.unwrap_err()
.to_string()
.contains("Unsafe ruleset URL rejected"));
}
#[tokio::test]
async fn test_fetch_from_url_blocks_localhost() {
let result = fetch_from_url("http://localhost/technologies.json").await;
assert!(result.is_err());
assert!(result
.unwrap_err()
.to_string()
.contains("Unsafe ruleset URL rejected"));
}
#[tokio::test]
async fn test_fetch_from_url_blocks_unsafe_scheme() {
let result = fetch_from_url("file:///etc/passwd").await;
assert!(result.is_err());
assert!(result
.unwrap_err()
.to_string()
.contains("Unsafe ruleset URL rejected"));
}
#[tokio::test]
async fn test_fetch_from_url_allows_public_urls() {
use crate::security::validate_url_safe;
assert!(validate_url_safe("https://example.com/technologies.json").is_ok());
assert!(validate_url_safe("http://8.8.8.8/technologies.json").is_ok()); }
#[tokio::test]
async fn test_fetch_single_file_with_size_limit_enforces_max_size() {
let server = Server::run();
let url = server.url("/large.json").to_string();
let large_body = vec![0u8; MAX_RULESET_DOWNLOAD_SIZE + 1];
server.expect(
Expectation::matching(request::method_path("GET", "/large.json")).respond_with(
status_code(200)
.insert_header(
"Content-Length",
(MAX_RULESET_DOWNLOAD_SIZE + 1).to_string(),
)
.body(large_body),
),
);
let client = reqwest::Client::builder()
.timeout(Duration::from_secs(60))
.build()
.unwrap();
let result = fetch_single_file_with_size_limit(&client, &url).await;
assert!(result.is_err());
assert!(result.unwrap_err().to_string().contains("too large"));
}
#[tokio::test]
async fn test_fetch_single_file_content_length_mismatch() {
let server = Server::run();
let url = server.url("/mismatch.json").to_string();
let large_body = vec![0u8; MAX_RULESET_DOWNLOAD_SIZE + 1];
server.expect(
Expectation::matching(request::method_path("GET", "/mismatch.json")).respond_with(
status_code(200)
.body(large_body),
),
);
let client = reqwest::Client::builder()
.timeout(Duration::from_secs(60))
.build()
.unwrap();
let result = fetch_single_file_with_size_limit(&client, &url).await;
assert!(result.is_err());
assert!(result.unwrap_err().to_string().contains("too large"));
}
#[test]
fn test_retry_exponential_backoff_calculation() {
assert_eq!(1 << (1 - 1), 1);
assert_eq!(1 << (2 - 1), 2);
assert_eq!(1 << (3 - 1), 4);
}
#[tokio::test]
async fn test_fetch_single_file_invalid_utf8() {
let server = Server::run();
let url = server.url("/invalid-utf8.json").to_string();
let invalid_utf8 = vec![0xFF, 0xFE, 0xFD];
server.expect(
Expectation::matching(request::method_path("GET", "/invalid-utf8.json")).respond_with(
status_code(200)
.insert_header("Content-Length", "3")
.body(invalid_utf8),
),
);
let client = reqwest::Client::builder()
.timeout(Duration::from_secs(60))
.build()
.unwrap();
let result = fetch_single_file_with_size_limit(&client, &url).await;
assert!(result.is_err());
assert!(result.unwrap_err().to_string().contains("invalid UTF-8"));
}
#[tokio::test]
async fn test_fetch_single_file_invalid_json() {
let server = Server::run();
let url = server.url("/invalid.json").to_string();
let invalid_json = "{invalid json}";
server.expect(
Expectation::matching(request::method_path("GET", "/invalid.json")).respond_with(
status_code(200)
.insert_header("Content-Length", invalid_json.len().to_string()) .body(invalid_json),
),
);
let client = reqwest::Client::builder()
.timeout(Duration::from_secs(60))
.build()
.unwrap();
let result = fetch_single_file_with_size_limit(&client, &url).await;
assert!(result.is_err());
assert!(result
.unwrap_err()
.to_string()
.contains("parse technologies JSON"));
}
}