use anyhow::Error;
use log::debug;
use crate::config::MAX_REDIRECT_HOPS;
use crate::error_handling::update_error_stats;
use crate::fetch::request::RequestHeaders;
use crate::fetch::UrlProcessOutcome;
use crate::fetch::{resolve_redirect_chain, ProcessingContext};
#[allow(clippy::too_many_lines)] pub async fn handle_http_request(
ctx: &ProcessingContext,
url: &str,
start_time: std::time::Instant,
) -> Result<UrlProcessOutcome, Error> {
debug!("Resolving redirects for {url}");
let (final_url_string, redirect_chain, reused_response) = resolve_redirect_chain(
url,
MAX_REDIRECT_HOPS,
&ctx.network.redirect_client,
ctx.runtime.allow_localhost_for_tests,
)
.await?;
if redirect_chain.len() > 1 {
ctx.runtime
.error_stats
.increment_info(crate::error_handling::InfoType::HttpRedirect);
let original_scheme = url::Url::parse(url)
.ok()
.map(|u| u.scheme().to_string())
.unwrap_or_default();
let final_scheme = url::Url::parse(&final_url_string)
.ok()
.map(|u| u.scheme().to_string())
.unwrap_or_default();
if original_scheme == "http" && final_scheme == "https" {
ctx.runtime
.error_stats
.increment_info(crate::error_handling::InfoType::HttpsRedirect);
}
if redirect_chain.len() > 2 {
ctx.runtime
.error_stats
.increment_info(crate::error_handling::InfoType::MultipleRedirects);
}
}
let res = if let Some(response) = reused_response {
debug!("Reusing response from redirect resolution for {final_url_string}");
Ok(response)
} else {
debug!("Sending request to final URL {final_url_string}");
RequestHeaders::apply_to_request_builder(ctx.network.client.get(&final_url_string))
.send()
.await
};
match res {
Ok(response) => {
let header_count = response.headers().len();
let response_headers: Vec<(String, String)> = response
.headers()
.iter()
.take(crate::config::MAX_HEADER_COUNT)
.map(|(name, value)| (name.to_string(), value.to_str().unwrap_or("").to_string()))
.collect();
if header_count > crate::config::MAX_HEADER_COUNT {
log::warn!(
"Response from {} has {} headers (limit: {}), ignoring excess headers (potential header bomb attack)",
url,
header_count,
crate::config::MAX_HEADER_COUNT
);
}
let status_code = response.status().as_u16();
let failure_match = status_code == 429 || (500..600).contains(&status_code);
if failure_match {
match response.error_for_status() {
Err(e) => {
update_error_stats(&ctx.runtime.error_stats, &e);
log::error!(
"HTTP request error for {}: {} (status: {:?})",
url,
e,
e.status()
);
let failure_context = crate::storage::failure::FailureContext {
final_url: Some(final_url_string.clone()),
redirect_chain: redirect_chain
.iter()
.map(|(url, _)| url.clone())
.collect(),
response_headers: response_headers.clone(),
};
let error = Error::from(e);
Err(crate::storage::failure::attach_failure_context(
error.context(format!("HTTP request failed for {url}")),
failure_context,
))
}
Ok(_) => {
let failure_context = crate::storage::failure::FailureContext {
final_url: Some(final_url_string.clone()),
redirect_chain: redirect_chain
.iter()
.map(|(url, _)| url.clone())
.collect(),
response_headers: response_headers.clone(),
};
Err(crate::storage::failure::attach_failure_context(
anyhow::anyhow!(
"HTTP request for {url} returned status {status_code} (treated as failure)"
),
failure_context,
))
}
}
} else {
if status_code == 403 {
ctx.runtime
.error_stats
.increment_info(crate::error_handling::InfoType::BotDetection403);
}
let elapsed = start_time.elapsed().as_secs_f64();
super::response::handle_response(
response,
url,
&final_url_string,
ctx,
elapsed,
Some(redirect_chain),
start_time,
)
.await
}
}
Err(e) => {
update_error_stats(&ctx.runtime.error_stats, &e);
log::error!("HTTP request error for {}: {} (status: {:?}, is_timeout: {}, is_connect: {}, is_request: {})",
url, e, e.status(), e.is_timeout(), e.is_connect(), e.is_request());
let failure_context = crate::storage::failure::FailureContext {
final_url: Some(final_url_string),
redirect_chain: redirect_chain.iter().map(|(url, _)| url.clone()).collect(),
response_headers: Vec::new(), };
let error = Error::from(e);
Err(crate::storage::failure::attach_failure_context(
error.context(format!("HTTP request failed for {url}")),
failure_context,
))
}
}
}
#[cfg(test)]
mod tests {
use super::*;
use crate::error_handling::ProcessingStats;
use crate::fetch::{NetworkContext, ProcessingContext, RuntimeContext};
use crate::initialization::test_resolver;
use crate::utils::TimingStats;
use httptest::{matchers::*, responders::*, Expectation, Server};
use std::sync::Arc;
async fn create_test_context(server: &Server) -> ProcessingContext {
server.expect(
Expectation::matching(request::method_path("GET", "/favicon.ico"))
.times(0..)
.respond_with(status_code(404)),
);
server.expect(
Expectation::matching(request::method_path("GET", "/.well-known/security.txt"))
.times(0..)
.respond_with(status_code(404)),
);
server.expect(
Expectation::matching(request::method_path("GET", "/security.txt"))
.times(0..)
.respond_with(status_code(404)),
);
server.expect(
Expectation::matching(request::method_path("GET", "/robots.txt"))
.times(0..)
.respond_with(status_code(404)),
);
let client = Arc::new(
reqwest::Client::builder()
.timeout(std::time::Duration::from_secs(5))
.build()
.expect("Failed to create HTTP client"),
);
let redirect_client = Arc::new(
reqwest::Client::builder()
.redirect(reqwest::redirect::Policy::none())
.timeout(std::time::Duration::from_secs(5))
.build()
.expect("Failed to create redirect client"),
);
let resolver = test_resolver();
let error_stats = Arc::new(ProcessingStats::new());
let timing_stats = Arc::new(TimingStats::new());
let pool = Arc::new(
sqlx::SqlitePool::connect("sqlite::memory:")
.await
.expect("Failed to create test pool"),
);
ProcessingContext::new(
NetworkContext::new(client, redirect_client, resolver),
pool,
RuntimeContext::new(
error_stats,
timing_stats,
None,
false,
std::path::PathBuf::from("/tmp/domain_status_whois_test"),
false,
Arc::new(crate::runtime_metrics::RuntimeMetrics::default()),
true,
),
Arc::new(crate::fingerprint::FingerprintRuleset::empty_for_tests()),
)
}
fn assert_db_insert_failed(result: Result<crate::fetch::UrlProcessOutcome, anyhow::Error>) {
let error_msg = result
.expect_err("expected SQLite insert failure without migrations")
.to_string();
assert!(
error_msg.contains("Database")
|| error_msg.contains("migration")
|| error_msg.contains("no such table")
|| error_msg.contains("url_status"),
"Expected database/schema error, got: {error_msg}"
);
}
#[tokio::test]
async fn test_handle_http_request_success() {
let server = Server::run();
let url = server.url("/success").to_string();
server.expect(
Expectation::matching(request::method_path("GET", "/success"))
.respond_with(status_code(200).body("<html><title>Success</title></html>")),
);
let ctx = create_test_context(&server).await;
let start_time = std::time::Instant::now();
assert_db_insert_failed(handle_http_request(&ctx, &url, start_time).await);
}
#[tokio::test]
async fn test_handle_http_request_403_bot_detection() {
let server = Server::run();
let url = server.url("/forbidden").to_string();
server.expect(
Expectation::matching(request::method_path("GET", "/forbidden"))
.respond_with(status_code(403).body("Forbidden")),
);
let ctx = create_test_context(&server).await;
let start_time = std::time::Instant::now();
assert_db_insert_failed(handle_http_request(&ctx, &url, start_time).await);
assert_eq!(
ctx.runtime
.error_stats
.get_info_count(crate::error_handling::InfoType::BotDetection403),
1
);
}
#[tokio::test]
async fn test_handle_http_request_404_not_found() {
let server = Server::run();
let url = server.url("/notfound").to_string();
server.expect(
Expectation::matching(request::method_path("GET", "/notfound"))
.respond_with(status_code(404).body("Not Found")),
);
let ctx = create_test_context(&server).await;
let start_time = std::time::Instant::now();
assert_db_insert_failed(handle_http_request(&ctx, &url, start_time).await);
assert_eq!(
ctx.runtime
.error_stats
.get_info_count(crate::error_handling::InfoType::BotDetection403),
0
);
}
#[tokio::test]
async fn test_handle_http_request_500_server_error() {
let server = Server::run();
let url = server.url("/error").to_string();
server.expect(
Expectation::matching(request::method_path("GET", "/error"))
.respond_with(status_code(500).body("Internal Server Error")),
);
let ctx = create_test_context(&server).await;
let start_time = std::time::Instant::now();
let result = handle_http_request(&ctx, &url, start_time).await;
assert!(result.is_err());
let error = result.unwrap_err();
use crate::storage::failure::extract_http_status;
let status = extract_http_status(&error);
use crate::storage::failure::extract_failure_context;
let context = extract_failure_context(&error);
let has_context = context.final_url.is_some();
let error_msg = error.to_string();
let chain_msgs: Vec<String> = error.chain().map(|e| e.to_string()).collect();
let full_chain = chain_msgs.join(" | ");
let has_500 = status.map(|s| s == 500).unwrap_or(false)
|| has_context
|| error_msg.contains("500")
|| error_msg.contains("Internal Server Error")
|| full_chain.contains("500")
|| full_chain.contains("Internal Server Error");
assert!(
has_500,
"Expected 500 error context or structured context (status={:?}, has_context={}), got: {} | Chain: {}",
status,
has_context,
error_msg,
full_chain
);
}
#[tokio::test]
async fn test_handle_http_request_connection_error() {
let url = "http://127.0.0.1:1/".to_string();
let client = Arc::new(
reqwest::Client::builder()
.timeout(std::time::Duration::from_millis(100))
.build()
.expect("Failed to create HTTP client"),
);
let redirect_client = Arc::new(
reqwest::Client::builder()
.redirect(reqwest::redirect::Policy::none())
.timeout(std::time::Duration::from_millis(100))
.build()
.expect("Failed to create redirect client"),
);
let resolver = test_resolver();
let error_stats = Arc::new(ProcessingStats::new());
let timing_stats = Arc::new(TimingStats::new());
let pool = Arc::new(
sqlx::SqlitePool::connect("sqlite::memory:")
.await
.expect("Failed to create test pool"),
);
let ctx = ProcessingContext::new(
NetworkContext::new(client, redirect_client, resolver),
pool,
RuntimeContext::new(
error_stats,
timing_stats,
None,
false,
std::path::PathBuf::from("/tmp/domain_status_whois_test"),
false,
Arc::new(crate::runtime_metrics::RuntimeMetrics::default()),
true,
),
Arc::new(crate::fingerprint::FingerprintRuleset::empty_for_tests()),
);
let start_time = std::time::Instant::now();
let result = handle_http_request(&ctx, &url, start_time).await;
assert!(result.is_err());
let error_msg = result.unwrap_err().to_string();
assert!(
error_msg.contains("HTTP request failed")
|| error_msg.contains("Connection")
|| error_msg.contains("timeout")
|| error_msg.contains("refused")
|| error_msg.contains("Request failed")
|| error_msg.contains("error sending request")
|| error_msg.contains("127.0.0.1:1"), "Expected connection error context, got: {}",
error_msg
);
}
#[tokio::test]
async fn test_handle_http_request_with_redirect_tracks_metrics() {
let server = Server::run();
let final_url = server.url("/final").to_string();
let start_url = server.url("/redirect").to_string();
server.expect(
Expectation::matching(request::method_path("GET", "/redirect")).respond_with(
status_code(302)
.insert_header("Location", final_url.as_str())
.body("Redirect"),
),
);
server.expect(
Expectation::matching(request::method_path("GET", "/final"))
.respond_with(status_code(200).body("<html><title>Final</title></html>")),
);
let ctx = create_test_context(&server).await;
let start_time = std::time::Instant::now();
assert_db_insert_failed(handle_http_request(&ctx, &start_url, start_time).await);
assert_eq!(
ctx.runtime
.error_stats
.get_info_count(crate::error_handling::InfoType::HttpRedirect),
1
);
}
#[tokio::test]
async fn test_handle_http_request_same_scheme_does_not_count_https_redirect() {
let server = Server::run();
let url = server.url("/secure").to_string();
server.expect(
Expectation::matching(request::method_path("GET", "/secure"))
.respond_with(status_code(200).body("<html><title>Secure</title></html>")),
);
let ctx = create_test_context(&server).await;
let start_time = std::time::Instant::now();
assert_db_insert_failed(handle_http_request(&ctx, &url, start_time).await);
assert_eq!(
ctx.runtime
.error_stats
.get_info_count(crate::error_handling::InfoType::HttpsRedirect),
0
);
}
#[tokio::test]
async fn test_handle_http_request_redirect_to_4xx_returns_error_and_tracks_redirect() {
let server = Server::run();
let start_url = server.url("/start").to_string();
let final_url = server.url("/final").to_string();
server.expect(
Expectation::matching(request::method_path("GET", "/start")).respond_with(
status_code(302)
.insert_header("Location", final_url.as_str())
.body("Redirect"),
),
);
server.expect(
Expectation::matching(request::method_path("GET", "/final"))
.respond_with(status_code(404).body("Not Found")),
);
let ctx = create_test_context(&server).await;
let start_time = std::time::Instant::now();
let result = handle_http_request(&ctx, &start_url, start_time).await;
assert!(result.is_err(), "redirect to 4xx must yield Err");
assert_eq!(
ctx.runtime
.error_stats
.get_info_count(crate::error_handling::InfoType::HttpRedirect),
1,
"redirect must be tracked when chain ends in 4xx"
);
}
#[tokio::test]
async fn test_handle_http_request_failure_context_attached() {
let server = Server::run();
let url = server.url("/error").to_string();
server.expect(
Expectation::matching(request::method_path("GET", "/error")).respond_with(
status_code(500)
.insert_header("X-Custom-Header", "test-value")
.body("Internal Server Error"),
),
);
let ctx = create_test_context(&server).await;
let start_time = std::time::Instant::now();
let result = handle_http_request(&ctx, &url, start_time).await;
assert!(result.is_err());
let error = result.unwrap_err();
let error_msg = error.to_string();
let extracted_context = crate::storage::failure::extract_failure_context(&error);
let has_structured_context =
extracted_context.final_url.is_some() || !extracted_context.redirect_chain.is_empty();
let has_context = has_structured_context
|| error_msg.contains("FINAL_URL")
|| error_msg.contains("REDIRECT_CHAIN")
|| error_msg.contains("Request failed")
|| error_msg.contains("HTTP request failed");
assert!(
has_context,
"Expected failure context in error message, got: {}",
error_msg
);
}
#[tokio::test]
async fn test_handle_http_request_multiple_redirects_tracked() {
let server = Server::run();
let final_url = server.url("/final").to_string();
let intermediate2 = server.url("/intermediate2").to_string();
let intermediate1 = server.url("/intermediate1").to_string();
let start_url = server.url("/start").to_string();
server.expect(
Expectation::matching(request::method_path("GET", "/start")).respond_with(
status_code(302)
.insert_header("Location", intermediate1.as_str())
.body("Redirect 1"),
),
);
server.expect(
Expectation::matching(request::method_path("GET", "/intermediate1")).respond_with(
status_code(302)
.insert_header("Location", intermediate2.as_str())
.body("Redirect 2"),
),
);
server.expect(
Expectation::matching(request::method_path("GET", "/intermediate2")).respond_with(
status_code(302)
.insert_header("Location", final_url.as_str())
.body("Redirect 3"),
),
);
server.expect(
Expectation::matching(request::method_path("GET", "/final"))
.respond_with(status_code(200).body("<html><title>Final</title></html>")),
);
let ctx = create_test_context(&server).await;
let start_time = std::time::Instant::now();
assert_db_insert_failed(handle_http_request(&ctx, &start_url, start_time).await);
assert_eq!(
ctx.runtime
.error_stats
.get_info_count(crate::error_handling::InfoType::MultipleRedirects),
1,
"Multiple redirects should be tracked"
);
assert_eq!(
ctx.runtime
.error_stats
.get_info_count(crate::error_handling::InfoType::HttpRedirect),
1,
"HttpRedirect should be tracked for multiple redirects"
);
}
#[tokio::test]
async fn test_handle_http_request_request_headers_applied() {
let server = Server::run();
let url = server.url("/headers").to_string();
server.expect(
Expectation::matching(request::method_path("GET", "/headers"))
.respond_with(status_code(200).body("<html><title>Test</title></html>")),
);
let ctx = create_test_context(&server).await;
let start_time = std::time::Instant::now();
assert_db_insert_failed(handle_http_request(&ctx, &url, start_time).await);
}
#[tokio::test]
async fn test_handle_http_request_error_stats_updated() {
let server = Server::run();
let url = server.url("/error").to_string();
server.expect(
Expectation::matching(request::method_path("GET", "/error"))
.respond_with(status_code(500).body("Internal Server Error")),
);
let ctx = create_test_context(&server).await;
let start_time = std::time::Instant::now();
let initial_error_count = ctx
.runtime
.error_stats
.get_error_count(crate::error_handling::ErrorType::HttpRequestInternalServerError);
let result = handle_http_request(&ctx, &url, start_time).await;
assert!(result.is_err());
let final_error_count = ctx
.runtime
.error_stats
.get_error_count(crate::error_handling::ErrorType::HttpRequestInternalServerError);
assert!(
final_error_count > initial_error_count,
"HTTP 500 should increment HttpRequestInternalServerError"
);
}
#[tokio::test]
async fn test_handle_http_request_hop_alt_svc_is_not_stored() {
let server = Server::run();
let final_url = server.url("/final").to_string();
let start_url = server.url("/redirect").to_string();
server.expect(
Expectation::matching(request::method_path("GET", "/redirect")).respond_with(
status_code(302)
.insert_header("Location", final_url.as_str())
.insert_header("Alt-Svc", "h3=\":443\"; ma=86400")
.body("Redirect"),
),
);
server.expect(
Expectation::matching(request::method_path("GET", "/final"))
.respond_with(status_code(200).body("<html><title>Final</title></html>")),
);
let ctx = create_test_context(&server).await;
crate::storage::run_migrations(&ctx.pool)
.await
.expect("migrate");
let start_time = std::time::Instant::now();
handle_http_request(&ctx, &start_url, start_time)
.await
.expect("scan should persist");
let rows: i64 = sqlx::query_scalar("SELECT COUNT(*) FROM url_status")
.fetch_one(ctx.pool.as_ref())
.await
.expect("count url_status");
assert_eq!(rows, 1, "final response should persist");
let alt_svc: Vec<String> = sqlx::query_scalar(
"SELECT header_value FROM url_http_headers WHERE lower(header_name) = 'alt-svc'",
)
.fetch_all(ctx.pool.as_ref())
.await
.expect("query");
assert!(
alt_svc.is_empty(),
"hop Alt-Svc must not be copied onto the final observation, got {alt_svc:?}"
);
}
#[tokio::test]
async fn test_handle_http_request_final_alt_svc_is_stored() {
let server = Server::run();
let url = server.url("/test").to_string();
server.expect(
Expectation::matching(request::method_path("GET", "/test")).respond_with(
status_code(200)
.insert_header("Alt-Svc", "h3=\":443\"; ma=86400")
.body("<html><title>Test</title></html>"),
),
);
let ctx = create_test_context(&server).await;
crate::storage::run_migrations(&ctx.pool)
.await
.expect("migrate");
let start_time = std::time::Instant::now();
handle_http_request(&ctx, &url, start_time)
.await
.expect("scan should persist");
let rows: i64 = sqlx::query_scalar("SELECT COUNT(*) FROM url_status")
.fetch_one(ctx.pool.as_ref())
.await
.expect("count url_status");
assert_eq!(rows, 1, "final response should persist");
let alt_svc: Vec<String> = sqlx::query_scalar(
"SELECT header_value FROM url_http_headers WHERE lower(header_name) = 'alt-svc'",
)
.fetch_all(ctx.pool.as_ref())
.await
.expect("query");
assert_eq!(
alt_svc,
vec!["h3=\":443\"; ma=86400".to_string()],
"native final Alt-Svc should still be stored"
);
}
}