use crate::domain::extract_domain;
use crate::storage::insert::insert_url_failure;
use crate::storage::models::UrlFailureRecord;
use anyhow::Error;
use sqlx::SqlitePool;
use super::context::FailureContext;
#[cfg(test)]
use super::context::attach_failure_context;
use super::error::{extract_error_type, extract_http_status};
fn truncate_header_value(value: String) -> String {
if value.len() <= crate::config::MAX_HEADER_VALUE_LENGTH {
return value;
}
crate::utils::sanitize::truncate_utf8_with_suffix(
&value,
crate::config::MAX_HEADER_VALUE_LENGTH,
"... (truncated)",
)
}
pub struct FailureRecordParams<'a> {
pub pool: &'a SqlitePool,
pub url: &'a str,
pub error: &'a Error,
pub context: FailureContext,
pub retry_count: u32,
pub elapsed_time: f64,
pub run_id: Option<&'a str>,
}
#[allow(clippy::too_many_lines)] pub async fn record_url_failure(params: FailureRecordParams<'_>) {
let attempted_url = params.url.to_string();
let extracted_context = super::context::extract_failure_context(params.error);
let final_url = params.context.final_url.or(extracted_context.final_url);
let redirect_chain: Vec<String> = if params.context.redirect_chain.is_empty() {
extracted_context.redirect_chain
} else {
params.context.redirect_chain
};
let domain = extract_domain(params.url).unwrap_or_else(|e| {
log::debug!(
"Failed to extract domain from URL {}: {}. Using 'unknown' as fallback.",
params.url,
e
);
"unknown".to_string()
});
let final_domain = final_url.as_ref().and_then(|u| extract_domain(u).ok());
let error_type = extract_error_type(params.error);
let error_message = {
let mut found_reqwest = false;
let mut reqwest_msg = String::new();
for cause in params.error.chain() {
if let Some(reqwest_err) = cause.downcast_ref::<reqwest::Error>() {
reqwest_msg = reqwest_err.to_string();
use std::error::Error as StdError;
if let Some(source) = reqwest_err.source() {
reqwest_msg = format!("{reqwest_msg}: {source}");
}
found_reqwest = true;
break;
}
}
let msg = if found_reqwest {
let chain_count = params.error.chain().count();
if chain_count > 3 {
let chain_summary: Vec<String> = params
.error
.chain()
.map(std::string::ToString::to_string)
.filter(|s| s.as_str() != reqwest_msg.as_str())
.take(3)
.collect();
if chain_summary.is_empty() {
reqwest_msg
} else {
format!(
"{} (error chain: {} -> ...)",
reqwest_msg,
chain_summary.join(" -> ")
)
}
} else {
reqwest_msg
}
} else {
let chain: Vec<String> = params
.error
.chain()
.map(std::string::ToString::to_string)
.collect();
if chain.len() > 1 {
format!(
"{} (error chain: {})",
chain.first().unwrap_or(¶ms.error.to_string()),
chain[1..].join(" -> ")
)
} else {
params.error.to_string()
}
};
crate::utils::sanitize::sanitize_and_truncate_error_message(&msg)
};
let http_status = extract_http_status(params.error);
let response_headers = if params.context.response_headers.is_empty() {
extracted_context.response_headers
} else {
params.context.response_headers
};
if final_url.is_none() && !redirect_chain.is_empty() {
log::debug!("Could not extract final_url from error, using last redirect chain URL");
}
let response_headers: Vec<(String, String)> = response_headers
.into_iter()
.map(|(name, value)| (name, truncate_header_value(value)))
.collect();
let failure = UrlFailureRecord {
url: attempted_url,
final_url,
domain,
final_domain,
error_type,
error_message,
http_status,
retry_count: params.retry_count,
elapsed_time_seconds: Some(params.elapsed_time),
timestamp: chrono::Utc::now().timestamp_millis(),
run_id: params.run_id.map(std::string::ToString::to_string),
redirect_chain,
response_headers,
};
if let Err(e) = insert_url_failure(params.pool, &failure).await {
log::error!(
"SQLite write failed while recording URL failure for {}: {}. Check disk space and permissions.",
failure.url,
e
);
}
}
#[cfg(test)]
mod tests {
use super::*;
use sqlx::Row;
use crate::storage::test_helpers::create_test_pool;
#[tokio::test]
async fn test_record_url_failure_domain_extraction_fallback() {
let pool = create_test_pool().await;
let error = anyhow::anyhow!("Test error");
let context = FailureContext::default();
let params = FailureRecordParams {
pool: &pool,
url: "not-a-valid-url", error: &error,
context,
retry_count: 0,
elapsed_time: 1.0,
run_id: None,
};
record_url_failure(params).await;
let row = sqlx::query(
"SELECT initial_domain FROM url_failures WHERE attempted_url = 'not-a-valid-url'",
)
.fetch_optional(&pool)
.await
.expect("Failed to query failures");
if let Some(row) = row {
assert_eq!(row.get::<String, _>("initial_domain"), "unknown");
}
}
#[tokio::test]
async fn test_record_url_failure_context_extraction() {
let pool = create_test_pool().await;
let context = FailureContext {
final_url: Some("https://example.com".to_string()),
redirect_chain: vec!["https://example.org".to_string()],
response_headers: vec![("server".to_string(), "nginx".to_string())],
};
let error = attach_failure_context(anyhow::anyhow!("HTTP request failed"), context.clone());
let params = FailureRecordParams {
pool: &pool,
url: "https://example.org",
error: &error,
context: FailureContext::default(), retry_count: 0,
elapsed_time: 1.0,
run_id: None,
};
record_url_failure(params).await;
let row = sqlx::query(
"SELECT final_url FROM url_failures WHERE attempted_url = 'https://example.org'",
)
.fetch_optional(&pool)
.await
.expect("Failed to query failures");
if let Some(row) = row {
let final_url: Option<String> = row.get::<Option<String>, _>("final_url");
assert_eq!(final_url, Some("https://example.com".to_string()));
}
}
#[tokio::test]
async fn test_record_url_failure_header_truncation() {
let pool = create_test_pool().await;
let long_header_value = "x".repeat(2000); let context = FailureContext {
final_url: None,
redirect_chain: vec![],
response_headers: vec![("server".to_string(), long_header_value.clone())],
};
let error = anyhow::anyhow!("Test error");
let params = FailureRecordParams {
pool: &pool,
url: "https://example.com",
error: &error,
context,
retry_count: 0,
elapsed_time: 1.0,
run_id: None,
};
record_url_failure(params).await;
let row = sqlx::query(
"SELECT header_value FROM url_failure_response_headers WHERE header_name = 'server'",
)
.fetch_optional(&pool)
.await
.expect("Failed to query headers");
if let Some(row) = row {
let value: String = row.get::<String, _>("header_value");
assert!(value.len() <= crate::config::MAX_HEADER_VALUE_LENGTH + 50); assert!(
value.contains("truncated")
|| value.len() <= crate::config::MAX_HEADER_VALUE_LENGTH
);
}
}
#[tokio::test]
async fn test_record_url_failure_header_truncation_handles_unicode() {
let pool = create_test_pool().await;
let long_header_value =
"测试🚀".repeat((crate::config::MAX_HEADER_VALUE_LENGTH / "测试🚀".len()) + 10);
let context = FailureContext {
final_url: None,
redirect_chain: vec![],
response_headers: vec![("server".to_string(), long_header_value)],
};
let error = anyhow::anyhow!("unicode header test");
let params = FailureRecordParams {
pool: &pool,
url: "https://example.com",
error: &error,
context,
retry_count: 0,
elapsed_time: 1.0,
run_id: None,
};
record_url_failure(params).await;
let row = sqlx::query(
"SELECT header_value FROM url_failure_response_headers WHERE header_name = 'server'",
)
.fetch_one(&pool)
.await
.expect("query header");
let value: String = row.get("header_value");
assert!(std::str::from_utf8(value.as_bytes()).is_ok());
assert!(value.contains("truncated"));
}
#[tokio::test]
async fn test_record_url_failure_error_message_sanitization() {
let pool = create_test_pool().await;
let error_msg = "Error with control chars: \x00\x01\x02\x03".to_string();
let error = anyhow::anyhow!(error_msg);
let context = FailureContext::default();
let params = FailureRecordParams {
pool: &pool,
url: "https://example.com",
error: &error,
context,
retry_count: 0,
elapsed_time: 1.0,
run_id: None,
};
record_url_failure(params).await;
let row = sqlx::query(
"SELECT error_message FROM url_failures WHERE attempted_url = 'https://example.com'",
)
.fetch_optional(&pool)
.await
.expect("Failed to query failures");
if let Some(row) = row {
let msg: String = row.get::<String, _>("error_message");
assert!(!msg.contains('\x00'));
assert!(!msg.contains('\x01'));
}
}
#[tokio::test]
async fn test_record_url_failure_provided_context_takes_precedence() {
let pool = create_test_pool().await;
let provided_context = FailureContext {
final_url: Some("https://provided.com".to_string()),
redirect_chain: vec!["https://provided.org".to_string()],
response_headers: vec![],
};
let extracted_context = FailureContext {
final_url: Some("https://extracted.com".to_string()),
redirect_chain: vec!["https://extracted.org".to_string()],
response_headers: vec![],
};
let error = anyhow::anyhow!("Test error").context(
crate::storage::failure::context::FailureContextError {
context: extracted_context,
},
);
let params = FailureRecordParams {
pool: &pool,
url: "https://example.com",
error: &error,
context: provided_context, retry_count: 0,
elapsed_time: 1.0,
run_id: None,
};
record_url_failure(params).await;
let row = sqlx::query(
"SELECT final_url FROM url_failures WHERE attempted_url = 'https://example.com'",
)
.fetch_optional(&pool)
.await
.expect("Failed to query failures");
if let Some(row) = row {
let final_url: Option<String> = row.get::<Option<String>, _>("final_url");
assert_eq!(final_url, Some("https://provided.com".to_string()));
}
}
#[tokio::test]
async fn test_record_url_failure_empty_context_uses_extracted() {
let pool = create_test_pool().await;
let extracted_context = FailureContext {
final_url: Some("https://extracted.com".to_string()),
redirect_chain: vec!["https://extracted.org".to_string()],
response_headers: vec![],
};
let error =
attach_failure_context(anyhow::anyhow!("Test error"), extracted_context.clone());
let params = FailureRecordParams {
pool: &pool,
url: "https://example.com",
error: &error,
context: FailureContext::default(), retry_count: 0,
elapsed_time: 1.0,
run_id: None,
};
record_url_failure(params).await;
let row = sqlx::query(
"SELECT final_url FROM url_failures WHERE attempted_url = 'https://example.com'",
)
.fetch_optional(&pool)
.await
.expect("Failed to query failures");
if let Some(row) = row {
let final_url: Option<String> = row.get::<Option<String>, _>("final_url");
assert_eq!(final_url, extracted_context.final_url);
}
}
#[tokio::test]
async fn test_record_url_failure_error_message_chain_handling() {
let pool = create_test_pool().await;
let error = anyhow::anyhow!("Root cause")
.context("Context 1")
.context("Context 2")
.context("Context 3")
.context("Context 4")
.context("Context 5");
let context = FailureContext::default();
let params = FailureRecordParams {
pool: &pool,
url: "https://example.com",
error: &error,
context,
retry_count: 0,
elapsed_time: 1.0,
run_id: None,
};
record_url_failure(params).await;
let row = sqlx::query(
"SELECT error_message FROM url_failures WHERE attempted_url = 'https://example.com'",
)
.fetch_optional(&pool)
.await
.expect("Failed to query failures");
if let Some(row) = row {
let msg: String = row.get::<String, _>("error_message");
assert!(!msg.is_empty());
}
}
#[tokio::test]
async fn test_record_url_failure_run_id_propagation() {
let pool = create_test_pool().await;
sqlx::query("INSERT INTO runs (run_id, start_time_ms) VALUES (?, ?)")
.bind("test-run-123")
.bind(chrono::Utc::now().timestamp_millis())
.execute(&pool)
.await
.expect("Failed to create test run");
let error = anyhow::anyhow!("Test error");
let context = FailureContext::default();
let run_id = Some("test-run-123");
let params = FailureRecordParams {
pool: &pool,
url: "https://example.com",
error: &error,
context,
retry_count: 0,
elapsed_time: 1.0,
run_id,
};
record_url_failure(params).await;
let row = sqlx::query(
"SELECT run_id FROM url_failures WHERE attempted_url = 'https://example.com'",
)
.fetch_optional(&pool)
.await
.expect("Failed to query failures");
if let Some(row) = row {
let stored_run_id: Option<String> = row.get::<Option<String>, _>("run_id");
assert_eq!(stored_run_id, Some("test-run-123".to_string()));
}
}
}