use crate::fetch::dns::{AdditionalDnsData, TlsDnsData};
use crate::fetch::response::{HtmlData, ResponseData};
use crate::storage::PersistedUrlRecord;
use super::builder::{build_persisted_url_record, build_url_record};
pub struct RecordPreparationParams<'a> {
pub resp_data: ResponseData,
pub html_data: HtmlData,
pub tls_dns_data: TlsDnsData,
pub additional_dns: AdditionalDnsData,
pub technologies_vec: Vec<crate::fingerprint::DetectedTechnology>,
pub partial_failures: Vec<(crate::error_handling::ErrorType, String)>,
pub redirect_chain: Vec<(String, u16)>,
pub elapsed: f64,
pub timestamp: i64,
pub ctx: &'a crate::fetch::ProcessingContext,
pub favicon: Option<crate::fetch::favicon::FaviconData>,
pub well_known: crate::fetch::well_known::WellKnownData,
}
#[allow(clippy::too_many_lines)] pub async fn prepare_record_for_insertion(
params: RecordPreparationParams<'_>,
) -> (PersistedUrlRecord, (u64, u64)) {
use crate::utils::duration_to_us;
use std::time::Instant;
let record = build_url_record(
¶ms.resp_data,
¶ms.html_data,
¶ms.tls_dns_data,
¶ms.additional_dns,
params.elapsed,
params.timestamp,
params.ctx.runtime.run_id.as_ref(),
);
let (geoip_data, whois_data) = tokio::join!(
async {
let geoip_start = Instant::now();
let geoip_data = params
.tls_dns_data
.ip_address
.as_ref()
.and_then(|ip_address| {
let ip_addr = std::hint::black_box(ip_address.as_str());
crate::geoip::lookup_ip(ip_addr).map(|result| (ip_address.clone(), result))
});
let geoip_elapsed = geoip_start.elapsed();
let geoip_lookup_us = duration_to_us(geoip_elapsed);
if geoip_lookup_us == 0 && geoip_data.is_some() {
log::debug!(
"GeoIP lookup returned data but timing is 0ms (elapsed: {:?}, micros: {}, nanos: {})",
geoip_elapsed,
geoip_elapsed.as_micros(),
geoip_elapsed.as_nanos()
);
}
(geoip_data, geoip_lookup_us)
},
async {
if params.ctx.runtime.enable_whois {
let whois_start = Instant::now();
log::debug!(
"Performing WHOIS lookup for domain: {}",
params.resp_data.final_domain
);
let result = match crate::whois::lookup_whois_with_client(
¶ms.resp_data.final_domain,
Some(params.ctx.runtime.whois_cache_dir.as_path()),
params.ctx.runtime.whois_client.as_deref(),
)
.await
{
Ok(Some(whois_result)) => {
log::debug!(
"WHOIS lookup successful for {}: registrar={:?}, creation={:?}, expiration={:?}",
params.resp_data.final_domain,
whois_result.registrar,
whois_result.creation_date,
whois_result.expiration_date
);
Some(whois_result)
}
Ok(None) => {
log::debug!(
"WHOIS lookup returned no data for {}",
params.resp_data.final_domain
);
None
}
Err(e) => {
log::warn!(
"WHOIS lookup failed for {}: {}",
params.resp_data.final_domain,
e
);
None
}
};
let whois_lookup_us = duration_to_us(whois_start.elapsed());
(result, whois_lookup_us)
} else {
(None, 0)
}
}
);
let (geoip_data, geoip_lookup_us) = geoip_data;
let (whois_data, whois_lookup_us) = whois_data;
let persisted_record = build_persisted_url_record(super::builder::PersistedUrlRecordParams {
record,
resp_data: params.resp_data,
html_data: params.html_data,
tls_dns_data: params.tls_dns_data,
technologies_vec: params.technologies_vec,
redirect_chain: params.redirect_chain,
partial_failures: params.partial_failures,
geoip_data,
whois_data,
timestamp: params.timestamp,
run_id: params.ctx.runtime.run_id.clone(),
favicon: params.favicon,
additional_dns: params.additional_dns,
well_known: params.well_known,
});
(persisted_record, (geoip_lookup_us, whois_lookup_us))
}
#[cfg(test)]
mod tests {
use super::*;
use crate::error_handling::ProcessingStats;
use crate::fetch::dns::{AdditionalDnsData, TlsDnsData};
use crate::fetch::response::{HtmlData, ResponseData};
use crate::fetch::{NetworkContext, ProcessingContext, RuntimeContext};
use crate::initialization::test_resolver;
use crate::utils::TimingStats;
use std::sync::Arc;
const GEOIP_TEST_TIMEOUT_US: u64 = 8_000_000;
async fn create_test_context() -> ProcessingContext {
let client = Arc::new(
reqwest::Client::builder()
.build()
.expect("Failed to create HTTP client"),
);
let redirect_client = Arc::new(
reqwest::Client::builder()
.redirect(reqwest::redirect::Policy::none())
.build()
.expect("Failed to create redirect client"),
);
let resolver = test_resolver();
let error_stats = Arc::new(ProcessingStats::new());
let timing_stats = Arc::new(TimingStats::new());
let run_id = Some("test-run".to_string());
let enable_whois = false; let pool = Arc::new(
sqlx::SqlitePool::connect("sqlite::memory:")
.await
.expect("Failed to create test pool"),
);
ProcessingContext::new(
NetworkContext::new(client, redirect_client, resolver),
pool,
RuntimeContext::new(
error_stats,
timing_stats,
run_id,
enable_whois,
std::path::PathBuf::from("/tmp/domain_status_whois_test"),
false,
Arc::new(crate::runtime_metrics::RuntimeMetrics::default()),
true,
),
Arc::new(crate::fingerprint::FingerprintRuleset::empty_for_tests()),
)
}
fn create_minimal_resp_data() -> ResponseData {
ResponseData {
initial_url: "https://example.com".to_string(),
final_url: "https://example.com".to_string(),
initial_domain: "example.com".to_string(),
final_domain: "example.com".to_string(),
host: "example.com".to_string(),
status: 200,
status_desc: "OK".to_string(),
headers: reqwest::header::HeaderMap::new(),
security_headers: std::collections::HashMap::new(),
http_headers: std::collections::HashMap::new(),
body: std::sync::Arc::<str>::from("<html><body>Test</body></html>"),
body_sha256: None,
body_truncated: false,
content_length: None,
http_version: None,
content_type: None,
}
}
fn create_minimal_html_data() -> HtmlData {
HtmlData {
title: "Test".to_string(),
description: None,
structured_data: crate::parse::StructuredData::default(),
social_media_links: Vec::new(),
contact_links: Vec::new(),
exposed_secrets: Vec::new(),
analytics_ids: Vec::new(),
meta_tags: std::collections::HashMap::new(),
script_sources: Vec::new(),
script_tag_ids: std::collections::HashSet::new(),
inline_script_text: String::new(),
external_scripts_eligible: 0,
external_scripts_scanned: 0,
favicon_url: None,
canonical_url: None,
meta_refresh_url: None,
meta_robots: None,
resource_hints: Vec::new(),
}
}
fn create_minimal_tls_dns_data() -> TlsDnsData {
TlsDnsData {
ip_address: Some("8.8.8.8".to_string()),
tls_version: None,
subject: None,
issuer: None,
valid_from: None,
valid_to: None,
oids: None,
cipher_suite: None,
key_algorithm: None,
subject_alternative_names: None,
reverse_dns_name: None,
cert_fingerprint_sha256: None,
cert_serial_number: None,
cert_is_self_signed: None,
cert_is_wildcard: None,
}
}
fn create_minimal_additional_dns_data() -> AdditionalDnsData {
AdditionalDnsData {
nameservers: None,
txt_records: None,
mx_records: None,
spf_record: None,
dmarc_record: None,
mta_sts_record: None,
tls_rpt_record: None,
bimi_record: None,
cname_chain: None,
aaaa_records: None,
caa_records: None,
}
}
#[tokio::test]
async fn test_prepare_record_for_insertion_basic_success() {
let ctx = create_test_context().await;
let resp_data = create_minimal_resp_data();
let html_data = create_minimal_html_data();
let tls_dns_data = create_minimal_tls_dns_data();
let additional_dns = create_minimal_additional_dns_data();
let (persisted_record, (geoip_us, whois_us)) =
prepare_record_for_insertion(RecordPreparationParams {
resp_data,
html_data,
tls_dns_data,
additional_dns,
technologies_vec: Vec::new(),
partial_failures: Vec::new(),
redirect_chain: Vec::new(),
elapsed: 1.0,
timestamp: chrono::Utc::now().timestamp_millis(),
ctx: &ctx,
favicon: None,
well_known: crate::fetch::well_known::WellKnownData::default(),
})
.await;
assert_eq!(persisted_record.url_record.final_domain, "example.com");
assert!(
geoip_us < GEOIP_TEST_TIMEOUT_US,
"GeoIP lookup took {geoip_us}us, expected < {GEOIP_TEST_TIMEOUT_US}us"
);
assert_eq!(whois_us, 0); }
#[tokio::test]
async fn test_prepare_record_for_insertion_geoip_lookup_failure_handled() {
let ctx = create_test_context().await;
let resp_data = create_minimal_resp_data();
let html_data = create_minimal_html_data();
let mut tls_dns_data = create_minimal_tls_dns_data();
tls_dns_data.ip_address = Some("invalid.ip.address".to_string());
let additional_dns = create_minimal_additional_dns_data();
let (persisted_record, (geoip_us, _whois_us)) =
prepare_record_for_insertion(RecordPreparationParams {
resp_data,
html_data,
tls_dns_data,
additional_dns,
technologies_vec: Vec::new(),
partial_failures: Vec::new(),
redirect_chain: Vec::new(),
elapsed: 1.0,
timestamp: chrono::Utc::now().timestamp_millis(),
ctx: &ctx,
favicon: None,
well_known: crate::fetch::well_known::WellKnownData::default(),
})
.await;
assert_eq!(persisted_record.url_record.final_domain, "example.com");
assert!(
geoip_us < GEOIP_TEST_TIMEOUT_US,
"GeoIP lookup took {geoip_us}us, expected < {GEOIP_TEST_TIMEOUT_US}us"
);
}
#[tokio::test]
async fn test_prepare_record_for_insertion_parallel_tasks_complete() {
let ctx = create_test_context().await;
let resp_data = create_minimal_resp_data();
let html_data = create_minimal_html_data();
let tls_dns_data = create_minimal_tls_dns_data();
let additional_dns = create_minimal_additional_dns_data();
let start = std::time::Instant::now();
let (persisted_record, (geoip_us, whois_us)) =
prepare_record_for_insertion(RecordPreparationParams {
resp_data,
html_data,
tls_dns_data,
additional_dns,
technologies_vec: Vec::new(),
partial_failures: Vec::new(),
redirect_chain: Vec::new(),
elapsed: 1.0,
timestamp: chrono::Utc::now().timestamp_millis(),
ctx: &ctx,
favicon: None,
well_known: crate::fetch::well_known::WellKnownData::default(),
})
.await;
let elapsed = start.elapsed();
assert_eq!(persisted_record.url_record.final_domain, "example.com");
assert!(
elapsed.as_millis() < 10000,
"Total elapsed time {}ms, expected < 10000ms",
elapsed.as_millis()
); assert!(
geoip_us < GEOIP_TEST_TIMEOUT_US,
"GeoIP lookup took {geoip_us}us, expected < {GEOIP_TEST_TIMEOUT_US}us"
);
assert_eq!(whois_us, 0); }
#[tokio::test]
async fn test_prepare_record_for_insertion_with_technologies() {
let ctx = create_test_context().await;
let resp_data = create_minimal_resp_data();
let html_data = create_minimal_html_data();
let tls_dns_data = create_minimal_tls_dns_data();
let additional_dns = create_minimal_additional_dns_data();
let technologies = vec![
crate::fingerprint::DetectedTechnology {
name: "WordPress".to_string(),
version: None,
category: None,
is_implied: false,
detection_source: None,
},
crate::fingerprint::DetectedTechnology {
name: "PHP".to_string(),
version: None,
category: None,
is_implied: false,
detection_source: None,
},
];
let technologies_count = technologies.len();
let (persisted_record, _) = prepare_record_for_insertion(RecordPreparationParams {
resp_data,
html_data,
tls_dns_data,
additional_dns,
technologies_vec: technologies,
partial_failures: Vec::new(),
redirect_chain: Vec::new(),
elapsed: 1.0,
timestamp: chrono::Utc::now().timestamp_millis(),
ctx: &ctx,
favicon: None,
well_known: crate::fetch::well_known::WellKnownData::default(),
})
.await;
assert_eq!(persisted_record.technologies.len(), technologies_count);
}
#[tokio::test]
async fn test_prepare_record_for_insertion_with_partial_failures() {
let ctx = create_test_context().await;
let resp_data = create_minimal_resp_data();
let html_data = create_minimal_html_data();
let tls_dns_data = create_minimal_tls_dns_data();
let additional_dns = create_minimal_additional_dns_data();
let partial_failures = vec![(
crate::error_handling::ErrorType::HttpRequestOtherError,
"DNS lookup failed".to_string(),
)];
let partial_failures_count = partial_failures.len();
let (persisted_record, _) = prepare_record_for_insertion(RecordPreparationParams {
resp_data,
html_data,
tls_dns_data,
additional_dns,
technologies_vec: Vec::new(),
partial_failures,
redirect_chain: Vec::new(),
elapsed: 1.0,
timestamp: chrono::Utc::now().timestamp_millis(),
ctx: &ctx,
favicon: None,
well_known: crate::fetch::well_known::WellKnownData::default(),
})
.await;
assert_eq!(
persisted_record.partial_failures.len(),
partial_failures_count
);
}
#[tokio::test]
async fn test_prepare_record_for_insertion_whois_when_enabled() {
let cache_dir = tempfile::TempDir::new().expect("whois cache");
crate::whois::seed_whois_cache(
cache_dir.path(),
"example.com",
&crate::whois::WhoisResult {
registrar: Some("Fixture Registrar".to_string()),
registrant_org: Some("Fixture Org".to_string()),
raw_text: Some("seeded".to_string()),
..crate::whois::WhoisResult::default()
},
)
.await
.expect("seed whois cache");
let mut ctx = create_test_context().await;
ctx.runtime.enable_whois = true;
ctx.runtime.whois_cache_dir = cache_dir.path().to_path_buf();
let resp_data = create_minimal_resp_data();
let html_data = create_minimal_html_data();
let tls_dns_data = create_minimal_tls_dns_data();
let additional_dns = create_minimal_additional_dns_data();
let (persisted_record, (_geoip_us, _whois_us)) =
prepare_record_for_insertion(RecordPreparationParams {
resp_data,
html_data,
tls_dns_data,
additional_dns,
technologies_vec: Vec::new(),
partial_failures: Vec::new(),
redirect_chain: Vec::new(),
elapsed: 1.0,
timestamp: chrono::Utc::now().timestamp_millis(),
ctx: &ctx,
favicon: None,
well_known: crate::fetch::well_known::WellKnownData::default(),
})
.await;
assert_eq!(persisted_record.url_record.final_domain, "example.com");
let whois = persisted_record
.whois
.expect("WHOIS cache hit must populate");
assert_eq!(whois.registrar.as_deref(), Some("Fixture Registrar"));
}
#[tokio::test]
async fn test_prepare_record_for_insertion_enrichment_failures_handled_gracefully() {
let ctx = create_test_context().await;
let resp_data = create_minimal_resp_data();
let html_data = create_minimal_html_data();
let mut tls_dns_data = create_minimal_tls_dns_data();
tls_dns_data.ip_address = Some("999.999.999.999".to_string());
let additional_dns = create_minimal_additional_dns_data();
let (persisted_record, (geoip_us, whois_us)) =
prepare_record_for_insertion(RecordPreparationParams {
resp_data,
html_data,
tls_dns_data,
additional_dns,
technologies_vec: Vec::new(),
partial_failures: Vec::new(),
redirect_chain: Vec::new(),
elapsed: 1.0,
timestamp: chrono::Utc::now().timestamp_millis(),
ctx: &ctx,
favicon: None,
well_known: crate::fetch::well_known::WellKnownData::default(),
})
.await;
assert_eq!(persisted_record.url_record.final_domain, "example.com");
assert!(
geoip_us < GEOIP_TEST_TIMEOUT_US,
"GeoIP lookup took {geoip_us}us, expected < {GEOIP_TEST_TIMEOUT_US}us"
);
assert_eq!(whois_us, 0); assert!(persisted_record.geoip.is_none());
}
}