use sqlx::SqlitePool;
use crate::error_handling::DatabaseError;
use crate::storage::insert::retry::with_sqlite_retry;
use crate::storage::insert::utils::naive_datetime_to_millis;
use crate::storage::models::UrlRecord;
use super::columns::upsert_url_status_row;
use super::core_satellites::URL_STATUS_CORE_SATELLITE_TABLES;
use super::satellite::{
insert_caa_records, insert_certificate_sans, insert_cname_records, insert_cookies,
insert_csp_domains, insert_http_headers, insert_ipv6_addresses, insert_mx_records,
insert_nameservers, insert_oids, insert_redirect_chain, insert_resource_hints,
insert_robots_txt, insert_script_hosts, insert_security_headers, insert_security_txt,
insert_technologies, insert_txt_records,
};
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct UrlUpsertOutcome {
pub id: i64,
pub inserted: bool,
pub satellite_insert_failures: Vec<SatelliteWriteFailure>,
pub partial_failures_inserted: usize,
pub satellite_insert_errors_inserted: usize,
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct SatelliteWriteFailure {
pub table: &'static str,
pub message: String,
}
#[derive(Debug)]
pub struct UrlRecordInsertParams<'a> {
pub pool: &'a SqlitePool,
pub record: &'a UrlRecord,
pub security_headers: &'a std::collections::HashMap<String, String>,
pub http_headers: &'a std::collections::HashMap<String, String>,
pub oids: &'a std::collections::HashSet<String>,
pub redirect_chain: &'a [(String, u16)],
pub technologies: &'a [crate::fingerprint::DetectedTechnology],
pub subject_alternative_names: &'a [String],
pub cname_records: Option<&'a String>,
pub aaaa_records: Option<&'a String>,
pub caa_records: Option<&'a String>,
pub csp_domains: &'a [(String, String, Option<String>)],
pub cookies: &'a [crate::storage::CookieInfo],
pub resource_hints: &'a [(String, String)],
pub script_hosts: &'a [crate::storage::ScriptHostInfo],
pub security_txt: Option<&'a crate::fetch::well_known::SecurityTxtData>,
pub robots_txt: Option<&'a crate::fetch::well_known::RobotsTxtData>,
}
impl<'a> UrlRecordInsertParams<'a> {
#[must_use]
pub(crate) fn from_persisted_record(
pool: &'a sqlx::SqlitePool,
persisted: &'a crate::storage::record::PersistedUrlRecord,
) -> Self {
Self {
pool,
record: &persisted.url_record,
security_headers: &persisted.security_headers,
http_headers: &persisted.http_headers,
oids: &persisted.oids,
redirect_chain: &persisted.redirect_chain,
technologies: &persisted.technologies,
subject_alternative_names: &persisted.subject_alternative_names,
cname_records: persisted.cname_records.as_ref(),
aaaa_records: persisted.aaaa_records.as_ref(),
caa_records: persisted.caa_records.as_ref(),
csp_domains: &persisted.csp_domains,
cookies: &persisted.cookies,
resource_hints: &persisted.resource_hints,
script_hosts: &persisted.script_hosts,
security_txt: persisted.security_txt.as_ref(),
robots_txt: persisted.robots_txt.as_ref(),
}
}
#[must_use]
#[cfg_attr(not(any(test, feature = "test-utils")), allow(dead_code))]
pub fn with_empty_satellites(
pool: &'a SqlitePool,
record: &'a UrlRecord,
security_headers: &'a std::collections::HashMap<String, String>,
http_headers: &'a std::collections::HashMap<String, String>,
oids: &'a std::collections::HashSet<String>,
) -> Self {
Self {
pool,
record,
security_headers,
http_headers,
oids,
redirect_chain: &[],
technologies: &[],
subject_alternative_names: &[],
cname_records: None,
aaaa_records: None,
caa_records: None,
csp_domains: &[],
cookies: &[],
resource_hints: &[],
script_hosts: &[],
security_txt: None,
robots_txt: None,
}
}
}
#[cfg_attr(not(any(test, feature = "test-utils")), allow(dead_code))]
pub async fn insert_url_record(params: UrlRecordInsertParams<'_>) -> Result<i64, DatabaseError> {
insert_url_record_with_outcome(params)
.await
.map(|outcome| outcome.id)
}
pub async fn insert_url_record_with_outcome(
params: UrlRecordInsertParams<'_>,
) -> Result<UrlUpsertOutcome, DatabaseError> {
with_sqlite_retry(|| insert_url_record_impl(¶ms)).await
}
#[allow(clippy::too_many_lines)] #[allow(clippy::cognitive_complexity)] async fn insert_url_record_impl(
params: &UrlRecordInsertParams<'_>,
) -> Result<UrlUpsertOutcome, DatabaseError> {
let valid_from_millis = naive_datetime_to_millis(params.record.ssl_cert_valid_from.as_ref());
let valid_to_millis = naive_datetime_to_millis(params.record.ssl_cert_valid_to.as_ref());
log::debug!(
"Inserting UrlRecord: initial_domain={}",
params.record.initial_domain
);
let mut tx = params.pool.begin().await.map_err(DatabaseError::SqlError)?;
let (url_status_id, inserted) =
upsert_url_status_row(&mut tx, params.record, valid_from_millis, valid_to_millis).await?;
let mut satellite_insert_failures = Vec::new();
let core_cleanup_ok = match crate::storage::insert::utils::delete_child_rows(
&mut tx,
URL_STATUS_CORE_SATELLITE_TABLES,
url_status_id,
)
.await
{
Ok(()) => true,
Err(e) => {
log::warn!(
"Failed to clean stale core satellite rows for url_status_id {url_status_id}: {e}"
);
satellite_insert_failures.push(SatelliteWriteFailure {
table: "core_satellites",
message: format!("Failed to clean stale core satellite rows: {e}"),
});
false
}
};
if core_cleanup_ok {
record_satellite_write(
"url_technologies",
url_status_id,
&mut satellite_insert_failures,
insert_technologies(&mut tx, url_status_id, params.technologies).await,
);
record_satellite_write(
"url_nameservers",
url_status_id,
&mut satellite_insert_failures,
insert_nameservers(&mut tx, url_status_id, params.record.nameservers.as_ref()).await,
);
record_satellite_write(
"url_txt_records",
url_status_id,
&mut satellite_insert_failures,
insert_txt_records(&mut tx, url_status_id, params.record.txt_records.as_ref()).await,
);
record_satellite_write(
"url_mx_records",
url_status_id,
&mut satellite_insert_failures,
insert_mx_records(&mut tx, url_status_id, params.record.mx_records.as_ref()).await,
);
record_satellite_write(
"url_security_headers",
url_status_id,
&mut satellite_insert_failures,
insert_security_headers(&mut tx, url_status_id, params.security_headers).await,
);
record_satellite_write(
"url_http_headers",
url_status_id,
&mut satellite_insert_failures,
insert_http_headers(&mut tx, url_status_id, params.http_headers).await,
);
record_satellite_write(
"url_certificate_oids",
url_status_id,
&mut satellite_insert_failures,
insert_oids(&mut tx, url_status_id, params.oids).await,
);
record_satellite_write(
"url_redirect_chain",
url_status_id,
&mut satellite_insert_failures,
insert_redirect_chain(&mut tx, url_status_id, params.redirect_chain).await,
);
record_satellite_write(
"url_certificate_sans",
url_status_id,
&mut satellite_insert_failures,
insert_certificate_sans(&mut tx, url_status_id, params.subject_alternative_names).await,
);
record_satellite_write(
"url_cname_records",
url_status_id,
&mut satellite_insert_failures,
insert_cname_records(&mut tx, url_status_id, params.cname_records).await,
);
record_satellite_write(
"url_ipv6_addresses",
url_status_id,
&mut satellite_insert_failures,
insert_ipv6_addresses(&mut tx, url_status_id, params.aaaa_records).await,
);
record_satellite_write(
"url_caa_records",
url_status_id,
&mut satellite_insert_failures,
insert_caa_records(&mut tx, url_status_id, params.caa_records).await,
);
record_satellite_write(
"url_csp_domains",
url_status_id,
&mut satellite_insert_failures,
insert_csp_domains(&mut tx, url_status_id, params.csp_domains).await,
);
record_satellite_write(
"url_cookies",
url_status_id,
&mut satellite_insert_failures,
insert_cookies(&mut tx, url_status_id, params.cookies).await,
);
record_satellite_write(
"url_resource_hints",
url_status_id,
&mut satellite_insert_failures,
insert_resource_hints(&mut tx, url_status_id, params.resource_hints).await,
);
record_satellite_write(
"url_script_hosts",
url_status_id,
&mut satellite_insert_failures,
insert_script_hosts(&mut tx, url_status_id, params.script_hosts).await,
);
record_satellite_write(
"url_security_txt",
url_status_id,
&mut satellite_insert_failures,
insert_security_txt(&mut tx, url_status_id, params.security_txt).await,
);
record_satellite_write(
"url_robots_txt",
url_status_id,
&mut satellite_insert_failures,
insert_robots_txt(&mut tx, url_status_id, params.robots_txt).await,
);
}
tx.commit().await.map_err(|e| {
log::error!(
"Failed to commit transaction for url_status_id {} (domain: {}): {}",
url_status_id,
params.record.initial_domain,
e
);
DatabaseError::SqlError(e)
})?;
Ok(UrlUpsertOutcome {
id: url_status_id,
inserted,
satellite_insert_failures,
partial_failures_inserted: 0,
satellite_insert_errors_inserted: 0,
})
}
fn record_satellite_write(
table: &'static str,
url_status_id: i64,
failures: &mut Vec<SatelliteWriteFailure>,
result: Result<(), sqlx::Error>,
) {
if let Err(e) = result {
log::warn!("Failed to insert {table} for url_status_id {url_status_id}: {e}");
failures.push(SatelliteWriteFailure {
table,
message: e.to_string(),
});
}
}