use chrono::NaiveDateTime;
pub(crate) fn naive_datetime_to_millis(datetime: Option<&NaiveDateTime>) -> Option<i64> {
datetime.map(|dt| dt.and_utc().timestamp_millis())
}
pub(crate) fn parse_json_array(json_str: Option<&String>) -> Option<Vec<String>> {
let json_str = json_str?;
if json_str.is_empty() {
return None;
}
serde_json::from_str::<Vec<String>>(json_str).ok()
}
pub(crate) fn detect_txt_type(txt: &str) -> &'static str {
if crate::dns::is_spf_txt(txt) {
"SPF"
} else if crate::dns::is_dmarc_txt(txt) {
"DMARC"
} else if crate::dns::is_mta_sts_txt(txt) {
"MTA-STS"
} else if crate::dns::is_tls_rpt_txt(txt) {
"TLS-RPT"
} else if crate::dns::is_bimi_txt(txt) {
"BIMI"
} else {
let txt_lower = txt.to_lowercase();
if txt_lower.contains("google-site-verification")
|| txt_lower.contains("ms-verify")
|| txt_lower.contains("facebook-domain-verification")
|| txt_lower.contains("atlassian-domain-verification")
{
"VERIFICATION"
} else {
"OTHER"
}
}
}
pub(crate) fn parse_mx_record(mx: &str) -> Option<(i32, String)> {
let parts: Vec<&str> = mx.split_whitespace().collect();
if parts.is_empty() {
return None;
}
if let Ok(priority) = parts[0].parse::<i32>() {
if parts.len() >= 2 {
Some((priority, parts[1].to_string()))
} else {
None
}
} else {
Some((0, parts[0].to_string()))
}
}
pub(crate) fn parse_mx_json_array(json_str: Option<&String>) -> Option<Vec<(i32, String)>> {
let json_str = json_str?;
if json_str.is_empty() {
return None;
}
if let Ok(mx_objects) = serde_json::from_str::<Vec<serde_json::Value>>(json_str) {
let mut result = Vec::new();
for obj in mx_objects {
#[allow(clippy::cast_possible_truncation)]
if let (Some(priority), Some(hostname)) = (
obj.get("priority")
.and_then(|v| v.as_i64().map(|p| p as i32)),
obj.get("hostname").and_then(|v| v.as_str()),
) {
result.push((priority, hostname.to_string()));
}
}
if !result.is_empty() {
return Some(result);
}
}
if let Ok(mx_strings) = serde_json::from_str::<Vec<String>>(json_str) {
let mut result = Vec::new();
for mx_str in mx_strings {
if let Some((priority, hostname)) = parse_mx_record(&mx_str) {
result.push((priority, hostname));
}
}
if !result.is_empty() {
return Some(result);
}
}
None
}
pub(crate) fn build_batch_insert_query(
table_name: &str,
columns: &[&str],
row_count: usize,
conflict_clause: Option<&str>,
) -> String {
if row_count == 0 {
return String::new();
}
let num_columns = columns.len();
let placeholder = format!(
"({})",
(0..num_columns).map(|_| "?").collect::<Vec<_>>().join(", ")
);
let placeholders: Vec<String> = (0..row_count).map(|_| placeholder.clone()).collect();
let mut query = format!(
"INSERT INTO {} ({}) VALUES {}",
table_name,
columns.join(", "),
placeholders.join(", ")
);
if let Some(conflict) = conflict_clause {
query.push(' ');
query.push_str(conflict);
}
query
}
#[allow(clippy::too_many_arguments)] pub(crate) async fn insert_key_value_batch<K, V>(
tx: &mut sqlx::Transaction<'_, sqlx::Sqlite>,
table_name: &str,
parent_id_column: &str,
key_column: &str,
value_column: &str,
parent_id: i64,
data: &[(K, V)],
conflict_clause: Option<&str>,
) -> Result<(), sqlx::Error>
where
K: for<'q> sqlx::Encode<'q, sqlx::Sqlite> + sqlx::Type<sqlx::Sqlite> + Clone,
V: for<'q> sqlx::Encode<'q, sqlx::Sqlite> + sqlx::Type<sqlx::Sqlite> + Clone,
{
if data.is_empty() {
return Ok(());
}
let query = build_batch_insert_query(
table_name,
&[parent_id_column, key_column, value_column],
data.len(),
conflict_clause,
);
let mut query_builder = crate::sql::query(query);
for (key, value) in data {
query_builder = query_builder
.bind(parent_id)
.bind(key.clone())
.bind(value.clone());
}
query_builder.execute(&mut **tx).await?;
Ok(())
}
pub(crate) async fn insert_single_column_batch<V>(
tx: &mut sqlx::Transaction<'_, sqlx::Sqlite>,
table_name: &str,
parent_id_column: &str,
value_column: &str,
parent_id: i64,
data: &[V],
conflict_clause: Option<&str>,
) -> Result<(), sqlx::Error>
where
V: for<'q> sqlx::Encode<'q, sqlx::Sqlite> + sqlx::Type<sqlx::Sqlite> + Clone,
{
if data.is_empty() {
return Ok(());
}
let query = build_batch_insert_query(
table_name,
&[parent_id_column, value_column],
data.len(),
conflict_clause,
);
let mut query_builder = crate::sql::query(query);
for value in data {
query_builder = query_builder.bind(parent_id).bind(value.clone());
}
query_builder.execute(&mut **tx).await?;
Ok(())
}
pub(crate) async fn delete_child_rows(
tx: &mut sqlx::Transaction<'_, sqlx::Sqlite>,
tables: &[&str],
url_status_id: i64,
) -> Result<(), sqlx::Error> {
for table in tables {
let sql = format!("DELETE FROM {table} WHERE url_status_id = ?");
crate::sql::query(sql)
.bind(url_status_id)
.execute(&mut **tx)
.await?;
}
Ok(())
}
#[cfg(test)]
mod tests {
use super::*;
use chrono::NaiveDate;
#[test]
fn test_naive_datetime_to_millis_some() {
let dt = NaiveDate::from_ymd_opt(2024, 1, 1)
.unwrap()
.and_hms_opt(12, 0, 0)
.unwrap();
let result = naive_datetime_to_millis(Some(&dt));
assert!(result.is_some());
let millis = result.unwrap();
assert!(millis > 1_700_000_000_000); assert!(millis < 2_000_000_000_000); }
#[test]
fn test_naive_datetime_to_millis_none() {
assert_eq!(naive_datetime_to_millis(None), None);
}
#[test]
fn test_parse_json_array_valid() {
let json = Some(r#"["item1", "item2", "item3"]"#.to_string());
let result = parse_json_array(json.as_ref());
assert_eq!(
result,
Some(vec![
"item1".to_string(),
"item2".to_string(),
"item3".to_string()
])
);
}
#[test]
fn test_parse_json_array_empty_string() {
let json = Some("".to_string());
assert_eq!(parse_json_array(json.as_ref()), None);
}
#[test]
fn test_parse_json_array_none() {
assert_eq!(parse_json_array(None), None);
}
#[test]
fn test_parse_json_array_invalid_json() {
let json = Some("not valid json".to_string());
assert_eq!(parse_json_array(json.as_ref()), None);
}
#[test]
fn test_parse_json_array_not_array() {
let json = Some(r#"{"key": "value"}"#.to_string());
assert_eq!(parse_json_array(json.as_ref()), None);
}
#[test]
fn test_detect_txt_type_spf() {
assert_eq!(
detect_txt_type("v=spf1 include:_spf.google.com ~all"),
"SPF"
);
assert_eq!(
detect_txt_type("V=SPF1 include:_spf.google.com ~all"),
"SPF"
); }
#[test]
fn test_detect_txt_type_dmarc() {
assert_eq!(detect_txt_type("v=dmarc1; p=none"), "DMARC");
assert_eq!(detect_txt_type("V=DMARC1; p=quarantine"), "DMARC"); }
#[test]
fn test_detect_txt_type_verification_google() {
assert_eq!(
detect_txt_type("google-site-verification=abc123"),
"VERIFICATION"
);
}
#[test]
fn test_detect_txt_type_verification_ms() {
assert_eq!(detect_txt_type("ms-verify=xyz789"), "VERIFICATION");
}
#[test]
fn test_detect_txt_type_verification_facebook() {
assert_eq!(
detect_txt_type("facebook-domain-verification=def456"),
"VERIFICATION"
);
}
#[test]
fn test_detect_txt_type_verification_atlassian() {
assert_eq!(
detect_txt_type("atlassian-domain-verification=ghi012"),
"VERIFICATION"
);
}
#[test]
fn test_detect_txt_type_other() {
assert_eq!(detect_txt_type("some other text record"), "OTHER");
assert!(!crate::dns::is_storable_txt(""));
assert!(!crate::dns::is_storable_txt(" "));
}
#[test]
fn test_parse_mx_record_with_priority() {
assert_eq!(
parse_mx_record("10 mail.example.com"),
Some((10, "mail.example.com".to_string()))
);
assert_eq!(
parse_mx_record("0 mail.example.com"),
Some((0, "mail.example.com".to_string()))
);
assert_eq!(
parse_mx_record("100 smtp.example.com"),
Some((100, "smtp.example.com".to_string()))
);
}
#[test]
fn test_parse_mx_record_without_priority() {
assert_eq!(
parse_mx_record("mail.example.com"),
Some((0, "mail.example.com".to_string()))
);
}
#[test]
fn test_parse_mx_record_multiple_spaces() {
assert_eq!(
parse_mx_record("10 mail.example.com"),
Some((10, "mail.example.com".to_string()))
);
}
#[test]
fn test_parse_mx_record_invalid() {
assert_eq!(parse_mx_record(""), None);
assert_eq!(parse_mx_record("10"), None); }
#[test]
fn test_parse_mx_json_array_objects_format() {
let json = Some(r#"[{"priority": 10, "hostname": "mail1.example.com"}, {"priority": 20, "hostname": "mail2.example.com"}]"#.to_string());
let result = parse_mx_json_array(json.as_ref());
assert_eq!(
result,
Some(vec![
(10, "mail1.example.com".to_string()),
(20, "mail2.example.com".to_string())
])
);
}
#[test]
fn test_parse_mx_json_array_strings_format() {
let json = Some(r#"["10 mail1.example.com", "20 mail2.example.com"]"#.to_string());
let result = parse_mx_json_array(json.as_ref());
assert_eq!(
result,
Some(vec![
(10, "mail1.example.com".to_string()),
(20, "mail2.example.com".to_string())
])
);
}
#[test]
fn test_parse_mx_json_array_strings_without_priority() {
let json = Some(r#"["mail1.example.com", "mail2.example.com"]"#.to_string());
let result = parse_mx_json_array(json.as_ref());
assert_eq!(
result,
Some(vec![
(0, "mail1.example.com".to_string()),
(0, "mail2.example.com".to_string())
])
);
}
#[test]
fn test_parse_mx_json_array_empty() {
assert_eq!(parse_mx_json_array(None), None);
assert_eq!(parse_mx_json_array(Some(&"".to_string())), None);
assert_eq!(parse_mx_json_array(Some(&"[]".to_string())), None);
}
#[test]
fn test_parse_mx_json_array_invalid_json() {
assert_eq!(parse_mx_json_array(Some(&"not json".to_string())), None);
}
#[test]
fn test_parse_mx_json_array_mixed_formats() {
let json = Some(
r#"[{"priority": 10, "hostname": "mail.example.com"}, "20 mail2.example.com"]"#
.to_string(),
);
let result = parse_mx_json_array(json.as_ref());
assert!(result.is_some());
assert_eq!(result.unwrap().len(), 1); }
#[test]
fn test_parse_mx_json_array_incomplete_object() {
let json = Some(r#"[{"priority": 10}, {"hostname": "mail.example.com"}]"#.to_string());
let result = parse_mx_json_array(json.as_ref());
assert_eq!(result, None); }
}