use anyhow::{bail, Context, Result};
use arrow::array::{
ArrayRef, BooleanBuilder, Float64Builder, Int32Builder, Int64Builder, ListBuilder, RecordBatch,
StringBuilder, StructBuilder,
};
use arrow::datatypes::{DataType, Field, Schema};
use parquet::arrow::ArrowWriter;
use parquet::basic::Compression;
use parquet::file::properties::WriterProperties;
use std::fs::File;
use std::sync::Arc;
use crate::utils::IoErrorContext;
use super::bootstrap::for_each_export_row;
const BATCH_SIZE: usize = 10_000;
pub(crate) fn build_schema() -> Schema {
super::fields::build_parquet_schema()
}
fn new_two_string_list_builder(f1: &str, f2: &str) -> ListBuilder<StructBuilder> {
let fields = vec![
Field::new(f1, DataType::Utf8, false),
Field::new(f2, DataType::Utf8, false),
];
let builder = StructBuilder::new(
fields.clone(),
vec![
Box::new(StringBuilder::new()),
Box::new(StringBuilder::new()),
],
);
ListBuilder::new(builder)
}
fn new_three_string_list_builder(f1: &str, f2: &str, f3: &str) -> ListBuilder<StructBuilder> {
let fields = vec![
Field::new(f1, DataType::Utf8, false),
Field::new(f2, DataType::Utf8, false),
Field::new(f3, DataType::Utf8, false),
];
let builder = StructBuilder::new(
fields.clone(),
vec![
Box::new(StringBuilder::new()),
Box::new(StringBuilder::new()),
Box::new(StringBuilder::new()),
],
);
ListBuilder::new(builder)
}
use super::row::ExportRow;
fn append_opt_str(builder: &mut StringBuilder, value: Option<&String>) {
match value {
Some(v) => builder.append_value(v),
None => builder.append_null(),
}
}
fn append_opt_i64(builder: &mut Int64Builder, value: Option<i64>) {
match value {
Some(v) => builder.append_value(v),
None => builder.append_null(),
}
}
fn append_opt_f64(builder: &mut Float64Builder, value: Option<f64>) {
match value {
Some(v) => builder.append_value(v),
None => builder.append_null(),
}
}
fn append_opt_i32(builder: &mut Int32Builder, value: Option<i32>) {
match value {
Some(v) => builder.append_value(v),
None => builder.append_null(),
}
}
#[allow(clippy::too_many_lines, clippy::cognitive_complexity)] fn write_batch(
writer: &mut ArrowWriter<File>,
schema: &Arc<Schema>,
rows: &[ExportRow],
) -> Result<()> {
let mut url_b = StringBuilder::new();
let mut initial_domain_b = StringBuilder::new();
let mut final_domain_b = StringBuilder::new();
let mut initial_url_b = StringBuilder::new();
let mut final_url_b = StringBuilder::new();
let mut ip_address_b = StringBuilder::new();
let mut reverse_dns_b = StringBuilder::new();
let mut http_status_b = Int32Builder::new();
let mut http_status_text_b = StringBuilder::new();
let mut response_time_b = Float64Builder::new();
let mut title_b = StringBuilder::new();
let mut description_b = StringBuilder::new();
let mut meta_robots_b = StringBuilder::new();
let mut body_sha256_b = StringBuilder::new();
let mut content_length_b = Int64Builder::new();
let mut http_version_b = StringBuilder::new();
let mut content_type_b = StringBuilder::new();
let mut canonical_url_b = StringBuilder::new();
let mut body_truncated_b = BooleanBuilder::new();
let mut redirect_count_b = Int32Builder::new();
let mut final_redirect_url_b = StringBuilder::new();
let mut technology_count_b = Int32Builder::new();
let mut tls_version_b = StringBuilder::new();
let mut ssl_cert_subject_b = StringBuilder::new();
let mut ssl_cert_issuer_b = StringBuilder::new();
let mut ssl_cert_valid_to_b = Int64Builder::new();
let mut cipher_suite_b = StringBuilder::new();
let mut key_algorithm_b = StringBuilder::new();
let mut cert_fingerprint_sha256_b = StringBuilder::new();
let mut cname_records_b = ListBuilder::new(StringBuilder::new());
let mut ipv6_addresses_b = ListBuilder::new(StringBuilder::new());
let mut caa_records_b = ListBuilder::new(StructBuilder::from_fields(
vec![
Field::new("flag", DataType::Int64, false),
Field::new("tag", DataType::Utf8, false),
Field::new("value", DataType::Utf8, false),
],
0,
));
let mut spf_record_b = StringBuilder::new();
let mut dmarc_record_b = StringBuilder::new();
let mut mta_sts_record_b = StringBuilder::new();
let mut tls_rpt_record_b = StringBuilder::new();
let mut bimi_record_b = StringBuilder::new();
let mut hsts_max_age_b = Int64Builder::new();
let mut hsts_include_subdomains_b = arrow::array::BooleanBuilder::new();
let mut hsts_preload_b = arrow::array::BooleanBuilder::new();
let mut cdn_provider_b = StringBuilder::new();
let mut script_hosts_b = ListBuilder::new(StructBuilder::from_fields(
vec![
Field::new("host", DataType::Utf8, false),
Field::new("registrable_domain", DataType::Utf8, true),
Field::new("is_first_party", DataType::Boolean, false),
],
0,
));
let mut geoip_cc_b = StringBuilder::new();
let mut geoip_cn_b = StringBuilder::new();
let mut geoip_region_b = StringBuilder::new();
let mut geoip_city_b = StringBuilder::new();
let mut geoip_lat_b = Float64Builder::new();
let mut geoip_lon_b = Float64Builder::new();
let mut geoip_asn_b = Int64Builder::new();
let mut geoip_asn_org_b = StringBuilder::new();
let mut whois_registrar_b = StringBuilder::new();
let mut whois_creation_b = Int64Builder::new();
let mut whois_expiration_b = Int64Builder::new();
let mut whois_country_b = StringBuilder::new();
let mut favicon_hash_b = Int32Builder::new();
let mut favicon_url_b = StringBuilder::new();
let mut contact_link_count_b = Int32Builder::new();
let mut contact_links_b = new_two_string_list_builder("contact_type", "contact_value");
let mut exposed_secret_count_b = Int32Builder::new();
let mut exposed_secrets_b = {
let fields = vec![
Field::new("secret_type", DataType::Utf8, false),
Field::new("matched_value", DataType::Utf8, false),
Field::new("severity", DataType::Utf8, false),
Field::new("location", DataType::Utf8, false),
Field::new("context", DataType::Utf8, true),
Field::new("jwt_algorithm", DataType::Utf8, true),
Field::new("jwt_issuer", DataType::Utf8, true),
Field::new("jwt_subject", DataType::Utf8, true),
Field::new("jwt_audience", DataType::Utf8, true),
Field::new("jwt_expiration_ms", DataType::Int64, true),
Field::new("jwt_issued_at_ms", DataType::Int64, true),
Field::new("jwt_header_json", DataType::Utf8, true),
Field::new("jwt_payload_json", DataType::Utf8, true),
];
let builder = StructBuilder::new(
fields.clone(),
vec![
Box::new(StringBuilder::new()), Box::new(StringBuilder::new()), Box::new(StringBuilder::new()), Box::new(StringBuilder::new()), Box::new(StringBuilder::new()), Box::new(StringBuilder::new()), Box::new(StringBuilder::new()), Box::new(StringBuilder::new()), Box::new(StringBuilder::new()), Box::new(Int64Builder::new()), Box::new(Int64Builder::new()), Box::new(StringBuilder::new()), Box::new(StringBuilder::new()), ],
);
ListBuilder::new(builder)
};
let mut observed_at_b = Int64Builder::new();
let mut run_id_b = StringBuilder::new();
let redirect_struct_fields = vec![
Field::new("redirect_url", DataType::Utf8, false),
Field::new("sequence_order", DataType::Int64, false),
];
let redirect_struct_builder = StructBuilder::new(
redirect_struct_fields,
vec![
Box::new(StringBuilder::new()),
Box::new(Int64Builder::new()),
],
);
let mut redirect_chain_b = ListBuilder::new(redirect_struct_builder);
let tech_fields = vec![
Field::new("name", DataType::Utf8, false),
Field::new("version", DataType::Utf8, true),
Field::new("category", DataType::Utf8, true),
Field::new("is_implied", DataType::Boolean, false),
Field::new("detection_source", DataType::Utf8, true),
];
let tech_struct_builder = StructBuilder::new(
tech_fields,
vec![
Box::new(StringBuilder::new()),
Box::new(StringBuilder::new()),
Box::new(StringBuilder::new()),
Box::new(BooleanBuilder::new()),
Box::new(StringBuilder::new()),
],
);
let mut technologies_b = ListBuilder::new(tech_struct_builder);
let mut cert_sans_b = ListBuilder::new(StringBuilder::new());
let mut cert_oids_b = ListBuilder::new(StringBuilder::new());
let mut nameservers_b = ListBuilder::new(StringBuilder::new());
let mut txt_records_b = new_two_string_list_builder("record_type", "content");
let mx_fields = vec![
Field::new("priority", DataType::Int64, false),
Field::new("mail_exchange", DataType::Utf8, false),
];
let mx_struct_builder = StructBuilder::new(
mx_fields,
vec![
Box::new(Int64Builder::new()),
Box::new(StringBuilder::new()),
],
);
let mut mx_records_b = ListBuilder::new(mx_struct_builder);
let mut analytics_ids_b = new_two_string_list_builder("provider", "tracking_id");
let social_fields = vec![
Field::new("platform", DataType::Utf8, false),
Field::new("url", DataType::Utf8, false),
Field::new("identifier", DataType::Utf8, true),
];
let social_struct_builder = StructBuilder::new(
social_fields,
vec![
Box::new(StringBuilder::new()),
Box::new(StringBuilder::new()),
Box::new(StringBuilder::new()),
],
);
let mut social_media_b = ListBuilder::new(social_struct_builder);
let mut structured_data_b =
new_three_string_list_builder("data_type", "property_name", "property_value");
let mut http_headers_b = new_two_string_list_builder("name", "value");
let mut security_headers_b = new_two_string_list_builder("name", "value");
let mut partial_failures_b = new_two_string_list_builder("error_type", "error_message");
for row in rows {
let url = super::row::build_url(&row.main.final_domain);
url_b.append_value(&url);
initial_domain_b.append_value(&row.main.initial_domain);
final_domain_b.append_value(&row.main.final_domain);
append_opt_str(&mut initial_url_b, row.main.initial_url.as_ref());
append_opt_str(&mut final_url_b, row.main.final_url.as_ref());
ip_address_b.append_value(&row.main.ip_address);
append_opt_str(&mut reverse_dns_b, row.main.reverse_dns.as_ref());
#[allow(clippy::cast_possible_wrap)]
http_status_b.append_value(i32::from(row.main.status));
http_status_text_b.append_value(&row.main.status_desc);
response_time_b.append_value(row.main.response_time);
title_b.append_value(&row.main.title);
append_opt_str(&mut description_b, row.main.description.as_ref());
append_opt_str(&mut meta_robots_b, row.main.meta_robots.as_ref());
append_opt_str(&mut body_sha256_b, row.main.body_sha256.as_ref());
append_opt_i64(&mut content_length_b, row.main.content_length);
append_opt_str(&mut http_version_b, row.main.http_version.as_ref());
append_opt_str(&mut content_type_b, row.main.content_type.as_ref());
append_opt_str(&mut canonical_url_b, row.main.canonical_url.as_ref());
body_truncated_b.append_value(row.main.body_truncated);
#[allow(clippy::cast_possible_truncation, clippy::cast_possible_wrap)]
redirect_count_b.append_value(row.redirect_count as i32);
final_redirect_url_b.append_value(&row.final_redirect_url);
for r in &row.redirects {
redirect_chain_b
.values()
.field_builder::<StringBuilder>(0)
.unwrap()
.append_value(&r.redirect_url);
redirect_chain_b
.values()
.field_builder::<Int64Builder>(1)
.unwrap()
.append_value(r.sequence_order);
redirect_chain_b.values().append(true);
}
redirect_chain_b.append(true);
#[allow(clippy::cast_possible_truncation, clippy::cast_possible_wrap)]
technology_count_b.append_value(row.technology_count as i32);
for tech in &row.technologies {
technologies_b
.values()
.field_builder::<StringBuilder>(0)
.unwrap()
.append_value(&tech.name);
append_opt_str(
technologies_b
.values()
.field_builder::<StringBuilder>(1)
.unwrap(),
tech.version.as_ref(),
);
append_opt_str(
technologies_b
.values()
.field_builder::<StringBuilder>(2)
.unwrap(),
tech.category.as_ref(),
);
technologies_b
.values()
.field_builder::<BooleanBuilder>(3)
.unwrap()
.append_value(tech.is_implied);
append_opt_str(
technologies_b
.values()
.field_builder::<StringBuilder>(4)
.unwrap(),
tech.detection_source.as_ref(),
);
technologies_b.values().append(true);
}
technologies_b.append(true);
append_opt_str(&mut tls_version_b, row.main.tls_version.as_ref());
append_opt_str(&mut ssl_cert_subject_b, row.main.ssl_cert_subject.as_ref());
append_opt_str(&mut ssl_cert_issuer_b, row.main.ssl_cert_issuer.as_ref());
append_opt_i64(&mut ssl_cert_valid_to_b, row.main.ssl_cert_valid_to_ms);
append_opt_str(&mut cipher_suite_b, row.main.cipher_suite.as_ref());
append_opt_str(&mut key_algorithm_b, row.main.key_algorithm.as_ref());
append_opt_str(
&mut cert_fingerprint_sha256_b,
row.main.cert_fingerprint_sha256.as_ref(),
);
for san in super::row::parse_string_list(&row.certificate_sans_str) {
cert_sans_b.values().append_value(&san);
}
cert_sans_b.append(true);
for oid in super::row::parse_string_list(&row.oids_str) {
cert_oids_b.values().append_value(&oid);
}
cert_oids_b.append(true);
for cname in &row.cname_records {
cname_records_b.values().append_value(cname);
}
cname_records_b.append(true);
for addr in &row.ipv6_addresses {
ipv6_addresses_b.values().append_value(addr);
}
ipv6_addresses_b.append(true);
for caa in &row.caa_records {
caa_records_b
.values()
.field_builder::<Int64Builder>(0)
.unwrap()
.append_value(caa.flag);
caa_records_b
.values()
.field_builder::<StringBuilder>(1)
.unwrap()
.append_value(&caa.tag);
caa_records_b
.values()
.field_builder::<StringBuilder>(2)
.unwrap()
.append_value(&caa.value);
caa_records_b.values().append(true);
}
caa_records_b.append(true);
append_opt_str(&mut spf_record_b, row.main.spf_record.as_ref());
append_opt_str(&mut dmarc_record_b, row.main.dmarc_record.as_ref());
append_opt_str(&mut mta_sts_record_b, row.main.mta_sts_record.as_ref());
append_opt_str(&mut tls_rpt_record_b, row.main.tls_rpt_record.as_ref());
append_opt_str(&mut bimi_record_b, row.main.bimi_record.as_ref());
match row.main.hsts_max_age {
Some(v) => hsts_max_age_b.append_value(v),
None => hsts_max_age_b.append_null(),
}
match row.main.hsts_include_subdomains {
Some(v) => hsts_include_subdomains_b.append_value(v),
None => hsts_include_subdomains_b.append_null(),
}
match row.main.hsts_preload {
Some(v) => hsts_preload_b.append_value(v),
None => hsts_preload_b.append_null(),
}
append_opt_str(&mut cdn_provider_b, row.main.cdn_provider.as_ref());
for host in &row.script_hosts {
script_hosts_b
.values()
.field_builder::<StringBuilder>(0)
.unwrap()
.append_value(&host.host);
match &host.registrable_domain {
Some(d) => script_hosts_b
.values()
.field_builder::<StringBuilder>(1)
.unwrap()
.append_value(d),
None => script_hosts_b
.values()
.field_builder::<StringBuilder>(1)
.unwrap()
.append_null(),
}
script_hosts_b
.values()
.field_builder::<arrow::array::BooleanBuilder>(2)
.unwrap()
.append_value(host.is_first_party);
script_hosts_b.values().append(true);
}
script_hosts_b.append(true);
for ns in &row.nameservers {
nameservers_b.values().append_value(&ns.nameserver);
}
nameservers_b.append(true);
for txt in &row.txt_records {
txt_records_b
.values()
.field_builder::<StringBuilder>(0)
.unwrap()
.append_value(&txt.record_type);
txt_records_b
.values()
.field_builder::<StringBuilder>(1)
.unwrap()
.append_value(&txt.record_value);
txt_records_b.values().append(true);
}
txt_records_b.append(true);
for mx in &row.mx_records {
mx_records_b
.values()
.field_builder::<Int64Builder>(0)
.unwrap()
.append_value(mx.priority);
mx_records_b
.values()
.field_builder::<StringBuilder>(1)
.unwrap()
.append_value(&mx.mail_exchange);
mx_records_b.values().append(true);
}
mx_records_b.append(true);
for aid in &row.analytics_ids {
analytics_ids_b
.values()
.field_builder::<StringBuilder>(0)
.unwrap()
.append_value(&aid.provider);
analytics_ids_b
.values()
.field_builder::<StringBuilder>(1)
.unwrap()
.append_value(&aid.tracking_id);
analytics_ids_b.values().append(true);
}
analytics_ids_b.append(true);
for link in &row.social_media_links {
social_media_b
.values()
.field_builder::<StringBuilder>(0)
.unwrap()
.append_value(&link.platform);
social_media_b
.values()
.field_builder::<StringBuilder>(1)
.unwrap()
.append_value(&link.profile_url);
append_opt_str(
social_media_b
.values()
.field_builder::<StringBuilder>(2)
.unwrap(),
link.identifier.as_ref(),
);
social_media_b.values().append(true);
}
social_media_b.append(true);
for entry in &row.structured_data_entries {
structured_data_b
.values()
.field_builder::<StringBuilder>(0)
.unwrap()
.append_value(&entry.data_type);
structured_data_b
.values()
.field_builder::<StringBuilder>(1)
.unwrap()
.append_value(&entry.property_name);
structured_data_b
.values()
.field_builder::<StringBuilder>(2)
.unwrap()
.append_value(&entry.property_value);
structured_data_b.values().append(true);
}
structured_data_b.append(true);
for h in &row.all_http_headers {
http_headers_b
.values()
.field_builder::<StringBuilder>(0)
.unwrap()
.append_value(&h.name);
http_headers_b
.values()
.field_builder::<StringBuilder>(1)
.unwrap()
.append_value(&h.value);
http_headers_b.values().append(true);
}
http_headers_b.append(true);
for h in &row.all_security_headers {
security_headers_b
.values()
.field_builder::<StringBuilder>(0)
.unwrap()
.append_value(&h.name);
security_headers_b
.values()
.field_builder::<StringBuilder>(1)
.unwrap()
.append_value(&h.value);
security_headers_b.values().append(true);
}
security_headers_b.append(true);
for f in &row.partial_failures {
partial_failures_b
.values()
.field_builder::<StringBuilder>(0)
.unwrap()
.append_value(&f.error_type);
partial_failures_b
.values()
.field_builder::<StringBuilder>(1)
.unwrap()
.append_value(&f.error_message);
partial_failures_b.values().append(true);
}
partial_failures_b.append(true);
append_opt_str(&mut geoip_cc_b, row.geoip.country_code.as_ref());
append_opt_str(&mut geoip_cn_b, row.geoip.country_name.as_ref());
append_opt_str(&mut geoip_region_b, row.geoip.region.as_ref());
append_opt_str(&mut geoip_city_b, row.geoip.city.as_ref());
append_opt_f64(&mut geoip_lat_b, row.geoip.latitude);
append_opt_f64(&mut geoip_lon_b, row.geoip.longitude);
append_opt_i64(&mut geoip_asn_b, row.geoip.asn);
append_opt_str(&mut geoip_asn_org_b, row.geoip.asn_org.as_ref());
append_opt_str(&mut whois_registrar_b, row.whois.registrar.as_ref());
append_opt_i64(&mut whois_creation_b, row.whois.creation_date_ms);
append_opt_i64(&mut whois_expiration_b, row.whois.expiration_date_ms);
append_opt_str(&mut whois_country_b, row.whois.registrant_country.as_ref());
append_opt_i32(&mut favicon_hash_b, row.favicon_hash);
append_opt_str(&mut favicon_url_b, row.favicon_url.as_ref());
#[allow(clippy::cast_possible_truncation, clippy::cast_possible_wrap)]
contact_link_count_b.append_value(row.contact_link_count as i32);
for c in &row.contact_links {
contact_links_b
.values()
.field_builder::<StringBuilder>(0)
.unwrap()
.append_value(&c.contact_type);
contact_links_b
.values()
.field_builder::<StringBuilder>(1)
.unwrap()
.append_value(&c.contact_value);
contact_links_b.values().append(true);
}
contact_links_b.append(true);
#[allow(clippy::cast_possible_truncation, clippy::cast_possible_wrap)]
exposed_secret_count_b.append_value(row.exposed_secret_count as i32);
for s in &row.exposed_secrets {
exposed_secrets_b
.values()
.field_builder::<StringBuilder>(0)
.unwrap()
.append_value(&s.secret_type);
exposed_secrets_b
.values()
.field_builder::<StringBuilder>(1)
.unwrap()
.append_value(&s.matched_value);
exposed_secrets_b
.values()
.field_builder::<StringBuilder>(2)
.unwrap()
.append_value(&s.severity);
exposed_secrets_b
.values()
.field_builder::<StringBuilder>(3)
.unwrap()
.append_value(&s.location);
append_opt_str(
exposed_secrets_b
.values()
.field_builder::<StringBuilder>(4)
.unwrap(),
s.context.as_ref(),
);
append_opt_str(
exposed_secrets_b
.values()
.field_builder::<StringBuilder>(5)
.unwrap(),
s.jwt_algorithm.as_ref(),
);
append_opt_str(
exposed_secrets_b
.values()
.field_builder::<StringBuilder>(6)
.unwrap(),
s.jwt_issuer.as_ref(),
);
append_opt_str(
exposed_secrets_b
.values()
.field_builder::<StringBuilder>(7)
.unwrap(),
s.jwt_subject.as_ref(),
);
append_opt_str(
exposed_secrets_b
.values()
.field_builder::<StringBuilder>(8)
.unwrap(),
s.jwt_audience.as_ref(),
);
append_opt_i64(
exposed_secrets_b
.values()
.field_builder::<Int64Builder>(9)
.unwrap(),
s.jwt_expiration_ms,
);
append_opt_i64(
exposed_secrets_b
.values()
.field_builder::<Int64Builder>(10)
.unwrap(),
s.jwt_issued_at_ms,
);
append_opt_str(
exposed_secrets_b
.values()
.field_builder::<StringBuilder>(11)
.unwrap(),
s.jwt_header_json.as_ref(),
);
append_opt_str(
exposed_secrets_b
.values()
.field_builder::<StringBuilder>(12)
.unwrap(),
s.jwt_payload_json.as_ref(),
);
exposed_secrets_b.values().append(true);
}
exposed_secrets_b.append(true);
observed_at_b.append_value(row.main.timestamp);
append_opt_str(&mut run_id_b, row.main.run_id.as_ref());
}
let mut finished: Vec<ArrayRef> = Vec::with_capacity(75);
finished.push(Arc::new(url_b.finish()));
finished.push(Arc::new(initial_domain_b.finish()));
finished.push(Arc::new(final_domain_b.finish()));
finished.push(Arc::new(initial_url_b.finish()));
finished.push(Arc::new(final_url_b.finish()));
finished.push(Arc::new(ip_address_b.finish()));
finished.push(Arc::new(reverse_dns_b.finish()));
finished.push(Arc::new(http_status_b.finish()));
finished.push(Arc::new(http_status_text_b.finish()));
finished.push(Arc::new(response_time_b.finish()));
finished.push(Arc::new(title_b.finish()));
finished.push(Arc::new(description_b.finish()));
finished.push(Arc::new(meta_robots_b.finish()));
finished.push(Arc::new(body_sha256_b.finish()));
finished.push(Arc::new(content_length_b.finish()));
finished.push(Arc::new(http_version_b.finish()));
finished.push(Arc::new(content_type_b.finish()));
finished.push(Arc::new(canonical_url_b.finish()));
finished.push(Arc::new(body_truncated_b.finish()));
finished.push(Arc::new(redirect_count_b.finish()));
finished.push(Arc::new(final_redirect_url_b.finish()));
finished.push(Arc::new(redirect_chain_b.finish()));
finished.push(Arc::new(technology_count_b.finish()));
finished.push(Arc::new(technologies_b.finish()));
finished.push(Arc::new(tls_version_b.finish()));
finished.push(Arc::new(ssl_cert_subject_b.finish()));
finished.push(Arc::new(ssl_cert_issuer_b.finish()));
finished.push(Arc::new(ssl_cert_valid_to_b.finish()));
finished.push(Arc::new(cipher_suite_b.finish()));
finished.push(Arc::new(key_algorithm_b.finish()));
finished.push(Arc::new(cert_fingerprint_sha256_b.finish()));
finished.push(Arc::new(cert_sans_b.finish()));
finished.push(Arc::new(cert_oids_b.finish()));
finished.push(Arc::new(cname_records_b.finish()));
finished.push(Arc::new(ipv6_addresses_b.finish()));
finished.push(Arc::new(caa_records_b.finish()));
finished.push(Arc::new(spf_record_b.finish()));
finished.push(Arc::new(dmarc_record_b.finish()));
finished.push(Arc::new(mta_sts_record_b.finish()));
finished.push(Arc::new(tls_rpt_record_b.finish()));
finished.push(Arc::new(bimi_record_b.finish()));
finished.push(Arc::new(hsts_max_age_b.finish()));
finished.push(Arc::new(hsts_include_subdomains_b.finish()));
finished.push(Arc::new(hsts_preload_b.finish()));
finished.push(Arc::new(cdn_provider_b.finish()));
finished.push(Arc::new(script_hosts_b.finish()));
finished.push(Arc::new(nameservers_b.finish()));
finished.push(Arc::new(txt_records_b.finish()));
finished.push(Arc::new(mx_records_b.finish()));
finished.push(Arc::new(analytics_ids_b.finish()));
finished.push(Arc::new(social_media_b.finish()));
finished.push(Arc::new(structured_data_b.finish()));
finished.push(Arc::new(http_headers_b.finish()));
finished.push(Arc::new(security_headers_b.finish()));
finished.push(Arc::new(partial_failures_b.finish()));
finished.push(Arc::new(geoip_cc_b.finish()));
finished.push(Arc::new(geoip_cn_b.finish()));
finished.push(Arc::new(geoip_region_b.finish()));
finished.push(Arc::new(geoip_city_b.finish()));
finished.push(Arc::new(geoip_lat_b.finish()));
finished.push(Arc::new(geoip_lon_b.finish()));
finished.push(Arc::new(geoip_asn_b.finish()));
finished.push(Arc::new(geoip_asn_org_b.finish()));
finished.push(Arc::new(whois_registrar_b.finish()));
finished.push(Arc::new(whois_creation_b.finish()));
finished.push(Arc::new(whois_expiration_b.finish()));
finished.push(Arc::new(whois_country_b.finish()));
finished.push(Arc::new(favicon_hash_b.finish()));
finished.push(Arc::new(favicon_url_b.finish()));
finished.push(Arc::new(contact_link_count_b.finish()));
finished.push(Arc::new(contact_links_b.finish()));
finished.push(Arc::new(exposed_secret_count_b.finish()));
finished.push(Arc::new(exposed_secrets_b.finish()));
finished.push(Arc::new(observed_at_b.finish()));
finished.push(Arc::new(run_id_b.finish()));
let mut by_name: std::collections::HashMap<&'static str, ArrayRef> =
std::collections::HashMap::with_capacity(75);
for (name, array) in super::fields::parquet_column_names().zip(finished.drain(..)) {
by_name.insert(name, array);
}
let columns: Vec<ArrayRef> = super::fields::parquet_column_names()
.map(|name| {
by_name
.remove(name)
.unwrap_or_else(|| panic!("missing Parquet column builder for {name}"))
})
.collect();
anyhow::ensure!(
columns.len() == schema.fields().len(),
"Parquet columns ({}) must match schema fields ({})",
columns.len(),
schema.fields().len()
);
let batch = RecordBatch::try_new(Arc::clone(schema), columns)
.context("Failed to create RecordBatch")?;
writer
.write(&batch)
.context("Failed to write RecordBatch to Parquet")?;
Ok(())
}
pub async fn export_parquet(opts: &super::ExportOptions) -> Result<usize> {
let output_path = opts.output.as_ref().ok_or_else(|| {
anyhow::anyhow!(
"Parquet is a binary format and cannot be written to stdout. Use --output <path> to specify a file."
)
})?;
if output_path.as_os_str() == "-" {
bail!("Parquet is a binary format and cannot be written to stdout. Use --output <path> to specify a file.");
}
let schema = Arc::new(build_schema());
let props = WriterProperties::builder()
.set_compression(Compression::SNAPPY)
.build();
let file = File::create(output_path).with_path(output_path)?;
let mut writer = ArrowWriter::try_new(file, Arc::clone(&schema), Some(props))
.context("Failed to create Parquet writer")?;
let mut batch_rows: Vec<super::row::ExportRow> = Vec::with_capacity(BATCH_SIZE);
let record_count = for_each_export_row(opts, |export_row| {
batch_rows.push(export_row);
if batch_rows.len() >= BATCH_SIZE {
write_batch(&mut writer, &schema, &batch_rows)?;
batch_rows.clear();
}
Ok(())
})
.await?;
if !batch_rows.is_empty() {
write_batch(&mut writer, &schema, &batch_rows)?;
}
writer.close().context("Failed to finalize Parquet file")?;
Ok(record_count)
}
#[cfg(test)]
mod tests {
use super::super::fields::{field_by_id, PARQUET_FIELD_ORDER};
use super::super::types::{ExportFormat, ExportOptions};
use super::{build_schema, export_parquet};
use crate::storage::migrations::run_migrations;
use arrow::array::{
Array, BooleanArray, Int32Array, Int64Array, ListArray, RecordBatch, StringArray,
StructArray,
};
use parquet::arrow::arrow_reader::ParquetRecordBatchReaderBuilder;
use sqlx::{Row, SqlitePool};
use std::path::{Path, PathBuf};
use tempfile::NamedTempFile;
async fn create_test_run(pool: &SqlitePool, run_id: &str) {
sqlx::query(
"INSERT INTO runs (run_id, start_time_ms) VALUES (?, ?)
ON CONFLICT(run_id) DO NOTHING",
)
.bind(run_id)
.bind(1704067200000i64)
.execute(pool)
.await
.expect("Failed to insert test run");
}
async fn create_test_url_status(pool: &SqlitePool, domain: &str) -> i64 {
create_test_run(pool, "test-run-1").await;
sqlx::query(
"INSERT INTO url_status (
initial_domain, final_domain, ip_address, http_status, http_status_text,
response_time_seconds, title, observed_at_ms, run_id
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
RETURNING id",
)
.bind(domain)
.bind(domain)
.bind("192.0.2.1")
.bind(200)
.bind("OK")
.bind(1.5f64)
.bind("Test Page")
.bind(1704067200000i64)
.bind("test-run-1")
.fetch_one(pool)
.await
.expect("Failed to insert test URL status")
.get::<i64, _>(0)
}
#[tokio::test]
async fn test_parquet_export_rejects_stdout() {
let temp_db = NamedTempFile::new().expect("temp DB");
let db_path = temp_db.path();
let pool = SqlitePool::connect(&format!("sqlite:{}", db_path.display()))
.await
.unwrap();
run_migrations(&pool).await.unwrap();
drop(pool);
let result = export_parquet(&ExportOptions {
db_path: db_path.to_path_buf(),
output: None, format: ExportFormat::Parquet,
run_id: None,
domain: None,
status: None,
since: None,
include_implied_tech: false,
})
.await;
assert!(result.is_err(), "Parquet to stdout should fail");
assert!(
result.unwrap_err().to_string().contains("binary format"),
"Error should mention binary format"
);
}
#[tokio::test]
async fn test_parquet_export_rejects_dash_stdout() {
let temp_db = NamedTempFile::new().expect("temp DB");
let db_path = temp_db.path();
let pool = SqlitePool::connect(&format!("sqlite:{}", db_path.display()))
.await
.unwrap();
run_migrations(&pool).await.unwrap();
drop(pool);
let result = export_parquet(&ExportOptions {
db_path: db_path.to_path_buf(),
output: Some(PathBuf::from("-")),
format: ExportFormat::Parquet,
run_id: None,
domain: None,
status: None,
since: None,
include_implied_tech: false,
})
.await;
assert!(result.is_err(), "Parquet to '-' (stdout) should fail");
}
#[tokio::test]
async fn test_parquet_export_empty_database() {
let temp_db = NamedTempFile::new().expect("temp DB");
let db_path = temp_db.path();
let pool = SqlitePool::connect(&format!("sqlite:{}", db_path.display()))
.await
.unwrap();
run_migrations(&pool).await.unwrap();
drop(pool);
let temp_file = NamedTempFile::new().expect("temp output");
let output_path = temp_file.path().to_path_buf();
let count = export_parquet(&ExportOptions {
db_path: db_path.to_path_buf(),
output: Some(output_path.clone()),
format: ExportFormat::Parquet,
run_id: None,
domain: None,
status: None,
since: None,
include_implied_tech: false,
})
.await
.expect("Should succeed on empty database");
assert_eq!(count, 0, "Should export 0 records");
let file = std::fs::File::open(&output_path).expect("open");
let reader =
ParquetRecordBatchReaderBuilder::try_new(file).expect("Should parse as valid Parquet");
let schema = reader.schema();
assert!(
schema.field_with_name("url").is_ok(),
"Schema should contain 'url' column"
);
assert!(
schema.field_with_name("nameservers").is_ok(),
"Schema should contain 'nameservers' column"
);
}
#[tokio::test]
async fn test_parquet_export_round_trip() {
let temp_db = NamedTempFile::new().expect("temp DB");
let db_path = temp_db.path();
let pool = SqlitePool::connect(&format!("sqlite:{}", db_path.display()))
.await
.unwrap();
run_migrations(&pool).await.unwrap();
let url_id = create_test_url_status(&pool, "example.com").await;
sqlx::query("INSERT INTO url_nameservers (url_status_id, nameserver) VALUES (?, ?)")
.bind(url_id)
.bind("ns1.example.com")
.execute(&pool)
.await
.unwrap();
sqlx::query(
"INSERT INTO url_technologies (url_status_id, technology_name, technology_version) VALUES (?, ?, ?)",
)
.bind(url_id)
.bind("WordPress")
.bind(Some("6.8"))
.execute(&pool)
.await
.unwrap();
sqlx::query(
"INSERT INTO url_partial_failures (url_status_id, error_type, error_message, observed_at_ms, run_id) VALUES (?, ?, ?, ?, ?)",
)
.bind(url_id)
.bind("DNS error")
.bind("NS timeout")
.bind(1704067200000i64)
.bind("test-run-1")
.execute(&pool)
.await
.unwrap();
drop(pool);
let temp_file = NamedTempFile::new().expect("temp output");
let output_path = temp_file.path().to_path_buf();
let count = export_parquet(&ExportOptions {
db_path: db_path.to_path_buf(),
output: Some(output_path.clone()),
format: ExportFormat::Parquet,
run_id: None,
domain: None,
status: None,
since: None,
include_implied_tech: false,
})
.await
.expect("Should export successfully");
assert_eq!(count, 1, "Should export 1 record");
let file = std::fs::File::open(&output_path).expect("open");
let builder = ParquetRecordBatchReaderBuilder::try_new(file).expect("parse Parquet");
let mut reader = builder.build().expect("build reader");
let batch = reader
.next()
.expect("Should have a batch")
.expect("batch ok");
assert_eq!(batch.num_rows(), 1);
assert_eq!(
batch.num_columns(),
build_schema().fields().len(),
"Parquet file column count must match build_schema()"
);
let url_col = batch.column_by_name("url").expect("Should have url column");
let url_arr = url_col
.as_any()
.downcast_ref::<arrow::array::StringArray>()
.expect("url should be StringArray");
assert_eq!(url_arr.value(0), "https://example.com");
let tls_col = batch
.column_by_name("tls_version")
.expect("Should have tls_version column");
let tls_arr = tls_col
.as_any()
.downcast_ref::<arrow::array::StringArray>()
.expect("tls_version should be StringArray");
assert!(
tls_arr.is_null(0),
"tls_version should be null for our test data"
);
let body_truncated_col = batch
.column_by_name("body_truncated")
.expect("Should have body_truncated column");
let body_truncated_arr = body_truncated_col
.as_any()
.downcast_ref::<arrow::array::BooleanArray>()
.expect("body_truncated should be BooleanArray");
assert!(
!body_truncated_arr.value(0),
"body_truncated should default to false"
);
}
#[tokio::test]
async fn test_parquet_export_with_filter() {
let temp_db = NamedTempFile::new().expect("temp DB");
let db_path = temp_db.path();
let pool = SqlitePool::connect(&format!("sqlite:{}", db_path.display()))
.await
.unwrap();
run_migrations(&pool).await.unwrap();
create_test_url_status(&pool, "example.com").await;
create_test_url_status(&pool, "other.com").await;
drop(pool);
let temp_file = NamedTempFile::new().expect("temp output");
let output_path = temp_file.path().to_path_buf();
let count = export_parquet(&ExportOptions {
db_path: db_path.to_path_buf(),
output: Some(output_path.clone()),
format: ExportFormat::Parquet,
run_id: None,
domain: Some("example.com".to_string()),
status: None,
since: None,
include_implied_tech: false,
})
.await
.expect("Should export with filter");
assert_eq!(count, 1, "Filter should select only 1 record");
}
fn parquet_tech_flags(path: &Path) -> Vec<(String, bool)> {
let file = std::fs::File::open(path).expect("open");
let builder = ParquetRecordBatchReaderBuilder::try_new(file).expect("parse Parquet");
let mut reader = builder.build().expect("build reader");
let batch = reader.next().expect("batch").expect("batch ok");
let col = batch
.column_by_name("technologies")
.expect("technologies column");
let list = col
.as_any()
.downcast_ref::<ListArray>()
.expect("technologies should be ListArray");
let values = list.value(0);
if values.is_empty() {
return Vec::new();
}
let st = values
.as_any()
.downcast_ref::<StructArray>()
.expect("list values should be StructArray");
let names = st
.column(0)
.as_any()
.downcast_ref::<StringArray>()
.expect("name");
let implied = st
.column(3)
.as_any()
.downcast_ref::<BooleanArray>()
.expect("is_implied");
(0..st.len())
.map(|i| (names.value(i).to_string(), implied.value(i)))
.collect()
}
#[tokio::test]
async fn test_parquet_include_implied_tech_on_off() {
let temp_db = NamedTempFile::new().expect("temp DB");
let db_path = temp_db.path();
let pool = SqlitePool::connect(&format!("sqlite:{}", db_path.display()))
.await
.unwrap();
run_migrations(&pool).await.unwrap();
let url_id = create_test_url_status(&pool, "implied.com").await;
sqlx::query(
"INSERT INTO url_technologies (url_status_id, technology_name, is_implied) VALUES (?, ?, ?)",
)
.bind(url_id)
.bind("nginx")
.bind(0i64)
.execute(&pool)
.await
.unwrap();
sqlx::query(
"INSERT INTO url_technologies (url_status_id, technology_name, is_implied) VALUES (?, ?, ?)",
)
.bind(url_id)
.bind("PHP")
.bind(1i64)
.execute(&pool)
.await
.unwrap();
drop(pool);
let off_file = NamedTempFile::new().expect("temp output");
let off_path = off_file.path().to_path_buf();
export_parquet(&ExportOptions {
db_path: db_path.to_path_buf(),
output: Some(off_path.clone()),
format: ExportFormat::Parquet,
run_id: None,
domain: None,
status: None,
since: None,
include_implied_tech: false,
})
.await
.expect("export off");
let off = parquet_tech_flags(&off_path);
assert_eq!(off, vec![("nginx".to_string(), false)]);
let on_file = NamedTempFile::new().expect("temp output");
let on_path = on_file.path().to_path_buf();
export_parquet(&ExportOptions {
db_path: db_path.to_path_buf(),
output: Some(on_path.clone()),
format: ExportFormat::Parquet,
run_id: None,
domain: None,
status: None,
since: None,
include_implied_tech: true,
})
.await
.expect("export on");
let mut on = parquet_tech_flags(&on_path);
on.sort_unstable();
assert_eq!(
on,
vec![("PHP".to_string(), true), ("nginx".to_string(), false)]
);
}
fn read_parquet_batch(path: &Path) -> RecordBatch {
let file = std::fs::File::open(path).expect("open parquet");
let builder = ParquetRecordBatchReaderBuilder::try_new(file).expect("parse Parquet");
let mut reader = builder.build().expect("build reader");
reader.next().expect("batch").expect("batch ok")
}
fn row_for_domain(batch: &RecordBatch, domain: &str) -> usize {
let arr = batch
.column_by_name("final_domain")
.expect("final_domain")
.as_any()
.downcast_ref::<StringArray>()
.expect("final_domain StringArray");
(0..arr.len())
.find(|&i| arr.value(i) == domain)
.unwrap_or_else(|| panic!("missing row for {domain}"))
}
fn list_len(batch: &RecordBatch, name: &str, row: usize) -> usize {
let list = batch
.column_by_name(name)
.unwrap_or_else(|| panic!("missing list column {name}"))
.as_any()
.downcast_ref::<ListArray>()
.unwrap_or_else(|| panic!("{name} should be ListArray"));
usize::try_from(list.value_length(row)).expect("list length")
}
fn utf8_opt(batch: &RecordBatch, name: &str, row: usize) -> Option<String> {
let arr = batch
.column_by_name(name)
.unwrap_or_else(|| panic!("missing utf8 column {name}"))
.as_any()
.downcast_ref::<StringArray>()
.unwrap_or_else(|| panic!("{name} should be StringArray"));
if arr.is_null(row) {
None
} else {
Some(arr.value(row).to_string())
}
}
fn i32_val(batch: &RecordBatch, name: &str, row: usize) -> i32 {
batch
.column_by_name(name)
.unwrap_or_else(|| panic!("missing i32 column {name}"))
.as_any()
.downcast_ref::<Int32Array>()
.unwrap_or_else(|| panic!("{name} should be Int32Array"))
.value(row)
}
fn i32_opt(batch: &RecordBatch, name: &str, row: usize) -> Option<i32> {
let arr = batch
.column_by_name(name)
.unwrap_or_else(|| panic!("missing i32 column {name}"))
.as_any()
.downcast_ref::<Int32Array>()
.unwrap_or_else(|| panic!("{name} should be Int32Array"));
if arr.is_null(row) {
None
} else {
Some(arr.value(row))
}
}
fn i64_opt(batch: &RecordBatch, name: &str, row: usize) -> Option<i64> {
let arr = batch
.column_by_name(name)
.unwrap_or_else(|| panic!("missing i64 column {name}"))
.as_any()
.downcast_ref::<Int64Array>()
.unwrap_or_else(|| panic!("{name} should be Int64Array"));
if arr.is_null(row) {
None
} else {
Some(arr.value(row))
}
}
async fn populate_full_satellites(pool: &SqlitePool, url_id: i64) {
sqlx::query("UPDATE url_status SET tls_version = ?, ssl_cert_subject = ? WHERE id = ?")
.bind("TLSv1.3")
.bind("CN=full.example.com")
.bind(url_id)
.execute(pool)
.await
.unwrap();
sqlx::query("INSERT INTO url_nameservers (url_status_id, nameserver) VALUES (?, ?)")
.bind(url_id)
.bind("ns1.example.com")
.execute(pool)
.await
.unwrap();
sqlx::query(
"INSERT INTO url_technologies (url_status_id, technology_name, technology_version)
VALUES (?, ?, ?)",
)
.bind(url_id)
.bind("WordPress")
.bind("6.8")
.execute(pool)
.await
.unwrap();
sqlx::query("INSERT INTO url_ipv6_addresses (url_status_id, ipv6_address) VALUES (?, ?)")
.bind(url_id)
.bind("2001:db8::1")
.execute(pool)
.await
.unwrap();
sqlx::query(
"INSERT INTO url_caa_records (url_status_id, flag, tag, value) VALUES (?, ?, ?, ?)",
)
.bind(url_id)
.bind(0i64)
.bind("issue")
.bind("ca.example.com")
.execute(pool)
.await
.unwrap();
sqlx::query("INSERT INTO url_certificate_sans (url_status_id, san_value) VALUES (?, ?)")
.bind(url_id)
.bind("full.example.com")
.execute(pool)
.await
.unwrap();
sqlx::query(
"INSERT INTO url_geoip (url_status_id, country_code, country_name, asn)
VALUES (?, ?, ?, ?)",
)
.bind(url_id)
.bind("US")
.bind("United States")
.bind(64496i64)
.execute(pool)
.await
.unwrap();
sqlx::query(
"INSERT INTO url_whois (url_status_id, registrar, registrant_country) VALUES (?, ?, ?)",
)
.bind(url_id)
.bind("Example Registrar")
.bind("US")
.execute(pool)
.await
.unwrap();
sqlx::query("INSERT INTO url_favicons (url_status_id, favicon_url, hash) VALUES (?, ?, ?)")
.bind(url_id)
.bind("https://full.example.com/favicon.ico")
.bind(12345i64)
.execute(pool)
.await
.unwrap();
}
fn parquet_name(id: &str) -> &'static str {
field_by_id(id)
.parquet
.as_ref()
.unwrap_or_else(|| panic!("{id} missing parquet spec"))
.name
}
fn assert_every_column_type(batch: &RecordBatch) {
let expected = build_schema();
assert_eq!(
batch.num_columns(),
expected.fields().len(),
"column count must match build_schema()"
);
for id in PARQUET_FIELD_ORDER {
let spec = field_by_id(id)
.parquet
.as_ref()
.unwrap_or_else(|| panic!("{id} missing parquet spec"));
let col = batch
.column_by_name(spec.name)
.unwrap_or_else(|| panic!("missing Parquet column {} (id {id})", spec.name));
let want = (spec.data_type)();
assert_eq!(
col.data_type(),
&want,
"Arrow type mismatch for id {id} / {}",
spec.name
);
let field = expected
.field_with_name(spec.name)
.unwrap_or_else(|_| panic!("schema missing {}", spec.name));
assert_eq!(field.data_type(), &want, "schema type for {}", spec.name);
assert_eq!(
field.is_nullable(),
spec.nullable,
"nullability for {}",
spec.name
);
}
}
fn assert_sparse_vs_full_core(batch: &RecordBatch, sparse: usize, full: usize) {
assert_eq!(
utf8_opt(batch, "url", sparse).as_deref(),
Some("https://sparse.example.com")
);
assert_eq!(
utf8_opt(batch, "url", full).as_deref(),
Some("https://full.example.com")
);
assert_eq!(list_len(batch, "nameservers", sparse), 0);
assert_eq!(list_len(batch, "nameservers", full), 1);
assert_eq!(list_len(batch, "technologies", sparse), 0);
assert_eq!(list_len(batch, "technologies", full), 1);
assert_eq!(i32_val(batch, "technology_count", sparse), 0);
assert_eq!(i32_val(batch, "technology_count", full), 1);
assert!(utf8_opt(batch, "tls_version", sparse).is_none());
assert_eq!(
utf8_opt(batch, "tls_version", full).as_deref(),
Some("TLSv1.3")
);
assert!(utf8_opt(batch, "ssl_cert_subject", sparse).is_none());
assert_eq!(
utf8_opt(batch, "ssl_cert_subject", full).as_deref(),
Some("CN=full.example.com")
);
assert_eq!(list_len(batch, "certificate_sans", sparse), 0);
assert_eq!(list_len(batch, "certificate_sans", full), 1);
assert_eq!(list_len(batch, "ipv6_addresses", sparse), 0);
assert_eq!(list_len(batch, "ipv6_addresses", full), 1);
assert_eq!(list_len(batch, "caa_records", sparse), 0);
assert_eq!(list_len(batch, "caa_records", full), 1);
}
fn assert_sparse_vs_full_enrichment(batch: &RecordBatch, sparse: usize, full: usize) {
assert!(utf8_opt(batch, "geoip_country_code", sparse).is_none());
assert_eq!(
utf8_opt(batch, "geoip_country_code", full).as_deref(),
Some("US")
);
assert!(utf8_opt(batch, "whois_registrar", sparse).is_none());
assert_eq!(
utf8_opt(batch, "whois_registrar", full).as_deref(),
Some("Example Registrar")
);
assert!(i32_opt(batch, "favicon_hash", sparse).is_none());
assert_eq!(i32_opt(batch, "favicon_hash", full), Some(12345));
assert!(utf8_opt(batch, "favicon_url", sparse).is_none());
assert_eq!(
utf8_opt(batch, "favicon_url", full).as_deref(),
Some("https://full.example.com/favicon.ico")
);
assert_eq!(
i64_opt(batch, "observed_at_ms", sparse),
Some(1_704_067_200_000)
);
assert_eq!(
utf8_opt(batch, "run_id", sparse).as_deref(),
Some("test-run-1")
);
for id in [
"redirect_chain",
"oids",
"cname_records",
"script_hosts",
"txt_records",
"mx_records",
"analytics_ids",
"social_media_links",
"structured_data_entries",
"http_headers",
"security_headers",
"partial_failures",
"contact_links",
"exposed_secrets",
] {
let name = parquet_name(id);
assert_eq!(list_len(batch, name, sparse), 0, "sparse {name} empty list");
assert_eq!(list_len(batch, name, full), 0, "full leftover {name} empty");
}
}
#[tokio::test]
async fn test_parquet_every_field_order_column_type_and_null() {
let temp_db = NamedTempFile::new().expect("temp DB");
let db_path = temp_db.path();
let pool = SqlitePool::connect(&format!("sqlite:{}", db_path.display()))
.await
.unwrap();
run_migrations(&pool).await.unwrap();
create_test_url_status(&pool, "sparse.example.com").await;
let full_id = create_test_url_status(&pool, "full.example.com").await;
populate_full_satellites(&pool, full_id).await;
drop(pool);
let temp_file = NamedTempFile::new().expect("temp output");
let output_path = temp_file.path().to_path_buf();
let count = export_parquet(&ExportOptions {
db_path: db_path.to_path_buf(),
output: Some(output_path.clone()),
format: ExportFormat::Parquet,
run_id: None,
domain: None,
status: None,
since: None,
include_implied_tech: false,
})
.await
.expect("export");
assert_eq!(count, 2);
let batch = read_parquet_batch(&output_path);
assert_eq!(batch.num_rows(), 2);
assert_every_column_type(&batch);
let sparse = row_for_domain(&batch, "sparse.example.com");
let full = row_for_domain(&batch, "full.example.com");
assert_sparse_vs_full_core(&batch, sparse, full);
assert_sparse_vs_full_enrichment(&batch, sparse, full);
}
}