use crate::storage::insert::utils::build_batch_insert_query;
pub(crate) async fn insert_csp_domains(
tx: &mut sqlx::Transaction<'_, sqlx::Sqlite>,
url_status_id: i64,
domains: &[(String, String, Option<String>)],
) -> Result<(), sqlx::Error> {
if domains.is_empty() {
return Ok(());
}
let query = build_batch_insert_query(
"url_csp_domains",
&["url_status_id", "directive", "fqdn", "registrable_domain"],
domains.len(),
Some("ON CONFLICT(url_status_id, directive, fqdn) DO NOTHING"),
);
let mut qb = crate::sql::query(query);
for (directive, fqdn, reg_domain) in domains {
qb = qb
.bind(url_status_id)
.bind(directive)
.bind(fqdn)
.bind(reg_domain);
}
qb.execute(&mut **tx).await?;
Ok(())
}
pub(crate) async fn insert_cookies(
tx: &mut sqlx::Transaction<'_, sqlx::Sqlite>,
url_status_id: i64,
cookies: &[crate::storage::CookieInfo],
) -> Result<(), sqlx::Error> {
if cookies.is_empty() {
return Ok(());
}
let query = build_batch_insert_query(
"url_cookies",
&[
"url_status_id",
"cookie_name",
"secure",
"http_only",
"same_site",
"domain",
"path",
],
cookies.len(),
Some(
"ON CONFLICT(url_status_id, cookie_name) DO UPDATE SET \
secure=excluded.secure, http_only=excluded.http_only, \
same_site=excluded.same_site, domain=excluded.domain, path=excluded.path",
),
);
let mut qb = crate::sql::query(query);
for c in cookies {
qb = qb
.bind(url_status_id)
.bind(&c.name)
.bind(c.secure)
.bind(c.http_only)
.bind(&c.same_site)
.bind(&c.domain)
.bind(&c.path);
}
qb.execute(&mut **tx).await?;
Ok(())
}
pub(crate) async fn insert_security_txt(
tx: &mut sqlx::Transaction<'_, sqlx::Sqlite>,
url_status_id: i64,
data: Option<&crate::fetch::well_known::SecurityTxtData>,
) -> Result<(), sqlx::Error> {
let Some(data) = data else {
return Ok(());
};
sqlx::query(
"INSERT INTO url_security_txt (
url_status_id, source_url, http_status, contacts, expires, encryption,
acknowledgments, preferred_languages, canonical, policy, hiring, raw_body
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
ON CONFLICT(url_status_id) DO UPDATE SET
source_url=excluded.source_url,
http_status=excluded.http_status,
contacts=excluded.contacts,
expires=excluded.expires,
encryption=excluded.encryption,
acknowledgments=excluded.acknowledgments,
preferred_languages=excluded.preferred_languages,
canonical=excluded.canonical,
policy=excluded.policy,
hiring=excluded.hiring,
raw_body=excluded.raw_body",
)
.bind(url_status_id)
.bind(&data.source_url)
.bind(i64::from(data.http_status))
.bind(data.contacts.join("\n"))
.bind(&data.expires)
.bind(data.encryption.join("\n"))
.bind(data.acknowledgments.join("\n"))
.bind(&data.preferred_languages)
.bind(data.canonical.join("\n"))
.bind(data.policy.join("\n"))
.bind(data.hiring.join("\n"))
.bind(&data.raw_body)
.execute(&mut **tx)
.await?;
Ok(())
}
pub(crate) async fn insert_robots_txt(
tx: &mut sqlx::Transaction<'_, sqlx::Sqlite>,
url_status_id: i64,
data: Option<&crate::fetch::well_known::RobotsTxtData>,
) -> Result<(), sqlx::Error> {
let Some(data) = data else {
return Ok(());
};
sqlx::query(
"INSERT INTO url_robots_txt (url_status_id, http_status, raw_body)
VALUES (?, ?, ?)
ON CONFLICT(url_status_id) DO UPDATE SET
http_status=excluded.http_status,
raw_body=excluded.raw_body",
)
.bind(url_status_id)
.bind(i64::from(data.http_status))
.bind(&data.raw_body)
.execute(&mut **tx)
.await?;
if data.directives.is_empty() {
return Ok(());
}
let query = build_batch_insert_query(
"url_robots_directives",
&["url_status_id", "directive", "value"],
data.directives.len(),
Some("ON CONFLICT(url_status_id, directive, value) DO NOTHING"),
);
let mut qb = crate::sql::query(query);
for (directive, value) in &data.directives {
qb = qb.bind(url_status_id).bind(directive).bind(value);
}
qb.execute(&mut **tx).await?;
Ok(())
}
pub(crate) async fn insert_script_hosts(
tx: &mut sqlx::Transaction<'_, sqlx::Sqlite>,
url_status_id: i64,
hosts: &[crate::storage::ScriptHostInfo],
) -> Result<(), sqlx::Error> {
if hosts.is_empty() {
return Ok(());
}
let query = build_batch_insert_query(
"url_script_hosts",
&[
"url_status_id",
"host",
"registrable_domain",
"is_first_party",
],
hosts.len(),
Some(
"ON CONFLICT(url_status_id, host) DO UPDATE SET \
registrable_domain=excluded.registrable_domain, \
is_first_party=excluded.is_first_party",
),
);
let mut qb = crate::sql::query(query);
for h in hosts {
qb = qb
.bind(url_status_id)
.bind(&h.host)
.bind(&h.registrable_domain)
.bind(h.is_first_party);
}
qb.execute(&mut **tx).await?;
Ok(())
}
pub(crate) async fn insert_resource_hints(
tx: &mut sqlx::Transaction<'_, sqlx::Sqlite>,
url_status_id: i64,
hints: &[(String, String)],
) -> Result<(), sqlx::Error> {
if hints.is_empty() {
return Ok(());
}
let query = build_batch_insert_query(
"url_resource_hints",
&["url_status_id", "hint_type", "href"],
hints.len(),
Some("ON CONFLICT(url_status_id, hint_type, href) DO NOTHING"),
);
let mut qb = crate::sql::query(query);
for (hint_type, href) in hints {
qb = qb
.bind(url_status_id)
.bind(hint_type.to_ascii_lowercase())
.bind(href);
}
qb.execute(&mut **tx).await?;
Ok(())
}