use std::collections::HashSet;
use log::{debug, error, info, warn};
use crate::config::SearxngCollectorSettings;
use crate::domain::repositories::searxng_instance_repository::{
normalize_instance_url, SearxngInstance, SearxngInstanceRepository, SearxngInstanceSource,
UpsertOutcome,
};
use crate::domain::repositories::task_repository::RepositoryError;
use super::dataset::{admit, parse_dataset, Dataset};
use super::{CollectorReport, SearxngError, SearxngTransport};
const UNBOUNDED_LIMIT: u64 = i64::MAX as u64;
pub(super) async fn fetch_dataset(
transport: &dyn SearxngTransport,
collector: &SearxngCollectorSettings,
) -> Result<Dataset, SearxngError> {
let timeout = std::time::Duration::from_secs(collector.request_timeout_seconds);
let (status, payload) = transport
.get_json(&collector.data_url, timeout)
.await
.map_err(|e| {
error!(
"searxng dataset fetch failed url={} class={} error={}",
collector.data_url,
e.class(),
e
);
SearxngError::DatasetFetch(format!("{}: {e}", collector.data_url))
})?;
if !(200..300).contains(&status) {
let err =
SearxngError::DatasetFetch(format!("{} returned HTTP {status}", collector.data_url));
error!("{err}");
return Err(err);
}
parse_dataset(&payload).map_err(|e| {
error!("searxng dataset parse failed: {e}");
SearxngError::DatasetFetch(e.to_string())
})
}
pub(super) async fn sync_instances(
repo: &dyn SearxngInstanceRepository,
dataset: &Dataset,
collector: &SearxngCollectorSettings,
report: &mut CollectorReport,
present: &mut HashSet<String>,
) {
for (raw_url, entry) in &dataset.instances {
if let Ok(url) = normalize_instance_url(raw_url) {
present.insert(url);
}
let admitted = match admit(raw_url, entry, collector.min_uptime_day) {
Ok(admitted) => admitted,
Err(reject) => {
report.errors += 1;
debug!("searxng admission rejected: {reject}");
continue;
}
};
report.seen += 1;
match repo.upsert_from_dataset(&admitted).await {
Ok(UpsertOutcome::Inserted) => report.admitted_new += 1,
Ok(UpsertOutcome::Updated) => report.updated += 1,
Err(e) => {
report.errors += 1;
warn!("searxng upsert failed url={} error={e}", admitted.url);
}
}
}
}
pub(super) async fn retire_absent(
repo: &dyn SearxngInstanceRepository,
present: &HashSet<String>,
collector: &SearxngCollectorSettings,
report: &mut CollectorReport,
) -> Result<(), SearxngError> {
for instance in list_tracked_instances(repo).await? {
retire_instance_if_absent(
repo,
&instance,
present,
collector.retire_miss_cycles,
report,
)
.await;
}
Ok(())
}
async fn list_tracked_instances(
repo: &dyn SearxngInstanceRepository,
) -> Result<Vec<SearxngInstance>, SearxngError> {
repo.list_paginated(UNBOUNDED_LIMIT, 0).await.map_err(|e| {
error!("searxng registry listing failed: {e}");
SearxngError::Repository(e.to_string())
})
}
async fn retire_instance_if_absent(
repo: &dyn SearxngInstanceRepository,
instance: &SearxngInstance,
present: &HashSet<String>,
retire_miss_cycles: i32,
report: &mut CollectorReport,
) {
if instance.source != SearxngInstanceSource::SearxSpace {
return;
}
if present.contains(&instance.url) {
return;
}
match repo
.record_dataset_miss(&instance.url, retire_miss_cycles)
.await
{
Ok(true) => {
report.retired += 1;
info!(
"searxng instance retired for dataset absence url={}",
instance.url
);
}
Ok(false) => {}
Err(RepositoryError::NotFound) => {
debug!(
"searxng instance vanished during collection url={}",
instance.url
);
}
Err(e) => {
report.errors += 1;
warn!(
"searxng dataset-miss record failed url={} error={e}",
instance.url
);
}
}
}