use std::collections::HashMap;
use log::debug;
use serde::{Deserialize, Serialize};
use thiserror::Error;
use url::Url;
use crate::domain::repositories::searxng_instance_repository::{
normalize_instance_url, AdmittedInstance, InstanceUrlError,
};
#[derive(Debug, Error, PartialEq, Eq)]
pub enum DatasetParseError {
#[error("invalid dataset payload: {0}")]
InvalidPayload(String),
}
#[derive(Debug, Clone, Default, Serialize, Deserialize)]
pub struct Dataset {
#[serde(default)]
pub instances: HashMap<String, DatasetEntry>,
}
#[derive(Debug, Clone, Default, Serialize, Deserialize)]
pub struct DatasetEntry {
#[serde(default)]
pub network_type: Option<String>,
#[serde(default)]
pub version: Option<String>,
#[serde(default)]
pub api: Option<serde_json::Value>,
#[serde(default)]
pub http: Option<GradedStatus>,
#[serde(default)]
pub tls: Option<GradedStatus>,
#[serde(default)]
pub cdn: Option<GradedStatus>,
#[serde(default)]
pub uptime: Option<Uptime>,
}
#[derive(Debug, Clone, Default, Serialize, Deserialize)]
pub struct GradedStatus {
#[serde(default)]
pub status_code: Option<u16>,
#[serde(default)]
pub grade: Option<String>,
}
#[derive(Debug, Clone, Default, Serialize, Deserialize)]
pub struct Uptime {
#[serde(default)]
pub day: Option<f64>,
#[serde(default)]
pub month: Option<f64>,
}
#[derive(Debug, Clone, PartialEq, Eq, Error)]
pub enum AdmissionReject {
#[error("not https: {0}")]
NotHttps(String),
#[error("unsafe or invalid url: {0}")]
UnsafeUrl(String),
#[error("unsupported network type: {0}")]
NonNormalNetwork(String),
#[error("json api unavailable: {0}")]
ApiUnavailable(String),
#[error("low uptime: {0}")]
LowUptime(String),
}
pub fn parse_dataset(payload: &serde_json::Value) -> Result<Dataset, DatasetParseError> {
let object = payload
.as_object()
.ok_or_else(|| DatasetParseError::InvalidPayload("top level is not an object".into()))?;
let Some(instances) = object.get("instances") else {
debug!("searx.space dataset has no `instances` key; treating as empty");
return Ok(Dataset::default());
};
let Some(instances) = instances.as_object() else {
return Err(DatasetParseError::InvalidPayload(
"`instances` is not an object".into(),
));
};
let mut dataset = Dataset::default();
for (url, raw_entry) in instances {
let entry = match DatasetEntry::deserialize(raw_entry) {
Ok(entry) => entry,
Err(e) => {
debug!("searx.space entry {url} metadata unparsable, using defaults: {e}");
DatasetEntry::default()
}
};
dataset.instances.insert(url.clone(), entry);
}
Ok(dataset)
}
pub fn admit(
url: &str,
entry: &DatasetEntry,
min_uptime_day: f64,
) -> Result<AdmittedInstance, AdmissionReject> {
let normalized = validate_admission_url(url)?;
check_admission_entry(&normalized, entry, min_uptime_day)?;
Ok(AdmittedInstance {
url: normalized,
network_type: entry.network_type.clone(),
version: entry.version.clone(),
api_grade: api_grade(entry.api.as_ref()),
http_grade: entry.http.as_ref().and_then(|g| g.grade.clone()),
tls_grade: entry.tls.as_ref().and_then(|g| g.grade.clone()),
cdn_grade: entry.cdn.as_ref().and_then(|g| g.grade.clone()),
uptime_day: entry.uptime.as_ref().and_then(|u| u.day),
uptime_month: entry.uptime.as_ref().and_then(|u| u.month),
})
}
fn validate_admission_url(url: &str) -> Result<String, AdmissionReject> {
match Url::parse(url).map(|u| u.scheme().to_string()) {
Ok(scheme) if scheme == "https" => {}
Ok(scheme) => return Err(AdmissionReject::NotHttps(format!("{url} ({scheme})"))),
Err(e) => return Err(AdmissionReject::UnsafeUrl(format!("{url} ({e})"))),
}
normalize_instance_url(url).map_err(|e| match e {
InstanceUrlError::Unsafe(_) | InstanceUrlError::Invalid(_) => {
AdmissionReject::UnsafeUrl(format!("{url} ({e})"))
}
InstanceUrlError::UnknownVariant(_) => AdmissionReject::UnsafeUrl(format!("{url} ({e})")),
})
}
fn check_admission_entry(
normalized: &str,
entry: &DatasetEntry,
min_uptime_day: f64,
) -> Result<(), AdmissionReject> {
if let Some(network_type) = entry.network_type.as_deref() {
if !network_type.eq_ignore_ascii_case("normal") {
return Err(AdmissionReject::NonNormalNetwork(format!(
"{normalized} ({network_type})"
)));
}
}
if let Some(api) = &entry.api {
if !api_flag_available(api) {
return Err(AdmissionReject::ApiUnavailable(normalized.to_string()));
}
}
if let Some(day) = entry.uptime.as_ref().and_then(|u| u.day) {
if day < min_uptime_day {
return Err(AdmissionReject::LowUptime(format!(
"{normalized} (day={day} < {min_uptime_day})"
)));
}
}
Ok(())
}
fn api_flag_available(api: &serde_json::Value) -> bool {
match api {
serde_json::Value::Bool(flag) => *flag,
serde_json::Value::Object(map) => match map.get("status_code").and_then(|v| v.as_u64()) {
Some(code) => code == 200,
None => true,
},
_ => true,
}
}
fn api_grade(api: Option<&serde_json::Value>) -> Option<String> {
api.and_then(|v| v.as_object())
.and_then(|m| m.get("grade"))
.and_then(|v| v.as_str())
.map(|s| s.to_string())
}
#[cfg(test)]
mod tests {
use super::*;
fn sample_dataset() -> Dataset {
let raw = include_str!("../../../../tests/fixtures/searxng/instances_sample.json");
let payload: serde_json::Value = serde_json::from_str(raw).expect("fixture is valid json");
parse_dataset(&payload).expect("fixture parses")
}
fn admit_url(dataset: &Dataset, url: &str) -> Result<AdmittedInstance, AdmissionReject> {
let entry = dataset
.instances
.get(url)
.unwrap_or_else(|| panic!("fixture missing entry {url}"));
admit(url, entry, 95.0)
}
#[test]
fn test_parse_fixture_collects_all_entries() {
let dataset = sample_dataset();
assert_eq!(dataset.instances.len(), 10);
}
#[test]
fn test_parse_unknown_extra_fields_ignored() {
let dataset = sample_dataset();
let entry = dataset
.instances
.get("https://searx.unknown-fields.example")
.expect("entry present");
assert_eq!(entry.network_type.as_deref(), Some("normal"));
assert_eq!(entry.uptime.as_ref().and_then(|u| u.day), Some(99.0));
}
#[test]
fn test_parse_bad_metadata_entry_falls_back_to_defaults() {
let dataset = sample_dataset();
let entry = dataset
.instances
.get("https://searx.bad-metadata.example")
.expect("entry present");
assert!(entry.http.is_none());
assert!(admit("https://searx.bad-metadata.example", entry, 95.0).is_ok());
}
#[test]
fn test_parse_missing_instances_key_yields_empty() {
let payload = serde_json::json!({ "meta": "no instances here" });
let dataset = parse_dataset(&payload).expect("tolerated");
assert!(dataset.instances.is_empty());
}
#[test]
fn test_parse_non_object_payload_rejected() {
let payload = serde_json::json!([1, 2, 3]);
assert!(matches!(
parse_dataset(&payload),
Err(DatasetParseError::InvalidPayload(_))
));
}
#[test]
fn test_parse_instances_not_object_rejected() {
let payload = serde_json::json!({ "instances": 42 });
assert!(matches!(
parse_dataset(&payload),
Err(DatasetParseError::InvalidPayload(_))
));
}
#[test]
fn test_admit_accepts_healthy_https_instance() {
let dataset = sample_dataset();
let admitted = admit_url(&dataset, "https://searx.working.example").expect("admitted");
assert_eq!(admitted.url, "https://searx.working.example");
assert_eq!(admitted.http_grade.as_deref(), Some("A+"));
assert_eq!(admitted.tls_grade.as_deref(), Some("A+"));
assert_eq!(admitted.cdn_grade.as_deref(), Some("A"));
assert_eq!(admitted.uptime_day, Some(99.5));
assert_eq!(admitted.uptime_month, Some(98.2));
assert_eq!(admitted.version.as_deref(), Some("2024.1.1"));
}
#[test]
fn test_admit_accepts_api_status_block() {
let dataset = sample_dataset();
let admitted = admit_url(&dataset, "https://searx.api-detail.example")
.expect("api block 200 admitted");
assert_eq!(admitted.api_grade.as_deref(), Some("A"));
}
#[test]
fn test_admit_rejects_api_disabled() {
let dataset = sample_dataset();
assert!(matches!(
admit_url(&dataset, "https://searx.api-off.example"),
Err(AdmissionReject::ApiUnavailable(_))
));
}
#[test]
fn test_admit_rejects_plain_http() {
let dataset = sample_dataset();
assert!(matches!(
admit_url(&dataset, "http://searx.plain-http.example"),
Err(AdmissionReject::NotHttps(_))
));
}
#[test]
fn test_admit_rejects_low_uptime() {
let dataset = sample_dataset();
assert!(matches!(
admit_url(&dataset, "https://searx.unstable.example"),
Err(AdmissionReject::LowUptime(_))
));
}
#[test]
fn test_admit_rejects_non_normal_network() {
let dataset = sample_dataset();
assert!(matches!(
admit_url(&dataset, "https://searx.tor.example"),
Err(AdmissionReject::NonNormalNetwork(_))
));
}
#[test]
fn test_admit_rejects_internal_address() {
let dataset = sample_dataset();
assert!(matches!(
admit_url(&dataset, "https://127.0.0.1:8888"),
Err(AdmissionReject::UnsafeUrl(_))
));
}
#[test]
fn test_admit_missing_optional_fields_passes() {
let dataset = sample_dataset();
let admitted = admit_url(&dataset, "https://searx.minimal.example").expect("admitted");
assert!(admitted.uptime_day.is_none());
assert!(admitted.api_grade.is_none());
}
#[test]
fn test_admit_rejects_garbage_url() {
let entry = DatasetEntry::default();
assert!(matches!(
admit("not-a-url", &entry, 95.0),
Err(AdmissionReject::UnsafeUrl(_))
));
}
#[test]
fn test_admit_normalizes_url() {
let entry = DatasetEntry::default();
let admitted = admit("https://EXAMPLE.com/searx/", &entry, 95.0).expect("admitted");
assert_eq!(admitted.url, "https://example.com/searx");
}
#[test]
fn test_reject_display_is_non_empty() {
let rejects = [
AdmissionReject::NotHttps("u".into()),
AdmissionReject::UnsafeUrl("u".into()),
AdmissionReject::NonNormalNetwork("u".into()),
AdmissionReject::ApiUnavailable("u".into()),
AdmissionReject::LowUptime("u".into()),
];
for reject in rejects {
assert!(!reject.to_string().is_empty(), "empty display: {reject:?}");
}
}
}