use std::collections::{HashMap, VecDeque};
use std::sync::atomic::{AtomicU32, AtomicUsize, Ordering};
use std::sync::Arc;
use std::time::Duration;
use async_trait::async_trait;
use chrono::Utc;
use parking_lot::Mutex;
use serde_json::Value;
use crate::domain::repositories::searxng_instance_repository::{
AdmittedInstance, CollectLockGuard, SearxngInstance, SearxngInstanceRepository,
SearxngInstanceSource, SearxngInstanceState, UpsertOutcome,
};
use crate::domain::repositories::task_repository::RepositoryError;
use super::{SearxngError, SearxngTransport};
struct InMemoryLockGuard {
locked: Arc<Mutex<bool>>,
}
#[async_trait]
impl CollectLockGuard for InMemoryLockGuard {
async fn release(&self) -> Result<(), RepositoryError> {
*self.locked.lock() = false;
Ok(())
}
}
#[derive(Default)]
pub struct InMemorySearxngRepo {
instances: Mutex<HashMap<String, SearxngInstance>>,
collect_locked: Arc<Mutex<bool>>,
pub lock_attempts: AtomicU32,
pub lock_acquisitions: AtomicU32,
pub dataset_upserts: AtomicU32,
pub seed_upserts: AtomicU32,
pub probe_records: AtomicU32,
}
impl InMemorySearxngRepo {
pub fn new() -> Self {
Self::default()
}
pub fn insert_raw(&self, instance: SearxngInstance) {
self.instances.lock().insert(instance.url.clone(), instance);
}
pub fn seed_instance(url: &str, state: SearxngInstanceState) -> SearxngInstance {
Self::instance(url, state, SearxngInstanceSource::Seed)
}
pub fn dataset_instance(url: &str, state: SearxngInstanceState) -> SearxngInstance {
Self::instance(url, state, SearxngInstanceSource::SearxSpace)
}
fn instance(
url: &str,
state: SearxngInstanceState,
source: SearxngInstanceSource,
) -> SearxngInstance {
let mut instance = SearxngInstance::new(url, source, state);
instance.network_type = Some("normal".to_string());
instance.uptime_day = Some(99.0);
instance
}
pub fn len(&self) -> usize {
self.instances.lock().len()
}
pub fn is_empty(&self) -> bool {
self.instances.lock().is_empty()
}
pub fn get(&self, url: &str) -> Option<SearxngInstance> {
self.instances.lock().get(url).cloned()
}
pub fn all(&self) -> Vec<SearxngInstance> {
let mut all: Vec<SearxngInstance> = self.instances.lock().values().cloned().collect();
all.sort_by(|a, b| a.url.cmp(&b.url));
all
}
}
#[async_trait]
impl SearxngInstanceRepository for InMemorySearxngRepo {
async fn try_acquire_collect_lock(
&self,
_key: i64,
) -> Result<Option<Box<dyn CollectLockGuard>>, RepositoryError> {
self.lock_attempts.fetch_add(1, Ordering::SeqCst);
let mut locked = self.collect_locked.lock();
if *locked {
return Ok(None);
}
*locked = true;
self.lock_acquisitions.fetch_add(1, Ordering::SeqCst);
Ok(Some(Box::new(InMemoryLockGuard {
locked: Arc::clone(&self.collect_locked),
})))
}
async fn upsert_from_dataset(
&self,
instance: &AdmittedInstance,
) -> Result<UpsertOutcome, RepositoryError> {
self.dataset_upserts.fetch_add(1, Ordering::SeqCst);
let mut map = self.instances.lock();
match map.get_mut(&instance.url) {
Some(existing) => {
existing.dataset_miss_count = 0;
existing.uptime_day = instance.uptime_day;
existing.uptime_month = instance.uptime_month;
existing.network_type = instance.network_type.clone();
if existing.state == SearxngInstanceState::Retired {
existing.state = SearxngInstanceState::Candidate;
}
existing.updated_at = Utc::now();
Ok(UpsertOutcome::Updated)
}
None => {
let mut row = Self::instance(
&instance.url,
SearxngInstanceState::Candidate,
SearxngInstanceSource::SearxSpace,
);
row.uptime_day = instance.uptime_day;
row.uptime_month = instance.uptime_month;
row.network_type = instance.network_type.clone();
map.insert(instance.url.clone(), row);
Ok(UpsertOutcome::Inserted)
}
}
}
async fn upsert_seed(&self, url: &str) -> Result<UpsertOutcome, RepositoryError> {
self.seed_upserts.fetch_add(1, Ordering::SeqCst);
let mut map = self.instances.lock();
match map.get_mut(url) {
Some(existing) => {
existing.source = SearxngInstanceSource::Seed;
existing.updated_at = Utc::now();
Ok(UpsertOutcome::Updated)
}
None => {
map.insert(
url.to_string(),
Self::seed_instance(url, SearxngInstanceState::Candidate),
);
Ok(UpsertOutcome::Inserted)
}
}
}
async fn list_candidates(&self, limit: u64) -> Result<Vec<SearxngInstance>, RepositoryError> {
let mut candidates: Vec<SearxngInstance> = self
.instances
.lock()
.values()
.filter(|i| i.state == SearxngInstanceState::Candidate)
.cloned()
.collect();
candidates.sort_by(|a, b| {
let uptime = b.uptime_day.unwrap_or(f64::NEG_INFINITY);
let lhs = a.uptime_day.unwrap_or(f64::NEG_INFINITY);
uptime
.partial_cmp(&lhs)
.unwrap_or(std::cmp::Ordering::Equal)
.then_with(|| match (a.latency_ms, b.latency_ms) {
(Some(x), Some(y)) => x.cmp(&y),
(Some(_), None) => std::cmp::Ordering::Less,
(None, Some(_)) => std::cmp::Ordering::Greater,
(None, None) => std::cmp::Ordering::Equal,
})
});
candidates.truncate(limit as usize);
Ok(candidates)
}
async fn list_by_state(
&self,
state: SearxngInstanceState,
limit: u64,
) -> Result<Vec<SearxngInstance>, RepositoryError> {
let mut rows: Vec<SearxngInstance> = self
.instances
.lock()
.values()
.filter(|i| i.state == state)
.cloned()
.collect();
rows.truncate(limit as usize);
Ok(rows)
}
async fn list_paginated(
&self,
limit: u64,
offset: u64,
) -> Result<Vec<SearxngInstance>, RepositoryError> {
let mut rows = self.all();
rows.sort_by(|a, b| {
a.state.as_str().cmp(b.state.as_str()).then_with(|| {
b.uptime_day
.unwrap_or(f64::NEG_INFINITY)
.partial_cmp(&a.uptime_day.unwrap_or(f64::NEG_INFINITY))
.unwrap_or(std::cmp::Ordering::Equal)
})
});
Ok(rows
.into_iter()
.skip(offset as usize)
.take(limit as usize)
.collect())
}
async fn record_probe_result(
&self,
url: &str,
ok: bool,
latency_ms: Option<i64>,
error: Option<&str>,
) -> Result<(), RepositoryError> {
self.probe_records.fetch_add(1, Ordering::SeqCst);
let mut map = self.instances.lock();
let instance = map.get_mut(url).ok_or(RepositoryError::NotFound)?;
if ok {
instance.consecutive_failures = 0;
instance.latency_ms = latency_ms;
instance.last_success_at = Some(Utc::now());
instance.last_error = None;
} else {
instance.consecutive_failures += 1;
instance.last_failure_at = Some(Utc::now());
instance.last_error = error.map(|e| e.to_string());
}
instance.updated_at = Utc::now();
Ok(())
}
async fn transition_state(
&self,
url: &str,
to: SearxngInstanceState,
) -> Result<(), RepositoryError> {
let mut map = self.instances.lock();
let instance = map.get_mut(url).ok_or(RepositoryError::NotFound)?;
if !instance.state.can_transition_to(to) {
return Err(RepositoryError::Database(anyhow::anyhow!(
"illegal searxng instance state transition for {url}: {} -> {to}",
instance.state
)));
}
instance.state = to;
instance.updated_at = Utc::now();
Ok(())
}
async fn record_dataset_miss(
&self,
url: &str,
retire_after: i32,
) -> Result<bool, RepositoryError> {
let mut map = self.instances.lock();
let instance = map.get_mut(url).ok_or(RepositoryError::NotFound)?;
if instance.source != SearxngInstanceSource::SearxSpace {
return Ok(false);
}
instance.dataset_miss_count += 1;
if instance.dataset_miss_count > retire_after {
if instance
.state
.can_transition_to(SearxngInstanceState::Retired)
{
instance.state = SearxngInstanceState::Retired;
return Ok(true);
}
if instance.state == SearxngInstanceState::Active {
instance.state = SearxngInstanceState::Cooldown;
}
}
Ok(false)
}
async fn get_by_url(&self, url: &str) -> Result<Option<SearxngInstance>, RepositoryError> {
Ok(self.instances.lock().get(url).cloned())
}
async fn count_by_state(&self, state: SearxngInstanceState) -> Result<u64, RepositoryError> {
Ok(self
.instances
.lock()
.values()
.filter(|i| i.state == state)
.count() as u64)
}
}
#[derive(Debug, Clone)]
pub enum Scripted {
Json {
status: u16,
body: Value,
},
Network(String),
Timeout,
}
pub struct ScriptedTransport {
queue: Mutex<VecDeque<Scripted>>,
pub calls: AtomicUsize,
pub urls: Mutex<Vec<String>>,
}
impl ScriptedTransport {
pub fn new(script: Vec<Scripted>) -> Self {
Self {
queue: Mutex::new(script.into_iter().collect()),
calls: AtomicUsize::new(0),
urls: Mutex::new(Vec::new()),
}
}
pub fn repeat(response: Scripted, count: usize) -> Self {
Self::new(std::iter::repeat_n(response, count).collect())
}
pub fn received_urls(&self) -> Vec<String> {
self.urls.lock().clone()
}
}
#[async_trait]
impl SearxngTransport for ScriptedTransport {
async fn get_json(&self, url: &str, timeout: Duration) -> Result<(u16, Value), SearxngError> {
self.calls.fetch_add(1, Ordering::SeqCst);
self.urls.lock().push(url.to_string());
let scripted = self.queue.lock().pop_front().unwrap_or_else(|| {
Scripted::Network(format!("scripted transport exhausted for {url}"))
});
match scripted {
Scripted::Json { status, body } => Ok((status, body)),
Scripted::Network(message) => Err(SearxngError::Network(message)),
Scripted::Timeout => Err(SearxngError::Timeout(timeout)),
}
}
}
pub fn test_settings() -> crate::config::SearxngSearchSettings {
crate::config::SearxngSearchSettings::default()
}
pub fn shared_repo() -> Arc<InMemorySearxngRepo> {
Arc::new(InMemorySearxngRepo::new())
}