use async_trait::async_trait;
use chrono::{DateTime, FixedOffset, Utc};
use dbnexus::sea_orm::sea_query::{Expr, ExprTrait, NullOrdering};
use dbnexus::sea_orm::{
ActiveModelTrait, ColumnTrait, ConnectionTrait, DatabaseBackend, EntityTrait, Order,
PaginatorTrait, QueryFilter, QueryOrder, QuerySelect, Set, Statement, TransactionTrait, Value,
};
use dbnexus::DbPool;
use std::sync::Arc;
use crate::domain::id::Id;
use crate::domain::repositories::searxng_instance_repository::{
AdmittedInstance, CollectLockGuard, SearxngInstance, SearxngInstanceRepository,
SearxngInstanceState, UpsertOutcome,
};
use crate::domain::repositories::task_repository::RepositoryError;
use crate::infrastructure::database::entities::searxng_instance;
const LAST_ERROR_MAX_CHARS: usize = 512;
struct PgCollectLockGuard {
tx: parking_lot::Mutex<Option<dbnexus::sea_orm::DatabaseTransaction>>,
}
#[async_trait]
impl CollectLockGuard for PgCollectLockGuard {
async fn release(&self) -> Result<(), RepositoryError> {
let tx = self.tx.lock().take();
let Some(tx) = tx else {
return Ok(());
};
tx.rollback()
.await
.map_err(|e| RepositoryError::Database(e.into()))?;
Ok(())
}
}
#[derive(Clone)]
pub struct SearxngInstanceRepositoryImpl {
pool: Arc<DbPool>,
}
impl SearxngInstanceRepositoryImpl {
pub fn new(pool: Arc<DbPool>) -> Self {
Self { pool }
}
async fn admin_conn(&self) -> Result<dbnexus::sea_orm::DatabaseConnection, RepositoryError> {
let session = self
.pool
.get_session("admin")
.await
.map_err(|e| RepositoryError::Database(e.into()))?;
session
.connection()
.cloned()
.map_err(|e| RepositoryError::Database(e.into()))
}
async fn find_model(
&self,
url: &str,
) -> Result<Option<searxng_instance::Model>, RepositoryError> {
let conn = self.admin_conn().await?;
searxng_instance::Entity::find()
.filter(searxng_instance::Column::Url.eq(url))
.one(&conn)
.await
.map_err(|e| RepositoryError::Database(e.into()))
}
}
fn now_tz() -> DateTime<FixedOffset> {
Utc::now().fixed_offset()
}
const DATASET_UPSERT_SQL: &str = "\
INSERT INTO searxng_instances (\
id, url, source, state, network_type, version, api_grade, http_grade, \
tls_grade, cdn_grade, uptime_day, uptime_month, latency_ms, \
consecutive_failures, last_success_at, last_failure_at, last_error, \
dataset_miss_count, created_at, updated_at\
) VALUES (\
$1, $2, $3, $4, $5, $6, $7, $8, $9, $10, \
$11, $12, $13, $14, $15, $16, $17, $18, $19, $20\
) ON CONFLICT (url) DO UPDATE SET \
network_type = EXCLUDED.network_type, \
version = EXCLUDED.version, \
api_grade = EXCLUDED.api_grade, \
http_grade = EXCLUDED.http_grade, \
tls_grade = EXCLUDED.tls_grade, \
cdn_grade = EXCLUDED.cdn_grade, \
uptime_day = EXCLUDED.uptime_day, \
uptime_month = EXCLUDED.uptime_month, \
dataset_miss_count = 0, \
updated_at = EXCLUDED.updated_at, \
state = CASE \
WHEN searxng_instances.state = 'retired' THEN 'candidate' \
ELSE searxng_instances.state \
END \
RETURNING (searxng_instances.xmax = 0) AS inserted";
const SEED_INSERT_SQL: &str = "\
INSERT INTO searxng_instances (\
id, url, source, state, network_type, version, api_grade, http_grade, \
tls_grade, cdn_grade, uptime_day, uptime_month, latency_ms, \
consecutive_failures, last_success_at, last_failure_at, last_error, \
dataset_miss_count, created_at, updated_at\
) VALUES (\
$1, $2, $3, $4, $5, $6, $7, $8, $9, $10, \
$11, $12, $13, $14, $15, $16, $17, $18, $19, $20\
) ON CONFLICT (url) DO UPDATE SET \
source = EXCLUDED.source, \
updated_at = EXCLUDED.updated_at \
RETURNING (searxng_instances.xmax = 0) AS inserted";
fn dataset_insert_values(instance: &AdmittedInstance, now: DateTime<FixedOffset>) -> Vec<Value> {
vec![
Value::from(Id::generate().to_uuid_v8()),
Value::from(instance.url.clone()),
Value::from("searx_space".to_string()),
Value::from(SearxngInstanceState::Candidate.as_str().to_string()),
Value::from(instance.network_type.clone()),
Value::from(instance.version.clone()),
Value::from(instance.api_grade.clone()),
Value::from(instance.http_grade.clone()),
Value::from(instance.tls_grade.clone()),
Value::from(instance.cdn_grade.clone()),
Value::from(instance.uptime_day),
Value::from(instance.uptime_month),
Value::from(None::<i64>),
Value::from(0i32),
Value::from(None::<DateTime<FixedOffset>>),
Value::from(None::<DateTime<FixedOffset>>),
Value::from(None::<String>),
Value::from(0i32),
Value::from(now),
Value::from(now),
]
}
fn seed_insert_values(url: &str, now: DateTime<FixedOffset>) -> Vec<Value> {
vec![
Value::from(Id::generate().to_uuid_v8()),
Value::from(url.to_string()),
Value::from("seed".to_string()),
Value::from(SearxngInstanceState::Candidate.as_str().to_string()),
Value::from(None::<String>),
Value::from(None::<String>),
Value::from(None::<String>),
Value::from(None::<String>),
Value::from(None::<String>),
Value::from(None::<String>),
Value::from(None::<f64>),
Value::from(None::<f64>),
Value::from(None::<i64>),
Value::from(0i32),
Value::from(None::<DateTime<FixedOffset>>),
Value::from(None::<DateTime<FixedOffset>>),
Value::from(None::<String>),
Value::from(0i32),
Value::from(now),
Value::from(now),
]
}
fn inserted_flag(row: Option<dbnexus::sea_orm::QueryResult>) -> Result<bool, RepositoryError> {
let row = row.ok_or_else(|| {
RepositoryError::Database(anyhow::anyhow!("searxng upsert returned no row"))
})?;
row.try_get::<bool>("", "inserted")
.map_err(|e| RepositoryError::Database(e.into()))
}
fn truncate_error(error: Option<&str>) -> Option<String> {
error.map(|e| e.chars().take(LAST_ERROR_MAX_CHARS).collect())
}
async fn find_locked_instance(
tx: &dbnexus::sea_orm::DatabaseTransaction,
url: &str,
) -> Result<searxng_instance::Model, RepositoryError> {
searxng_instance::Entity::find()
.filter(searxng_instance::Column::Url.eq(url))
.lock_exclusive()
.one(tx)
.await
.map_err(|e| RepositoryError::Database(e.into()))?
.ok_or(RepositoryError::NotFound)
}
fn next_miss_state(
current_state: SearxngInstanceState,
miss_count: i32,
retire_after: i32,
) -> (SearxngInstanceState, bool) {
let mut retired = false;
let mut next_state = current_state;
if miss_count > retire_after {
if current_state.can_transition_to(SearxngInstanceState::Retired) {
next_state = SearxngInstanceState::Retired;
retired = true;
} else if current_state == SearxngInstanceState::Active {
next_state = SearxngInstanceState::Cooldown;
}
}
(next_state, retired)
}
async fn persist_dataset_miss(
tx: &dbnexus::sea_orm::DatabaseTransaction,
model: searxng_instance::Model,
miss_count: i32,
next_state: SearxngInstanceState,
) -> Result<(), RepositoryError> {
let mut active: searxng_instance::ActiveModel = model.into();
active.dataset_miss_count = Set(miss_count);
active.state = Set(next_state.as_str().to_string());
active.updated_at = Set(now_tz());
active
.update(tx)
.await
.map_err(|e| RepositoryError::Database(e.into()))?;
Ok(())
}
#[async_trait]
impl SearxngInstanceRepository for SearxngInstanceRepositoryImpl {
async fn try_acquire_collect_lock(
&self,
key: i64,
) -> Result<Option<Box<dyn CollectLockGuard>>, RepositoryError> {
let conn = self.admin_conn().await?;
let tx = conn
.begin()
.await
.map_err(|e| RepositoryError::Database(e.into()))?;
let row = tx
.query_one_raw(Statement::from_sql_and_values(
DatabaseBackend::Postgres,
"SELECT pg_try_advisory_xact_lock($1) AS locked",
[key.into()],
))
.await
.map_err(|e| RepositoryError::Database(e.into()))?;
let row = row.ok_or_else(|| {
RepositoryError::Database(anyhow::anyhow!("pg_try_advisory_xact_lock returned no row"))
})?;
let acquired = row
.try_get::<bool>("", "locked")
.map_err(|e| RepositoryError::Database(e.into()))?;
if acquired {
Ok(Some(Box::new(PgCollectLockGuard {
tx: parking_lot::Mutex::new(Some(tx)),
})))
} else {
tx.rollback()
.await
.map_err(|e| RepositoryError::Database(e.into()))?;
Ok(None)
}
}
async fn upsert_from_dataset(
&self,
instance: &AdmittedInstance,
) -> Result<UpsertOutcome, RepositoryError> {
let conn = self.admin_conn().await?;
let now = now_tz();
let values = dataset_insert_values(instance, now);
let row = conn
.query_one_raw(Statement::from_sql_and_values(
DatabaseBackend::Postgres,
DATASET_UPSERT_SQL,
values,
))
.await
.map_err(|e| RepositoryError::Database(e.into()))?;
let inserted = inserted_flag(row)?;
Ok(if inserted {
UpsertOutcome::Inserted
} else {
UpsertOutcome::Updated
})
}
async fn upsert_seed(&self, url: &str) -> Result<UpsertOutcome, RepositoryError> {
if !url.starts_with("https://") || url.len() <= "https://".len() {
return Err(RepositoryError::Database(anyhow::anyhow!(
"upsert_seed requires a normalized https url, got: {url}"
)));
}
let conn = self.admin_conn().await?;
let now = now_tz();
let row = conn
.query_one_raw(Statement::from_sql_and_values(
DatabaseBackend::Postgres,
SEED_INSERT_SQL,
seed_insert_values(url, now),
))
.await
.map_err(|e| RepositoryError::Database(e.into()))?;
let inserted = inserted_flag(row)?;
Ok(if inserted {
UpsertOutcome::Inserted
} else {
UpsertOutcome::Updated
})
}
async fn list_candidates(&self, limit: u64) -> Result<Vec<SearxngInstance>, RepositoryError> {
let conn = self.admin_conn().await?;
let models = searxng_instance::Entity::find()
.filter(searxng_instance::Column::State.eq(SearxngInstanceState::Candidate.as_str()))
.order_by_with_nulls(
searxng_instance::Column::UptimeDay,
Order::Desc,
NullOrdering::Last,
)
.order_by_with_nulls(
searxng_instance::Column::LatencyMs,
Order::Asc,
NullOrdering::Last,
)
.limit(limit)
.all(&conn)
.await
.map_err(|e| RepositoryError::Database(e.into()))?;
Ok(models.into_iter().map(SearxngInstance::from).collect())
}
async fn list_by_state(
&self,
state: SearxngInstanceState,
limit: u64,
) -> Result<Vec<SearxngInstance>, RepositoryError> {
let conn = self.admin_conn().await?;
let models = searxng_instance::Entity::find()
.filter(searxng_instance::Column::State.eq(state.as_str()))
.order_by_with_nulls(
searxng_instance::Column::UptimeDay,
Order::Desc,
NullOrdering::Last,
)
.limit(limit)
.all(&conn)
.await
.map_err(|e| RepositoryError::Database(e.into()))?;
Ok(models.into_iter().map(SearxngInstance::from).collect())
}
async fn list_paginated(
&self,
limit: u64,
offset: u64,
) -> Result<Vec<SearxngInstance>, RepositoryError> {
let conn = self.admin_conn().await?;
let models = searxng_instance::Entity::find()
.order_by_asc(searxng_instance::Column::State)
.order_by_with_nulls(
searxng_instance::Column::UptimeDay,
Order::Desc,
NullOrdering::Last,
)
.offset(offset)
.limit(limit)
.all(&conn)
.await
.map_err(|e| RepositoryError::Database(e.into()))?;
Ok(models.into_iter().map(SearxngInstance::from).collect())
}
async fn record_probe_result(
&self,
url: &str,
ok: bool,
latency_ms: Option<i64>,
error: Option<&str>,
) -> Result<(), RepositoryError> {
let conn = self.admin_conn().await?;
let now = now_tz();
let update =
searxng_instance::Entity::update_many().filter(searxng_instance::Column::Url.eq(url));
let update = if ok {
update
.col_expr(
searxng_instance::Column::ConsecutiveFailures,
Expr::value(0),
)
.col_expr(searxng_instance::Column::LatencyMs, Expr::value(latency_ms))
.col_expr(
searxng_instance::Column::LastSuccessAt,
Expr::value(Some(now)),
)
.col_expr(
searxng_instance::Column::LastError,
Expr::value(None::<String>),
)
} else {
update
.col_expr(
searxng_instance::Column::ConsecutiveFailures,
Expr::col(searxng_instance::Column::ConsecutiveFailures).add(1),
)
.col_expr(
searxng_instance::Column::LastFailureAt,
Expr::value(Some(now)),
)
.col_expr(
searxng_instance::Column::LastError,
Expr::value(truncate_error(error)),
)
};
let result = update
.col_expr(searxng_instance::Column::UpdatedAt, Expr::value(now))
.exec(&conn)
.await
.map_err(|e| RepositoryError::Database(e.into()))?;
if result.rows_affected == 0 {
return Err(RepositoryError::NotFound);
}
Ok(())
}
async fn transition_state(
&self,
url: &str,
to: SearxngInstanceState,
) -> Result<(), RepositoryError> {
let model = self
.find_model(url)
.await?
.ok_or(RepositoryError::NotFound)?;
let from: SearxngInstanceState = model
.state
.parse()
.map_err(|e| RepositoryError::Database(anyhow::anyhow!("{e}")))?;
if !from.can_transition_to(to) {
return Err(RepositoryError::Database(anyhow::anyhow!(
"illegal searxng instance state transition for {url}: {from} -> {to}"
)));
}
if from == to {
return Ok(());
}
let conn = self.admin_conn().await?;
let mut active: searxng_instance::ActiveModel = model.into();
active.state = Set(to.as_str().to_string());
active.updated_at = Set(now_tz());
active
.update(&conn)
.await
.map_err(|e| RepositoryError::Database(e.into()))?;
Ok(())
}
async fn record_dataset_miss(
&self,
url: &str,
retire_after: i32,
) -> Result<bool, RepositoryError> {
let conn = self.admin_conn().await?;
let tx = conn
.begin()
.await
.map_err(|e| RepositoryError::Database(e.into()))?;
let model = find_locked_instance(&tx, url).await?;
if model.source != "searx_space" {
tx.commit()
.await
.map_err(|e| RepositoryError::Database(e.into()))?;
return Ok(false);
}
let miss_count = model.dataset_miss_count.saturating_add(1);
let current_state: SearxngInstanceState = model
.state
.parse()
.map_err(|e| RepositoryError::Database(anyhow::anyhow!("{e}")))?;
let (next_state, retired) = next_miss_state(current_state, miss_count, retire_after);
persist_dataset_miss(&tx, model, miss_count, next_state).await?;
tx.commit()
.await
.map_err(|e| RepositoryError::Database(e.into()))?;
Ok(retired)
}
async fn get_by_url(&self, url: &str) -> Result<Option<SearxngInstance>, RepositoryError> {
Ok(self.find_model(url).await?.map(SearxngInstance::from))
}
async fn count_by_state(&self, state: SearxngInstanceState) -> Result<u64, RepositoryError> {
let conn = self.admin_conn().await?;
searxng_instance::Entity::find()
.filter(searxng_instance::Column::State.eq(state.as_str()))
.count(&conn)
.await
.map_err(|e| RepositoryError::Database(e.into()))
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_truncate_error_caps_length() {
let long = "x".repeat(LAST_ERROR_MAX_CHARS + 100);
let truncated = truncate_error(Some(&long)).expect("some");
assert_eq!(truncated.chars().count(), LAST_ERROR_MAX_CHARS);
}
#[test]
fn test_truncate_error_preserves_none_and_short() {
assert!(truncate_error(None).is_none());
assert_eq!(truncate_error(Some("boom")), Some("boom".to_string()));
}
#[test]
fn test_now_tz_is_utc() {
assert_eq!(now_tz().offset().local_minus_utc(), 0);
}
}