use crate::common::error::{RepositoryResultExt, WorkerError};
use crate::config::Settings;
use crate::domain::models::task_domain::TaskStatus;
use crate::domain::repositories::{
task_repository::TaskRepository,
tasks_backlog_repository::{TasksBacklog, TasksBacklogRepository},
};
use crate::domain::services::rate_limiting_service::{
ConcurrencyResult, RateLimitingError, RateLimitingService,
};
use crate::i18n::{tr_log, tr_log_args};
use crate::workers::worker::{ProcessResult, WorkerProcess};
use async_trait::async_trait;
use chrono::Utc;
use log::{error, info, warn};
use std::sync::atomic::{AtomicU64, Ordering};
use std::sync::Arc;
pub struct BacklogWorker {
tasks_backlog_repository: Arc<dyn TasksBacklogRepository>,
task_repository: Arc<dyn TaskRepository>,
rate_limiting_service: Arc<dyn RateLimitingService>,
settings: Arc<Settings>,
cleanup_cycle_counter: AtomicU64,
}
enum BacklogOutcome {
Reactivated,
Expired,
RetryExhausted,
Denied,
Queued,
Skipped,
}
#[derive(Default)]
struct BacklogCounts {
reactivated: usize,
expired: usize,
retry_exhausted: usize,
denied: usize,
queued: usize,
skipped: usize,
failed: usize,
}
impl BacklogCounts {
fn add(&mut self, other: Self) {
self.reactivated += other.reactivated;
self.expired += other.expired;
self.retry_exhausted += other.retry_exhausted;
self.denied += other.denied;
self.queued += other.queued;
self.skipped += other.skipped;
self.failed += other.failed;
}
}
impl BacklogWorker {
pub fn new(
tasks_backlog_repository: Arc<dyn TasksBacklogRepository>,
task_repository: Arc<dyn TaskRepository>,
rate_limiting_service: Arc<dyn RateLimitingService>,
settings: Arc<Settings>,
) -> Self {
Self {
tasks_backlog_repository,
task_repository,
rate_limiting_service,
settings,
cleanup_cycle_counter: AtomicU64::new(1),
}
}
async fn process_backlog(&self) -> Result<(), WorkerError> {
info!("{}", tr_log("backlog-processing-started"));
let batch_size = self.settings.concurrency.default_team_limit as usize;
let pending_backlogs = self
.tasks_backlog_repository
.get_pending_tasks(None, Some(batch_size as u64))
.await
.repo_err()?;
if pending_backlogs.is_empty() {
info!("{}", tr_log("backlog-none-pending"));
return Ok(());
}
info!(
"{}",
tr_log_args(
"backlog-pending-found",
&[("count", pending_backlogs.len().to_string())],
)
);
let backlogs_by_team = group_backlogs_by_team(pending_backlogs);
let mut counts = BacklogCounts::default();
for (team_id, team_backlogs) in backlogs_by_team {
counts.add(self.process_team_backlogs(team_id, team_backlogs).await);
}
log_backlog_summary(&counts);
Ok(())
}
async fn process_team_backlogs(
&self,
team_id: crate::domain::id::Id,
team_backlogs: Vec<TasksBacklog>,
) -> BacklogCounts {
info!(
"{}",
tr_log_args(
"backlog-team-processing",
&[
("team_id", team_id.to_string()),
("count", team_backlogs.len().to_string()),
],
)
);
let mut counts = BacklogCounts::default();
for backlog in team_backlogs {
match self.process_single_backlog(backlog).await {
Ok(BacklogOutcome::Reactivated) => counts.reactivated += 1,
Ok(BacklogOutcome::Expired) => counts.expired += 1,
Ok(BacklogOutcome::RetryExhausted) => counts.retry_exhausted += 1,
Ok(BacklogOutcome::Denied) => counts.denied += 1,
Ok(BacklogOutcome::Queued) => counts.queued += 1,
Ok(BacklogOutcome::Skipped) => counts.skipped += 1,
Err(e) => {
error!(
"{}",
tr_log_args("backlog-processing-failed", &[("error", e.to_string())],)
);
counts.failed += 1;
}
}
}
counts
}
async fn process_single_backlog(
&self,
backlog: TasksBacklog,
) -> Result<BacklogOutcome, WorkerError> {
if backlog.is_expired() {
return self.mark_backlog_expired(&backlog).await;
}
if !backlog.can_retry() {
return self.mark_backlog_retry_exhausted(&backlog).await;
}
let concurrency = self
.rate_limiting_service
.check_team_concurrency(backlog.team_id, backlog.task_id)
.await;
self.apply_concurrency_result(&backlog, concurrency).await
}
async fn mark_backlog_expired(
&self,
backlog: &TasksBacklog,
) -> Result<BacklogOutcome, WorkerError> {
info!(
"{}",
tr_log_args(
"backlog-expired-marked",
&[("task_id", backlog.id.to_string())],
)
);
let mut expired_backlog = backlog.clone();
expired_backlog
.mark_expired()
.map_err(WorkerError::DomainError)?;
self.tasks_backlog_repository
.update(&expired_backlog)
.await
.repo_err()?;
Ok(BacklogOutcome::Expired)
}
async fn mark_backlog_retry_exhausted(
&self,
backlog: &TasksBacklog,
) -> Result<BacklogOutcome, WorkerError> {
warn!(
"{}",
tr_log_args(
"backlog-retry-exhausted-marked",
&[("task_id", backlog.id.to_string())],
)
);
let mut failed_backlog = backlog.clone();
failed_backlog
.mark_failed()
.map_err(WorkerError::DomainError)?;
self.tasks_backlog_repository
.update(&failed_backlog)
.await
.repo_err()?;
Ok(BacklogOutcome::RetryExhausted)
}
async fn apply_concurrency_result(
&self,
backlog: &TasksBacklog,
concurrency: Result<ConcurrencyResult, RateLimitingError>,
) -> Result<BacklogOutcome, WorkerError> {
match concurrency {
Ok(ConcurrencyResult::Allowed) => self.reactivate_allowed_backlog(backlog).await,
Ok(ConcurrencyResult::Denied { reason }) => {
info!(
"{}",
tr_log_args(
"backlog-concurrency-denied",
&[
("team_id", backlog.team_id.to_string()),
("reason", reason.to_string()),
("task_id", backlog.id.to_string()),
],
)
);
Ok(BacklogOutcome::Denied)
}
Ok(ConcurrencyResult::Queued { .. }) => {
warn!(
"{}",
tr_log_args(
"backlog-unexpected-requeue",
&[("task_id", backlog.id.to_string())],
)
);
Ok(BacklogOutcome::Queued)
}
Err(e) => {
error!(
"{}",
tr_log_args(
"backlog-concurrency-check-failed",
&[("error", e.to_string())],
)
);
Err(WorkerError::ServiceError(e.to_string()))
}
}
}
async fn reactivate_allowed_backlog(
&self,
backlog: &TasksBacklog,
) -> Result<BacklogOutcome, WorkerError> {
log_backlog_concurrency_available(backlog);
let Some(processing_backlog) = begin_backlog_processing(backlog) else {
return Ok(BacklogOutcome::Skipped);
};
match self.reactivate_task(processing_backlog).await {
Ok(_) => {
info!(
"{}",
tr_log_args(
"backlog-reactivated",
&[("task_id", backlog.id.to_string())],
)
);
Ok(BacklogOutcome::Reactivated)
}
Err(e) => {
self.persist_retry_after_reactivation_failure(backlog, e)
.await
}
}
}
async fn persist_retry_after_reactivation_failure(
&self,
backlog: &TasksBacklog,
e: WorkerError,
) -> Result<BacklogOutcome, WorkerError> {
error!(
"{}",
tr_log_args("backlog-reactivation-failed", &[("error", e.to_string())],)
);
let mut retry_backlog = backlog.clone();
retry_backlog.increment_retry_count();
self.tasks_backlog_repository
.update(&retry_backlog)
.await
.repo_err()?;
Err(e)
}
async fn reactivate_task(
&self,
backlog: crate::domain::repositories::tasks_backlog_repository::TasksBacklog,
) -> Result<(), WorkerError> {
let task = self
.task_repository
.find_by_id(backlog.task_id)
.await
.repo_err()?
.ok_or_else(|| WorkerError::NotFound(format!("Task {} not found", backlog.task_id)))?;
if task.status != TaskStatus::Queued {
return self.complete_inactive_backlog(&task, backlog).await;
}
let mut updated_task = task.clone();
updated_task.status = TaskStatus::Queued;
updated_task.updated_at = Utc::now();
self.task_repository
.update(&updated_task)
.await
.repo_err()?;
Self::mark_backlog_completed(self.tasks_backlog_repository.as_ref(), backlog).await?;
info!(
"{}",
tr_log_args(
"backlog-task-reactivated",
&[("task_id", task.id.to_string())],
)
);
Ok(())
}
async fn complete_inactive_backlog(
&self,
task: &crate::domain::models::Task,
backlog: crate::domain::repositories::tasks_backlog_repository::TasksBacklog,
) -> Result<(), WorkerError> {
info!(
"{}",
tr_log_args(
"backlog-task-status-not-activatable",
&[
("task_id", task.id.to_string()),
("status", task.status.to_string()),
],
)
);
Self::mark_backlog_completed(self.tasks_backlog_repository.as_ref(), backlog).await
}
async fn mark_backlog_completed(
repository: &dyn crate::domain::repositories::tasks_backlog_repository::TasksBacklogRepository,
mut backlog: crate::domain::repositories::tasks_backlog_repository::TasksBacklog,
) -> Result<(), WorkerError> {
backlog
.mark_completed()
.map_err(|e| WorkerError::DomainError(e.to_string()))?;
repository.update(&backlog).await.repo_err()?;
Ok(())
}
async fn cleanup_expired_tasks(&self) -> Result<(), WorkerError> {
info!("{}", tr_log("backlog-cleanup-started"));
let expired_backlogs = self
.tasks_backlog_repository
.get_expired_tasks(Some(100))
.await
.repo_err()?;
if expired_backlogs.is_empty() {
info!("{}", tr_log("backlog-cleanup-none"));
return Ok(());
}
let mut cleaned_count = 0;
for backlog in expired_backlogs {
match self.process_expired_backlog(backlog).await {
Ok(_) => cleaned_count += 1,
Err(e) => {
error!(
"{}",
tr_log_args("backlog-cleanup-failed", &[("error", e.to_string())],)
);
}
}
}
info!(
"{}",
tr_log_args(
"backlog-cleanup-completed",
&[("count", cleaned_count.to_string())],
)
);
Ok(())
}
async fn process_expired_backlog(
&self,
backlog: crate::domain::repositories::tasks_backlog_repository::TasksBacklog,
) -> Result<(), WorkerError> {
info!(
"{}",
tr_log_args(
"backlog-cleanup-processing",
&[("task_id", backlog.id.to_string())],
)
);
let mut expired_backlog = backlog.clone();
expired_backlog
.mark_expired()
.map_err(WorkerError::DomainError)?;
self.tasks_backlog_repository
.update(&expired_backlog)
.await
.repo_err()?;
let task = self
.task_repository
.find_by_id(backlog.task_id)
.await
.repo_err()?;
if let Some(task) = task {
if task.status == TaskStatus::Queued {
let mut failed_task = task.clone();
failed_task.status = TaskStatus::Failed;
failed_task.updated_at = Utc::now();
self.task_repository.update(&failed_task).await.repo_err()?;
info!(
"{}",
tr_log_args(
"backlog-task-marked-failed-by-expiry",
&[("task_id", task.id.to_string())],
)
);
}
}
Ok(())
}
}
fn log_backlog_concurrency_available(backlog: &TasksBacklog) {
info!(
"{}",
tr_log_args(
"backlog-concurrency-available",
&[
("team_id", backlog.team_id.to_string()),
("task_id", backlog.id.to_string()),
],
)
);
}
fn begin_backlog_processing(backlog: &TasksBacklog) -> Option<TasksBacklog> {
let mut processing_backlog = backlog.clone();
if let Err(e) = processing_backlog.mark_processing() {
warn!(
"{}",
tr_log_args(
"backlog-state-transition-skipped",
&[
("task_id", backlog.id.to_string()),
("current", backlog.status.to_string()),
("error", e.to_string()),
],
)
);
return None;
}
Some(processing_backlog)
}
fn group_backlogs_by_team(
pending_backlogs: Vec<TasksBacklog>,
) -> std::collections::HashMap<crate::domain::id::Id, Vec<TasksBacklog>> {
let mut backlogs_by_team: std::collections::HashMap<crate::domain::id::Id, Vec<TasksBacklog>> =
std::collections::HashMap::with_capacity(32);
for backlog in pending_backlogs {
backlogs_by_team
.entry(backlog.team_id)
.or_default()
.push(backlog);
}
backlogs_by_team
}
fn log_backlog_summary(counts: &BacklogCounts) {
info!(
"{}",
tr_log_args(
"backlog-processing-summary",
&[
("reactivated", counts.reactivated.to_string()),
("expired", counts.expired.to_string()),
("retry_exhausted", counts.retry_exhausted.to_string()),
("denied", counts.denied.to_string()),
("queued", counts.queued.to_string()),
("skipped", counts.skipped.to_string()),
("failed", counts.failed.to_string()),
],
)
);
}
#[async_trait]
impl WorkerProcess for BacklogWorker {
fn name(&self) -> &str {
"backlog-worker"
}
async fn process(&self) -> ProcessResult {
if let Err(e) = self.process_backlog().await {
return ProcessResult::Error(tr_log_args(
"backlog-process-error",
&[("error", e.to_string())],
));
}
let counter = self.cleanup_cycle_counter.fetch_add(1, Ordering::SeqCst);
if counter.is_multiple_of(10) {
if let Err(e) = self.cleanup_expired_tasks().await {
return ProcessResult::Error(tr_log_args(
"backlog-cleanup-error",
&[("error", e.to_string())],
));
}
}
ProcessResult::Completed
}
}
#[cfg(test)]
mod tests {
use super::*;
use crate::domain::models::{CreditsTransactionType, Task, TaskType};
use crate::domain::repositories::task_repository::RepositoryError;
use crate::domain::repositories::tasks_backlog_repository::TasksBacklog;
use crate::domain::services::rate_limiting_service::{BacklogService, QuotaService};
use crate::domain::services::rate_limiting_service::{
ConcurrencyConfig, ConcurrencyControlService, ConcurrencyResult, RateLimitConfig,
RateLimitResult, RateLimitService, RateLimitingError,
};
use async_trait::async_trait;
use std::collections::HashSet;
use std::sync::Mutex;
fn make_test_settings(default_team_limit: u64) -> Arc<Settings> {
use crate::config::settings::*;
let settings = Settings {
server: ServerSettings::default(),
robots: RobotsSettings::default(),
database: DatabaseSettings::default(),
cors: CorsSettings::default(),
rate_limiting: RateLimitingSettings::default(),
retention: RetentionSettings::default(),
concurrency: ConcurrencySettings {
default_team_limit,
task_lock_duration_seconds: 300,
mem_pressure_threshold: 0.8,
mem_critical_threshold: 0.9,
critical_timeout_seconds: 30,
adaptive_enabled: false,
},
webhook: WebhookSettings::default(),
bing_search: BingSearchSettings::default(),
search: SearchSettings::default(),
llm: LLMSettings::default(),
rag: RagSettings::default(),
fluxweir: FluxweirSettings::default(),
proxy: ProxySettings::default(),
engines: EngineSettings::default(),
markdown: Default::default(),
logging: LoggingSettings::default(),
workers: WorkerSettings::default(),
sitemap: SitemapSettings::default(),
crawl: CrawlSettings::default(),
timeouts: TimeoutSettings::default(),
cache: CacheSettings::default(),
websocket: WebSocketSettings::default(),
trusted_proxies: TrustedProxySettings::default(),
observability: ObservabilitySettings::default(),
auth: AuthSettings::default(),
i18n: I18nSettings::default(),
politeness: PolitenessSettings::default(),
session: SessionSettings::default(),
content_extraction: ContentExtractionSettings::default(),
id: IdSettings::default(),
};
Arc::new(settings)
}
struct MockBacklogRepo {
pending: Mutex<Vec<TasksBacklog>>,
expired: Mutex<Vec<TasksBacklog>>,
updated: Mutex<Vec<TasksBacklog>>,
fail_get_pending: bool,
}
impl MockBacklogRepo {
fn new(pending: Vec<TasksBacklog>) -> Self {
Self {
pending: Mutex::new(pending),
expired: Mutex::new(vec![]),
updated: Mutex::new(vec![]),
fail_get_pending: false,
}
}
fn new_empty() -> Self {
Self::new(vec![])
}
fn new_failing() -> Self {
Self {
pending: Mutex::new(vec![]),
expired: Mutex::new(vec![]),
updated: Mutex::new(vec![]),
fail_get_pending: true,
}
}
fn new_with_expired(expired: Vec<TasksBacklog>) -> Self {
Self {
pending: Mutex::new(vec![]),
expired: Mutex::new(expired),
updated: Mutex::new(vec![]),
fail_get_pending: false,
}
}
fn updated(&self) -> Vec<TasksBacklog> {
self.updated.lock().unwrap().clone()
}
}
#[async_trait]
impl TasksBacklogRepository for MockBacklogRepo {
async fn create(&self, backlog: &TasksBacklog) -> Result<TasksBacklog, RepositoryError> {
Ok(backlog.clone())
}
async fn find_by_id(
&self,
_id: crate::domain::id::Id,
) -> Result<Option<TasksBacklog>, RepositoryError> {
Ok(None)
}
async fn find_by_task_id(
&self,
_task_id: crate::domain::id::Id,
) -> Result<Option<TasksBacklog>, RepositoryError> {
Ok(None)
}
async fn update(&self, backlog: &TasksBacklog) -> Result<TasksBacklog, RepositoryError> {
self.updated.lock().unwrap().push(backlog.clone());
Ok(backlog.clone())
}
async fn delete(&self, _id: crate::domain::id::Id) -> Result<(), RepositoryError> {
Ok(())
}
async fn get_pending_tasks(
&self,
_team_id: Option<crate::domain::id::Id>,
_limit: Option<u64>,
) -> Result<Vec<TasksBacklog>, RepositoryError> {
if self.fail_get_pending {
return Err(RepositoryError::Database(anyhow::anyhow!("db error")));
}
Ok(self.pending.lock().unwrap().drain(..).collect())
}
async fn get_expired_tasks(
&self,
_limit: Option<u64>,
) -> Result<Vec<TasksBacklog>, RepositoryError> {
Ok(self.expired.lock().unwrap().drain(..).collect())
}
async fn count_by_status(
&self,
_team_id: Option<crate::domain::id::Id>,
_status: crate::domain::repositories::tasks_backlog_repository::TasksBacklogStatus,
) -> Result<i64, RepositoryError> {
Ok(0)
}
async fn update_status_batch(
&self,
_ids: &[crate::domain::id::Id],
_status: crate::domain::repositories::tasks_backlog_repository::TasksBacklogStatus,
) -> Result<u64, RepositoryError> {
Ok(0)
}
}
struct MockTaskRepo {
tasks: Mutex<std::collections::HashMap<crate::domain::id::Id, Task>>,
updated_tasks: Mutex<Vec<Task>>,
expire_count: AtomicU64,
}
impl MockTaskRepo {
fn new() -> Self {
Self {
tasks: Mutex::new(std::collections::HashMap::new()),
updated_tasks: Mutex::new(vec![]),
expire_count: AtomicU64::new(0),
}
}
fn with_task(self, task: Task) -> Self {
self.tasks.lock().unwrap().insert(task.id, task);
self
}
fn updated_tasks(&self) -> Vec<Task> {
self.updated_tasks.lock().unwrap().clone()
}
}
#[async_trait]
impl TaskRepository for MockTaskRepo {
async fn create(&self, task: &Task) -> Result<Task, RepositoryError> {
Ok(task.clone())
}
async fn find_by_id(
&self,
id: crate::domain::id::Id,
) -> Result<Option<Task>, RepositoryError> {
Ok(self.tasks.lock().unwrap().get(&id).cloned())
}
async fn update(&self, task: &Task) -> Result<Task, RepositoryError> {
self.updated_tasks.lock().unwrap().push(task.clone());
Ok(task.clone())
}
async fn acquire_next(
&self,
_worker_id: crate::domain::id::Id,
) -> Result<Option<Task>, RepositoryError> {
Ok(None)
}
async fn mark_completed(
&self,
_id: crate::domain::id::Id,
_lock_token: Option<crate::domain::id::Id>,
) -> Result<u64, RepositoryError> {
Ok(1)
}
async fn mark_failed(
&self,
_id: crate::domain::id::Id,
_lock_token: Option<crate::domain::id::Id>,
) -> Result<u64, RepositoryError> {
Ok(1)
}
async fn mark_cancelled(&self, _id: crate::domain::id::Id) -> Result<u64, RepositoryError> {
Ok(1)
}
async fn exists_by_url(&self, _url: &str) -> Result<bool, RepositoryError> {
Ok(false)
}
async fn find_existing_urls(
&self,
_urls: &[String],
) -> Result<HashSet<String>, RepositoryError> {
Ok(HashSet::new())
}
async fn reset_stuck_tasks(
&self,
_timeout: chrono::Duration,
) -> Result<u64, RepositoryError> {
Ok(0)
}
async fn cancel_tasks_by_crawl_id(
&self,
_crawl_id: crate::domain::id::Id,
) -> Result<u64, RepositoryError> {
Ok(0)
}
async fn expire_tasks(&self) -> Result<u64, RepositoryError> {
self.expire_count.fetch_add(1, Ordering::SeqCst);
Ok(0)
}
async fn find_by_crawl_id(
&self,
_crawl_id: crate::domain::id::Id,
) -> Result<Vec<Task>, RepositoryError> {
Ok(vec![])
}
async fn query_tasks(
&self,
_params: crate::domain::repositories::task_repository::TaskQueryParams,
) -> Result<(Vec<Task>, u64), RepositoryError> {
Ok((vec![], 0))
}
async fn batch_cancel(
&self,
_task_ids: Vec<crate::domain::id::Id>,
_team_id: crate::domain::id::Id,
_force: bool,
) -> Result<
(
Vec<crate::domain::id::Id>,
Vec<(crate::domain::id::Id, String)>,
),
RepositoryError,
> {
Ok((vec![], vec![]))
}
async fn renew_lock(
&self,
_task_id: crate::domain::id::Id,
_worker_id: crate::domain::id::Id,
_extend_seconds: i64,
) -> Result<bool, RepositoryError> {
Ok(true)
}
}
struct MockRateLimitingService {
concurrency_result: Mutex<ConcurrencyResult>,
should_error: bool,
}
impl MockRateLimitingService {
fn new_allowed() -> Self {
Self {
concurrency_result: Mutex::new(ConcurrencyResult::Allowed),
should_error: false,
}
}
fn new_denied() -> Self {
Self {
concurrency_result: Mutex::new(ConcurrencyResult::Denied {
reason: "limit reached".to_string(),
}),
should_error: false,
}
}
fn new_queued() -> Self {
Self {
concurrency_result: Mutex::new(ConcurrencyResult::Queued {
backlog_id: crate::domain::id::Id::generate(),
}),
should_error: false,
}
}
fn new_error() -> Self {
Self {
concurrency_result: Mutex::new(ConcurrencyResult::Allowed),
should_error: true,
}
}
}
#[async_trait]
impl RateLimitService for MockRateLimitingService {
async fn check_rate_limit(
&self,
_api_key: &str,
_endpoint: &str,
) -> Result<RateLimitResult, RateLimitingError> {
Ok(RateLimitResult::Allowed)
}
async fn get_team_rate_limit_config(
&self,
_team_id: crate::domain::id::Id,
) -> Result<RateLimitConfig, RateLimitingError> {
Ok(RateLimitConfig::default())
}
async fn update_team_rate_limit_config(
&self,
_team_id: crate::domain::id::Id,
_config: RateLimitConfig,
) -> Result<(), RateLimitingError> {
Ok(())
}
async fn cleanup_expired_rate_limits(&self) -> Result<u64, RateLimitingError> {
Ok(0)
}
}
#[async_trait]
impl ConcurrencyControlService for MockRateLimitingService {
async fn check_team_concurrency(
&self,
_team_id: crate::domain::id::Id,
_task_id: crate::domain::id::Id,
) -> Result<ConcurrencyResult, RateLimitingError> {
if self.should_error {
return Err(RateLimitingError::Other(anyhow::anyhow!(
"concurrency check error"
)));
}
Ok(self.concurrency_result.lock().unwrap().clone())
}
async fn release_team_concurrency_slot(
&self,
_team_id: crate::domain::id::Id,
_task_id: crate::domain::id::Id,
) -> Result<(), RateLimitingError> {
Ok(())
}
async fn get_team_current_concurrency(
&self,
_team_id: crate::domain::id::Id,
) -> Result<u32, RateLimitingError> {
Ok(0)
}
async fn get_team_concurrency_config(
&self,
_team_id: crate::domain::id::Id,
) -> Result<ConcurrencyConfig, RateLimitingError> {
Ok(ConcurrencyConfig::default())
}
async fn update_team_concurrency_config(
&self,
_team_id: crate::domain::id::Id,
_config: ConcurrencyConfig,
) -> Result<(), RateLimitingError> {
Ok(())
}
}
#[async_trait]
impl BacklogService for MockRateLimitingService {
async fn process_backlog_tasks(
&self,
_team_id: crate::domain::id::Id,
) -> Result<u32, RateLimitingError> {
Ok(0)
}
}
#[async_trait]
impl QuotaService for MockRateLimitingService {
async fn check_and_deduct_quota(
&self,
_team_id: crate::domain::id::Id,
_amount: i64,
_transaction_type: CreditsTransactionType,
_description: String,
_reference_id: Option<crate::domain::id::Id>,
) -> Result<(), RateLimitingError> {
Ok(())
}
async fn get_quota_balance(
&self,
_team_id: crate::domain::id::Id,
) -> Result<i64, RateLimitingError> {
Ok(100)
}
async fn refund_quota(
&self,
_team_id: crate::domain::id::Id,
_amount: i64,
_description: String,
_reference_id: Option<crate::domain::id::Id>,
) -> Result<(), RateLimitingError> {
Ok(())
}
}
#[async_trait]
impl RateLimitingService for MockRateLimitingService {}
fn make_backlog(
team_id: crate::domain::id::Id,
task_id: crate::domain::id::Id,
) -> TasksBacklog {
TasksBacklog::new(
task_id,
team_id,
"scrape".to_string(),
0,
serde_json::json!({"url": "http://example.com"}),
None,
)
}
fn make_expired_backlog(
team_id: crate::domain::id::Id,
task_id: crate::domain::id::Id,
) -> TasksBacklog {
let mut backlog = make_backlog(team_id, task_id);
backlog.expires_at = Some(Utc::now() - chrono::Duration::hours(1));
backlog
}
fn make_max_retry_backlog(
team_id: crate::domain::id::Id,
task_id: crate::domain::id::Id,
) -> TasksBacklog {
let mut backlog = make_backlog(team_id, task_id);
backlog.retry_count = 3;
backlog.max_retries = 3;
backlog
}
fn make_processing_backlog(
team_id: crate::domain::id::Id,
task_id: crate::domain::id::Id,
) -> TasksBacklog {
let mut backlog = make_backlog(team_id, task_id);
backlog.mark_processing().unwrap();
backlog
}
fn make_task(id: crate::domain::id::Id, status: TaskStatus) -> Task {
let now = Utc::now();
Task {
id,
task_type: TaskType::Scrape,
status,
priority: 0,
team_id: crate::domain::id::Id::generate(),
api_key_id: crate::domain::id::Id::generate(),
url: "http://example.com".to_string(),
payload: serde_json::json!({}),
retry_count: 0,
attempt_count: 0,
max_retries: 3,
scheduled_at: None,
expires_at: None,
created_at: now,
started_at: None,
completed_at: None,
crawl_id: None,
updated_at: now,
lock_token: None,
lock_expires_at: None,
}
}
fn make_worker(
backlog_repo: Arc<dyn TasksBacklogRepository>,
task_repo: Arc<dyn TaskRepository>,
rate_limiting: Arc<dyn RateLimitingService>,
settings: Arc<Settings>,
) -> BacklogWorker {
BacklogWorker::new(backlog_repo, task_repo, rate_limiting, settings)
}
fn default_settings() -> Arc<Settings> {
make_test_settings(10)
}
#[test]
fn test_worker_name() {
let worker = make_worker(
Arc::new(MockBacklogRepo::new_empty()),
Arc::new(MockTaskRepo::new()),
Arc::new(MockRateLimitingService::new_allowed()),
default_settings(),
);
assert_eq!(worker.name(), "backlog-worker");
}
#[tokio::test]
async fn test_process_empty_backlog_returns_completed() {
let worker = make_worker(
Arc::new(MockBacklogRepo::new_empty()),
Arc::new(MockTaskRepo::new()),
Arc::new(MockRateLimitingService::new_allowed()),
default_settings(),
);
let result = worker.process().await;
assert_eq!(result, ProcessResult::Completed);
}
#[tokio::test]
async fn test_process_repo_failure_returns_error() {
let worker = make_worker(
Arc::new(MockBacklogRepo::new_failing()),
Arc::new(MockTaskRepo::new()),
Arc::new(MockRateLimitingService::new_allowed()),
default_settings(),
);
let result = worker.process().await;
match result {
ProcessResult::Error(msg) => {
assert!(
msg.contains("backlog-process-error"),
"tr_log falls back to the key when the startup bundle is not initialized: {msg}"
);
}
_ => panic!("Expected ProcessResult::Error, got {:?}", result),
}
}
#[tokio::test]
async fn test_process_expired_backlog_marks_expired() {
let team_id = crate::domain::id::Id::generate();
let task_id = crate::domain::id::Id::generate();
let backlog = make_expired_backlog(team_id, task_id);
let repo = Arc::new(MockBacklogRepo::new(vec![backlog]));
let updated_repo = repo.clone();
let worker = make_worker(
repo,
Arc::new(MockTaskRepo::new()),
Arc::new(MockRateLimitingService::new_allowed()),
default_settings(),
);
let result = worker.process().await;
assert_eq!(result, ProcessResult::Completed);
let updated = updated_repo.updated();
assert_eq!(updated.len(), 1);
assert_eq!(
updated[0].status,
crate::domain::repositories::tasks_backlog_repository::TasksBacklogStatus::Expired
);
}
#[tokio::test]
async fn test_process_max_retry_backlog_marks_failed() {
let team_id = crate::domain::id::Id::generate();
let task_id = crate::domain::id::Id::generate();
let backlog = make_max_retry_backlog(team_id, task_id);
let repo = Arc::new(MockBacklogRepo::new(vec![backlog]));
let updated_repo = repo.clone();
let worker = make_worker(
repo,
Arc::new(MockTaskRepo::new()),
Arc::new(MockRateLimitingService::new_allowed()),
default_settings(),
);
let result = worker.process().await;
assert_eq!(result, ProcessResult::Completed);
let updated = updated_repo.updated();
assert_eq!(updated.len(), 1);
assert_eq!(
updated[0].status,
crate::domain::repositories::tasks_backlog_repository::TasksBacklogStatus::Failed
);
}
#[tokio::test]
async fn test_process_concurrency_denied_keeps_backlog() {
let team_id = crate::domain::id::Id::generate();
let task_id = crate::domain::id::Id::generate();
let backlog = make_backlog(team_id, task_id);
let repo = Arc::new(MockBacklogRepo::new(vec![backlog]));
let worker = make_worker(
repo,
Arc::new(MockTaskRepo::new()),
Arc::new(MockRateLimitingService::new_denied()),
default_settings(),
);
let result = worker.process().await;
assert_eq!(result, ProcessResult::Completed);
}
#[tokio::test]
async fn test_process_concurrency_queued_returns_completed() {
let team_id = crate::domain::id::Id::generate();
let task_id = crate::domain::id::Id::generate();
let backlog = make_backlog(team_id, task_id);
let repo = Arc::new(MockBacklogRepo::new(vec![backlog]));
let worker = make_worker(
repo,
Arc::new(MockTaskRepo::new()),
Arc::new(MockRateLimitingService::new_queued()),
default_settings(),
);
let result = worker.process().await;
assert_eq!(result, ProcessResult::Completed);
}
#[tokio::test]
async fn test_process_allowed_task_not_found_returns_error() {
let team_id = crate::domain::id::Id::generate();
let task_id = crate::domain::id::Id::generate();
let backlog = make_backlog(team_id, task_id);
let repo = Arc::new(MockBacklogRepo::new(vec![backlog]));
let worker = make_worker(
repo,
Arc::new(MockTaskRepo::new()), Arc::new(MockRateLimitingService::new_allowed()),
default_settings(),
);
let result = worker.process().await;
match result {
ProcessResult::Completed => {}
ProcessResult::Error(msg) => {
panic!("Expected Completed but got Error: {}", msg);
}
_ => panic!("Unexpected result: {:?}", result),
}
}
#[tokio::test]
async fn test_process_allowed_task_not_queued_marks_completed() {
let team_id = crate::domain::id::Id::generate();
let task_id = crate::domain::id::Id::generate();
let backlog = make_backlog(team_id, task_id);
let task = make_task(task_id, TaskStatus::Completed);
let repo = Arc::new(MockBacklogRepo::new(vec![backlog]));
let updated_repo = repo.clone();
let task_repo = Arc::new(MockTaskRepo::new().with_task(task));
let worker = make_worker(
repo,
task_repo,
Arc::new(MockRateLimitingService::new_allowed()),
default_settings(),
);
let result = worker.process().await;
assert_eq!(result, ProcessResult::Completed);
let updated = updated_repo.updated();
assert_eq!(updated.len(), 1);
assert_eq!(
updated[0].status,
crate::domain::repositories::tasks_backlog_repository::TasksBacklogStatus::Completed
);
assert_eq!(updated[0].retry_count, 0);
}
#[tokio::test]
async fn test_process_allowed_task_queued_reactivates() {
let team_id = crate::domain::id::Id::generate();
let task_id = crate::domain::id::Id::generate();
let backlog = make_backlog(team_id, task_id);
let task = make_task(task_id, TaskStatus::Queued);
let repo = Arc::new(MockBacklogRepo::new(vec![backlog]));
let updated_repo = repo.clone();
let task_repo = Arc::new(MockTaskRepo::new().with_task(task));
let worker = make_worker(
repo,
task_repo,
Arc::new(MockRateLimitingService::new_allowed()),
default_settings(),
);
let result = worker.process().await;
assert_eq!(result, ProcessResult::Completed);
let updated = updated_repo.updated();
assert_eq!(updated.len(), 1);
assert_eq!(
updated[0].status,
crate::domain::repositories::tasks_backlog_repository::TasksBacklogStatus::Completed
);
assert_eq!(updated[0].retry_count, 0);
}
#[tokio::test]
async fn test_process_multiple_teams() {
let team1 = crate::domain::id::Id::generate();
let team2 = crate::domain::id::Id::generate();
let backlog1 = make_backlog(team1, crate::domain::id::Id::generate());
let backlog2 = make_backlog(team2, crate::domain::id::Id::generate());
let repo = Arc::new(MockBacklogRepo::new(vec![backlog1, backlog2]));
let worker = make_worker(
repo,
Arc::new(MockTaskRepo::new()),
Arc::new(MockRateLimitingService::new_denied()),
default_settings(),
);
let result = worker.process().await;
assert_eq!(result, ProcessResult::Completed);
}
#[tokio::test]
async fn test_cleanup_runs_on_10th_cycle() {
let worker = make_worker(
Arc::new(MockBacklogRepo::new_empty()),
Arc::new(MockTaskRepo::new()),
Arc::new(MockRateLimitingService::new_allowed()),
default_settings(),
);
for _ in 0..9 {
let r = worker.process().await;
assert_eq!(r, ProcessResult::Completed);
}
let result10 = worker.process().await;
assert_eq!(result10, ProcessResult::Completed);
}
#[tokio::test]
async fn test_cleanup_does_not_run_on_non_10th_cycle() {
let worker = make_worker(
Arc::new(MockBacklogRepo::new_empty()),
Arc::new(MockTaskRepo::new()),
Arc::new(MockRateLimitingService::new_allowed()),
default_settings(),
);
let _ = worker.process().await; let _ = worker.process().await; let _ = worker.process().await; let result = worker.process().await; assert_eq!(result, ProcessResult::Completed);
}
#[tokio::test]
async fn test_cleanup_expired_backlog_task_not_found() {
let team_id = crate::domain::id::Id::generate();
let task_id = crate::domain::id::Id::generate();
let expired_backlog = make_expired_backlog(team_id, task_id);
let repo = Arc::new(MockBacklogRepo::new_with_expired(vec![expired_backlog]));
let updated_repo = repo.clone();
let worker = make_worker(
repo,
Arc::new(MockTaskRepo::new()), Arc::new(MockRateLimitingService::new_allowed()),
default_settings(),
);
worker.cleanup_expired_tasks().await.unwrap();
let updated = updated_repo.updated();
assert_eq!(updated.len(), 1);
assert_eq!(
updated[0].status,
crate::domain::repositories::tasks_backlog_repository::TasksBacklogStatus::Expired
);
}
#[tokio::test]
async fn test_cleanup_expired_with_queued_task_marks_failed() {
let team_id = crate::domain::id::Id::generate();
let task_id = crate::domain::id::Id::generate();
let expired_backlog = make_expired_backlog(team_id, task_id);
let task = make_task(task_id, TaskStatus::Queued);
let repo = Arc::new(MockBacklogRepo::new_with_expired(vec![expired_backlog]));
let updated_repo = repo.clone();
let task_repo = Arc::new(MockTaskRepo::new().with_task(task));
let task_repo_for_check = task_repo.clone();
let worker = make_worker(
repo,
task_repo,
Arc::new(MockRateLimitingService::new_allowed()),
default_settings(),
);
worker.cleanup_expired_tasks().await.unwrap();
let updated = updated_repo.updated();
assert_eq!(updated.len(), 1);
assert_eq!(
updated[0].status,
crate::domain::repositories::tasks_backlog_repository::TasksBacklogStatus::Expired
);
let updated_tasks = task_repo_for_check.updated_tasks();
assert_eq!(updated_tasks.len(), 1);
assert_eq!(updated_tasks[0].status, TaskStatus::Failed);
}
#[tokio::test]
async fn test_cleanup_expired_with_non_queued_task_no_task_update() {
let team_id = crate::domain::id::Id::generate();
let task_id = crate::domain::id::Id::generate();
let expired_backlog = make_expired_backlog(team_id, task_id);
let task = make_task(task_id, TaskStatus::Completed);
let repo = Arc::new(MockBacklogRepo::new_with_expired(vec![expired_backlog]));
let updated_repo = repo.clone();
let worker = make_worker(
repo,
Arc::new(MockTaskRepo::new().with_task(task)),
Arc::new(MockRateLimitingService::new_allowed()),
default_settings(),
);
worker.cleanup_expired_tasks().await.unwrap();
let updated = updated_repo.updated();
assert_eq!(updated.len(), 1);
assert_eq!(
updated[0].status,
crate::domain::repositories::tasks_backlog_repository::TasksBacklogStatus::Expired
);
}
#[tokio::test]
async fn test_cleanup_multiple_expired_backlogs() {
let team1 = crate::domain::id::Id::generate();
let team2 = crate::domain::id::Id::generate();
let expired1 = make_expired_backlog(team1, crate::domain::id::Id::generate());
let expired2 = make_expired_backlog(team2, crate::domain::id::Id::generate());
let repo = Arc::new(MockBacklogRepo::new_with_expired(vec![expired1, expired2]));
let updated_repo = repo.clone();
let worker = make_worker(
repo,
Arc::new(MockTaskRepo::new()),
Arc::new(MockRateLimitingService::new_allowed()),
default_settings(),
);
worker.cleanup_expired_tasks().await.unwrap();
let updated = updated_repo.updated();
assert_eq!(updated.len(), 2);
for entry in &updated {
assert_eq!(
entry.status,
crate::domain::repositories::tasks_backlog_repository::TasksBacklogStatus::Expired
);
}
}
#[tokio::test]
async fn test_process_success_reactivates_queued_task() {
let team_id = crate::domain::id::Id::generate();
let task_id = crate::domain::id::Id::generate();
let backlog = make_backlog(team_id, task_id);
let task = make_task(task_id, TaskStatus::Queued);
let repo = Arc::new(MockBacklogRepo::new(vec![backlog]));
let updated_repo = repo.clone();
let task_repo = Arc::new(MockTaskRepo::new().with_task(task));
let task_repo_for_check = task_repo.clone();
let worker = make_worker(
repo,
task_repo,
Arc::new(MockRateLimitingService::new_allowed()),
default_settings(),
);
let result = worker.process().await;
assert_eq!(result, ProcessResult::Completed);
let updated = updated_repo.updated();
assert_eq!(updated.len(), 1);
assert_eq!(
updated[0].status,
crate::domain::repositories::tasks_backlog_repository::TasksBacklogStatus::Completed
);
let updated_tasks = task_repo_for_check.updated_tasks();
assert_eq!(updated_tasks.len(), 1);
assert_eq!(updated_tasks[0].status, TaskStatus::Queued);
}
#[tokio::test]
async fn test_process_task_not_queued_marks_backlog_completed() {
let team_id = crate::domain::id::Id::generate();
let task_id = crate::domain::id::Id::generate();
let backlog = make_backlog(team_id, task_id);
let task = make_task(task_id, TaskStatus::Completed);
let repo = Arc::new(MockBacklogRepo::new(vec![backlog]));
let updated_repo = repo.clone();
let worker = make_worker(
repo,
Arc::new(MockTaskRepo::new().with_task(task)),
Arc::new(MockRateLimitingService::new_allowed()),
default_settings(),
);
let result = worker.process().await;
assert_eq!(result, ProcessResult::Completed);
let updated = updated_repo.updated();
assert_eq!(updated.len(), 1);
assert_eq!(
updated[0].status,
crate::domain::repositories::tasks_backlog_repository::TasksBacklogStatus::Completed
);
}
#[tokio::test]
async fn test_process_already_processing_backlog_is_skipped() {
let team_id = crate::domain::id::Id::generate();
let task_id = crate::domain::id::Id::generate();
let backlog = make_processing_backlog(team_id, task_id);
let task = make_task(task_id, TaskStatus::Queued);
let repo = Arc::new(MockBacklogRepo::new(vec![backlog]));
let updated_repo = repo.clone();
let worker = make_worker(
repo,
Arc::new(MockTaskRepo::new().with_task(task)),
Arc::new(MockRateLimitingService::new_allowed()),
default_settings(),
);
let result = worker.process().await;
assert_eq!(result, ProcessResult::Completed);
let updated = updated_repo.updated();
assert_eq!(updated.len(), 0, "skipped backlog must not be updated");
}
#[tokio::test]
async fn test_process_concurrency_check_error_returns_completed() {
let team_id = crate::domain::id::Id::generate();
let task_id = crate::domain::id::Id::generate();
let backlog = make_backlog(team_id, task_id);
let repo = Arc::new(MockBacklogRepo::new(vec![backlog]));
let worker = make_worker(
repo,
Arc::new(MockTaskRepo::new()),
Arc::new(MockRateLimitingService::new_error()),
default_settings(),
);
let result = worker.process().await;
assert_eq!(result, ProcessResult::Completed);
}
#[tokio::test]
async fn test_process_mixed_backlogs_expired_and_normal() {
let team_id = crate::domain::id::Id::generate();
let task_id1 = crate::domain::id::Id::generate();
let task_id2 = crate::domain::id::Id::generate();
let expired = make_expired_backlog(team_id, task_id1);
let normal = make_backlog(team_id, task_id2);
let repo = Arc::new(MockBacklogRepo::new(vec![expired, normal]));
let updated_repo = repo.clone();
let worker = make_worker(
repo,
Arc::new(MockTaskRepo::new()),
Arc::new(MockRateLimitingService::new_denied()),
default_settings(),
);
let result = worker.process().await;
assert_eq!(result, ProcessResult::Completed);
let updated = updated_repo.updated();
assert_eq!(updated.len(), 1);
assert_eq!(
updated[0].status,
crate::domain::repositories::tasks_backlog_repository::TasksBacklogStatus::Expired
);
}
struct FailingUpdateBacklogRepo {
expired: Mutex<Vec<TasksBacklog>>,
}
impl FailingUpdateBacklogRepo {
fn new_with_expired(expired: Vec<TasksBacklog>) -> Self {
Self {
expired: Mutex::new(expired),
}
}
}
#[async_trait]
impl TasksBacklogRepository for FailingUpdateBacklogRepo {
async fn create(&self, backlog: &TasksBacklog) -> Result<TasksBacklog, RepositoryError> {
Ok(backlog.clone())
}
async fn find_by_id(
&self,
_id: crate::domain::id::Id,
) -> Result<Option<TasksBacklog>, RepositoryError> {
Ok(None)
}
async fn find_by_task_id(
&self,
_task_id: crate::domain::id::Id,
) -> Result<Option<TasksBacklog>, RepositoryError> {
Ok(None)
}
async fn update(&self, _backlog: &TasksBacklog) -> Result<TasksBacklog, RepositoryError> {
Err(RepositoryError::Database(anyhow::anyhow!("update failed")))
}
async fn delete(&self, _id: crate::domain::id::Id) -> Result<(), RepositoryError> {
Ok(())
}
async fn get_pending_tasks(
&self,
_team_id: Option<crate::domain::id::Id>,
_limit: Option<u64>,
) -> Result<Vec<TasksBacklog>, RepositoryError> {
Ok(vec![])
}
async fn get_expired_tasks(
&self,
_limit: Option<u64>,
) -> Result<Vec<TasksBacklog>, RepositoryError> {
Ok(self.expired.lock().unwrap().drain(..).collect())
}
async fn count_by_status(
&self,
_team_id: Option<crate::domain::id::Id>,
_status: crate::domain::repositories::tasks_backlog_repository::TasksBacklogStatus,
) -> Result<i64, RepositoryError> {
Ok(0)
}
async fn update_status_batch(
&self,
_ids: &[crate::domain::id::Id],
_status: crate::domain::repositories::tasks_backlog_repository::TasksBacklogStatus,
) -> Result<u64, RepositoryError> {
Ok(0)
}
}
#[tokio::test]
async fn test_cleanup_expired_tasks_continues_after_process_error() {
let team_id = crate::domain::id::Id::generate();
let task_id1 = crate::domain::id::Id::generate();
let task_id2 = crate::domain::id::Id::generate();
let expired1 = make_expired_backlog(team_id, task_id1);
let expired2 = make_expired_backlog(team_id, task_id2);
let repo = Arc::new(FailingUpdateBacklogRepo::new_with_expired(vec![
expired1, expired2,
]));
let worker = make_worker(
repo,
Arc::new(MockTaskRepo::new()),
Arc::new(MockRateLimitingService::new_allowed()),
default_settings(),
);
let result = worker.cleanup_expired_tasks().await;
assert!(result.is_ok());
}
#[tokio::test]
async fn test_cleanup_expired_tasks_single_failure_returns_completed() {
let team_id = crate::domain::id::Id::generate();
let task_id = crate::domain::id::Id::generate();
let expired = make_expired_backlog(team_id, task_id);
let repo = Arc::new(FailingUpdateBacklogRepo::new_with_expired(vec![expired]));
let worker = make_worker(
repo,
Arc::new(MockTaskRepo::new()),
Arc::new(MockRateLimitingService::new_allowed()),
default_settings(),
);
let result = worker.cleanup_expired_tasks().await;
assert!(result.is_ok());
}
#[tokio::test]
async fn test_cleanup_expired_tasks_empty_returns_completed() {
let repo = Arc::new(FailingUpdateBacklogRepo::new_with_expired(vec![]));
let worker = make_worker(
repo,
Arc::new(MockTaskRepo::new()),
Arc::new(MockRateLimitingService::new_allowed()),
default_settings(),
);
let result = worker.cleanup_expired_tasks().await;
assert!(result.is_ok());
}
}