crawlrs 0.2.0

crawlrs - Enterprise-grade web scraping and crawling platform built with Rust
Documentation
// Copyright (c) 2025-2026 Kirky.X🌠
// SPDX-License-Identifier: Apache-2.0

//! Worker 启动去重 — 提取 `start_api_service` 和 `start_worker_service` 中
//! 重复的 worker spawn 代码。

use crate::config::settings::Settings;
use crate::di::CrawlRsState;
use crate::di::CrawlRsStateExt;
use crate::workers::searxng_collector_worker::SearxngCollectorWorker;
use crate::workers::searxng_health_worker::SearxngHealthWorker;
use crate::workers::{AbstractWorker, Worker};
use std::sync::Arc;

/// 启动所有通用 worker(webhook / backlog / expiration)。
///
/// `start_api_service()` 和 `start_worker_service()` 共享的 worker spawn 逻辑
/// 统一由此函数管理,避免重复代码。
///
/// 返回所有 worker 的 `JoinHandle`,调用方应在关闭时 await
/// 并记录 panic。
pub async fn spawn_common_workers(
    app_state: &CrawlRsState,
    settings: &Arc<Settings>,
) -> Vec<tokio::task::JoinHandle<()>> {
    let mut handles: Vec<tokio::task::JoinHandle<()>> = Vec::new();

    // webhook worker 仅在 webhook feature 启用时启动
    #[cfg(feature = "webhook")]
    {
        let webhook_worker = AbstractWorker::new(
            app_state.webhook_worker(),
            std::time::Duration::from_secs(5),
        );
        handles.push(tokio::spawn(async move {
            webhook_worker.run().await;
        }));
    }

    let backlog_worker = AbstractWorker::new(
        app_state.backlog_worker(),
        std::time::Duration::from_secs(settings.timeouts.workers.backlog_interval_seconds),
    );
    handles.push(tokio::spawn(async move {
        backlog_worker.run().await;
    }));

    let expiration_worker = AbstractWorker::new(
        app_state.expiration_worker(),
        std::time::Duration::from_secs(3600), // Run every hour
    );
    handles.push(tokio::spawn(async move {
        expiration_worker.run().await;
    }));

    // SearXNG 双周期 worker:仅在实例生命周期服务存在(enabled=true)时拉起。
    // api 与 worker 双进程都会拉起:采集轮由 PG advisory lock 防重,探活幂等。
    if let Some(searxng) = app_state.searxng_instances() {
        let collector = AbstractWorker::new(
            Arc::new(SearxngCollectorWorker::new(searxng.clone())),
            std::time::Duration::from_secs(settings.search.searxng.collector.interval_seconds),
        );
        handles.push(tokio::spawn(async move {
            collector.run().await;
        }));

        let health = AbstractWorker::new(
            Arc::new(SearxngHealthWorker::new(searxng)),
            std::time::Duration::from_secs(settings.search.searxng.health.interval_seconds),
        );
        handles.push(tokio::spawn(async move {
            health.run().await;
        }));
    }

    handles
}