kreuzcrawl 0.2.0

High-performance web crawling engine
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32

//! Shared API state.

use std::sync::Arc;
use std::time::Duration;

use crate::engine::CrawlEngine;

use super::jobs::JobRegistry;

/// Default time-to-live for jobs (1 hour).
const DEFAULT_JOB_MAX_AGE: Duration = Duration::from_secs(3600);

/// Shared state for all API handlers.
#[derive(Clone)]
pub struct ApiState {
    /// The crawl engine used to execute scrape, crawl, and map operations.
    pub engine: Arc<CrawlEngine>,
    /// Registry of asynchronous crawl and batch scrape jobs.
    pub jobs: Arc<JobRegistry>,
}

impl ApiState {
    /// Create a new `ApiState` wrapping the given engine.
    ///
    /// This also spawns a background task that periodically evicts expired jobs
    /// from the registry (default TTL: 1 hour).
    pub fn new(engine: Arc<CrawlEngine>) -> Self {
        let jobs = Arc::new(JobRegistry::new());
        jobs.spawn_eviction_task(DEFAULT_JOB_MAX_AGE);
        Self { engine, jobs }
    }
}