spider_lib/
state.rs

1use std::sync::atomic::{AtomicUsize, Ordering};
2use std::sync::Arc;
3
4/// Represents the shared state of the crawler's various actors.
5#[derive(Debug, Default)]
6pub struct CrawlerState {
7    /// The number of requests currently being downloaded.
8    pub in_flight_requests: AtomicUsize,
9    /// The number of responses currently being parsed.
10    pub parsing_responses: AtomicUsize,
11    /// The number of items currently being processed by pipelines.
12    pub processing_items: AtomicUsize,
13}
14
15impl CrawlerState {
16    /// Creates a new, atomically reference-counted `CrawlerState`.
17    pub fn new() -> Arc<Self> {
18        Arc::new(Self::default())
19    }
20
21    /// Checks if all crawler activities are idle.
22    pub fn is_idle(&self) -> bool {
23        self.in_flight_requests.load(Ordering::SeqCst) == 0
24            && self.parsing_responses.load(Ordering::SeqCst) == 0
25            && self.processing_items.load(Ordering::SeqCst) == 0
26    }
27}
spider_lib/state.rs

spider_lib/
state.rs
