Struct StatCollector 

pub struct StatCollector {

    pub start_time: Instant,
    pub requests_enqueued: AtomicUsize,
    pub requests_sent: AtomicUsize,
    pub requests_succeeded: AtomicUsize,
    pub requests_failed: AtomicUsize,
    pub requests_retried: AtomicUsize,
    pub requests_dropped: AtomicUsize,
    pub responses_received: AtomicUsize,
    pub responses_from_cache: AtomicUsize,
    pub response_status_counts: Arc<DashMap<u16, usize>>,
    pub total_bytes_downloaded: AtomicUsize,
    pub items_scraped: AtomicUsize,
    pub items_processed: AtomicUsize,
    pub items_dropped_by_pipeline: AtomicUsize,
    pub request_times: Arc<DashMap<String, Duration>>,
    pub parsing_times: Arc<DashMap<String, Duration>>,
    /* private fields */
}

Collects and stores various statistics about the crawler’s operation.

Fields

start_time: Instant

requests_enqueued: AtomicUsize

requests_sent: AtomicUsize

requests_succeeded: AtomicUsize

requests_failed: AtomicUsize

requests_retried: AtomicUsize

requests_dropped: AtomicUsize

responses_received: AtomicUsize

responses_from_cache: AtomicUsize

response_status_counts: Arc<DashMap<u16, usize>>

total_bytes_downloaded: AtomicUsize

items_scraped: AtomicUsize

items_processed: AtomicUsize

items_dropped_by_pipeline: AtomicUsize

request_times: Arc<DashMap<String, Duration>>

parsing_times: Arc<DashMap<String, Duration>>

Implementations

impl StatCollector

pub fn record_request_time(&self, url: &str, duration: Duration)

Records the time taken for a request.

pub fn average_request_time(&self) -> Option<Duration>

Calculates the average request time across all recorded requests.

pub fn fastest_request_time(&self) -> Option<Duration>

Gets the fastest request time among all recorded requests.

pub fn slowest_request_time(&self) -> Option<Duration>

Gets the slowest request time among all recorded requests.

pub fn request_time_count(&self) -> usize

Gets the total number of recorded request times.

pub fn get_request_time(&self, url: &str) -> Option<Duration>

Gets the request time for a specific URL.

pub fn get_all_request_times(&self) -> Vec<(String, Duration)>

Gets all recorded request times as a vector of (URL, Duration) pairs.

pub fn record_parsing_time(&self, duration: Duration)

Records the time taken for parsing a response.

pub fn average_parsing_time(&self) -> Option<Duration>

Calculates the average parsing time across all recorded parses.

pub fn fastest_parsing_time(&self) -> Option<Duration>

Gets the fastest parsing time among all recorded parses.

pub fn slowest_parsing_time(&self) -> Option<Duration>

Gets the slowest parsing time among all recorded parses.

pub fn parsing_time_count(&self) -> usize

Gets the total number of recorded parsing times.

pub fn clear_request_times(&self)

Clears all recorded request times.

pub fn to_json_string(&self) -> Result<String, SpiderError>

Converts the snapshot into a JSON string.

pub fn to_json_string_pretty(&self) -> Result<String, SpiderError>

Converts the snapshot into a pretty-printed JSON string.

pub fn to_markdown_string(&self) -> String

Exports the current statistics to a Markdown formatted string.

Trait Implementations

impl Debug for StatCollector

fn fmt(&self, f: &mut Formatter<'_>) -> Result<(), Error>

Formats the value using the given formatter.

impl Default for StatCollector

fn default() -> StatCollector

Returns the “default value” for a type.

impl Display for StatCollector

fn fmt(&self, f: &mut Formatter<'_>) -> Result<(), Error>

Formats the value using the given formatter.

impl Serialize for StatCollector

fn serialize<__S>( &self, __serializer: __S, ) -> Result<<__S as Serializer>::Ok, <__S as Serializer>::Error>

where __S: Serializer,

Serialize this value into the given Serde serializer.