Struct StatCollector 

pub struct StatCollector {

    pub start_time: Instant,
    pub requests_enqueued: Atomic<usize>,
    pub requests_sent: Atomic<usize>,
    pub requests_succeeded: Atomic<usize>,
    pub requests_failed: Atomic<usize>,
    pub requests_retried: Atomic<usize>,
    pub requests_dropped: Atomic<usize>,
    pub responses_received: Atomic<usize>,
    pub responses_from_cache: Atomic<usize>,
    pub response_status_counts: Arc<DashMap<u16, usize>>,
    pub total_bytes_downloaded: Atomic<usize>,
    pub items_scraped: Atomic<usize>,
    pub items_processed: Atomic<usize>,
    pub items_dropped_by_pipeline: Atomic<usize>,
    pub request_times: Cache<String, Duration>,
    pub parsing_times: Cache<String, Duration>,
    /* private fields */
}

Collects and stores various statistics about the crawler’s operation.

Fields

start_time: Instant

requests_enqueued: Atomic<usize>

requests_sent: Atomic<usize>

requests_succeeded: Atomic<usize>

requests_failed: Atomic<usize>

requests_retried: Atomic<usize>

requests_dropped: Atomic<usize>

responses_received: Atomic<usize>

responses_from_cache: Atomic<usize>

response_status_counts: Arc<DashMap<u16, usize>>

total_bytes_downloaded: Atomic<usize>

items_scraped: Atomic<usize>

items_processed: Atomic<usize>

items_dropped_by_pipeline: Atomic<usize>

request_times: Cache<String, Duration>

parsing_times: Cache<String, Duration>

Implementations

impl StatCollector

pub fn record_request_time(&self, url: &str, duration: Duration)

Records the time taken for a request.

pub fn average_request_time(&self) -> Option<Duration>

Calculates the average request time across all recorded requests.

pub fn fastest_request_time(&self) -> Option<Duration>

Gets the fastest request time among all recorded requests.

pub fn slowest_request_time(&self) -> Option<Duration>

Gets the slowest request time among all recorded requests.

pub fn request_time_count(&self) -> usize

Gets the total number of recorded request times.

pub fn get_request_time(&self, url: &str) -> Option<Duration>

Gets the request time for a specific URL.

pub fn get_all_request_times(&self) -> Vec<(String, Duration)>

Gets all recorded request times as a vector of (URL, Duration) pairs.

pub fn record_parsing_time(&self, duration: Duration)

Records the time taken for parsing a response.

pub fn average_parsing_time(&self) -> Option<Duration>

Calculates the average parsing time across all recorded parses.

pub fn fastest_parsing_time(&self) -> Option<Duration>

Gets the fastest parsing time among all recorded parses.

pub fn slowest_parsing_time(&self) -> Option<Duration>

Gets the slowest parsing time among all recorded parses.

pub fn parsing_time_count(&self) -> usize

Gets the total number of recorded parsing times.

pub fn clear_request_times(&self)

Clears all recorded request times.

pub fn clear_parsing_times(&self)

Clears all recorded parsing times.

pub fn to_json_string(&self) -> Result<String, SpiderError>

Converts the snapshot into a JSON string.

pub fn to_json_string_pretty(&self) -> Result<String, SpiderError>

Converts the snapshot into a pretty-printed JSON string.

pub fn to_markdown_string(&self) -> String

Exports the current statistics to a Markdown formatted string.

pub fn to_live_report_string(&self) -> String

Exports current statistics to the text layout used for terminal output.

Trait Implementations

impl Debug for StatCollector

fn fmt(&self, f: &mut Formatter<'_>) -> Result<(), Error>

Formats the value using the given formatter.

impl Default for StatCollector

fn default() -> StatCollector

Returns the “default value” for a type.

impl Display for StatCollector

fn fmt(&self, f: &mut Formatter<'_>) -> Result<(), Error>

Formats the value using the given formatter.

impl Serialize for StatCollector

fn serialize<__S>( &self, __serializer: __S, ) -> Result<<__S as Serializer>::Ok, <__S as Serializer>::Error>

where __S: Serializer,

Serialize this value into the given Serde serializer.