Struct StatCollector

Source

pub struct StatCollector {Show 15 fields
    pub start_time: Instant,
    pub requests_enqueued: AtomicUsize,
    pub requests_sent: AtomicUsize,
    pub requests_succeeded: AtomicUsize,
    pub requests_failed: AtomicUsize,
    pub requests_retried: AtomicUsize,
    pub requests_dropped: AtomicUsize,
    pub responses_received: AtomicUsize,
    pub responses_from_cache: AtomicUsize,
    pub response_status_counts: Arc<DashMap<u16, usize>>,
    pub total_bytes_downloaded: AtomicUsize,
    pub items_scraped: AtomicUsize,
    pub items_processed: AtomicUsize,
    pub items_dropped_by_pipeline: AtomicUsize,
    pub request_times: Arc<DashMap<String, Duration>>,
    /* private fields */
}

Expand description

Collects and stores various statistics about the crawler’s operation.

Fields§

§start_time: Instant§requests_enqueued: AtomicUsize§requests_sent: AtomicUsize§requests_succeeded: AtomicUsize§requests_failed: AtomicUsize§requests_retried: AtomicUsize§requests_dropped: AtomicUsize§responses_received: AtomicUsize§responses_from_cache: AtomicUsize§response_status_counts: Arc<DashMap<u16, usize>>§total_bytes_downloaded: AtomicUsize§items_scraped: AtomicUsize§items_processed: AtomicUsize§items_dropped_by_pipeline: AtomicUsize§request_times: Arc<DashMap<String, Duration>>

Struct StatCollector Copy item path

Fields§

Implementations§

impl StatCollector

pub fn record_request_time(&self, url: &str, duration: Duration)

pub fn average_request_time(&self) -> Option<Duration>

pub fn fastest_request_time(&self) -> Option<Duration>

pub fn slowest_request_time(&self) -> Option<Duration>

pub fn request_time_count(&self) -> usize

pub fn get_request_time(&self, url: &str) -> Option<Duration>

pub fn get_all_request_times(&self) -> Vec<(String, Duration)>

pub fn clear_request_times(&self)

pub fn to_json_string(&self) -> Result<String, SpiderError>

pub fn to_json_string_pretty(&self) -> Result<String, SpiderError>

pub fn to_markdown_string(&self) -> String

Trait Implementations§

impl Debug for StatCollector

fn fmt(&self, f: &mut Formatter<'_>) -> Result<(), Error>

impl Default for StatCollector

fn default() -> StatCollector

impl Display for StatCollector

fn fmt(&self, f: &mut Formatter<'_>) -> Result<(), Error>

impl Serialize for StatCollector

fn serialize<__S>( &self, __serializer: __S, ) -> Result<<__S as Serializer>::Ok, <__S as Serializer>::Error>where __S: Serializer,

Auto Trait Implementations§

impl !Freeze for StatCollector

impl !RefUnwindSafe for StatCollector

impl Send for StatCollector

impl Sync for StatCollector

impl Unpin for StatCollector

impl !UnwindSafe for StatCollector

Blanket Implementations§

impl<T> Any for Twhere T: 'static + ?Sized,

fn type_id(&self) -> TypeId

impl<T> Borrow<T> for Twhere T: ?Sized,

fn borrow(&self) -> &T

impl<T> BorrowMut<T> for Twhere T: ?Sized,

fn borrow_mut(&mut self) -> &mut T

impl<T> From<T> for T

fn from(t: T) -> T

impl<T> Instrument for T

fn instrument(self, span: Span) -> Instrumented<Self>

fn in_current_span(self) -> Instrumented<Self>

impl<T, U> Into<U> for Twhere U: From<T>,

fn into(self) -> U

impl<T> Pointable for T

const ALIGN: usize

type Init = T

unsafe fn init(init: <T as Pointable>::Init) -> usize

unsafe fn deref<'a>(ptr: usize) -> &'a T

unsafe fn deref_mut<'a>(ptr: usize) -> &'a mut T

unsafe fn drop(ptr: usize)

impl<T> PolicyExt for Twhere T: ?Sized,

fn and<P, B, E>(self, other: P) -> And<T, P>where T: Policy<B, E>, P: Policy<B, E>,

fn or<P, B, E>(self, other: P) -> Or<T, P>where T: Policy<B, E>, P: Policy<B, E>,

impl<T> Same for T

type Output = T

impl<T> ToString for Twhere T: Display + ?Sized,

fn to_string(&self) -> String

impl<T> ToStringFallible for Twhere T: Display,

fn try_to_string(&self) -> Result<String, TryReserveError>

impl<T, U> TryFrom<U> for Twhere U: Into<T>,

type Error = Infallible

fn try_from(value: U) -> Result<T, <T as TryFrom<U>>::Error>

impl<T, U> TryInto<U> for Twhere U: TryFrom<T>,

type Error = <U as TryFrom<T>>::Error

fn try_into(self) -> Result<U, <U as TryFrom<T>>::Error>

impl<V, T> VZip<V> for Twhere V: MultiLane<T>,

fn vzip(self) -> V

impl<T> WithSubscriber for T

fn with_subscriber<S>(self, subscriber: S) -> WithDispatch<Self>where S: Into<Dispatch>,

fn with_current_subscriber(self) -> WithDispatch<Self>

Struct StatCollector

fn serialize<S>( &self, serializer: S, ) -> Result<<S as Serializer>::Ok, <S as Serializer>::Error>
where S: Serializer,

impl<T> Any for T
where T: 'static + ?Sized,

impl<T> Borrow<T> for T
where T: ?Sized,

impl<T> BorrowMut<T> for T
where T: ?Sized,

impl<T, U> Into<U> for T
where U: From<T>,

impl<T> PolicyExt for T
where T: ?Sized,

fn and<P, B, E>(self, other: P) -> And<T, P>
where T: Policy<B, E>, P: Policy<B, E>,

fn or<P, B, E>(self, other: P) -> Or<T, P>
where T: Policy<B, E>, P: Policy<B, E>,

impl<T> ToString for T
where T: Display + ?Sized,

impl<T> ToStringFallible for T
where T: Display,

impl<T, U> TryFrom<U> for T
where U: Into<T>,

impl<T, U> TryInto<U> for T
where U: TryFrom<T>,

impl<V, T> VZip<V> for T
where V: MultiLane<T>,

fn with_subscriber<S>(self, subscriber: S) -> WithDispatch<Self>
where S: Into<Dispatch>,