Struct UserAgentMiddleware 

pub struct UserAgentMiddleware { /* private fields */ }

Implementations

impl UserAgentMiddleware

pub fn builder() -> UserAgentMiddlewareBuilder

Creates a new UserAgentMiddlewareBuilder to start building a UserAgentMiddleware.

Examples found in repository

examples/quotes_scraper.rs (line 99)

async fn main() -> Result<(), SpiderError> {
    tracing_subscriber::fmt()
        .with_env_filter("info,spider_lib=debug")
        .without_time()
        .init();

    let crawler = CrawlerBuilder::<_, ReqwestClientDownloader>::new(QuotesSpider)
        .add_pipeline(DeduplicationPipeline::new(&["text"]))
        .add_pipeline(CsvExporterPipeline::<QuoteItem>::new("output/quotes.csv")?)
        .add_middleware(HttpCacheMiddleware::builder().build()?)
        .add_middleware(
            RateLimitMiddleware::builder()
                .use_token_bucket_limiter(5)
                .build(),
        )
        .add_middleware(RetryMiddleware::default().max_retries(2))
        .add_middleware(UserAgentMiddleware::builder().build()?)
        .add_middleware(RobotsTxtMiddleware::new())
        .add_middleware(
            RefererMiddleware::new()
                .same_origin_only(true)
                .max_chain_length(100)
                .include_fragment(false),
        )
        .with_checkpoint_path("output/quotes.bin")
        .with_checkpoint_interval(Duration::from_secs(15))
        .max_concurrent_downloads(5)
        .max_parser_workers(2)
        .max_concurrent_pipelines(2)
        .build()
        .await?;

    crawler.start_crawl().await?;

    Ok(())
}

Trait Implementations

impl Debug for UserAgentMiddleware

fn fmt(&self, f: &mut Formatter<'_>) -> Result

Formats the value using the given formatter.

impl<C: Send + Sync> Middleware<C> for UserAgentMiddleware

fn name(&self) -> &str

fn process_request<'life0, 'life1, 'async_trait>( &'life0 mut self, _client: &'life1 C, request: Request, ) -> Pin<Box<dyn Future<Output = Result<MiddlewareAction<Request>, SpiderError>> + Send + 'async_trait>>

where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait,

fn process_response<'life0, 'async_trait>( &'life0 mut self, response: Response, ) -> Pin<Box<dyn Future<Output = Result<MiddlewareAction<Response>, SpiderError>> + Send + 'async_trait>>

where Self: 'async_trait, 'life0: 'async_trait,

fn handle_error<'life0, 'life1, 'life2, 'async_trait>( &'life0 mut self, _request: &'life1 Request, error: &'life2 SpiderError, ) -> Pin<Box<dyn Future<Output = Result<MiddlewareAction<Request>, SpiderError>> + Send + 'async_trait>>

where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait, 'life2: 'async_trait,