Struct RefererMiddleware

Source

pub struct RefererMiddleware {
    pub same_origin_only: bool,
    pub max_chain_length: usize,
    pub include_fragment: bool,
    /* private fields */
}

Expand description

Referer middleware that automatically sets Referer headers based on the navigation chain

Fields§

§same_origin_only: bool

Whether to use same-origin only referer

§max_chain_length: usize

Maximum referer chain length to keep in memory

§include_fragment: bool

Whether to include fragment in referer URL

Implementations§

Source §

impl RefererMiddleware

Source

pub fn new() -> Self

Create a new RefererMiddleware with default config

Examples found in repository ?

examples/quotes_scraper.rs (line 102)

83async fn main() -> Result<(), SpiderError> {
84    tracing_subscriber::fmt()
85        .with_env_filter("info,spider_lib=debug")
86        .without_time()
87        .init();
88
89    let crawler = CrawlerBuilder::<_, ReqwestClientDownloader>::new(QuotesSpider)
90        .add_pipeline(DeduplicationPipeline::new(&["text"]))
91        .add_pipeline(CsvExporterPipeline::<QuoteItem>::new("output/quotes.csv")?)
92        .add_middleware(HttpCacheMiddleware::builder().build()?)
93        .add_middleware(
94            RateLimitMiddleware::builder()
95                .use_token_bucket_limiter(5)
96                .build(),
97        )
98        .add_middleware(RetryMiddleware::default().max_retries(2))
99        .add_middleware(UserAgentMiddleware::builder().build()?)
100        .add_middleware(RobotsTxtMiddleware::new())
101        .add_middleware(
102            RefererMiddleware::new()
103                .same_origin_only(true)
104                .max_chain_length(100)
105                .include_fragment(false),
106        )
107        .with_checkpoint_path("output/quotes.bin")
108        .with_checkpoint_interval(Duration::from_secs(15))
109        .max_concurrent_downloads(5)
110        .max_parser_workers(2)
111        .max_concurrent_pipelines(2)
112        .build()
113        .await?;
114
115    crawler.start_crawl().await?;
116
117    Ok(())
118}

Source

pub fn same_origin_only(self, same_origin_only: bool) -> Self

Set whether to use same-origin only referer.

Examples found in repository ?

examples/quotes_scraper.rs (line 103)

83async fn main() -> Result<(), SpiderError> {
84    tracing_subscriber::fmt()
85        .with_env_filter("info,spider_lib=debug")
86        .without_time()
87        .init();
88
89    let crawler = CrawlerBuilder::<_, ReqwestClientDownloader>::new(QuotesSpider)
90        .add_pipeline(DeduplicationPipeline::new(&["text"]))
91        .add_pipeline(CsvExporterPipeline::<QuoteItem>::new("output/quotes.csv")?)
92        .add_middleware(HttpCacheMiddleware::builder().build()?)
93        .add_middleware(
94            RateLimitMiddleware::builder()
95                .use_token_bucket_limiter(5)
96                .build(),
97        )
98        .add_middleware(RetryMiddleware::default().max_retries(2))
99        .add_middleware(UserAgentMiddleware::builder().build()?)
100        .add_middleware(RobotsTxtMiddleware::new())
101        .add_middleware(
102            RefererMiddleware::new()
103                .same_origin_only(true)
104                .max_chain_length(100)
105                .include_fragment(false),
106        )
107        .with_checkpoint_path("output/quotes.bin")
108        .with_checkpoint_interval(Duration::from_secs(15))
109        .max_concurrent_downloads(5)
110        .max_parser_workers(2)
111        .max_concurrent_pipelines(2)
112        .build()
113        .await?;
114
115    crawler.start_crawl().await?;
116
117    Ok(())
118}

Source

pub fn max_chain_length(self, max_chain_length: usize) -> Self

Set the maximum referer chain length to keep in memory.

Examples found in repository ?

examples/quotes_scraper.rs (line 104)

83async fn main() -> Result<(), SpiderError> {
84    tracing_subscriber::fmt()
85        .with_env_filter("info,spider_lib=debug")
86        .without_time()
87        .init();
88
89    let crawler = CrawlerBuilder::<_, ReqwestClientDownloader>::new(QuotesSpider)
90        .add_pipeline(DeduplicationPipeline::new(&["text"]))
91        .add_pipeline(CsvExporterPipeline::<QuoteItem>::new("output/quotes.csv")?)
92        .add_middleware(HttpCacheMiddleware::builder().build()?)
93        .add_middleware(
94            RateLimitMiddleware::builder()
95                .use_token_bucket_limiter(5)
96                .build(),
97        )
98        .add_middleware(RetryMiddleware::default().max_retries(2))
99        .add_middleware(UserAgentMiddleware::builder().build()?)
100        .add_middleware(RobotsTxtMiddleware::new())
101        .add_middleware(
102            RefererMiddleware::new()
103                .same_origin_only(true)
104                .max_chain_length(100)
105                .include_fragment(false),
106        )
107        .with_checkpoint_path("output/quotes.bin")
108        .with_checkpoint_interval(Duration::from_secs(15))
109        .max_concurrent_downloads(5)
110        .max_parser_workers(2)
111        .max_concurrent_pipelines(2)
112        .build()
113        .await?;
114
115    crawler.start_crawl().await?;
116
117    Ok(())
118}

Source

pub fn include_fragment(self, include_fragment: bool) -> Self

Set whether to include the fragment in the referer URL.

Examples found in repository ?

examples/quotes_scraper.rs (line 105)

83async fn main() -> Result<(), SpiderError> {
84    tracing_subscriber::fmt()
85        .with_env_filter("info,spider_lib=debug")
86        .without_time()
87        .init();
88
89    let crawler = CrawlerBuilder::<_, ReqwestClientDownloader>::new(QuotesSpider)
90        .add_pipeline(DeduplicationPipeline::new(&["text"]))
91        .add_pipeline(CsvExporterPipeline::<QuoteItem>::new("output/quotes.csv")?)
92        .add_middleware(HttpCacheMiddleware::builder().build()?)
93        .add_middleware(
94            RateLimitMiddleware::builder()
95                .use_token_bucket_limiter(5)
96                .build(),
97        )
98        .add_middleware(RetryMiddleware::default().max_retries(2))
99        .add_middleware(UserAgentMiddleware::builder().build()?)
100        .add_middleware(RobotsTxtMiddleware::new())
101        .add_middleware(
102            RefererMiddleware::new()
103                .same_origin_only(true)
104                .max_chain_length(100)
105                .include_fragment(false),
106        )
107        .with_checkpoint_path("output/quotes.bin")
108        .with_checkpoint_interval(Duration::from_secs(15))
109        .max_concurrent_downloads(5)
110        .max_parser_workers(2)
111        .max_concurrent_pipelines(2)
112        .build()
113        .await?;
114
115    crawler.start_crawl().await?;
116
117    Ok(())
118}