Struct voyager::Crawler

pub struct Crawler<T: Scraper> { /* fields omitted */ }

The crawler that is responsible for driving the requests to completion and providing the crawl response for the Scraper.

Implementations

impl<T: Scraper> Crawler<T>

pub fn new(config: CrawlerConfig) -> Self

Create a new crawler following the config

pub fn max_depth(&self) -> usize

The maximum allowed depth of requests

pub fn respects_robots_txt(&self) -> bool

Whether this crawler respects the domains robots.txt rules

pub fn skips_non_successful_responses(&self) -> bool

Whether non 2xx responses are treated as failures and are not being scraped

impl<T> Crawler<T> where
    T: Scraper + Unpin + 'static,
    <T as Scraper>::State: Unpin + Send + Sync + 'static,
    <T as Scraper>::Output: Unpin, 

pub fn crawl<TCrawlFunction, TCrawlFuture>(&mut self, fun: TCrawlFunction) where
    TCrawlFunction: FnOnce(&Client) -> TCrawlFuture,
    TCrawlFuture: Future<Output = Result<(Response, Option<T::State>)>> + 'static, 

Send a crawling request whose html response and context is returned to the scraper again

pub fn complete<TCrawlFunction, TCrawlFuture>(&mut self, fun: TCrawlFunction) where
    TCrawlFunction: FnOnce(&Client) -> TCrawlFuture,
    TCrawlFuture: Future<Output = Result<Option<T::Output>>> + 'static, 

Submit a complete crawling job that is driven to completion and directly returned once finished.

pub fn visit(&mut self, url: impl IntoUrl)

This queues in a GET request for the url, without any state attached

pub fn visit_with_state(&mut self, url: impl IntoUrl, state: T::State)

This queues in a GET request for the url with state attached

pub fn request(&mut self, req: RequestBuilder)

This queues in a whole request with no state attached

pub fn request_with_state(&mut self, req: RequestBuilder, state: T::State)

This queues in a whole request with a state attached

pub fn client(&self) -> &Client

The client that performs all request