Struct spider::website::Website

pub struct Website {
    pub configuration: Box<Configuration>,
    pub on_link_find_callback: Option<fn(_: CaseInsensitiveString) -> CaseInsensitiveString>,
    /* private fields */
}

Represents a website to crawl and gather all links.

use spider::website::Website;
let mut website = Website::new("http://example.com");
website.crawl();
// `Website` will be filled with `Pages` when crawled. To get them, just use
while let Some(page) = website.get_pages() {
    // do something
}

Fields

configuration: Box<Configuration>

configuration properties for website.

on_link_find_callback: Option<fn(_: CaseInsensitiveString) -> CaseInsensitiveString>

callback when a link is found.

Implementations

impl Website

pub fn new(domain: &str) -> Self

Initialize Website object with a start link to crawl.

pub fn is_allowed( &self, link: &CaseInsensitiveString, blacklist_url: &Box<Vec<CompactString>> ) -> bool

return true if URL:

• is not already crawled
• is not blacklisted
• is not forbidden in robot.txt file (if parameter is defined)

pub fn is_allowed_default( &self, link: &CompactString, blacklist_url: &Box<Vec<CompactString>> ) -> bool

return true if URL:

• is not blacklisted
• is not forbidden in robot.txt file (if parameter is defined)

pub fn is_allowed_robots(&self, link: &str) -> bool

return true if URL:

• is not forbidden in robot.txt file (if parameter is defined)

pub fn get_pages(&self) -> Option<&Box<Vec<Page>>>

page getter

pub fn get_links(&self) -> &HashSet<CaseInsensitiveString>

links visited getter

pub fn get_absolute_path(&self, domain: Option<&str>) -> Option<Url>

absolute base url of crawl

pub async fn configure_robots_parser(&mut self, client: Client) -> Client

configure the robots parser on initial crawl attempt and run.

pub fn configure_http_client(&mut self) -> Client

configure http client

pub async fn setup<T>(&mut self) -> (Client, Option<T>)

setup config for crawl

pub async fn crawl(&mut self)

Start to crawl website with async conccurency

pub async fn crawl_sync(&mut self)

Start to crawl website in sync

pub async fn scrape(&mut self)

Start to scrape/download website with async conccurency

Trait Implementations

impl Clone for Website

fn clone(&self) -> Website

Returns a copy of the value.

fn clone_from(&mut self, source: &Self)

Performs copy-assignment from source.

impl Debug for Website

fn fmt(&self, f: &mut Formatter<'_>) -> Result

Formats the value using the given formatter.