Struct spider::configuration::Configuration[−][src]

pub struct Configuration {
    pub respect_robots_txt: bool,
    pub verbose: bool,
    pub blacklist_url: Vec<String>,
    pub user_agent: &'static str,
    pub delay: u64,
}

Structure to configure Website crawler

let mut website: Website = Website::new("https://choosealicense.com");
website.configuration.blacklist_url.push("https://choosealicense.com/licenses/".to_string());
website.configuration.respect_robots_txt = true;
website.configuration.verbose = true;
localhost.crawl();

Fields

respect_robots_txt: bool

Respect robots.txt file and not scrape not allowed files (not implemented)

verbose: bool

Print page visited on standart output

blacklist_url: Vec<String>

List of page to not crawl

user_agent: &'static str

User-Agent

delay: u64

Polite crawling delay in milli seconds

Implementations

impl Configuration[src]

pub fn new() -> Self[src]

Trait Implementations

impl Debug for Configuration[src]

Auto Trait Implementations

Blanket Implementations

impl<T> Any for T where
    T: 'static + ?Sized, [src]

impl<T> Borrow<T> for T where
    T: ?Sized, [src]

impl<T> BorrowMut<T> for T where
    T: ?Sized, [src]

impl<T> From<T> for T[src]

impl<T> Instrument for T[src]

impl<T, U> Into<U> for T where
    U: From<T>, [src]

impl<T, U> TryFrom<U> for T where
    U: Into<T>, [src]

type Error = Infallible

The type returned in the event of a conversion error.

impl<T, U> TryInto<U> for T where
    U: TryFrom<T>, [src]

type Error = <U as TryFrom<T>>::Error

The type returned in the event of a conversion error.

impl<T, U> TryInto<U> for T where
    U: TryFrom<T>, 

type Err = <U as TryFrom<T>>::Err