Struct CrawlOptions

Source

pub struct CrawlOptions {
    pub limit: Option<u32>,
    pub max_depth: Option<u8>,
    pub exclude_paths: Option<Vec<String>>,
    pub include_paths: Option<Vec<String>>,
    pub follow_external: Option<bool>,
    pub delay_between_requests: Option<u32>,
    pub parallel_requests: Option<u32>,
}

Fields§

§limit: Option<u32>§max_depth: Option<u8>§exclude_paths: Option<Vec<String>>§include_paths: Option<Vec<String>>§follow_external: Option<bool>§delay_between_requests: Option<u32>§parallel_requests: Option<u32>

Implementations§

Source §

impl CrawlOptions

Source

pub fn new() -> Self

Examples found in repository ?

examples/web-scrape.rs (line 70)

64fn example_crawling() {
65    println!("--- Example 3: Recursive Website Crawling ---");
66
67    let url = "https://example.com";
68    println!("Crawling website: {}", url);
69
70    let options = CrawlOptions::new()
71        .with_max_depth(2)
72        .with_limit(10)
73        .with_include_paths(vec!["/".to_string()])
74        .with_exclude_paths(vec!["/admin/".to_string(), "/api/".to_string()])
75        .with_follow_external(false)
76        .with_delay_between_requests(1000)
77        .with_parallel_requests(3);
78
79    let response = BlessCrawl::default()
80        .crawl(url, Some(options))
81        .expect("Failed to crawl");
82    println!("response: {:?}", response);
83    println!();
84    println!(
85        "------------ pages ------------\n{:?}\n------------------------------",
86        response.data.pages
87    );
88    println!();
89    println!(
90        "------------ total pages ------------\n{}\n------------------------------",
91        response.data.total_pages
92    );
93}

Source

pub fn with_limit(self, limit: u32) -> Self

Examples found in repository ?

examples/web-scrape.rs (line 72)

64fn example_crawling() {
65    println!("--- Example 3: Recursive Website Crawling ---");
66
67    let url = "https://example.com";
68    println!("Crawling website: {}", url);
69
70    let options = CrawlOptions::new()
71        .with_max_depth(2)
72        .with_limit(10)
73        .with_include_paths(vec!["/".to_string()])
74        .with_exclude_paths(vec!["/admin/".to_string(), "/api/".to_string()])
75        .with_follow_external(false)
76        .with_delay_between_requests(1000)
77        .with_parallel_requests(3);
78
79    let response = BlessCrawl::default()
80        .crawl(url, Some(options))
81        .expect("Failed to crawl");
82    println!("response: {:?}", response);
83    println!();
84    println!(
85        "------------ pages ------------\n{:?}\n------------------------------",
86        response.data.pages
87    );
88    println!();
89    println!(
90        "------------ total pages ------------\n{}\n------------------------------",
91        response.data.total_pages
92    );
93}

Source

pub fn with_max_depth(self, max_depth: u8) -> Self

Examples found in repository ?

examples/web-scrape.rs (line 71)

64fn example_crawling() {
65    println!("--- Example 3: Recursive Website Crawling ---");
66
67    let url = "https://example.com";
68    println!("Crawling website: {}", url);
69
70    let options = CrawlOptions::new()
71        .with_max_depth(2)
72        .with_limit(10)
73        .with_include_paths(vec!["/".to_string()])
74        .with_exclude_paths(vec!["/admin/".to_string(), "/api/".to_string()])
75        .with_follow_external(false)
76        .with_delay_between_requests(1000)
77        .with_parallel_requests(3);
78
79    let response = BlessCrawl::default()
80        .crawl(url, Some(options))
81        .expect("Failed to crawl");
82    println!("response: {:?}", response);
83    println!();
84    println!(
85        "------------ pages ------------\n{:?}\n------------------------------",
86        response.data.pages
87    );
88    println!();
89    println!(
90        "------------ total pages ------------\n{}\n------------------------------",
91        response.data.total_pages
92    );
93}

Source

pub fn with_exclude_paths(self, paths: Vec<String>) -> Self

Examples found in repository ?

examples/web-scrape.rs (line 74)

64fn example_crawling() {
65    println!("--- Example 3: Recursive Website Crawling ---");
66
67    let url = "https://example.com";
68    println!("Crawling website: {}", url);
69
70    let options = CrawlOptions::new()
71        .with_max_depth(2)
72        .with_limit(10)
73        .with_include_paths(vec!["/".to_string()])
74        .with_exclude_paths(vec!["/admin/".to_string(), "/api/".to_string()])
75        .with_follow_external(false)
76        .with_delay_between_requests(1000)
77        .with_parallel_requests(3);
78
79    let response = BlessCrawl::default()
80        .crawl(url, Some(options))
81        .expect("Failed to crawl");
82    println!("response: {:?}", response);
83    println!();
84    println!(
85        "------------ pages ------------\n{:?}\n------------------------------",
86        response.data.pages
87    );
88    println!();
89    println!(
90        "------------ total pages ------------\n{}\n------------------------------",
91        response.data.total_pages
92    );
93}

Source

pub fn with_include_paths(self, paths: Vec<String>) -> Self

Examples found in repository ?

examples/web-scrape.rs (line 73)

64fn example_crawling() {
65    println!("--- Example 3: Recursive Website Crawling ---");
66
67    let url = "https://example.com";
68    println!("Crawling website: {}", url);
69
70    let options = CrawlOptions::new()
71        .with_max_depth(2)
72        .with_limit(10)
73        .with_include_paths(vec!["/".to_string()])
74        .with_exclude_paths(vec!["/admin/".to_string(), "/api/".to_string()])
75        .with_follow_external(false)
76        .with_delay_between_requests(1000)
77        .with_parallel_requests(3);
78
79    let response = BlessCrawl::default()
80        .crawl(url, Some(options))
81        .expect("Failed to crawl");
82    println!("response: {:?}", response);
83    println!();
84    println!(
85        "------------ pages ------------\n{:?}\n------------------------------",
86        response.data.pages
87    );
88    println!();
89    println!(
90        "------------ total pages ------------\n{}\n------------------------------",
91        response.data.total_pages
92    );
93}

Source

pub fn with_follow_external(self, follow: bool) -> Self

Examples found in repository ?

examples/web-scrape.rs (line 75)

64fn example_crawling() {
65    println!("--- Example 3: Recursive Website Crawling ---");
66
67    let url = "https://example.com";
68    println!("Crawling website: {}", url);
69
70    let options = CrawlOptions::new()
71        .with_max_depth(2)
72        .with_limit(10)
73        .with_include_paths(vec!["/".to_string()])
74        .with_exclude_paths(vec!["/admin/".to_string(), "/api/".to_string()])
75        .with_follow_external(false)
76        .with_delay_between_requests(1000)
77        .with_parallel_requests(3);
78
79    let response = BlessCrawl::default()
80        .crawl(url, Some(options))
81        .expect("Failed to crawl");
82    println!("response: {:?}", response);
83    println!();
84    println!(
85        "------------ pages ------------\n{:?}\n------------------------------",
86        response.data.pages
87    );
88    println!();
89    println!(
90        "------------ total pages ------------\n{}\n------------------------------",
91        response.data.total_pages
92    );
93}

Source

pub fn with_delay_between_requests(self, delay: u32) -> Self

Examples found in repository ?

examples/web-scrape.rs (line 76)

64fn example_crawling() {
65    println!("--- Example 3: Recursive Website Crawling ---");
66
67    let url = "https://example.com";
68    println!("Crawling website: {}", url);
69
70    let options = CrawlOptions::new()
71        .with_max_depth(2)
72        .with_limit(10)
73        .with_include_paths(vec!["/".to_string()])
74        .with_exclude_paths(vec!["/admin/".to_string(), "/api/".to_string()])
75        .with_follow_external(false)
76        .with_delay_between_requests(1000)
77        .with_parallel_requests(3);
78
79    let response = BlessCrawl::default()
80        .crawl(url, Some(options))
81        .expect("Failed to crawl");
82    println!("response: {:?}", response);
83    println!();
84    println!(
85        "------------ pages ------------\n{:?}\n------------------------------",
86        response.data.pages
87    );
88    println!();
89    println!(
90        "------------ total pages ------------\n{}\n------------------------------",
91        response.data.total_pages
92    );
93}

Source

pub fn with_parallel_requests(self, parallel: u32) -> Self

Examples found in repository ?

examples/web-scrape.rs (line 77)

64fn example_crawling() {
65    println!("--- Example 3: Recursive Website Crawling ---");
66
67    let url = "https://example.com";
68    println!("Crawling website: {}", url);
69
70    let options = CrawlOptions::new()
71        .with_max_depth(2)
72        .with_limit(10)
73        .with_include_paths(vec!["/".to_string()])
74        .with_exclude_paths(vec!["/admin/".to_string(), "/api/".to_string()])
75        .with_follow_external(false)
76        .with_delay_between_requests(1000)
77        .with_parallel_requests(3);
78
79    let response = BlessCrawl::default()
80        .crawl(url, Some(options))
81        .expect("Failed to crawl");
82    println!("response: {:?}", response);
83    println!();
84    println!(
85        "------------ pages ------------\n{:?}\n------------------------------",
86        response.data.pages
87    );
88    println!();
89    println!(
90        "------------ total pages ------------\n{}\n------------------------------",
91        response.data.total_pages
92    );
93}