Struct BlessCrawl

pub struct BlessCrawl { /* private fields */ }

BlessCrawl client for distributed web scraping operations.

Implementations

impl BlessCrawl

pub const DEFAULT_TIMEOUT_MS: u32 = 15_000u32

Default timeout in milliseconds (15 seconds)

pub const DEFAULT_WAIT_TIME_MS: u32 = 3_000u32

Default wait time in milliseconds (3 seconds)

pub const MAX_TIMEOUT_MS: u32 = 120_000u32

Maximum timeout in milliseconds (2 minutes)

pub const MAX_WAIT_TIME_MS: u32 = 20_000u32

Maximum wait time in milliseconds (20 seconds)

pub const MAX_SCRAPE_BUFFER_SIZE: usize = 2_097_152usize

Maximum result buffer size in bytes (2MB)

pub const MAX_MAP_BUFFER_SIZE: usize = 1_048_576usize

Maximum result buffer size in bytes (1MB)

pub const MAX_CRAWL_BUFFER_SIZE: usize = 8_388_608usize

Maximum result buffer size in bytes (8MB)

pub fn with_config(config: ScrapeOptions) -> Result<Self, WebScrapeErrorKind>

Creates a new BlessCrawl instance with the given configuration.

pub fn get_config(&self) -> &ScrapeOptions

Returns a reference to the current configuration.

pub fn handle(&self) -> u32

pub fn scrape( &self, url: &str, options: Option<ScrapeOptions>, ) -> Result<Response<ScrapeData>, WebScrapeErrorKind>

Scrapes webpage content and returns it as markdown with metadata.

Examples found in repository

examples/web-scrape.rs (line 27)

fn example_scraping() {
    println!("--- Example 1: Basic Web Scraping ---");

    let url = "https://example.com";
    println!("scraping: {}...", url);

    // First scrape with default config
    let response = BlessCrawl::default()
        .scrape(url, None)
        .expect("Failed to scrape");
    println!("response with default config: {:?}", response);
    println!();
    println!(
        "---------- markdown ----------\n{}\n------------------------------",
        response.data.content
    );
}

pub fn map( &self, url: &str, options: Option<MapOptions>, ) -> Result<Response<MapData>, WebScrapeErrorKind>

Extracts all links from a webpage, categorized by type.

Examples found in repository

examples/web-scrape.rs (line 49)

fn example_mapping() {
    println!("--- Example 2: Link Mapping/Discovery ---");

    let url = "https://example.com";
    println!("Mapping links from: {}", url);

    let options = MapOptions::new()
        .with_link_types(vec!["internal".to_string(), "external".to_string()])
        .with_base_url(url.to_string())
        .with_filter_extensions(vec![".html".to_string(), ".htm".to_string()]);

    let response = BlessCrawl::default()
        .map(url, Some(options))
        .expect("Failed to map");
    println!("response: {:?}", response);
    println!();
    println!(
        "------------ links ------------\n{:?}\n------------------------------",
        response.data.links
    );
    println!();
    println!(
        "------------ total links ------------\n{}\n------------------------------",
        response.data.total_links
    );
}

pub fn crawl( &self, url: &str, options: Option<CrawlOptions>, ) -> Result<Response<CrawlData<ScrapeData>>, WebScrapeErrorKind>

Recursively crawls a website with configurable depth and filtering.

Examples found in repository

examples/web-scrape.rs (line 80)

fn example_crawling() {
    println!("--- Example 3: Recursive Website Crawling ---");

    let url = "https://example.com";
    println!("Crawling website: {}", url);

    let options = CrawlOptions::new()
        .with_max_depth(2)
        .with_limit(10)
        .with_include_paths(vec!["/".to_string()])
        .with_exclude_paths(vec!["/admin/".to_string(), "/api/".to_string()])
        .with_follow_external(false)
        .with_delay_between_requests(1000)
        .with_parallel_requests(3);

    let response = BlessCrawl::default()
        .crawl(url, Some(options))
        .expect("Failed to crawl");
    println!("response: {:?}", response);
    println!();
    println!(
        "------------ pages ------------\n{:?}\n------------------------------",
        response.data.pages
    );
    println!();
    println!(
        "------------ total pages ------------\n{}\n------------------------------",
        response.data.total_pages
    );
}

Trait Implementations

impl Clone for BlessCrawl

fn clone(&self) -> BlessCrawl

Returns a duplicate of the value.

fn clone_from(&mut self, source: &Self)

Performs copy-assignment from source.

impl Debug for BlessCrawl

fn fmt(&self, f: &mut Formatter<'_>) -> Result

Formats the value using the given formatter.

impl Default for BlessCrawl

fn default() -> BlessCrawl

Returns the “default value” for a type.

impl Drop for BlessCrawl

fn drop(&mut self)

Executes the destructor for this type.