spider-lib 3.0.4

A Rust-based web scraping framework inspired by Scrapy (Python).
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33

use spider_lib::prelude::*;

#[tokio::main]
async fn main() -> Result<(), SpiderError> {
    let (scheduler, request_rx) = Scheduler::new(None, 16);

    scheduler
        .enqueue_requests_batch(vec![
            Request::try_new("https://example.com/list")?,
            Request::try_new("https://example.com/archive")?.with_priority(-5),
            Request::try_new("https://example.com/detail/important")?.with_priority(10),
            Request::try_new("https://example.com/detail/also-important")?.with_priority(4),
        ])
        .await?;

    println!("Dequeued order:");

    for index in 1..=4 {
        let request = request_rx.recv().await.map_err(|err| {
            SpiderError::GeneralError(format!("scheduler receiver closed unexpectedly: {}", err))
        })?;
        println!(
            "{}. priority={} url={}",
            index,
            request.priority(),
            request.url
        );
        scheduler.complete_request();
    }

    scheduler.shutdown().await?;
    Ok(())
}