medina 0.0.1

An asynchronous web crawling engine
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45

use async_std::task;
use async_trait::async_trait;
use medina::{Crawler, CrawlerRunner, Error, Page, PageResult};

const ITEMS_PER_HOST: usize = 50;

pub struct FakePage {
    pub url: String,
}

#[async_trait]
impl Page for FakePage {
    async fn process_request(
        &self,
    ) -> Result<(Vec<Box<dyn Page>>, Vec<Box<dyn PageResult>>), Error> {
        let mut vec = Vec::<Box<dyn Page>>::new();
        vec.reserve_exact(ITEMS_PER_HOST);

        let ch = self.url.as_bytes()[8];
        if ch >= b'a' && ch <= b'y' {
            for i in 1..ITEMS_PER_HOST + 1 {
                vec.push(Box::new(FakePage {
                    url: format!("https://{}.com/{}", (ch + 1) as char, i),
                }));
            }
        }

        return Ok((vec, Vec::new()));
    }

    fn get_url<'a>(&'a self) -> &'a String {
        return &self.url;
    }
}

fn main() {
    task::block_on(async {
        let mut crawler = Crawler::new();
        crawler.push_item(Box::new(FakePage {
            url: "https://a.com".to_string(),
        }));

        CrawlerRunner::run(crawler, 50, 30).await;
    })
}