voyager 0.2.1

Web crawler and scraper
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135

#![allow(dead_code)]

use anyhow::Result;
use futures::StreamExt;
use reqwest::Url;
use std::time::Duration;
use voyager::scraper::Selector;
use voyager::{Collector, Crawler, CrawlerConfig, RequestDelay, Response, Scraper};

#[tokio::main]
async fn main() -> Result<(), Box<dyn std::error::Error>> {
    struct HackernewsScraper {
        post_selector: Selector,
        author_selector: Selector,
        title_selector: Selector,
        comment_selector: Selector,
        max_page: usize,
    }

    impl Default for HackernewsScraper {
        fn default() -> Self {
            Self {
                post_selector: Selector::parse("table.itemlist tr.athing").unwrap(),
                author_selector: Selector::parse("a.hnuser").unwrap(),
                title_selector: Selector::parse("td.title a").unwrap(),
                comment_selector: Selector::parse(".comment-tree tr.athing").unwrap(),
                max_page: 1,
            }
        }
    }

    #[derive(Debug)]
    enum HackernewsState {
        Page(usize),
        Post,
    }

    #[derive(Debug)]
    struct Entry {
        author: String,
        url: Url,
        link: Option<String>,
        title: String,
        replies: Vec<Reply>,
    }

    #[derive(Debug)]
    struct Reply {
        author: String,
        url: Url,
        comment: String,
        replies: Vec<Reply>,
    }

    impl Scraper for HackernewsScraper {
        type Output = Entry;
        type State = HackernewsState;

        fn scrape(
            &mut self,
            response: Response<Self::State>,
            crawler: &mut Crawler<Self>,
        ) -> Result<Option<Self::Output>> {
            let html = response.html();

            if let Some(state) = response.state {
                match state {
                    HackernewsState::Page(page) => {
                        // find all entries
                        for id in html
                            .select(&self.post_selector)
                            .filter_map(|el| el.value().attr("id"))
                        {
                            crawler.visit_with_state(
                                &format!("https://news.ycombinator.com/item?id={}", id),
                                HackernewsState::Post,
                            );
                        }
                        if page < self.max_page {
                            // queue in next page
                            crawler.visit_with_state(
                                &format!("https://news.ycombinator.com/news?p={}", page + 1),
                                HackernewsState::Page(page + 1),
                            );
                        }
                    }
                    HackernewsState::Post => {
                        let el_title = html.select(&self.title_selector).next().unwrap();
                        let author = html
                            .select(&self.author_selector)
                            .map(|el| el.inner_html())
                            .next();
                        if author.is_none() {
                            // promo
                            return Ok(None);
                        }
                        // scrape the post
                        let entry = Entry {
                            author: author.unwrap(),
                            url: response.response_url,
                            link: el_title.value().attr("href").map(str::to_string),
                            title: el_title.inner_html(),
                            replies: Vec::new(),
                        };

                        // scrape replies...

                        return Ok(Some(entry));
                    }
                }
            }

            Ok(None)
        }
    }

    let config = CrawlerConfig::default().allow_domain_with_delay(
        "news.ycombinator.com",
        RequestDelay::Fixed(Duration::from_millis(2_000)),
    );
    let mut collector = Collector::new(HackernewsScraper::default(), config);

    collector.crawler_mut().visit_with_state(
        "https://news.ycombinator.com/news",
        HackernewsState::Page(1),
    );

    while let Some(output) = collector.next().await {
        if let Ok(post) = output {
            println!("Post: {:?}", post)
        }
    }

    Ok(())
}