dachterasse 0.1.4

A crate for scraping the HPI lecture overview site
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105

use crate::lectures::entities::StaticDegree;
use crate::lectures::entities::Lecture;
use regex::Regex;
use reqwest::Client;
use scraper::element_ref::Text;
use scraper::{Html, Selector};
use std::collections::HashMap;

#[derive(Default)]
pub struct LectureScraper {
    client: Client,
}

pub type Error = reqwest::Error;

impl LectureScraper {
    pub fn new() -> Self {
        LectureScraper {
            client: Client::new(),
        }
    }

    // TODO: Extract document fetching and parsing into separate method for better testability
    pub async fn fetch_lectures(&self, degree: &StaticDegree) -> Result<Vec<Lecture>, Error> {
        let url = degree.url;
        let document = get_text(url, &self.client).await?;
        let fragment = Html::parse_document(&document);
        let selector = Selector::parse("a.courselink").unwrap();
        let link_regex = Regex::new(r"/studium.*\.html").unwrap();

        Ok(fragment
            .select(&selector)
            .map(|element| Lecture {
                title: clean(element.text()),
                url: String::from("https://hpi.de")
                    + &link_regex.captures(&*element.html()).unwrap()[0],
                categories: None,
            })
            .collect())
    }

    // TODO: Extract document fetching and parsing into separate method for better testability
    pub async fn fetch_lecture_details(&self, degree: &StaticDegree) -> Result<Vec<Lecture>, Error> {
        let mut lectures = self.fetch_lectures(degree).await?;
        for mut lecture in &mut lectures {
            let document = get_text(&lecture.url, &self.client).await?;

            if let Some(inner_fragment) = self.scrape_modules(document.as_str(), degree.name) {
                let module_list = Html::parse_fragment(inner_fragment.as_str());
                let item_selector = Selector::parse("li").unwrap();
                let categories: Vec<(String, Vec<String>)> = module_list
                    .select(&item_selector)
                    .map(|element| {
                        (
                            clean(element.text()),
                            Html::parse_fragment(element.inner_html().as_str())
                                .select(&item_selector)
                                .map(|child| clean(child.text()))
                                .collect::<Vec<_>>(),
                        )
                    })
                    .filter(|(_, children)| !children.is_empty())
                    .collect();

                let mut category_map = HashMap::<String, Vec<String>>::new();
                for mut category in categories {
                    category_map
                        .entry(category.0)
                        .or_insert(Vec::new())
                        .append(&mut category.1);
                }
                lecture.categories = Some(category_map);
            }
        }

        Ok(lectures)
    }

    fn scrape_modules(&self, document: &str, degree: &str) -> Option<String> {
        let fragment = Html::parse_document(document);
        let selector = Selector::parse("div.tx_dscclipclap").unwrap();
        let header_selector = Selector::parse("div.tx_dscclipclap_header").unwrap();

        fragment
            .select(&selector)
            .find(|element| {
                Html::parse_fragment(element.inner_html().as_str())
                    .select(&header_selector)
                    .any(|inner| clean(inner.text()) == degree)
            })
            .map(|optional| optional.inner_html())
    }
}

fn clean(text: Text) -> String {
    String::from(clean_str(text))
}

fn clean_str(text: Text) -> &str {
    text.collect::<Vec<_>>().first().unwrap().trim()
}

async fn get_text(url: &str, client: &Client) -> Result<String, reqwest::Error> {
    client.get(url).send().await?.text().await
}