sc-scraping 0.2.1

Utility functions and structures to scrape data from the Roberts Space Industries website
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45

use scraper::{ElementRef, Selector};
use tracing::error;

use crate::error::ScScrapingError;

pub(crate) struct WebBio {
    pub website: Option<String>,
    pub bio: Vec<String>,
}

pub(crate) fn parse_web_bio<'a>(element: &ElementRef) -> Result<WebBio, ScScrapingError<'a>> {
    let selector = Selector::parse("div.profile-content > div.right-col").unwrap();

    let fragment = element.select(&selector).next().ok_or_else(|| {
        let message = "Web and bio parent could not be found";
        error!("{message}");
        ScScrapingError::citizen(message)
    })?;

    let web_bio = WebBio {
        website: parse_website(&fragment),
        bio: parse_bio(&fragment),
    };

    Ok(web_bio)
}

fn parse_website(element: &ElementRef) -> Option<String> {
    let selector = Selector::parse("p.website > a").unwrap();

    element.select(&selector).map(|w| w.inner_html()).next()
}

fn parse_bio(element: &ElementRef) -> Vec<String> {
    let selector = Selector::parse("div.entry.bio > div.value").unwrap();

    element
        .select(&selector)
        .flat_map(|e| e.children())
        .filter_map(|e| e.value().as_text())
        .map(|t| t.trim())
        .filter(|t| !t.is_empty())
        .map(std::string::ToString::to_string)
        .collect::<Vec<_>>()
}