sc_scraping/
lib.rs

1use std::collections::HashMap;
2
3use jiff::civil::{Date, DateTime};
4use scraper::{Html, Selector};
5use serde::{Deserialize, Serialize};
6use tracing::error;
7use url::Url;
8
9mod error;
10mod parse_citizen_other;
11mod parse_citizen_profile;
12mod parse_org;
13mod parse_web_bio;
14
15pub use error::*;
16
17use crate::{
18    parse_citizen_other::{parse_citizen_probation, parse_citizen_record_id, parse_other},
19    parse_citizen_profile::parse_profile_info,
20    parse_org::parse_org,
21    parse_web_bio::parse_web_bio,
22};
23
24#[derive(Debug, Serialize, Deserialize)]
25#[serde(rename_all = "camelCase")]
26pub struct Citizen {
27    pub moniker: String,
28    pub handle: String,
29    pub title: Title,
30    pub enlisted: Date,
31    #[serde(skip_serializing_if = "Option::is_none")]
32    pub citizen_record_number: Option<u64>,
33    pub avatar: Url,
34    #[serde(default, skip_serializing_if = "Vec::is_empty")]
35    pub location: Vec<String>,
36    #[serde(default, skip_serializing_if = "Vec::is_empty")]
37    pub fluency: Vec<String>,
38    #[serde(skip_serializing_if = "Option::is_none")]
39    pub website: Option<String>,
40    #[serde(default, skip_serializing_if = "Vec::is_empty")]
41    pub bio: Vec<String>,
42    #[serde(skip_serializing_if = "Option::is_none")]
43    pub probation: Option<DateTime>,
44}
45
46#[derive(Debug, Serialize, Deserialize)]
47pub struct Title {
48    pub icon: Url,
49    pub value: String,
50}
51
52#[derive(Debug, Serialize, Deserialize)]
53#[serde(rename_all = "camelCase")]
54pub enum Organization {
55    Visible {
56        logo: Box<Url>,
57        name: String,
58        sid: String,
59        url: Box<Url>,
60        rank: OrganizationRank,
61        member_count: u64,
62    },
63    Redacted,
64}
65
66#[derive(Debug, Serialize, Deserialize)]
67pub struct OrganizationRank {
68    pub name: String,
69    pub value: u8,
70}
71
72#[derive(Debug, Serialize, Deserialize)]
73pub struct Organizations {
74    #[serde(skip_serializing_if = "Option::is_none")]
75    pub main: Option<Organization>,
76    #[serde(default, skip_serializing_if = "Vec::is_empty")]
77    pub affiliates: Vec<Organization>,
78}
79
80#[derive(PartialEq, Eq, Hash, Clone)]
81pub(crate) enum OrganizationType {
82    Main,
83    Affiliate,
84}
85
86pub fn parse_citizen_record<'a>(html_data: &str) -> Result<Citizen, ScScrapingError<'a>> {
87    let html = Html::parse_document(html_data);
88
89    let citizen_selector = Selector::parse("div.public-profile").unwrap();
90    let citizen = html.select(&citizen_selector).next().ok_or_else(|| {
91        let message = "Citizen content is empty";
92        error!("{message}");
93        ScScrapingError::citizen(message)
94    })?;
95
96    let web_bio = parse_web_bio(&citizen)?;
97    let profile_info = parse_profile_info(&citizen)?;
98    let other_data = parse_other(&citizen)?;
99
100    let record = Citizen {
101        moniker: profile_info.moniker,
102        handle: profile_info.handle,
103        title: profile_info.title,
104        avatar: profile_info.avatar,
105
106        website: web_bio.website,
107        bio: web_bio.bio,
108
109        enlisted: other_data.enlisted,
110        location: other_data.location,
111        fluency: other_data.languages,
112
113        citizen_record_number: parse_citizen_record_id(&citizen),
114
115        probation: parse_citizen_probation(&citizen),
116    };
117
118    Ok(record)
119}
120
121pub fn parse_organizations<'a>(
122    html_data: &str,
123) -> Result<Option<Organizations>, ScScrapingError<'a>> {
124    let html = Html::parse_document(html_data);
125
126    let org_selector = Selector::parse("div.box-content.org").unwrap();
127
128    let mut map = HashMap::new();
129
130    for e in html.select(&org_selector) {
131        let (org_type, org) = parse_org(&e)?;
132
133        if !map.contains_key(&org_type) {
134            map.insert(org_type.clone(), Vec::new());
135        }
136
137        map.get_mut(&org_type).unwrap().push(org);
138    }
139
140    if map.is_empty() {
141        return Ok(None);
142    }
143
144    if map.contains_key(&OrganizationType::Main)
145        && let Some(main) = map.get(&OrganizationType::Main)
146        && main.len() > 1
147    {
148        let message = "Cannot have multiple main organizations";
149        error!("{message}");
150        return Err(ScScrapingError::organization(message));
151    }
152
153    let orgs = Organizations {
154        main: map
155            .remove(&OrganizationType::Main)
156            .map(|mut o| o.pop().unwrap()),
157
158        affiliates: map.remove(&OrganizationType::Affiliate).unwrap_or_default(),
159    };
160
161    Ok(Some(orgs))
162}
163
164#[cfg(test)]
165mod tests {
166    use super::*;
167
168    #[test]
169    fn test() {
170        let test_profile_data = include_str!("./test-data/test-profile-probation.html");
171        let test_org_data = include_str!("./test-data/test-orgs.html");
172
173        let citizen = parse_citizen_record(test_profile_data).unwrap();
174        let orgs = parse_organizations(test_org_data).unwrap();
175
176        // TODO: make this test not suck
177
178        println!("{citizen:#?}");
179        println!("{orgs:#?}");
180    }
181}
sc_scraping/lib.rs

sc_scraping/
lib.rs
