reqwest-scraper 0.7.1

Web scraping integration with reqwest
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121

//!  Use JsonPath to select fields in json response
//!
use crate::error::{Result, ScraperError};
use jsonpath_lib as jsonpath;
use serde::de::DeserializeOwned;

/// Json Response
#[derive(Debug)]
pub struct Json {
    value: serde_json::Value,
}

impl Json {
    /// constructor
    pub fn new(json: &str) -> Result<Self> {
        let value = serde_json::from_str(json)?;
        Ok(Self { value })
    }

    /// Use jsonpath to select json fragments and convert them into structures
    pub fn select<T: DeserializeOwned>(&self, path: &str) -> Result<Vec<T>> {
        jsonpath::Selector::new()
            .str_path(path)?
            .value(&self.value)
            .select_as()
            .map_err(ScraperError::from)
    }

    /// Use jsonpath to select json string fields
    pub fn select_one<T: DeserializeOwned>(&self, path: &str) -> Result<T> {
        let result = jsonpath::Selector::new()
            .str_path(path)?
            .value(&self.value)
            .select()?;
        let v = result
            .first()
            .ok_or_else(|| {
                ScraperError::JsonPathMatchError(format!(
                    "The \"{}\" jsonpath did not find data in json",
                    path
                ))
            })?
            .to_owned();
        Ok(serde_json::from_value::<T>(v.to_owned())?)
    }

    /// Use jsonpath to select json fields as string
    pub fn select_as_str(&self, path: &str) -> Result<String> {
        jsonpath::Selector::new()
            .str_path(path)?
            .value(&self.value)
            .select_as_str()
            .map_err(ScraperError::from)
    }

    /// Use jsonpath to select one json fields as string
    pub fn select_one_as_str(&self, path: &str) -> Result<String> {
        let result = jsonpath::Selector::new()
            .str_path(path)?
            .value(&self.value)
            .select()?;
        let v = result
            .first()
            .ok_or_else(|| {
                ScraperError::JsonPathMatchError(format!(
                    "The \"{}\" jsonpath did not find data in json",
                    path
                ))
            })?
            .to_owned();
        Ok(v.to_string())
    }
}

#[cfg(test)]
mod tests {

    #[test]
    #[allow(clippy::get_first)]
    fn test_jsonpath() {
        use super::Json;
        use serde::Deserialize;
        let json = r#"
        {
            "time":"2020.10.12 21:22:34",
            "data":[
                {"a":"A1","B":"b1","c":1},
                {"a":"A2","B":"b2","c":2}
            ]
        }
        "#;

        let json = Json::new(json).unwrap();

        assert_eq!(
            json.select_one_as_str("$.time").unwrap(),
            r#""2020.10.12 21:22:34""#
        );

        #[derive(Deserialize)]
        struct DataItem {
            a: String,
            #[serde(rename = "B")]
            b: String,
            c: i32,
        }

        let data: Vec<DataItem> = json.select("$.data[*]").unwrap();
        assert_eq!(data.len(), 2);

        let d1 = data.get(0).unwrap();
        assert_eq!(d1.a, "A1");
        assert_eq!(d1.b, "b1");
        assert_eq!(d1.c, 1);

        let d2 = data.get(1).unwrap();
        assert_eq!(d2.a, "A2");
        assert_eq!(d2.b, "b2");
        assert_eq!(d2.c, 2);
    }
}