dyer 3.3.2

dyer is designed for reliable, flexible and fast Request-Response based service, including data processing, web-crawling and so on, providing some friendly, flexible, comprehensive features without compromising speed.
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166

//! implement the xpath for the response
//! more to see [Response::xpath]

use super::Response;
use libxml::{
    parser::{ParseFormat, Parser},
    tree::Node,
    xpath::Context,
};

impl Response {
    #[cfg_attr(docsrs, doc(cfg(feature = "xpath")))]
    /// initialize the context for the root html
    fn xpath_context(&mut self) {
        if self.context.0.is_none() || self.context.1.is_none() {
            let input = self.body.bytes();
            assert!(input.len() != 0, "body must not be empty");
            let parser = Parser {
                format: ParseFormat::HTML,
            };
            let doc = parser.parse_string(input).unwrap();
            self.context.0 = Some(doc);
            let context = Context::new(self.context.0.as_ref().unwrap()).unwrap();
            self.context.1 = Some(context);
        }
    }

    #[cfg_attr(docsrs, doc(cfg(feature = "xpath")))]
    /// parse the ready response's body with xpath
    ///
    /// the feature `xpath` must be enabled
    /// ```rust
    /// let html = r#"<!doctype html>
    /// <html>
    ///     <title>Hello dyer!</title>
    ///     <body> ... </body>
    /// </html>"#;
    /// let body = Body::from(html);
    /// let mut res = Response::new(body);
    /// let title = res.xpath("//title/text()");
    /// assert_eq!(
    ///     title[0]
    ///         .get_content(),
    ///     "Hello dyer!"
    /// );
    /// ```
    pub fn xpath(&mut self, xpath: &str) -> Vec<Node> {
        self.xpath_context();
        self.context
            .1
            .as_ref()
            .unwrap()
            .evaluate(xpath)
            .unwrap()
            .get_nodes_as_vec()
    }
}

#[test]
fn test_xpath() {
    use super::*;
    let html = r#"<!doctype html>
<html>
<head>
    <title>Hello dyer!</title>
    <meta charset="utf-8" />
    <meta http-equiv="Content-type" content="text/html; charset=utf-8" />
    <meta name="viewport" content="width=device-width, initial-scale=1" />
</head>

<body>
    <h1>Hello dyer!</h1>
    <p>
        dyer is designed for reliable, flexible and fast web crawling, providing some high-level, comprehensive features without compromising speed.
    </p>
    <h2>Features</h2>
    <ol class="ol">
        <li>Async</li>
        <li class="class-wrapper">Concurrency</li>
        <li id="id-wrapper">Multi-Thread</li>
    </ol class="ol">
    <ol>
        <li>Command line Support</li>
        <li>Session Backup</li>
        <li>Signal Handling</li>
    </ol>
    <h2>Guide</h2>
    <p><strong>Get started</strong> by installing <a href="https://github.com/homelyguy/dyer-cli">dyer-cli</a> and looking over the <a href="https://github.com/HomelyGuy/dyer/tree/master/examples/">examples</a>.</p>
    <p>The <a href="https://homelyguy.github.io/dyer/">Cookbook</a> gives a detailed view of dyer.</p>
    <p><a href="https://github.com/homelyguy/dyer">See Repository...</a></p>
</body>
</html>
    "#;

    let body = Body::from(html);
    let mut res = Response::new(body);
    let title = res.xpath("/html/body/h1");
    assert_eq!(title.len(), 1);
    assert_eq!(title[0].get_content(), "Hello dyer!");
    let item1 = res.xpath("//*[@class=\"class-wrapper\"]");
    assert_eq!(item1[0].get_content(), "Concurrency");
    let h2 = res.xpath("//h2");
    assert_eq!(
        h2.iter()
            .map(|node| node.get_content())
            .collect::<Vec<String>>(),
        vec!["Features", "Guide"]
    );
    let li = res.xpath("//ol");
    let mut nodes = Vec::new();
    for node in li.iter() {
        let slice = node
            .findnodes("./li/text()")
            .unwrap()
            .into_iter()
            .map(|n| n.get_content())
            .collect::<Vec<_>>();
        assert_eq!(slice.len(), 3);
        nodes.extend(slice);
    }
    assert_eq!(
        nodes,
        vec![
            "Async",
            "Concurrency",
            "Multi-Thread",
            "Command line Support",
            "Session Backup",
            "Signal Handling"
        ]
    );
    let nodes = li
        .into_iter()
        .flat_map(|node| {
            node.findnodes("./li/text()")
                .unwrap()
                .into_iter()
                .map(|n| n.get_content())
        })
        .collect::<Vec<_>>();
    assert_eq!(
        nodes,
        vec![
            "Async",
            "Concurrency",
            "Multi-Thread",
            "Command line Support",
            "Session Backup",
            "Signal Handling"
        ]
    );
    let nodes = res
        .xpath("//body//a/@href")
        .iter()
        .map(|node| node.get_content())
        .collect::<Vec<_>>();
    assert_eq!(
        nodes,
        vec![
            "https://github.com/homelyguy/dyer-cli",
            "https://github.com/HomelyGuy/dyer/tree/master/examples/",
            "https://homelyguy.github.io/dyer/",
            "https://github.com/homelyguy/dyer",
        ]
    );
}