skyscraper 0.7.0

XPath for HTML web scraping
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120

use skyscraper::{
    html::{self, trim_internal_whitespace},
    xpath,
};

#[test]
fn contains_should_select_text() {
    // arrange
    let text = r###"
        <html>
            <div>
                hello world
            </div>
            <div>
                select me
            </div>
        </html>"###;

    let document = html::parse(&text).unwrap();
    let xpath = xpath::parse("//div[contains(text(),'select')]").unwrap();

    // act
    let nodes = xpath.apply(&document).unwrap();

    // assert
    assert_eq!(nodes.len(), 1);
    let mut nodes = nodes.into_iter();

    // assert node
    {
        let tree_node = nodes.next().unwrap().extract_into_node();

        let element = tree_node.extract_as_element_node();
        assert_eq!(element.name, "div");

        assert_eq!(
            trim_internal_whitespace(&tree_node.text(&document).unwrap()),
            "select me"
        );
    }
}

#[test]
fn contains_should_select_attribute() {
    // arrange
    let text = r###"
        <html>
            <div class="nah">
                hello world
            </div>
            <div class="select me">
                select me
            </div>
        </html>"###;

    let document = html::parse(&text).unwrap();
    let xpath = xpath::parse("//div[contains(@class,'select')]").unwrap();

    // act
    let nodes = xpath.apply(&document).unwrap();

    // assert
    assert_eq!(nodes.len(), 1);
    let mut nodes = nodes.into_iter();

    // assert node
    {
        let tree_node = nodes.next().unwrap().extract_into_node();

        let element = tree_node.extract_as_element_node();
        assert_eq!(element.name, "div");

        assert_eq!(
            trim_internal_whitespace(&tree_node.text(&document).unwrap()),
            "select me"
        );
    }
}

#[test]
fn contains_should_select_on_expression() {
    // arrange
    let text = r###"
        <html>
            <div>
                other
            </div>
            <div>
                hello world
            </div>
            <div class="hello" id="select"></div>
        
        </html>"###;

    let document = html::parse(&text).unwrap();

    // this expression first selects the class attribute of the div with id 'select',
    // then uses that result to find a div with text containing that class.
    let xpath = xpath::parse("//div[contains(text(),//div[@id='select']/@class)]").unwrap();

    // act
    let nodes = xpath.apply(&document).unwrap();

    // assert
    assert_eq!(nodes.len(), 1);
    let mut nodes = nodes.into_iter();

    // assert node
    {
        let tree_node = nodes.next().unwrap().extract_into_node();

        let element = tree_node.extract_as_element_node();
        assert_eq!(element.name, "div");

        assert_eq!(
            trim_internal_whitespace(&tree_node.text(&document).unwrap()),
            "hello world"
        );
    }
}