skyscraper 0.7.0

XPath for HTML web scraping
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110

use skyscraper::{html, xpath};

/// The `or` operator should return nodes matching either condition
/// (XPath 3.1 section 3.6).
#[test]
fn or_operator_matches_either_condition() {
    let text = r#"<html><body>
        <div class="a">first</div>
        <div class="b">second</div>
        <div class="c">third</div>
    </body></html>"#;

    let document = html::parse(text).unwrap();
    let xpath = xpath::parse("//div[@class='a' or @class='b']").unwrap();

    let nodes = xpath.apply(&document).unwrap();
    assert_eq!(nodes.len(), 2, "should match two divs: {nodes:?}");
}

/// The `or` operator should return true if the first operand is true
/// (XPath 3.1 section 3.6).
#[test]
fn or_operator_returns_true_for_first_true() {
    let text = r#"<html><body>
        <div class="a">found</div>
    </body></html>"#;

    let document = html::parse(text).unwrap();
    let xpath = xpath::parse("//div[@class='a' or @class='nonexistent']").unwrap();

    let nodes = xpath.apply(&document).unwrap();
    assert_eq!(nodes.len(), 1, "should match when first condition is true: {nodes:?}");
}

/// The `or` operator should return true if only the second operand is true
/// (XPath 3.1 section 3.6).
#[test]
fn or_operator_returns_true_for_second_true() {
    let text = r#"<html><body>
        <div class="a">found</div>
    </body></html>"#;

    let document = html::parse(text).unwrap();
    let xpath = xpath::parse("//div[@class='nonexistent' or @class='a']").unwrap();

    let nodes = xpath.apply(&document).unwrap();
    assert_eq!(nodes.len(), 1, "should match when second condition is true: {nodes:?}");
}

/// The `or` operator should return false when neither operand is true
/// (XPath 3.1 section 3.6).
#[test]
fn or_operator_returns_false_when_both_false() {
    let text = r#"<html><body>
        <div class="a">found</div>
    </body></html>"#;

    let document = html::parse(text).unwrap();
    let xpath = xpath::parse("//div[@class='x' or @class='y']").unwrap();

    let nodes = xpath.apply(&document).unwrap();
    assert_eq!(nodes.len(), 0, "should match nothing when both false: {nodes:?}");
}

/// The `and` operator should return nodes matching both conditions
/// (XPath 3.1 section 3.6).
#[test]
fn and_operator_matches_both_conditions() {
    let text = r#"<html><body>
        <div class="a" id="1">both</div>
        <div class="a">only class</div>
        <div id="1">only id</div>
    </body></html>"#;

    let document = html::parse(text).unwrap();
    let xpath = xpath::parse("//div[@class='a' and @id='1']").unwrap();

    let nodes = xpath.apply(&document).unwrap();
    assert_eq!(nodes.len(), 1, "should match only the div with both attributes: {nodes:?}");
}

/// The `and` operator should return false when the first operand is false
/// (XPath 3.1 section 3.6).
#[test]
fn and_operator_returns_false_when_first_false() {
    let text = r#"<html><body>
        <div class="a" id="1">match</div>
    </body></html>"#;

    let document = html::parse(text).unwrap();
    let xpath = xpath::parse("//div[@class='nonexistent' and @id='1']").unwrap();

    let nodes = xpath.apply(&document).unwrap();
    assert_eq!(nodes.len(), 0, "should match nothing when first is false: {nodes:?}");
}

/// The `and` operator should return false when the second operand is false
/// (XPath 3.1 section 3.6).
#[test]
fn and_operator_returns_false_when_second_false() {
    let text = r#"<html><body>
        <div class="a" id="1">match</div>
    </body></html>"#;

    let document = html::parse(text).unwrap();
    let xpath = xpath::parse("//div[@class='a' and @id='nonexistent']").unwrap();

    let nodes = xpath.apply(&document).unwrap();
    assert_eq!(nodes.len(), 0, "should match nothing when second is false: {nodes:?}");
}