skyscraper 0.7.0

XPath for HTML web scraping
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151

use skyscraper::{html, xpath};

/// `//div | //span` should return all div and span elements.
#[test]
fn union_bar_combines_results() {
    let text = r#"<html><body><div>a</div><span>b</span><p>c</p></body></html>"#;

    let document = html::parse(text).unwrap();
    let xpath = xpath::parse("//div | //span").unwrap();

    let items = xpath.apply(&document).unwrap();
    assert_eq!(items.len(), 2, "should return div and span: {items:?}");
}

/// `//div union //span` should work like `|`.
#[test]
fn union_keyword_combines_results() {
    let text = r#"<html><body><div>a</div><span>b</span><p>c</p></body></html>"#;

    let document = html::parse(text).unwrap();
    let xpath = xpath::parse("//div union //span").unwrap();

    let items = xpath.apply(&document).unwrap();
    assert_eq!(items.len(), 2, "should return div and span: {items:?}");
}

/// Union of overlapping sets removes duplicates.
#[test]
fn union_deduplicates() {
    let text = r#"<html><body><div class="a">x</div><div class="b">y</div></body></html>"#;

    let document = html::parse(text).unwrap();
    // Both sides select the same divs — result should still be 2, not 4.
    let xpath = xpath::parse("//div | //div").unwrap();

    let items = xpath.apply(&document).unwrap();
    assert_eq!(items.len(), 2, "should deduplicate: {items:?}");
}

/// Union of three sets: `//div | //span | //p`.
#[test]
fn union_three_sets() {
    let text = r#"<html><body><div>a</div><span>b</span><p>c</p></body></html>"#;

    let document = html::parse(text).unwrap();
    let xpath = xpath::parse("//div | //span | //p").unwrap();

    let items = xpath.apply(&document).unwrap();
    assert_eq!(items.len(), 3, "should return all three elements: {items:?}");
}

/// Union results should be in document order.
#[test]
fn union_document_order() {
    let text = r#"<html><body><span>first</span><div>second</div></body></html>"#;

    let document = html::parse(text).unwrap();
    // Select div first, then span — result should still be span before div (document order).
    let xpath = xpath::parse("//div | //span").unwrap();

    let items = xpath.apply(&document).unwrap();
    assert_eq!(items.len(), 2);
    let first = items[0].extract_as_node().extract_as_element_node();
    assert_eq!(first.name, "span", "span should come first in document order");
}

/// `//div intersect //div[@class]` keeps only divs that have a class attribute.
#[test]
fn intersect_filters_common() {
    let text =
        r#"<html><body><div class="a">x</div><div>y</div><div class="b">z</div></body></html>"#;

    let document = html::parse(text).unwrap();
    let xpath = xpath::parse("//div intersect //div[@class]").unwrap();

    let items = xpath.apply(&document).unwrap();
    assert_eq!(
        items.len(),
        2,
        "should return only divs with class attr: {items:?}"
    );
}

/// `//div except //div[@class]` removes divs that have a class attribute.
#[test]
fn except_removes_matching() {
    let text =
        r#"<html><body><div class="a">x</div><div>y</div><div class="b">z</div></body></html>"#;

    let document = html::parse(text).unwrap();
    let xpath = xpath::parse("//div except //div[@class]").unwrap();

    let items = xpath.apply(&document).unwrap();
    assert_eq!(
        items.len(),
        1,
        "should return only the div without class: {items:?}"
    );
}

/// Except with disjoint sets returns the LHS unchanged.
#[test]
fn except_disjoint_returns_lhs() {
    let text = r#"<html><body><div>a</div><span>b</span></body></html>"#;

    let document = html::parse(text).unwrap();
    let xpath = xpath::parse("//div except //span").unwrap();

    let items = xpath.apply(&document).unwrap();
    assert_eq!(items.len(), 1, "should return div: {items:?}");
}

/// Intersect with disjoint sets returns empty.
#[test]
fn intersect_disjoint_returns_empty() {
    let text = r#"<html><body><div>a</div><span>b</span></body></html>"#;

    let document = html::parse(text).unwrap();
    let xpath = xpath::parse("//div intersect //span").unwrap();

    let items = xpath.apply(&document).unwrap();
    assert_eq!(items.len(), 0, "should return empty: {items:?}");
}

/// Regression: XPath sequences (not node-sets) must preserve duplicate values.
/// A parenthesized sequence like `(1, 2, 2, 3)` should have length 4, not 3.
#[test]
fn sequence_preserves_duplicates() {
    let text = "<html><body></body></html>";
    let document = html::parse(text).unwrap();
    let xpath = xpath::parse("count((1, 2, 2, 3))").unwrap();
    let items = xpath.apply(&document).unwrap();
    assert_eq!(
        items[0],
        skyscraper::xpath::grammar::data_model::XpathItem::AnyAtomicType(
            skyscraper::xpath::grammar::data_model::AnyAtomicType::Integer(4)
        ),
        "Sequences must preserve duplicate values"
    );
}

/// Regression: union of node-sets must still deduplicate (node-set semantics).
#[test]
fn union_still_deduplicates_nodes() {
    let text = r#"<html><body><div>x</div></body></html>"#;
    let document = html::parse(text).unwrap();
    // //div | //div should return 1, not 2
    let xpath = xpath::parse("//div | //div").unwrap();
    let items = xpath.apply(&document).unwrap();
    assert_eq!(items.len(), 1, "Union of same nodes should deduplicate: {items:?}");
}