skyscraper 0.7.0

XPath for HTML web scraping
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119

use skyscraper::{
    html,
    xpath::{
        self,
        grammar::data_model::{AnyAtomicType, XpathItem},
    },
};

/// URI-qualified fn:contains should work like unprefixed contains.
#[test]
fn uri_qualified_contains() {
    let text = r#"<html><body><div>hello world</div><div>other</div></body></html>"#;

    let document = html::parse(text).unwrap();
    let xpath = xpath::parse(
        r#"//div[Q{http://www.w3.org/2005/xpath-functions}contains(text(), "hello")]"#,
    )
    .unwrap();

    let items = xpath.apply(&document).unwrap();
    assert_eq!(items.len(), 1, "should match 1 div: {items:?}");
}

/// fn:contains should work with fn: prefix (currently only unprefixed works).
#[test]
fn fn_prefixed_contains() {
    let text = r#"<html><body><div>hello world</div><div>other</div></body></html>"#;

    let document = html::parse(text).unwrap();
    let xpath = xpath::parse(r#"//div[fn:contains(text(), "hello")]"#).unwrap();

    let items = xpath.apply(&document).unwrap();
    assert_eq!(items.len(), 1, "should match 1 div: {items:?}");
}

/// URI-qualified fn:data should atomize the context item.
#[test]
fn uri_qualified_data() {
    let text = r#"<html><body><div>42</div></body></html>"#;

    let document = html::parse(text).unwrap();
    let xpath = xpath::parse(
        r#"Q{http://www.w3.org/2005/xpath-functions}data(//div/text())"#,
    )
    .unwrap();

    let items = xpath.apply(&document).unwrap();
    assert_eq!(items.len(), 1, "should return 1 item: {items:?}");
    assert_eq!(
        items[0],
        XpathItem::AnyAtomicType(AnyAtomicType::String("42".to_string()))
    );
}

/// fn:data with fn: prefix.
#[test]
fn fn_prefixed_data() {
    let text = r#"<html><body><div>hello</div></body></html>"#;

    let document = html::parse(text).unwrap();
    let xpath = xpath::parse(r#"fn:data(//div/text())"#).unwrap();

    let items = xpath.apply(&document).unwrap();
    assert_eq!(items.len(), 1, "should return 1 item: {items:?}");
    assert_eq!(
        items[0],
        XpathItem::AnyAtomicType(AnyAtomicType::String("hello".to_string()))
    );
}

/// fn:string with fn: prefix.
#[test]
fn fn_prefixed_string() {
    let text = r#"<html><body><div>hello</div></body></html>"#;

    let document = html::parse(text).unwrap();
    let xpath = xpath::parse(r#"fn:string(//div)"#).unwrap();

    let items = xpath.apply(&document).unwrap();
    assert_eq!(items.len(), 1, "should return 1 item: {items:?}");
    assert_eq!(
        items[0],
        XpathItem::AnyAtomicType(AnyAtomicType::String("hello".to_string()))
    );
}

/// URI-qualified fn:string.
#[test]
fn uri_qualified_string() {
    let text = r#"<html><body><div>world</div></body></html>"#;

    let document = html::parse(text).unwrap();
    let xpath = xpath::parse(
        r#"Q{http://www.w3.org/2005/xpath-functions}string(//div)"#,
    )
    .unwrap();

    let items = xpath.apply(&document).unwrap();
    assert_eq!(items.len(), 1, "should return 1 item: {items:?}");
    assert_eq!(
        items[0],
        XpathItem::AnyAtomicType(AnyAtomicType::String("world".to_string()))
    );
}

/// Unknown URI namespace should error.
#[test]
fn unknown_uri_namespace_errors() {
    let text = r#"<html><body></body></html>"#;

    let document = html::parse(text).unwrap();
    let xpath = xpath::parse(
        r#"Q{http://example.com/unknown}foo()"#,
    )
    .unwrap();

    let result = xpath.apply(&document);
    assert!(result.is_err(), "unknown namespace should error: {result:?}");
}