rust_scrapling 0.4.1

Modern web scraping: spiders, CSS selectors, robots.txt/AutoThrottle politeness, and LLM/RAG-ready Markdown extraction - a Rust port of Scrapling
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128

use rust_scrapling::parser::selector_generation::{generate_css_selector, generate_xpath_selector};
use rust_scrapling::parser::Selector;

const HTML: &str = r#"<html><body><div id="main"><ul><li class="item">First</li><li class="item">Second</li></ul><div><p>Nested</p></div></div></body></html>"#;

#[test]
fn test_css_selector_with_id_stops_at_id() {
    let sel = Selector::from_html(HTML);
    let div = sel.css("#main");
    assert_eq!(div.len(), 1);
    let css = generate_css_selector(&div[0], false);
    assert!(
        css.contains("[id=\"main\"]"),
        "Expected attribute-form id selector in: {}",
        css
    );
    // The generated selector must round-trip: running it selects the div.
    assert_eq!(sel.css(&css).len(), 1);
}

#[test]
fn css_selector_handles_ids_that_are_not_css_identifiers() {
    // A raw `#foo.bar:baz` would be an invalid/misparsed selector (`.` and
    // `:` are structural in CSS). The attribute form works for any value.
    let html = r#"<html><body><div id="foo.bar:baz x">content</div></body></html>"#;
    let sel = Selector::from_html(html);
    let div = sel.css("div");
    assert_eq!(div.len(), 1);
    let css = generate_css_selector(&div[0], false);
    assert_eq!(css, "[id=\"foo.bar:baz x\"]");
    assert_eq!(
        sel.css(&css).len(),
        1,
        "generated selector must be parseable and find the element"
    );

    // Quotes and backslashes in the id are escaped in the quoted string.
    let html = r#"<html><body><div id='he said "hi"'>q</div></body></html>"#;
    let sel = Selector::from_html(html);
    let div = sel.css("div");
    let css = generate_css_selector(&div[0], false);
    assert_eq!(css, "[id=\"he said \\\"hi\\\"\"]");
    assert_eq!(sel.css(&css).len(), 1);
}

#[test]
fn test_css_selector_for_li_includes_nth_of_type() {
    let sel = Selector::from_html(HTML);
    let li = sel.css("li.item");
    assert!(li.len() >= 2);
    // The second li should get nth-of-type since there are multiple siblings
    let css = generate_css_selector(&li[1], true);
    assert!(
        css.contains("nth-of-type"),
        "Expected 'nth-of-type' in: {}",
        css
    );
}

#[test]
fn test_xpath_selector_with_id() {
    let sel = Selector::from_html(HTML);
    let div = sel.css("#main");
    assert_eq!(div.len(), 1);
    let xpath = generate_xpath_selector(&div[0], false);
    assert!(
        xpath.contains("@id='main'"),
        "Expected \"@id='main'\" in: {}",
        xpath
    );
}

#[test]
fn test_full_css_selector_includes_body() {
    let sel = Selector::from_html(HTML);
    let div = sel.css("#main");
    assert_eq!(div.len(), 1);
    let css = generate_css_selector(&div[0], true);
    assert!(
        css.contains("body"),
        "Expected 'body' in full path: {}",
        css
    );
}

#[test]
fn identical_siblings_get_distinct_nth_of_type_positions() {
    // Three <li> with identical inner content. The bug compared outer_html
    // and returned position 1 for all of them. The fix uses NodeId.
    let html = r#"<html><body><ul><li>Apple</li><li>Apple</li><li>Apple</li></ul></body></html>"#;
    let sel = Selector::from_html(html);
    let items = sel.css("li");
    assert_eq!(items.len(), 3);

    let css1 = generate_css_selector(&items[0], true);
    let css2 = generate_css_selector(&items[1], true);
    let css3 = generate_css_selector(&items[2], true);
    assert!(css1.contains("nth-of-type(1)"), "got: {}", css1);
    assert!(css2.contains("nth-of-type(2)"), "got: {}", css2);
    assert!(css3.contains("nth-of-type(3)"), "got: {}", css3);

    let xp1 = generate_xpath_selector(&items[0], true);
    let xp2 = generate_xpath_selector(&items[1], true);
    let xp3 = generate_xpath_selector(&items[2], true);
    assert!(xp1.ends_with("li[1]"), "got: {}", xp1);
    assert!(xp2.ends_with("li[2]"), "got: {}", xp2);
    assert!(xp3.ends_with("li[3]"), "got: {}", xp3);
}

#[test]
fn xpath_selector_escapes_id_with_single_quote() {
    // Adversarial id containing a single quote. Without escaping the
    // generated XPath would be `div[@id='foo' or '1'='1']`, which is
    // syntactically valid but semantically attacker-controlled.
    let html = r#"<html><body><div id="foo' or '1'='1">x</div></body></html>"#;
    let sel = Selector::from_html(html);
    let div = sel.css("div");
    assert_eq!(div.len(), 1);
    let xp = generate_xpath_selector(&div[0], false);
    // Because the id contains a single quote, the generator switches to
    // double-quoted delimiters; the original `'` no longer terminates the
    // string.
    assert!(
        xp.contains("@id=\"foo' or '1'='1\""),
        "expected double-quoted id literal, got: {}",
        xp
    );
}