skyscraper 0.7.0

XPath for HTML web scraping
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143

use criterion::{BenchmarkId, Criterion};
use skyscraper::{html, xpath};

static HTML: &str = include_str!("../tests/samples/James-LG_Skyscraper.html");

fn tree() -> xpath::XpathItemTree {
    html::parse(HTML).unwrap()
}

/// Benchmark different XPath axis traversals on a real HTML document.
pub fn benchmark_axes(c: &mut Criterion) {
    let tree = tree();

    let cases = [
        ("child", "//body/div"),
        ("descendant", "//body//div"),
        ("descendant-or-self", "//div/descendant-or-self::div"),
        ("self", "//div/self::div"),
        ("attribute", "//div/@class"),
        ("parent", "//a/.."),
        ("ancestor", "//a/ancestor::div"),
        ("ancestor-or-self", "//div/ancestor-or-self::div"),
        ("following-sibling", "//div/following-sibling::div"),
        ("preceding-sibling", "//div/preceding-sibling::div"),
    ];

    let mut group = c.benchmark_group("axes");
    for (name, expr) in cases {
        let xpath = xpath::parse(expr).unwrap();
        group.bench_with_input(BenchmarkId::new("eval", name), &xpath, |b, xpath| {
            b.iter(|| xpath.apply(&tree).unwrap());
        });
    }
    group.finish();
}

/// Benchmark predicate evaluation: positional, attribute equality, and compound.
pub fn benchmark_predicates(c: &mut Criterion) {
    let tree = tree();

    let cases = [
        ("positional_first", "//div[1]"),
        ("positional_last", "//div[last()]"),
        ("attribute_eq", "//div[@class='repository-content']"),
        ("attribute_contains", "//div[contains(@class, 'Border')]"),
        ("compound_and", "//div[@class and @id]"),
        ("nested", "//div[div[@class]]"),
    ];

    let mut group = c.benchmark_group("predicates");
    for (name, expr) in cases {
        let xpath = xpath::parse(expr).unwrap();
        group.bench_with_input(BenchmarkId::new("eval", name), &xpath, |b, xpath| {
            b.iter(|| xpath.apply(&tree).unwrap());
        });
    }
    group.finish();
}

/// Benchmark builtin XPath functions.
pub fn benchmark_functions(c: &mut Criterion) {
    let tree = tree();

    let cases = [
        ("count", "count(//div)"),
        ("string-length", "string-length(//title)"),
        ("concat", "concat('hello', ' ', 'world')"),
        ("contains", "contains('skyscraper', 'sky')"),
        ("not", "not(false())"),
        ("sum", "sum((1, 2, 3, 4, 5))"),
        ("string-join", "string-join(('a','b','c'), '-')"),
        ("substring", "substring('skyscraper', 4, 3)"),
        ("reverse", "reverse((1, 2, 3, 4, 5))"),
        ("name", "name(//div[1])"),
    ];

    let mut group = c.benchmark_group("functions");
    for (name, expr) in cases {
        let xpath = xpath::parse(expr).unwrap();
        group.bench_with_input(BenchmarkId::new("eval", name), &xpath, |b, xpath| {
            b.iter(|| xpath.apply(&tree).unwrap());
        });
    }
    group.finish();
}

/// Benchmark XPath expression types: arithmetic, comparison, logical, conditional.
pub fn benchmark_expressions(c: &mut Criterion) {
    let tree = tree();

    let cases = [
        ("arithmetic", "3 * 4 + 2 div 1 - 1"),
        ("comparison_value", "count(//div) > 10"),
        ("logical_and_or", "true() and (false() or true())"),
        ("conditional", "if (count(//div) > 0) then 'yes' else 'no'"),
        ("range", "1 to 100"),
        ("for_expr", "for $x in (1, 2, 3) return $x * 2"),
        ("let_expr", "let $x := count(//div) return $x + 1"),
        ("quantified_some", "some $x in (1, 2, 3) satisfies $x > 2"),
    ];

    let mut group = c.benchmark_group("expressions");
    for (name, expr) in cases {
        let xpath = xpath::parse(expr).unwrap();
        group.bench_with_input(BenchmarkId::new("eval", name), &xpath, |b, xpath| {
            b.iter(|| xpath.apply(&tree).unwrap());
        });
    }
    group.finish();
}

/// Benchmark real-world-like compound XPath queries.
pub fn benchmark_real_world(c: &mut Criterion) {
    let tree = tree();

    let cases = [
        (
            "links_in_nav",
            "//nav//a/@href",
        ),
        (
            "deep_text_select",
            "//div[@class='repository-content']//span/text()",
        ),
        (
            "multi_predicate",
            "//div[@class='BorderGrid-cell']/div[@class=' text-small']/a",
        ),
        (
            "ancestor_filter",
            "//a[ancestor::div[@class='BorderGrid-cell']]",
        ),
    ];

    let mut group = c.benchmark_group("real_world");
    for (name, expr) in cases {
        let xpath = xpath::parse(expr).unwrap();
        group.bench_with_input(BenchmarkId::new("eval", name), &xpath, |b, xpath| {
            b.iter(|| xpath.apply(&tree).unwrap());
        });
    }
    group.finish();
}