legible 0.4.2

A Rust port of Mozilla's Readability.js for extracting readable content from web pages
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128

//! Cached CSS selector matchers for performance optimization.
//!
//! This module provides pre-compiled `Matcher` objects for frequently-used CSS selectors.
//! The `dom_query` crate parses selectors on every `select()` call. By caching `Matcher`
//! objects in a static `Lazy<Selectors>`, we eliminate redundant parsing entirely.

use dom_query::Matcher;
use once_cell::sync::Lazy;

/// Global pre-compiled selectors, initialized once on first use.
pub static SELECTORS: Lazy<Selectors> = Lazy::new(Selectors::new);

/// Pre-compiled CSS selector matchers.
#[derive(Clone)]
pub struct Selectors {
    // Document-level selectors
    pub body: Matcher,
    pub html: Matcher,
    pub title: Matcher,
    pub meta: Matcher,

    // Common element selectors
    pub p: Matcher,
    pub a: Matcher,
    pub br: Matcher,
    pub hr: Matcher,
    pub h1: Matcher,
    pub h1_h2: Matcher,
    pub img: Matcher,
    pub li: Matcher,
    pub table: Matcher,

    // Compound selectors for hot paths
    pub ul_ol: Matcher,
    pub object_embed_iframe: Matcher,
    pub script_noscript: Matcher,
    pub style: Matcher,
    pub font: Matcher,
    pub noscript: Matcher,
    pub caption: Matcher,
    pub itemprop: Matcher,

    // Image/media related selectors
    pub img_picture_figure: Matcher,
    pub img_picture: Matcher,
    pub img_embed_object_iframe: Matcher,
    pub img_picture_figure_video_audio_source: Matcher,

    // Table data detection
    pub table_data_elements: Matcher,

    // Metadata selectors
    pub json_ld_script: Matcher,

    // Text density selectors (for clean_conditionally)
    pub headings: Matcher,
    pub textish_tags: Matcher,

    // Readerable check
    pub p_pre_article: Matcher,
    pub div_br: Matcher,
}

impl Selectors {
    /// Create a new set of pre-compiled selectors.
    pub fn new() -> Self {
        Self {
            // Document-level selectors
            body: Matcher::new("body").unwrap(),
            html: Matcher::new("html").unwrap(),
            title: Matcher::new("title").unwrap(),
            meta: Matcher::new("meta").unwrap(),

            // Common element selectors
            p: Matcher::new("p").unwrap(),
            a: Matcher::new("a").unwrap(),
            br: Matcher::new("br").unwrap(),
            hr: Matcher::new("hr").unwrap(),
            h1: Matcher::new("h1").unwrap(),
            h1_h2: Matcher::new("h1, h2").unwrap(),
            img: Matcher::new("img").unwrap(),
            li: Matcher::new("li").unwrap(),
            table: Matcher::new("table").unwrap(),

            // Compound selectors for hot paths
            ul_ol: Matcher::new("ul, ol").unwrap(),
            object_embed_iframe: Matcher::new("object, embed, iframe").unwrap(),
            script_noscript: Matcher::new("script, noscript").unwrap(),
            style: Matcher::new("style").unwrap(),
            font: Matcher::new("font").unwrap(),
            noscript: Matcher::new("noscript").unwrap(),
            caption: Matcher::new("caption").unwrap(),
            itemprop: Matcher::new("[itemprop*='name']").unwrap(),

            // Image/media related selectors
            img_picture_figure: Matcher::new("img, picture, figure").unwrap(),
            img_picture: Matcher::new("img, picture").unwrap(),
            img_embed_object_iframe: Matcher::new("img, embed, object, iframe").unwrap(),
            img_picture_figure_video_audio_source: Matcher::new(
                "img, picture, figure, video, audio, source",
            )
            .unwrap(),

            // Table data detection
            table_data_elements: Matcher::new("col, colgroup, tfoot, thead, th").unwrap(),

            // Metadata selectors
            json_ld_script: Matcher::new("script[type='application/ld+json']").unwrap(),

            // Text density selectors (for clean_conditionally)
            headings: Matcher::new("h1, h2, h3, h4, h5, h6").unwrap(),
            textish_tags: Matcher::new(
                "span, li, td, blockquote, dl, div, img, ol, p, pre, table, ul",
            )
            .unwrap(),

            // Readerable check
            p_pre_article: Matcher::new("p, pre, article").unwrap(),
            div_br: Matcher::new("div > br").unwrap(),
        }
    }
}

impl Default for Selectors {
    fn default() -> Self {
        Self::new()
    }
}