web_search/providers/
html_utils.rs

1//! Shared HTML text utilities and a generic anchor-list parser.
2//!
3//! Mirrors the JavaScript `src/providers/html-utils.js` and the
4//! `parseAnchorList` helper from `src/providers/html-engines.js`, so the
5//! descriptor-driven HTML engines behave identically across both language
6//! implementations (issue #3 parity requirement).
7
8use std::collections::HashSet;
9use std::sync::LazyLock;
10
11use regex::Regex;
12
13use super::base::SearchResult;
14
15/// Decode the small set of HTML entities that appear in SERP markup, including
16/// numeric (`&#160;`) and hex (`&#xA0;`) character references.
17pub fn decode_html_entities(input: &str) -> String {
18    if input.is_empty() {
19        return String::new();
20    }
21
22    let named = [
23        ("&amp;", "&"),
24        ("&lt;", "<"),
25        ("&gt;", ">"),
26        ("&quot;", "\""),
27        ("&#39;", "'"),
28        ("&apos;", "'"),
29        ("&nbsp;", " "),
30        ("&hellip;", "…"),
31        ("&mdash;", "—"),
32        ("&ndash;", "–"),
33    ];
34
35    let mut output = input.to_string();
36    for (entity, replacement) in named {
37        output = output.replace(entity, replacement);
38    }
39
40    static NUMERIC: LazyLock<Regex> = LazyLock::new(|| Regex::new(r"&#(x?[0-9a-fA-F]+);").unwrap());
41    NUMERIC
42        .replace_all(&output, |caps: &regex::Captures| {
43            let token = &caps[1];
44            let code =
45                if let Some(hex) = token.strip_prefix('x').or_else(|| token.strip_prefix('X')) {
46                    u32::from_str_radix(hex, 16).ok()
47                } else {
48                    token.parse::<u32>().ok()
49                };
50            code.and_then(char::from_u32)
51                .map(|c| c.to_string())
52                .unwrap_or_else(|| caps[0].to_string())
53        })
54        .into_owned()
55}
56
57/// Strip HTML tags from a fragment, leaving only its text content.
58pub fn strip_html(input: &str) -> String {
59    static TAG: LazyLock<Regex> = LazyLock::new(|| Regex::new(r"<[^>]*>").unwrap());
60    TAG.replace_all(input, "").into_owned()
61}
62
63/// Normalize an HTML fragment to clean display text: strip tags, decode
64/// entities, and collapse runs of whitespace.
65pub fn clean_text(input: &str) -> String {
66    static WS: LazyLock<Regex> = LazyLock::new(|| Regex::new(r"\s+").unwrap());
67    let stripped = strip_html(input);
68    let decoded = decode_html_entities(&stripped);
69    WS.replace_all(&decoded, " ").trim().to_string()
70}
71
72/// Configuration for [`parse_anchor_list`], mirroring the JS `parseAnchorList`
73/// per-engine descriptor.
74pub struct AnchorConfig {
75    /// Provider id recorded as the result `source`.
76    pub source: &'static str,
77    /// Max results to return.
78    pub limit: usize,
79    /// Compiled item regex; capture groups feed the fields below.
80    pub item_regex: &'static Regex,
81    /// Capture-group index for the URL.
82    pub url_group: usize,
83    /// Capture-group index for the title.
84    pub title_group: usize,
85    /// Optional capture-group index for the snippet.
86    pub snippet_group: Option<usize>,
87    /// Optional URL normalizer applied before dedup/skip checks.
88    pub url_transform: Option<fn(&str) -> String>,
89    /// Optional predicate; return true to drop a URL.
90    pub skip: Option<fn(&str) -> bool>,
91}
92
93/// Generic HTML result-list parser driven by a per-engine regex.
94pub fn parse_anchor_list(html: &str, config: &AnchorConfig) -> Vec<SearchResult> {
95    let mut results = Vec::new();
96    let mut seen = HashSet::new();
97
98    for caps in config.item_regex.captures_iter(html) {
99        if results.len() >= config.limit {
100            break;
101        }
102
103        let raw_url = match caps.get(config.url_group) {
104            Some(m) => m.as_str(),
105            None => continue,
106        };
107        let url = match config.url_transform {
108            Some(transform) => transform(raw_url),
109            None => raw_url.to_string(),
110        };
111
112        if url.is_empty() || seen.contains(&url) {
113            continue;
114        }
115        if let Some(skip) = config.skip {
116            if skip(&url) {
117                continue;
118            }
119        }
120        seen.insert(url.clone());
121
122        let title = caps
123            .get(config.title_group)
124            .map(|m| clean_text(m.as_str()))
125            .filter(|t| !t.is_empty())
126            .unwrap_or_else(|| "Untitled".to_string());
127        let snippet = config
128            .snippet_group
129            .and_then(|g| caps.get(g))
130            .map(|m| clean_text(m.as_str()))
131            .unwrap_or_default();
132
133        results.push(SearchResult {
134            title,
135            url,
136            snippet,
137            source: config.source.to_string(),
138            rank: results.len() + 1,
139            score: None,
140            sources: None,
141        });
142    }
143
144    results
145}
web_search/providers/html_utils.rs

web_search/providers/
html_utils.rs
