web_search/providers/engines/
mod.rs

1//! Search-engine descriptor catalog.
2//!
3//! A faithful Rust port of the JavaScript descriptor catalog
4//! (`src/providers/api-engines.js` and `src/providers/html-engines.js`). Each
5//! engine declares only its URL, request kind, and parser; the shared
6//! [`GenericProvider`](super::generic::GenericProvider) performs all fetch,
7//! decode, and error plumbing. Keeping both languages descriptor-driven is the
8//! issue #3 parity requirement: a new engine added in one place is added in all
9//! places.
10
11use std::sync::LazyLock;
12
13use regex::Regex;
14use serde_json::Value;
15
16use super::base::{SearchOptions, SearchResult};
17use super::html_utils::{clean_text, parse_anchor_list, AnchorConfig};
18
19/// How a response body is decoded before parsing.
20#[derive(Debug, Clone, Copy, PartialEq, Eq)]
21pub enum EngineKind {
22    /// `application/json` body parsed with serde_json.
23    Json,
24    /// Plain-text/XML body (e.g. arXiv Atom).
25    Text,
26    /// HTML SERP scraped with a regex.
27    Html,
28}
29
30/// HTTP method used for an engine request.
31#[derive(Debug, Clone, Copy, PartialEq, Eq)]
32pub enum HttpMethod {
33    /// HTTP GET.
34    Get,
35    /// HTTP POST with a form-encoded body.
36    Post,
37}
38
39/// Builds a request URL (or POST body) from the query and options.
40pub type BuildFn = fn(&str, &SearchOptions) -> String;
41
42/// Produces extra request headers (e.g. auth tokens) from the options.
43pub type HeadersFn = fn(&SearchOptions) -> Vec<(String, String)>;
44
45/// Parses a decoded response body into normalized results.
46pub type ParseFn = fn(&str, usize, &SearchOptions) -> Vec<SearchResult>;
47
48/// A declarative description of a single search engine.
49#[derive(Clone, Copy)]
50pub struct EngineDescriptor {
51    /// Stable provider id.
52    pub id: &'static str,
53    /// Human-readable label.
54    pub label: &'static str,
55    /// Provider category (one of [`super::registry::CATEGORIES`]).
56    pub category: &'static str,
57    /// How the response body is decoded.
58    pub kind: EngineKind,
59    /// Whether the endpoint is browser-CORS readable.
60    pub cors_readable: bool,
61    /// Whether this is its category's default provider.
62    pub default_for_category: bool,
63    /// HTTP method.
64    pub method: HttpMethod,
65    /// Build the request URL from the query and options.
66    pub build_url: BuildFn,
67    /// Build an optional POST body.
68    pub build_body: Option<BuildFn>,
69    /// Extra request headers (e.g. auth tokens).
70    pub headers: Option<HeadersFn>,
71    /// Parse the decoded body into results.
72    pub parse: ParseFn,
73}
74
75/// The descriptor `access` label derived from its [`EngineKind`].
76pub fn access_for(kind: EngineKind) -> &'static str {
77    match kind {
78        EngineKind::Json | EngineKind::Text => "api",
79        EngineKind::Html => "html",
80    }
81}
82
83fn limit_of(options: &SearchOptions, max: usize) -> usize {
84    options.limit.unwrap_or(10).min(max)
85}
86
87fn language_of(options: &SearchOptions) -> String {
88    let lang = options.language.clone().unwrap_or_else(|| "en".to_string());
89    lang.chars().take(12).collect()
90}
91
92fn make_result(source: &str, title: &str, url: &str, snippet: &str, rank: usize) -> SearchResult {
93    let title = clean_text(title);
94    SearchResult {
95        title: if title.is_empty() {
96            "Untitled".to_string()
97        } else {
98            title
99        },
100        url: url.to_string(),
101        snippet: clean_text(snippet),
102        source: source.to_string(),
103        rank,
104        score: None,
105        sources: None,
106    }
107}
108
109/// Reconstruct an abstract from OpenAlex's inverted-index representation.
110pub fn reconstruct_inverted_abstract(inverted: &Value) -> String {
111    let obj = match inverted.as_object() {
112        Some(obj) => obj,
113        None => return String::new(),
114    };
115    let mut slots: Vec<Option<&str>> = Vec::new();
116    for (word, positions) in obj {
117        if let Some(arr) = positions.as_array() {
118            for pos in arr.iter().filter_map(Value::as_u64) {
119                let idx = pos as usize;
120                if idx >= slots.len() {
121                    slots.resize(idx + 1, None);
122                }
123                slots[idx] = Some(word);
124            }
125        }
126    }
127    slots.into_iter().flatten().collect::<Vec<_>>().join(" ")
128}
129
130/// Decode a Yahoo redirect href (`.../RU=<encoded>/RK=...`) to its destination.
131pub fn resolve_yahoo_href(href: &str) -> String {
132    if href.is_empty() {
133        return String::new();
134    }
135    static RU: LazyLock<Regex> = LazyLock::new(|| Regex::new(r"/RU=([^/]+)/").unwrap());
136    if let Some(caps) = RU.captures(href) {
137        let encoded = &caps[1];
138        return urlencoding::decode(encoded)
139            .map(|c| c.into_owned())
140            .unwrap_or_else(|_| encoded.to_string());
141    }
142    href.to_string()
143}
144
145/// Parse an arXiv Atom feed into normalized results.
146pub fn parse_arxiv_atom(xml: &str, limit: usize) -> Vec<SearchResult> {
147    static ENTRY: LazyLock<Regex> =
148        LazyLock::new(|| Regex::new(r"(?s)<entry>(.*?)</entry>").unwrap());
149    static TITLE: LazyLock<Regex> =
150        LazyLock::new(|| Regex::new(r"(?s)<title>(.*?)</title>").unwrap());
151    static ID: LazyLock<Regex> = LazyLock::new(|| Regex::new(r"(?s)<id>(.*?)</id>").unwrap());
152    static SUMMARY: LazyLock<Regex> =
153        LazyLock::new(|| Regex::new(r"(?s)<summary>(.*?)</summary>").unwrap());
154
155    let mut results = Vec::new();
156    for caps in ENTRY.captures_iter(xml) {
157        if results.len() >= limit {
158            break;
159        }
160        let entry = &caps[1];
161        let id = ID
162            .captures(entry)
163            .map(|c| c[1].trim().to_string())
164            .unwrap_or_default();
165        if id.is_empty() {
166            continue;
167        }
168        let title = TITLE
169            .captures(entry)
170            .map(|c| c[1].to_string())
171            .unwrap_or_default();
172        let summary = SUMMARY
173            .captures(entry)
174            .map(|c| c[1].to_string())
175            .unwrap_or_default();
176        let rank = results.len() + 1;
177        results.push(make_result("arxiv", &title, &id, &summary, rank));
178    }
179    results
180}
181
182fn json(body: &str) -> Value {
183    serde_json::from_str(body).unwrap_or(Value::Null)
184}
185
186fn str_field<'a>(item: &'a Value, key: &str) -> &'a str {
187    item.get(key).and_then(Value::as_str).unwrap_or("")
188}
189
190/// Project a JSON array into normalized results.
191///
192/// Centralizes the slice/rank/filter loop (mirrors the JavaScript
193/// `listResults` helper): each engine only declares how to turn one raw item
194/// into `(title, url, snippet)`, and items whose `url` is empty are dropped.
195fn list_results<F>(
196    source: &str,
197    items: Option<&Value>,
198    limit: usize,
199    project: F,
200) -> Vec<SearchResult>
201where
202    F: Fn(&Value) -> (String, String, String),
203{
204    let arr = match items.and_then(Value::as_array) {
205        Some(arr) => arr,
206        None => return Vec::new(),
207    };
208    let mut out = Vec::new();
209    for item in arr {
210        if out.len() >= limit {
211            break;
212        }
213        let (title, url, snippet) = project(item);
214        if url.is_empty() {
215            continue;
216        }
217        let rank = out.len() + 1;
218        out.push(make_result(source, &title, &url, &snippet, rank));
219    }
220    out
221}
222
223/// Normalize a code-host repository list into results (mirrors the JavaScript
224/// `repoResults` helper).
225fn repo_results(
226    source: &str,
227    data: &Value,
228    limit: usize,
229    container: Option<&str>,
230    title_field: &str,
231    url_field: &str,
232) -> Vec<SearchResult> {
233    let items = match container {
234        Some(key) => data.get(key),
235        None => Some(data),
236    };
237    list_results(source, items, limit, |it| {
238        let title = if str_field(it, title_field).is_empty() {
239            str_field(it, "name")
240        } else {
241            str_field(it, title_field)
242        };
243        (
244            title.to_string(),
245            str_field(it, url_field).to_string(),
246            str_field(it, "description").to_string(),
247        )
248    })
249}
250
251mod api;
252mod code;
253mod html;
254
255pub use api::api_engines;
256pub use html::html_engines;
257
258/// All descriptor-driven engines (API + HTML) in catalog order.
259pub fn all_descriptor_engines() -> Vec<EngineDescriptor> {
260    let mut engines = api_engines();
261    engines.extend(html_engines());
262    engines
263}
web_search/providers/engines/mod.rs

web_search/providers/engines/
mod.rs
