web_search/providers/
engines.rs

1//! Search-engine descriptor catalog.
2//!
3//! A faithful Rust port of the JavaScript descriptor catalog
4//! (`src/providers/api-engines.js` and `src/providers/html-engines.js`). Each
5//! engine declares only its URL, request kind, and parser; the shared
6//! [`GenericProvider`](super::generic::GenericProvider) performs all fetch,
7//! decode, and error plumbing. Keeping both languages descriptor-driven is the
8//! issue #3 parity requirement: a new engine added in one place is added in all
9//! places.
10
11use std::sync::LazyLock;
12
13use regex::Regex;
14use serde_json::Value;
15
16use super::base::{SearchOptions, SearchResult};
17use super::html_utils::{clean_text, parse_anchor_list, AnchorConfig};
18
19/// How a response body is decoded before parsing.
20#[derive(Debug, Clone, Copy, PartialEq, Eq)]
21pub enum EngineKind {
22    /// `application/json` body parsed with serde_json.
23    Json,
24    /// Plain-text/XML body (e.g. arXiv Atom).
25    Text,
26    /// HTML SERP scraped with a regex.
27    Html,
28}
29
30/// HTTP method used for an engine request.
31#[derive(Debug, Clone, Copy, PartialEq, Eq)]
32pub enum HttpMethod {
33    /// HTTP GET.
34    Get,
35    /// HTTP POST with a form-encoded body.
36    Post,
37}
38
39/// Builds a request URL (or POST body) from the query and options.
40pub type BuildFn = fn(&str, &SearchOptions) -> String;
41
42/// Produces extra request headers (e.g. auth tokens) from the options.
43pub type HeadersFn = fn(&SearchOptions) -> Vec<(String, String)>;
44
45/// Parses a decoded response body into normalized results.
46pub type ParseFn = fn(&str, usize, &SearchOptions) -> Vec<SearchResult>;
47
48/// A declarative description of a single search engine.
49#[derive(Clone, Copy)]
50pub struct EngineDescriptor {
51    /// Stable provider id.
52    pub id: &'static str,
53    /// Human-readable label.
54    pub label: &'static str,
55    /// Provider category (one of [`super::registry::CATEGORIES`]).
56    pub category: &'static str,
57    /// How the response body is decoded.
58    pub kind: EngineKind,
59    /// Whether the endpoint is browser-CORS readable.
60    pub cors_readable: bool,
61    /// Whether this is its category's default provider.
62    pub default_for_category: bool,
63    /// HTTP method.
64    pub method: HttpMethod,
65    /// Build the request URL from the query and options.
66    pub build_url: BuildFn,
67    /// Build an optional POST body.
68    pub build_body: Option<BuildFn>,
69    /// Extra request headers (e.g. auth tokens).
70    pub headers: Option<HeadersFn>,
71    /// Parse the decoded body into results.
72    pub parse: ParseFn,
73}
74
75/// The descriptor `access` label derived from its [`EngineKind`].
76pub fn access_for(kind: EngineKind) -> &'static str {
77    match kind {
78        EngineKind::Json | EngineKind::Text => "api",
79        EngineKind::Html => "html",
80    }
81}
82
83fn limit_of(options: &SearchOptions, max: usize) -> usize {
84    options.limit.unwrap_or(10).min(max)
85}
86
87fn language_of(options: &SearchOptions) -> String {
88    let lang = options.language.clone().unwrap_or_else(|| "en".to_string());
89    lang.chars().take(12).collect()
90}
91
92fn make_result(source: &str, title: &str, url: &str, snippet: &str, rank: usize) -> SearchResult {
93    let title = clean_text(title);
94    SearchResult {
95        title: if title.is_empty() {
96            "Untitled".to_string()
97        } else {
98            title
99        },
100        url: url.to_string(),
101        snippet: clean_text(snippet),
102        source: source.to_string(),
103        rank,
104        score: None,
105        sources: None,
106    }
107}
108
109/// Reconstruct an abstract from OpenAlex's inverted-index representation.
110pub fn reconstruct_inverted_abstract(inverted: &Value) -> String {
111    let obj = match inverted.as_object() {
112        Some(obj) => obj,
113        None => return String::new(),
114    };
115    let mut slots: Vec<Option<&str>> = Vec::new();
116    for (word, positions) in obj {
117        if let Some(arr) = positions.as_array() {
118            for pos in arr.iter().filter_map(Value::as_u64) {
119                let idx = pos as usize;
120                if idx >= slots.len() {
121                    slots.resize(idx + 1, None);
122                }
123                slots[idx] = Some(word);
124            }
125        }
126    }
127    slots.into_iter().flatten().collect::<Vec<_>>().join(" ")
128}
129
130/// Decode a Yahoo redirect href (`.../RU=<encoded>/RK=...`) to its destination.
131pub fn resolve_yahoo_href(href: &str) -> String {
132    if href.is_empty() {
133        return String::new();
134    }
135    static RU: LazyLock<Regex> = LazyLock::new(|| Regex::new(r"/RU=([^/]+)/").unwrap());
136    if let Some(caps) = RU.captures(href) {
137        let encoded = &caps[1];
138        return urlencoding::decode(encoded)
139            .map(|c| c.into_owned())
140            .unwrap_or_else(|_| encoded.to_string());
141    }
142    href.to_string()
143}
144
145/// Parse an arXiv Atom feed into normalized results.
146pub fn parse_arxiv_atom(xml: &str, limit: usize) -> Vec<SearchResult> {
147    static ENTRY: LazyLock<Regex> =
148        LazyLock::new(|| Regex::new(r"(?s)<entry>(.*?)</entry>").unwrap());
149    static TITLE: LazyLock<Regex> =
150        LazyLock::new(|| Regex::new(r"(?s)<title>(.*?)</title>").unwrap());
151    static ID: LazyLock<Regex> = LazyLock::new(|| Regex::new(r"(?s)<id>(.*?)</id>").unwrap());
152    static SUMMARY: LazyLock<Regex> =
153        LazyLock::new(|| Regex::new(r"(?s)<summary>(.*?)</summary>").unwrap());
154
155    let mut results = Vec::new();
156    for caps in ENTRY.captures_iter(xml) {
157        if results.len() >= limit {
158            break;
159        }
160        let entry = &caps[1];
161        let id = ID
162            .captures(entry)
163            .map(|c| c[1].trim().to_string())
164            .unwrap_or_default();
165        if id.is_empty() {
166            continue;
167        }
168        let title = TITLE
169            .captures(entry)
170            .map(|c| c[1].to_string())
171            .unwrap_or_default();
172        let summary = SUMMARY
173            .captures(entry)
174            .map(|c| c[1].to_string())
175            .unwrap_or_default();
176        let rank = results.len() + 1;
177        results.push(make_result("arxiv", &title, &id, &summary, rank));
178    }
179    results
180}
181
182fn json(body: &str) -> Value {
183    serde_json::from_str(body).unwrap_or(Value::Null)
184}
185
186fn str_field<'a>(item: &'a Value, key: &str) -> &'a str {
187    item.get(key).and_then(Value::as_str).unwrap_or("")
188}
189
190// ---------------------------------------------------------------------------
191// API engines
192// ---------------------------------------------------------------------------
193
194fn wikipedia_url(query: &str, options: &SearchOptions) -> String {
195    let limit = limit_of(options, 100);
196    let lang = language_of(options);
197    format!(
198        "https://{lang}.wikipedia.org/w/rest.php/v1/search/page?q={}&limit={limit}",
199        urlencoding::encode(query)
200    )
201}
202
203fn wikipedia_parse(body: &str, limit: usize, options: &SearchOptions) -> Vec<SearchResult> {
204    let lang = language_of(options);
205    let data = json(body);
206    data.get("pages")
207        .and_then(Value::as_array)
208        .map(|pages| {
209            pages
210                .iter()
211                .take(limit)
212                .enumerate()
213                .map(|(i, p)| {
214                    let key = str_field(p, "key");
215                    let url = format!(
216                        "https://{lang}.wikipedia.org/wiki/{}",
217                        urlencoding::encode(key)
218                    );
219                    let snippet = if str_field(p, "excerpt").is_empty() {
220                        str_field(p, "description")
221                    } else {
222                        str_field(p, "excerpt")
223                    };
224                    make_result("wikipedia", str_field(p, "title"), &url, snippet, i + 1)
225                })
226                .collect()
227        })
228        .unwrap_or_default()
229}
230
231fn wikidata_url(query: &str, options: &SearchOptions) -> String {
232    let limit = limit_of(options, 50);
233    let lang = language_of(options);
234    format!(
235        "https://www.wikidata.org/w/api.php?action=wbsearchentities&format=json&language={lang}&uselang={lang}&limit={limit}&search={}",
236        urlencoding::encode(query)
237    )
238}
239
240fn wikidata_parse(body: &str, limit: usize, _options: &SearchOptions) -> Vec<SearchResult> {
241    json(body)
242        .get("search")
243        .and_then(Value::as_array)
244        .map(|entries| {
245            entries
246                .iter()
247                .take(limit)
248                .enumerate()
249                .map(|(i, e)| {
250                    let id = str_field(e, "id");
251                    let title = if str_field(e, "label").is_empty() {
252                        id
253                    } else {
254                        str_field(e, "label")
255                    };
256                    let url = if str_field(e, "concepturi").is_empty() {
257                        format!("https://www.wikidata.org/wiki/{id}")
258                    } else {
259                        str_field(e, "concepturi").to_string()
260                    };
261                    make_result("wikidata", title, &url, str_field(e, "description"), i + 1)
262                })
263                .collect()
264        })
265        .unwrap_or_default()
266}
267
268fn searx_url(query: &str, _options: &SearchOptions) -> String {
269    format!(
270        "https://searx.be/search?format=json&q={}",
271        urlencoding::encode(query)
272    )
273}
274
275fn searx_parse(body: &str, limit: usize, _options: &SearchOptions) -> Vec<SearchResult> {
276    json(body)
277        .get("results")
278        .and_then(Value::as_array)
279        .map(|entries| {
280            entries
281                .iter()
282                .take(limit)
283                .enumerate()
284                .map(|(i, e)| {
285                    make_result(
286                        "searx",
287                        str_field(e, "title"),
288                        str_field(e, "url"),
289                        str_field(e, "content"),
290                        i + 1,
291                    )
292                })
293                .collect()
294        })
295        .unwrap_or_default()
296}
297
298fn crossref_url(query: &str, options: &SearchOptions) -> String {
299    let rows = limit_of(options, 50);
300    format!(
301        "https://api.crossref.org/works?rows={rows}&query={}",
302        urlencoding::encode(query)
303    )
304}
305
306fn first_str(value: Option<&Value>) -> String {
307    match value {
308        Some(Value::Array(arr)) => arr
309            .first()
310            .and_then(Value::as_str)
311            .unwrap_or("")
312            .to_string(),
313        Some(Value::String(s)) => s.clone(),
314        _ => String::new(),
315    }
316}
317
318fn crossref_parse(body: &str, limit: usize, _options: &SearchOptions) -> Vec<SearchResult> {
319    json(body)
320        .get("message")
321        .and_then(|m| m.get("items"))
322        .and_then(Value::as_array)
323        .map(|items| {
324            items
325                .iter()
326                .take(limit)
327                .enumerate()
328                .filter_map(|(i, it)| {
329                    let title = first_str(it.get("title"));
330                    let url = if !str_field(it, "URL").is_empty() {
331                        str_field(it, "URL").to_string()
332                    } else {
333                        let doi = str_field(it, "DOI");
334                        if doi.is_empty() {
335                            String::new()
336                        } else {
337                            format!("https://doi.org/{doi}")
338                        }
339                    };
340                    if url.is_empty() {
341                        return None;
342                    }
343                    let snippet = if str_field(it, "abstract").is_empty() {
344                        first_str(it.get("container-title"))
345                    } else {
346                        str_field(it, "abstract").to_string()
347                    };
348                    Some(make_result("crossref", &title, &url, &snippet, i + 1))
349                })
350                .collect()
351        })
352        .unwrap_or_default()
353}
354
355fn openalex_url(query: &str, options: &SearchOptions) -> String {
356    let per_page = limit_of(options, 50);
357    format!(
358        "https://api.openalex.org/works?per-page={per_page}&search={}",
359        urlencoding::encode(query)
360    )
361}
362
363fn openalex_parse(body: &str, limit: usize, _options: &SearchOptions) -> Vec<SearchResult> {
364    json(body)
365        .get("results")
366        .and_then(Value::as_array)
367        .map(|items| {
368            items
369                .iter()
370                .take(limit)
371                .enumerate()
372                .filter_map(|(i, it)| {
373                    let title = if str_field(it, "title").is_empty() {
374                        str_field(it, "display_name")
375                    } else {
376                        str_field(it, "title")
377                    };
378                    let url = if str_field(it, "doi").is_empty() {
379                        str_field(it, "id")
380                    } else {
381                        str_field(it, "doi")
382                    };
383                    if url.is_empty() {
384                        return None;
385                    }
386                    let snippet = it
387                        .get("abstract_inverted_index")
388                        .map(reconstruct_inverted_abstract)
389                        .unwrap_or_default();
390                    Some(make_result("openalex", title, url, &snippet, i + 1))
391                })
392                .collect()
393        })
394        .unwrap_or_default()
395}
396
397fn github_url(query: &str, options: &SearchOptions) -> String {
398    let per_page = limit_of(options, 50);
399    format!(
400        "https://api.github.com/search/repositories?per_page={per_page}&q={}",
401        urlencoding::encode(query)
402    )
403}
404
405fn github_headers(_options: &SearchOptions) -> Vec<(String, String)> {
406    let mut headers = vec![
407        (
408            "Accept".to_string(),
409            "application/vnd.github+json".to_string(),
410        ),
411        ("X-GitHub-Api-Version".to_string(), "2022-11-28".to_string()),
412    ];
413    if let Ok(token) = std::env::var("GITHUB_TOKEN") {
414        if !token.is_empty() {
415            headers.push(("Authorization".to_string(), format!("Bearer {token}")));
416        }
417    }
418    headers
419}
420
421fn github_parse(body: &str, limit: usize, _options: &SearchOptions) -> Vec<SearchResult> {
422    json(body)
423        .get("items")
424        .and_then(Value::as_array)
425        .map(|items| {
426            items
427                .iter()
428                .take(limit)
429                .enumerate()
430                .map(|(i, it)| {
431                    let title = if str_field(it, "full_name").is_empty() {
432                        str_field(it, "name")
433                    } else {
434                        str_field(it, "full_name")
435                    };
436                    make_result(
437                        "github",
438                        title,
439                        str_field(it, "html_url"),
440                        str_field(it, "description"),
441                        i + 1,
442                    )
443                })
444                .collect()
445        })
446        .unwrap_or_default()
447}
448
449fn hackernews_url(query: &str, options: &SearchOptions) -> String {
450    let hits = limit_of(options, 50);
451    format!(
452        "https://hn.algolia.com/api/v1/search?hitsPerPage={hits}&query={}",
453        urlencoding::encode(query)
454    )
455}
456
457fn hackernews_parse(body: &str, limit: usize, _options: &SearchOptions) -> Vec<SearchResult> {
458    json(body)
459        .get("hits")
460        .and_then(Value::as_array)
461        .map(|hits| {
462            hits.iter()
463                .take(limit)
464                .enumerate()
465                .map(|(i, h)| {
466                    let title = if str_field(h, "title").is_empty() {
467                        str_field(h, "story_title")
468                    } else {
469                        str_field(h, "title")
470                    };
471                    let url = if !str_field(h, "url").is_empty() {
472                        str_field(h, "url").to_string()
473                    } else if !str_field(h, "story_url").is_empty() {
474                        str_field(h, "story_url").to_string()
475                    } else {
476                        format!(
477                            "https://news.ycombinator.com/item?id={}",
478                            str_field(h, "objectID")
479                        )
480                    };
481                    let snippet = if str_field(h, "story_text").is_empty() {
482                        str_field(h, "comment_text")
483                    } else {
484                        str_field(h, "story_text")
485                    };
486                    make_result("hackernews", title, &url, snippet, i + 1)
487                })
488                .collect()
489        })
490        .unwrap_or_default()
491}
492
493fn arxiv_url(query: &str, options: &SearchOptions) -> String {
494    let max = limit_of(options, 50);
495    format!(
496        "http://export.arxiv.org/api/query?max_results={max}&search_query={}",
497        urlencoding::encode(&format!("all:{query}"))
498    )
499}
500
501fn arxiv_parse(body: &str, limit: usize, _options: &SearchOptions) -> Vec<SearchResult> {
502    parse_arxiv_atom(body, limit)
503}
504
505// ---------------------------------------------------------------------------
506// HTML engines
507// ---------------------------------------------------------------------------
508
509fn skip_mojeek(url: &str) -> bool {
510    url.contains("mojeek.com")
511}
512
513fn skip_ecosia(url: &str) -> bool {
514    url.contains("ecosia.org")
515}
516
517fn skip_startpage(url: &str) -> bool {
518    url.contains("startpage.com")
519}
520
521fn skip_lite(url: &str) -> bool {
522    url.contains("duckduckgo.com")
523}
524
525fn brave_url(query: &str, options: &SearchOptions) -> String {
526    let mut url = format!(
527        "https://search.brave.com/search?q={}",
528        urlencoding::encode(query)
529    );
530    match options.safe_search {
531        Some(true) => url.push_str("&safesearch=strict"),
532        Some(false) => url.push_str("&safesearch=off"),
533        None => {}
534    }
535    url
536}
537
538static BRAVE_RE: LazyLock<Regex> = LazyLock::new(|| {
539    Regex::new(r#"(?s)<a[^>]+href="(https?://[^"]+)"[^>]*class="[^"]*result-header[^"]*"[^>]*>.*?<span[^>]*class="[^"]*title[^"]*"[^>]*>(.*?)</span>.*?<p[^>]*class="[^"]*snippet[^"]*"[^>]*>(.*?)</p>"#).unwrap()
540});
541
542fn brave_parse(html: &str, limit: usize, _options: &SearchOptions) -> Vec<SearchResult> {
543    parse_anchor_list(
544        html,
545        &AnchorConfig {
546            source: "brave",
547            limit,
548            item_regex: &BRAVE_RE,
549            url_group: 1,
550            title_group: 2,
551            snippet_group: Some(3),
552            url_transform: None,
553            skip: Some(|url| url.contains("search.brave.com")),
554        },
555    )
556}
557
558fn mojeek_url(query: &str, options: &SearchOptions) -> String {
559    let mut url = format!(
560        "https://www.mojeek.com/search?q={}",
561        urlencoding::encode(query)
562    );
563    if options.safe_search == Some(false) {
564        url.push_str("&safe=0");
565    }
566    url
567}
568
569static MOJEEK_RE: LazyLock<Regex> = LazyLock::new(|| {
570    Regex::new(r#"(?s)<a[^>]+href="(https?://[^"]+)"[^>]*class="[^"]*ob[^"]*"[^>]*>(.*?)</a>.*?<p[^>]*class="[^"]*s[^"]*"[^>]*>(.*?)</p>"#).unwrap()
571});
572
573fn mojeek_parse(html: &str, limit: usize, _options: &SearchOptions) -> Vec<SearchResult> {
574    parse_anchor_list(
575        html,
576        &AnchorConfig {
577            source: "mojeek",
578            limit,
579            item_regex: &MOJEEK_RE,
580            url_group: 1,
581            title_group: 2,
582            snippet_group: Some(3),
583            url_transform: None,
584            skip: Some(skip_mojeek),
585        },
586    )
587}
588
589fn ecosia_url(query: &str, options: &SearchOptions) -> String {
590    let mut url = format!(
591        "https://www.ecosia.org/search?q={}",
592        urlencoding::encode(query)
593    );
594    if let Some(ref lang) = options.language {
595        url.push_str(&format!("&hl={lang}"));
596    }
597    url
598}
599
600static ECOSIA_RE: LazyLock<Regex> = LazyLock::new(|| {
601    Regex::new(r#"(?s)<a[^>]+class="[^"]*result__link[^"]*"[^>]*href="(https?://[^"]+)"[^>]*>(.*?)</a>.*?<(?:p|div)[^>]*class="[^"]*result__description[^"]*"[^>]*>(.*?)</(?:p|div)>"#).unwrap()
602});
603
604fn ecosia_parse(html: &str, limit: usize, _options: &SearchOptions) -> Vec<SearchResult> {
605    parse_anchor_list(
606        html,
607        &AnchorConfig {
608            source: "ecosia",
609            limit,
610            item_regex: &ECOSIA_RE,
611            url_group: 1,
612            title_group: 2,
613            snippet_group: Some(3),
614            url_transform: None,
615            skip: Some(skip_ecosia),
616        },
617    )
618}
619
620fn startpage_url(query: &str, options: &SearchOptions) -> String {
621    let mut url = format!(
622        "https://www.startpage.com/sp/search?query={}",
623        urlencoding::encode(query)
624    );
625    if let Some(ref lang) = options.language {
626        url.push_str(&format!("&language={lang}"));
627    }
628    url
629}
630
631static STARTPAGE_RE: LazyLock<Regex> = LazyLock::new(|| {
632    Regex::new(r#"(?s)<a[^>]+class="[^"]*result-title[^"]*"[^>]*href="(https?://[^"]+)"[^>]*>(.*?)</a>.*?<p[^>]*class="[^"]*description[^"]*"[^>]*>(.*?)</p>"#).unwrap()
633});
634
635fn startpage_parse(html: &str, limit: usize, _options: &SearchOptions) -> Vec<SearchResult> {
636    parse_anchor_list(
637        html,
638        &AnchorConfig {
639            source: "startpage",
640            limit,
641            item_regex: &STARTPAGE_RE,
642            url_group: 1,
643            title_group: 2,
644            snippet_group: Some(3),
645            url_transform: None,
646            skip: Some(skip_startpage),
647        },
648    )
649}
650
651fn yahoo_url(query: &str, options: &SearchOptions) -> String {
652    let mut url = format!(
653        "https://search.yahoo.com/search?p={}",
654        urlencoding::encode(query)
655    );
656    if let Some(ref region) = options.region {
657        url.push_str(&format!("&vc={region}"));
658    }
659    url
660}
661
662static YAHOO_RE: LazyLock<Regex> = LazyLock::new(|| {
663    Regex::new(
664        r#"(?s)<h3[^>]*class="[^"]*title[^"]*"[^>]*>.*?<a[^>]+href="([^"]+)"[^>]*>(.*?)</a>"#,
665    )
666    .unwrap()
667});
668
669fn yahoo_parse(html: &str, limit: usize, _options: &SearchOptions) -> Vec<SearchResult> {
670    parse_anchor_list(
671        html,
672        &AnchorConfig {
673            source: "yahoo",
674            limit,
675            item_regex: &YAHOO_RE,
676            url_group: 1,
677            title_group: 2,
678            snippet_group: None,
679            url_transform: Some(resolve_yahoo_href),
680            skip: Some(|url| url.is_empty() || url.contains("yahoo.com")),
681        },
682    )
683}
684
685fn lite_url(_query: &str, _options: &SearchOptions) -> String {
686    "https://lite.duckduckgo.com/lite/".to_string()
687}
688
689fn lite_body(query: &str, options: &SearchOptions) -> String {
690    let mut body = format!("q={}", urlencoding::encode(query));
691    if let Some(ref region) = options.region {
692        body.push_str(&format!("&kl={region}"));
693    }
694    body
695}
696
697static LITE_RE: LazyLock<Regex> = LazyLock::new(|| {
698    Regex::new(
699        r#"(?s)<a[^>]+class="[^"]*result-link[^"]*"[^>]*href="(https?://[^"]+)"[^>]*>(.*?)</a>"#,
700    )
701    .unwrap()
702});
703
704fn lite_parse(html: &str, limit: usize, _options: &SearchOptions) -> Vec<SearchResult> {
705    parse_anchor_list(
706        html,
707        &AnchorConfig {
708            source: "lite",
709            limit,
710            item_regex: &LITE_RE,
711            url_group: 1,
712            title_group: 2,
713            snippet_group: None,
714            url_transform: None,
715            skip: Some(skip_lite),
716        },
717    )
718}
719
720/// All API-based engine descriptors, in catalog order.
721pub fn api_engines() -> Vec<EngineDescriptor> {
722    vec![
723        EngineDescriptor {
724            id: "wikipedia",
725            label: "Wikipedia",
726            category: "knowledge",
727            kind: EngineKind::Json,
728            cors_readable: true,
729            default_for_category: true,
730            method: HttpMethod::Get,
731            build_url: wikipedia_url,
732            build_body: None,
733            headers: None,
734            parse: wikipedia_parse,
735        },
736        EngineDescriptor {
737            id: "wikidata",
738            label: "Wikidata",
739            category: "knowledge",
740            kind: EngineKind::Json,
741            cors_readable: true,
742            default_for_category: false,
743            method: HttpMethod::Get,
744            build_url: wikidata_url,
745            build_body: None,
746            headers: None,
747            parse: wikidata_parse,
748        },
749        EngineDescriptor {
750            id: "searx",
751            label: "SearXNG",
752            category: "search",
753            kind: EngineKind::Json,
754            cors_readable: false,
755            default_for_category: false,
756            method: HttpMethod::Get,
757            build_url: searx_url,
758            build_body: None,
759            headers: None,
760            parse: searx_parse,
761        },
762        EngineDescriptor {
763            id: "crossref",
764            label: "Crossref",
765            category: "papers",
766            kind: EngineKind::Json,
767            cors_readable: true,
768            default_for_category: true,
769            method: HttpMethod::Get,
770            build_url: crossref_url,
771            build_body: None,
772            headers: None,
773            parse: crossref_parse,
774        },
775        EngineDescriptor {
776            id: "openalex",
777            label: "OpenAlex",
778            category: "papers",
779            kind: EngineKind::Json,
780            cors_readable: true,
781            default_for_category: false,
782            method: HttpMethod::Get,
783            build_url: openalex_url,
784            build_body: None,
785            headers: None,
786            parse: openalex_parse,
787        },
788        EngineDescriptor {
789            id: "github",
790            label: "GitHub",
791            category: "code",
792            kind: EngineKind::Json,
793            cors_readable: true,
794            default_for_category: true,
795            method: HttpMethod::Get,
796            build_url: github_url,
797            build_body: None,
798            headers: Some(github_headers),
799            parse: github_parse,
800        },
801        EngineDescriptor {
802            id: "hackernews",
803            label: "Hacker News",
804            category: "code",
805            kind: EngineKind::Json,
806            cors_readable: true,
807            default_for_category: false,
808            method: HttpMethod::Get,
809            build_url: hackernews_url,
810            build_body: None,
811            headers: None,
812            parse: hackernews_parse,
813        },
814        EngineDescriptor {
815            id: "arxiv",
816            label: "arXiv",
817            category: "papers",
818            kind: EngineKind::Text,
819            cors_readable: true,
820            default_for_category: false,
821            method: HttpMethod::Get,
822            build_url: arxiv_url,
823            build_body: None,
824            headers: None,
825            parse: arxiv_parse,
826        },
827    ]
828}
829
830/// All HTML-scraping engine descriptors, in catalog order.
831pub fn html_engines() -> Vec<EngineDescriptor> {
832    vec![
833        EngineDescriptor {
834            id: "brave",
835            label: "Brave Search",
836            category: "search",
837            kind: EngineKind::Html,
838            cors_readable: false,
839            default_for_category: false,
840            method: HttpMethod::Get,
841            build_url: brave_url,
842            build_body: None,
843            headers: None,
844            parse: brave_parse,
845        },
846        EngineDescriptor {
847            id: "mojeek",
848            label: "Mojeek",
849            category: "search",
850            kind: EngineKind::Html,
851            cors_readable: false,
852            default_for_category: false,
853            method: HttpMethod::Get,
854            build_url: mojeek_url,
855            build_body: None,
856            headers: None,
857            parse: mojeek_parse,
858        },
859        EngineDescriptor {
860            id: "ecosia",
861            label: "Ecosia",
862            category: "search",
863            kind: EngineKind::Html,
864            cors_readable: false,
865            default_for_category: false,
866            method: HttpMethod::Get,
867            build_url: ecosia_url,
868            build_body: None,
869            headers: None,
870            parse: ecosia_parse,
871        },
872        EngineDescriptor {
873            id: "startpage",
874            label: "Startpage",
875            category: "search",
876            kind: EngineKind::Html,
877            cors_readable: false,
878            default_for_category: false,
879            method: HttpMethod::Get,
880            build_url: startpage_url,
881            build_body: None,
882            headers: None,
883            parse: startpage_parse,
884        },
885        EngineDescriptor {
886            id: "yahoo",
887            label: "Yahoo Search",
888            category: "search",
889            kind: EngineKind::Html,
890            cors_readable: false,
891            default_for_category: false,
892            method: HttpMethod::Get,
893            build_url: yahoo_url,
894            build_body: None,
895            headers: None,
896            parse: yahoo_parse,
897        },
898        EngineDescriptor {
899            id: "lite",
900            label: "DuckDuckGo Lite",
901            category: "search",
902            kind: EngineKind::Html,
903            cors_readable: false,
904            default_for_category: false,
905            method: HttpMethod::Post,
906            build_url: lite_url,
907            build_body: Some(lite_body),
908            headers: None,
909            parse: lite_parse,
910        },
911    ]
912}
913
914/// All descriptor-driven engines (API + HTML) in catalog order.
915pub fn all_descriptor_engines() -> Vec<EngineDescriptor> {
916    let mut engines = api_engines();
917    engines.extend(html_engines());
918    engines
919}
web_search/providers/engines.rs

web_search/providers/
engines.rs
