Skip to main content

rustai_core/search/
mod.rs

1//! The collection router.
2//!
3//! Free sources only, queried concurrently, then fused. No provider here needs
4//! an API key or a credit card, which is the entire point: the cost of a
5//! research query should be bandwidth, not a per-call fee.
6//!
7//! Individual providers are flaky by nature — a public SearXNG instance may be
8//! down, DuckDuckGo may rate-limit — so the router treats a provider failure as
9//! a partial result rather than an error, and reports what failed alongside
10//! what worked.
11
12mod academic;
13mod community;
14mod duckduckgo;
15mod feeds;
16mod searxng;
17mod wikipedia;
18
19use std::collections::HashMap;
20
21use url::Url;
22
23use crate::error::Result;
24use crate::http::Fetcher;
25
26pub use feeds::{parse_rss, parse_sitemap};
27
28/// Reciprocal-rank-fusion constant. 60 is the value from the original paper and
29/// is not worth tuning: it only sets how fast rank influence decays.
30const RRF_K: f32 = 60.0;
31
32/// Query parameters that identify a campaign, not a document.
33const TRACKING_PARAMS: &[&str] = &[
34    "utm_source",
35    "utm_medium",
36    "utm_campaign",
37    "utm_term",
38    "utm_content",
39    "utm_id",
40    "fbclid",
41    "gclid",
42    "dclid",
43    "msclkid",
44    "mc_cid",
45    "mc_eid",
46    "igshid",
47    "ref",
48    "ref_src",
49    "spm",
50    "yclid",
51    "_ga",
52    "s_kwcid",
53];
54
55/// Where results can come from.
56#[derive(Debug, Clone, PartialEq, Eq)]
57pub enum Provider {
58    /// DuckDuckGo's HTML endpoint.
59    DuckDuckGo,
60    /// arXiv's Atom API — preprints in physics, maths, CS and related fields.
61    Arxiv,
62    /// OpenAlex, the open index of scholarly works across every discipline.
63    OpenAlex,
64    /// Crossref, the DOI registry's metadata index.
65    Crossref,
66    /// Europe PMC — the life-sciences literature, including PubMed, PMC and
67    /// preprints, in a single request.
68    EuropePmc,
69    /// Hacker News, through the Algolia index behind its own search.
70    HackerNews,
71    /// Stack Exchange search over one site, by site key (`stackoverflow` by
72    /// default).
73    StackExchange(String),
74    /// GitHub repository search.
75    GitHub,
76    /// The MediaWiki search API for a given language code.
77    Wikipedia(String),
78    /// A SearXNG instance, by base URL.
79    SearxNG(String),
80    /// An RSS or Atom feed, by URL.
81    Rss(String),
82    /// A `sitemap.xml` or sitemap index, by URL.
83    Sitemap(String),
84    /// An HTML listing page — a front page, archive or feed rendered for
85    /// people — harvested for the links it offers.
86    Index(String),
87}
88
89impl Provider {
90    /// Short stable name, as it appears on [`SearchResult::provider`].
91    pub fn name(&self) -> &'static str {
92        match self {
93            Provider::DuckDuckGo => "duckduckgo",
94            Provider::Arxiv => "arxiv",
95            Provider::OpenAlex => "openalex",
96            Provider::Crossref => "crossref",
97            Provider::EuropePmc => "europepmc",
98            Provider::HackerNews => "hackernews",
99            Provider::StackExchange(_) => "stackexchange",
100            Provider::GitHub => "github",
101            Provider::Wikipedia(_) => "wikipedia",
102            Provider::SearxNG(_) => "searxng",
103            Provider::Rss(_) => "rss",
104            Provider::Sitemap(_) => "sitemap",
105            Provider::Index(_) => "index",
106        }
107    }
108
109    /// Parse the string form used by the Python API.
110    ///
111    /// `"duckduckgo"`, `"wikipedia"`, `"wikipedia:ko"`, `"searxng:https://…"`,
112    /// `"rss:https://…"`, `"sitemap:https://…"`.
113    pub fn parse(spec: &str) -> Result<Provider> {
114        let spec = spec.trim();
115        let (kind, arg) = match spec.split_once(':') {
116            Some((k, a)) if !a.starts_with("//") => (k, a.trim()),
117            _ => (spec, ""),
118        };
119        Ok(match kind.to_ascii_lowercase().as_str() {
120            "duckduckgo" | "ddg" => Provider::DuckDuckGo,
121            "arxiv" => Provider::Arxiv,
122            "openalex" => Provider::OpenAlex,
123            "crossref" => Provider::Crossref,
124            "europepmc" | "pubmed" | "pmc" => Provider::EuropePmc,
125            "hackernews" | "hn" => Provider::HackerNews,
126            "stackexchange" | "stackoverflow" | "se" => {
127                Provider::StackExchange(if arg.is_empty() {
128                    "stackoverflow".into()
129                } else {
130                    arg.into()
131                })
132            }
133            "github" | "gh" => Provider::GitHub,
134            "wikipedia" | "wiki" => {
135                Provider::Wikipedia(if arg.is_empty() { "en".into() } else { arg.into() })
136            }
137            "searxng" | "searx" => {
138                if arg.is_empty() {
139                    return Err(crate::Error::Config(
140                        "searxng needs an instance URL, e.g. `searxng:https://searx.be`".into(),
141                    ));
142                }
143                Provider::SearxNG(arg.into())
144            }
145            "rss" | "feed" | "atom" => Provider::Rss(require_url(arg, "rss")?),
146            "sitemap" => Provider::Sitemap(require_url(arg, "sitemap")?),
147            "index" | "page" | "listing" => Provider::Index(require_url(arg, "index")?),
148            other => {
149                return Err(crate::Error::Config(format!("unknown provider `{other}`")));
150            }
151        })
152    }
153}
154
155fn require_url(arg: &str, kind: &str) -> Result<String> {
156    if arg.is_empty() {
157        return Err(crate::Error::Config(format!("{kind} needs a URL")));
158    }
159    Ok(arg.to_string())
160}
161
162/// One search hit.
163#[derive(Debug, Clone, PartialEq, serde::Serialize)]
164pub struct SearchResult {
165    /// Result title.
166    pub title: String,
167    /// Absolute URL.
168    pub url: String,
169    /// Provider-supplied snippet, plain text.
170    pub snippet: String,
171    /// Providers that returned this URL.
172    pub providers: Vec<String>,
173    /// Fused score. Higher is better; only meaningful relative to siblings.
174    pub score: f32,
175    /// Best rank this URL achieved at any single provider, zero-based.
176    pub rank: usize,
177}
178
179/// Router settings.
180#[derive(Debug, Clone)]
181pub struct SearchConfig {
182    /// Providers to query, in no particular order.
183    pub providers: Vec<Provider>,
184    /// Maximum results to return after fusion.
185    pub limit: usize,
186    /// Maximum results to request from each provider.
187    pub per_provider: usize,
188    /// Contact address for the OpenAlex and Crossref "polite pools".
189    ///
190    /// Both APIs route identified callers to a faster, more reliable pool.
191    /// Leaving this unset works, but is slower and more likely to be throttled.
192    pub contact_email: Option<String>,
193}
194
195impl Default for SearchConfig {
196    fn default() -> Self {
197        SearchConfig {
198            providers: vec![Provider::DuckDuckGo, Provider::Wikipedia("en".into())],
199            limit: 10,
200            per_provider: 15,
201            contact_email: None,
202        }
203    }
204}
205
206/// What a search returned, including what went wrong.
207#[derive(Debug, Clone, serde::Serialize)]
208pub struct SearchOutcome {
209    /// Fused, deduplicated results, best first.
210    pub results: Vec<SearchResult>,
211    /// `(provider, message)` for each provider that failed.
212    pub failures: Vec<(String, String)>,
213}
214
215/// Runs providers concurrently and fuses their rankings.
216#[derive(Debug, Clone)]
217pub struct Router {
218    fetcher: Fetcher,
219    cfg: SearchConfig,
220}
221
222impl Router {
223    /// Build a router over an existing fetcher.
224    pub fn new(fetcher: Fetcher, cfg: SearchConfig) -> Self {
225        Router { fetcher, cfg }
226    }
227
228    /// The fetcher this router uses, for reusing its connection pool.
229    pub fn fetcher(&self) -> &Fetcher {
230        &self.fetcher
231    }
232
233    /// Run the query against every configured provider.
234    pub async fn search(&self, query: &str) -> SearchOutcome {
235        let limit = self.cfg.per_provider;
236        let contact = self.cfg.contact_email.clone();
237        let tasks = self.cfg.providers.iter().map(|p| {
238            let fetcher = self.fetcher.clone();
239            let provider = p.clone();
240            let query = query.to_string();
241            let contact = contact.clone();
242            async move {
243                let out = match &provider {
244                    Provider::DuckDuckGo => duckduckgo::search(&fetcher, &query, limit).await,
245                    Provider::Arxiv => academic::arxiv(&fetcher, &query, limit).await,
246                    Provider::OpenAlex => {
247                        academic::openalex(&fetcher, &query, limit, contact.as_deref()).await
248                    }
249                    Provider::Crossref => {
250                        academic::crossref(&fetcher, &query, limit, contact.as_deref()).await
251                    }
252                    Provider::EuropePmc => academic::europe_pmc(&fetcher, &query, limit).await,
253                    Provider::HackerNews => community::hacker_news(&fetcher, &query, limit).await,
254                    Provider::StackExchange(site) => {
255                        community::stack_exchange(&fetcher, site, &query, limit).await
256                    }
257                    Provider::GitHub => community::github(&fetcher, &query, limit).await,
258                    Provider::Wikipedia(lang) => {
259                        wikipedia::search(&fetcher, &query, lang, limit).await
260                    }
261                    Provider::SearxNG(base) => searxng::search(&fetcher, base, &query, limit).await,
262                    Provider::Rss(url) => feeds::search_rss(&fetcher, url, &query, limit).await,
263                    Provider::Sitemap(url) => {
264                        feeds::search_sitemap(&fetcher, url, &query, limit).await
265                    }
266                    Provider::Index(url) => feeds::search_index(&fetcher, url, &query, limit).await,
267                };
268                (provider, out)
269            }
270        });
271
272        let outcomes = futures_util::future::join_all(tasks).await;
273        let mut failures = Vec::new();
274        let mut ranked: Vec<(&'static str, Vec<RawHit>)> = Vec::new();
275        for (provider, out) in &outcomes {
276            match out {
277                Ok(hits) => ranked.push((provider.name(), hits.clone())),
278                Err(e) => failures.push((provider.name().to_string(), e.to_string())),
279            }
280        }
281        SearchOutcome { results: fuse(&ranked, self.cfg.limit), failures }
282    }
283}
284
285/// A provider's raw output, before fusion.
286#[derive(Debug, Clone, PartialEq)]
287pub(crate) struct RawHit {
288    pub(crate) title: String,
289    pub(crate) url: String,
290    pub(crate) snippet: String,
291}
292
293/// Reciprocal rank fusion.
294///
295/// Chosen over score averaging because provider scores are not comparable —
296/// DuckDuckGo has no score at all, Wikipedia's is a relevance number on its own
297/// scale. Ranks are the only signal every provider actually shares.
298fn fuse(ranked: &[(&'static str, Vec<RawHit>)], limit: usize) -> Vec<SearchResult> {
299    struct Acc {
300        title: String,
301        url: String,
302        snippet: String,
303        providers: Vec<String>,
304        score: f32,
305        best_rank: usize,
306    }
307    let mut acc: HashMap<String, Acc> = HashMap::new();
308    let mut order: Vec<String> = Vec::new();
309
310    for (provider, hits) in ranked {
311        for (rank, hit) in hits.iter().enumerate() {
312            let Some(key) = canonical_key(&hit.url) else { continue };
313            let contribution = 1.0 / (RRF_K + rank as f32);
314            match acc.get_mut(&key) {
315                Some(existing) => {
316                    existing.score += contribution;
317                    existing.best_rank = existing.best_rank.min(rank);
318                    if !existing.providers.iter().any(|p| p == provider) {
319                        existing.providers.push(provider.to_string());
320                    }
321                    // Prefer the longest snippet: providers truncate differently
322                    // and a longer one is strictly more useful downstream.
323                    if hit.snippet.len() > existing.snippet.len() {
324                        existing.snippet = hit.snippet.clone();
325                    }
326                    if existing.title.is_empty() {
327                        existing.title = hit.title.clone();
328                    }
329                }
330                None => {
331                    order.push(key.clone());
332                    acc.insert(
333                        key,
334                        Acc {
335                            title: hit.title.clone(),
336                            url: hit.url.clone(),
337                            snippet: hit.snippet.clone(),
338                            providers: vec![provider.to_string()],
339                            score: contribution,
340                            best_rank: rank,
341                        },
342                    );
343                }
344            }
345        }
346    }
347
348    let mut out: Vec<SearchResult> = order
349        .into_iter()
350        .filter_map(|k| acc.remove(&k))
351        .map(|a| SearchResult {
352            title: a.title,
353            url: a.url,
354            snippet: a.snippet,
355            providers: a.providers,
356            score: a.score,
357            rank: a.best_rank,
358        })
359        .collect();
360    out.sort_by(|a, b| b.score.total_cmp(&a.score).then(a.rank.cmp(&b.rank)));
361    out.truncate(limit);
362    out
363}
364
365/// A stable identity for a URL, for deduplication across providers.
366///
367/// Scheme, `www.`, tracking parameters, fragments and a trailing slash are all
368/// noise: the same article arrives from three providers with three different
369/// decorations, and counting it once is what makes fusion mean anything.
370pub fn canonical_key(url: &str) -> Option<String> {
371    let mut parsed = Url::parse(url.trim()).ok()?;
372    if !matches!(parsed.scheme(), "http" | "https") {
373        return None;
374    }
375    parsed.set_fragment(None);
376
377    let kept: Vec<(String, String)> = parsed
378        .query_pairs()
379        .filter(|(k, _)| !TRACKING_PARAMS.contains(&k.as_ref()))
380        .map(|(k, v)| (k.into_owned(), v.into_owned()))
381        .collect();
382    if kept.is_empty() {
383        parsed.set_query(None);
384    } else {
385        let mut sorted = kept;
386        sorted.sort();
387        let mut q = parsed.query_pairs_mut();
388        q.clear();
389        for (k, v) in &sorted {
390            q.append_pair(k, v);
391        }
392        drop(q);
393    }
394
395    let host = parsed.host_str()?.trim_start_matches("www.").to_ascii_lowercase();
396    let path = parsed.path().trim_end_matches('/');
397    let query = parsed.query().map(|q| format!("?{q}")).unwrap_or_default();
398    Some(format!("{host}{path}{query}"))
399}
400
401#[cfg(test)]
402mod tests {
403    use super::*;
404
405    fn hit(url: &str) -> RawHit {
406        RawHit { title: "T".into(), url: url.into(), snippet: "s".into() }
407    }
408
409    #[test]
410    fn academic_provider_specs_parse() {
411        assert_eq!(Provider::parse("arxiv").unwrap(), Provider::Arxiv);
412        assert_eq!(Provider::parse("OpenAlex").unwrap(), Provider::OpenAlex);
413        assert_eq!(Provider::parse("crossref").unwrap(), Provider::Crossref);
414        assert_eq!(Provider::parse("arxiv").unwrap().name(), "arxiv");
415    }
416
417    #[test]
418    fn provider_specs_parse() {
419        assert_eq!(Provider::parse("ddg").unwrap(), Provider::DuckDuckGo);
420        assert_eq!(Provider::parse("wikipedia:ko").unwrap(), Provider::Wikipedia("ko".into()));
421        assert_eq!(Provider::parse("wiki").unwrap(), Provider::Wikipedia("en".into()));
422        assert_eq!(
423            Provider::parse("searxng:https://searx.be").unwrap(),
424            Provider::SearxNG("https://searx.be".into())
425        );
426        assert_eq!(
427            Provider::parse("index:https://news.example/").unwrap(),
428            Provider::Index("https://news.example/".into())
429        );
430        assert!(Provider::parse("searxng").is_err());
431        assert!(Provider::parse("index").is_err());
432        assert!(Provider::parse("tavily").is_err());
433    }
434
435    #[test]
436    fn canonicalisation_collapses_decorations() {
437        let a = canonical_key("https://www.Example.com/post/?utm_source=x&fbclid=y#top");
438        let b = canonical_key("http://example.com/post");
439        assert_eq!(a, b, "{a:?} vs {b:?}");
440    }
441
442    #[test]
443    fn canonicalisation_keeps_meaningful_query() {
444        let a = canonical_key("https://example.com/s?b=2&a=1").unwrap();
445        let b = canonical_key("https://example.com/s?a=1&b=2").unwrap();
446        assert_eq!(a, b);
447        assert!(a.contains("a=1"));
448        assert_ne!(a, canonical_key("https://example.com/s?a=9").unwrap());
449    }
450
451    #[test]
452    fn agreement_between_providers_wins() {
453        let ranked = vec![
454            ("duckduckgo", vec![hit("https://a.dev/1"), hit("https://b.dev/2")]),
455            ("wikipedia", vec![hit("https://c.dev/3"), hit("https://b.dev/2?utm_source=q")]),
456        ];
457        let fused = fuse(&ranked, 10);
458        assert_eq!(fused[0].url, "https://b.dev/2", "fused: {fused:#?}");
459        assert_eq!(fused[0].providers, ["duckduckgo", "wikipedia"]);
460        assert_eq!(fused.len(), 3, "the duplicate should have collapsed");
461    }
462
463    #[test]
464    fn limit_is_respected_and_bad_urls_dropped() {
465        let ranked = vec![(
466            "duckduckgo",
467            vec![hit("https://a.dev/1"), hit("javascript:void"), hit("https://b.dev/2")],
468        )];
469        let fused = fuse(&ranked, 1);
470        assert_eq!(fused.len(), 1);
471        assert_eq!(fused[0].url, "https://a.dev/1");
472    }
473}