1mod academic;
13mod community;
14mod duckduckgo;
15mod feeds;
16mod searxng;
17mod wikipedia;
18
19use std::collections::HashMap;
20
21use url::Url;
22
23use crate::error::Result;
24use crate::http::Fetcher;
25
26pub use feeds::{parse_rss, parse_sitemap};
27
28const RRF_K: f32 = 60.0;
31
32const TRACKING_PARAMS: &[&str] = &[
34 "utm_source",
35 "utm_medium",
36 "utm_campaign",
37 "utm_term",
38 "utm_content",
39 "utm_id",
40 "fbclid",
41 "gclid",
42 "dclid",
43 "msclkid",
44 "mc_cid",
45 "mc_eid",
46 "igshid",
47 "ref",
48 "ref_src",
49 "spm",
50 "yclid",
51 "_ga",
52 "s_kwcid",
53];
54
55#[derive(Debug, Clone, PartialEq, Eq)]
57pub enum Provider {
58 DuckDuckGo,
60 Arxiv,
62 OpenAlex,
64 Crossref,
66 EuropePmc,
69 HackerNews,
71 StackExchange(String),
74 GitHub,
76 Wikipedia(String),
78 SearxNG(String),
80 Rss(String),
82 Sitemap(String),
84 Index(String),
87}
88
89impl Provider {
90 pub fn name(&self) -> &'static str {
92 match self {
93 Provider::DuckDuckGo => "duckduckgo",
94 Provider::Arxiv => "arxiv",
95 Provider::OpenAlex => "openalex",
96 Provider::Crossref => "crossref",
97 Provider::EuropePmc => "europepmc",
98 Provider::HackerNews => "hackernews",
99 Provider::StackExchange(_) => "stackexchange",
100 Provider::GitHub => "github",
101 Provider::Wikipedia(_) => "wikipedia",
102 Provider::SearxNG(_) => "searxng",
103 Provider::Rss(_) => "rss",
104 Provider::Sitemap(_) => "sitemap",
105 Provider::Index(_) => "index",
106 }
107 }
108
109 pub fn parse(spec: &str) -> Result<Provider> {
114 let spec = spec.trim();
115 let (kind, arg) = match spec.split_once(':') {
116 Some((k, a)) if !a.starts_with("//") => (k, a.trim()),
117 _ => (spec, ""),
118 };
119 Ok(match kind.to_ascii_lowercase().as_str() {
120 "duckduckgo" | "ddg" => Provider::DuckDuckGo,
121 "arxiv" => Provider::Arxiv,
122 "openalex" => Provider::OpenAlex,
123 "crossref" => Provider::Crossref,
124 "europepmc" | "pubmed" | "pmc" => Provider::EuropePmc,
125 "hackernews" | "hn" => Provider::HackerNews,
126 "stackexchange" | "stackoverflow" | "se" => {
127 Provider::StackExchange(if arg.is_empty() {
128 "stackoverflow".into()
129 } else {
130 arg.into()
131 })
132 }
133 "github" | "gh" => Provider::GitHub,
134 "wikipedia" | "wiki" => {
135 Provider::Wikipedia(if arg.is_empty() { "en".into() } else { arg.into() })
136 }
137 "searxng" | "searx" => {
138 if arg.is_empty() {
139 return Err(crate::Error::Config(
140 "searxng needs an instance URL, e.g. `searxng:https://searx.be`".into(),
141 ));
142 }
143 Provider::SearxNG(arg.into())
144 }
145 "rss" | "feed" | "atom" => Provider::Rss(require_url(arg, "rss")?),
146 "sitemap" => Provider::Sitemap(require_url(arg, "sitemap")?),
147 "index" | "page" | "listing" => Provider::Index(require_url(arg, "index")?),
148 other => {
149 return Err(crate::Error::Config(format!("unknown provider `{other}`")));
150 }
151 })
152 }
153}
154
155fn require_url(arg: &str, kind: &str) -> Result<String> {
156 if arg.is_empty() {
157 return Err(crate::Error::Config(format!("{kind} needs a URL")));
158 }
159 Ok(arg.to_string())
160}
161
162#[derive(Debug, Clone, PartialEq, serde::Serialize)]
164pub struct SearchResult {
165 pub title: String,
167 pub url: String,
169 pub snippet: String,
171 pub providers: Vec<String>,
173 pub score: f32,
175 pub rank: usize,
177}
178
179#[derive(Debug, Clone)]
181pub struct SearchConfig {
182 pub providers: Vec<Provider>,
184 pub limit: usize,
186 pub per_provider: usize,
188 pub contact_email: Option<String>,
193}
194
195impl Default for SearchConfig {
196 fn default() -> Self {
197 SearchConfig {
198 providers: vec![Provider::DuckDuckGo, Provider::Wikipedia("en".into())],
199 limit: 10,
200 per_provider: 15,
201 contact_email: None,
202 }
203 }
204}
205
206#[derive(Debug, Clone, serde::Serialize)]
208pub struct SearchOutcome {
209 pub results: Vec<SearchResult>,
211 pub failures: Vec<(String, String)>,
213}
214
215#[derive(Debug, Clone)]
217pub struct Router {
218 fetcher: Fetcher,
219 cfg: SearchConfig,
220}
221
222impl Router {
223 pub fn new(fetcher: Fetcher, cfg: SearchConfig) -> Self {
225 Router { fetcher, cfg }
226 }
227
228 pub fn fetcher(&self) -> &Fetcher {
230 &self.fetcher
231 }
232
233 pub async fn search(&self, query: &str) -> SearchOutcome {
235 let limit = self.cfg.per_provider;
236 let contact = self.cfg.contact_email.clone();
237 let tasks = self.cfg.providers.iter().map(|p| {
238 let fetcher = self.fetcher.clone();
239 let provider = p.clone();
240 let query = query.to_string();
241 let contact = contact.clone();
242 async move {
243 let out = match &provider {
244 Provider::DuckDuckGo => duckduckgo::search(&fetcher, &query, limit).await,
245 Provider::Arxiv => academic::arxiv(&fetcher, &query, limit).await,
246 Provider::OpenAlex => {
247 academic::openalex(&fetcher, &query, limit, contact.as_deref()).await
248 }
249 Provider::Crossref => {
250 academic::crossref(&fetcher, &query, limit, contact.as_deref()).await
251 }
252 Provider::EuropePmc => academic::europe_pmc(&fetcher, &query, limit).await,
253 Provider::HackerNews => community::hacker_news(&fetcher, &query, limit).await,
254 Provider::StackExchange(site) => {
255 community::stack_exchange(&fetcher, site, &query, limit).await
256 }
257 Provider::GitHub => community::github(&fetcher, &query, limit).await,
258 Provider::Wikipedia(lang) => {
259 wikipedia::search(&fetcher, &query, lang, limit).await
260 }
261 Provider::SearxNG(base) => searxng::search(&fetcher, base, &query, limit).await,
262 Provider::Rss(url) => feeds::search_rss(&fetcher, url, &query, limit).await,
263 Provider::Sitemap(url) => {
264 feeds::search_sitemap(&fetcher, url, &query, limit).await
265 }
266 Provider::Index(url) => feeds::search_index(&fetcher, url, &query, limit).await,
267 };
268 (provider, out)
269 }
270 });
271
272 let outcomes = futures_util::future::join_all(tasks).await;
273 let mut failures = Vec::new();
274 let mut ranked: Vec<(&'static str, Vec<RawHit>)> = Vec::new();
275 for (provider, out) in &outcomes {
276 match out {
277 Ok(hits) => ranked.push((provider.name(), hits.clone())),
278 Err(e) => failures.push((provider.name().to_string(), e.to_string())),
279 }
280 }
281 SearchOutcome { results: fuse(&ranked, self.cfg.limit), failures }
282 }
283}
284
285#[derive(Debug, Clone, PartialEq)]
287pub(crate) struct RawHit {
288 pub(crate) title: String,
289 pub(crate) url: String,
290 pub(crate) snippet: String,
291}
292
293fn fuse(ranked: &[(&'static str, Vec<RawHit>)], limit: usize) -> Vec<SearchResult> {
299 struct Acc {
300 title: String,
301 url: String,
302 snippet: String,
303 providers: Vec<String>,
304 score: f32,
305 best_rank: usize,
306 }
307 let mut acc: HashMap<String, Acc> = HashMap::new();
308 let mut order: Vec<String> = Vec::new();
309
310 for (provider, hits) in ranked {
311 for (rank, hit) in hits.iter().enumerate() {
312 let Some(key) = canonical_key(&hit.url) else { continue };
313 let contribution = 1.0 / (RRF_K + rank as f32);
314 match acc.get_mut(&key) {
315 Some(existing) => {
316 existing.score += contribution;
317 existing.best_rank = existing.best_rank.min(rank);
318 if !existing.providers.iter().any(|p| p == provider) {
319 existing.providers.push(provider.to_string());
320 }
321 if hit.snippet.len() > existing.snippet.len() {
324 existing.snippet = hit.snippet.clone();
325 }
326 if existing.title.is_empty() {
327 existing.title = hit.title.clone();
328 }
329 }
330 None => {
331 order.push(key.clone());
332 acc.insert(
333 key,
334 Acc {
335 title: hit.title.clone(),
336 url: hit.url.clone(),
337 snippet: hit.snippet.clone(),
338 providers: vec![provider.to_string()],
339 score: contribution,
340 best_rank: rank,
341 },
342 );
343 }
344 }
345 }
346 }
347
348 let mut out: Vec<SearchResult> = order
349 .into_iter()
350 .filter_map(|k| acc.remove(&k))
351 .map(|a| SearchResult {
352 title: a.title,
353 url: a.url,
354 snippet: a.snippet,
355 providers: a.providers,
356 score: a.score,
357 rank: a.best_rank,
358 })
359 .collect();
360 out.sort_by(|a, b| b.score.total_cmp(&a.score).then(a.rank.cmp(&b.rank)));
361 out.truncate(limit);
362 out
363}
364
365pub fn canonical_key(url: &str) -> Option<String> {
371 let mut parsed = Url::parse(url.trim()).ok()?;
372 if !matches!(parsed.scheme(), "http" | "https") {
373 return None;
374 }
375 parsed.set_fragment(None);
376
377 let kept: Vec<(String, String)> = parsed
378 .query_pairs()
379 .filter(|(k, _)| !TRACKING_PARAMS.contains(&k.as_ref()))
380 .map(|(k, v)| (k.into_owned(), v.into_owned()))
381 .collect();
382 if kept.is_empty() {
383 parsed.set_query(None);
384 } else {
385 let mut sorted = kept;
386 sorted.sort();
387 let mut q = parsed.query_pairs_mut();
388 q.clear();
389 for (k, v) in &sorted {
390 q.append_pair(k, v);
391 }
392 drop(q);
393 }
394
395 let host = parsed.host_str()?.trim_start_matches("www.").to_ascii_lowercase();
396 let path = parsed.path().trim_end_matches('/');
397 let query = parsed.query().map(|q| format!("?{q}")).unwrap_or_default();
398 Some(format!("{host}{path}{query}"))
399}
400
401#[cfg(test)]
402mod tests {
403 use super::*;
404
405 fn hit(url: &str) -> RawHit {
406 RawHit { title: "T".into(), url: url.into(), snippet: "s".into() }
407 }
408
409 #[test]
410 fn academic_provider_specs_parse() {
411 assert_eq!(Provider::parse("arxiv").unwrap(), Provider::Arxiv);
412 assert_eq!(Provider::parse("OpenAlex").unwrap(), Provider::OpenAlex);
413 assert_eq!(Provider::parse("crossref").unwrap(), Provider::Crossref);
414 assert_eq!(Provider::parse("arxiv").unwrap().name(), "arxiv");
415 }
416
417 #[test]
418 fn provider_specs_parse() {
419 assert_eq!(Provider::parse("ddg").unwrap(), Provider::DuckDuckGo);
420 assert_eq!(Provider::parse("wikipedia:ko").unwrap(), Provider::Wikipedia("ko".into()));
421 assert_eq!(Provider::parse("wiki").unwrap(), Provider::Wikipedia("en".into()));
422 assert_eq!(
423 Provider::parse("searxng:https://searx.be").unwrap(),
424 Provider::SearxNG("https://searx.be".into())
425 );
426 assert_eq!(
427 Provider::parse("index:https://news.example/").unwrap(),
428 Provider::Index("https://news.example/".into())
429 );
430 assert!(Provider::parse("searxng").is_err());
431 assert!(Provider::parse("index").is_err());
432 assert!(Provider::parse("tavily").is_err());
433 }
434
435 #[test]
436 fn canonicalisation_collapses_decorations() {
437 let a = canonical_key("https://www.Example.com/post/?utm_source=x&fbclid=y#top");
438 let b = canonical_key("http://example.com/post");
439 assert_eq!(a, b, "{a:?} vs {b:?}");
440 }
441
442 #[test]
443 fn canonicalisation_keeps_meaningful_query() {
444 let a = canonical_key("https://example.com/s?b=2&a=1").unwrap();
445 let b = canonical_key("https://example.com/s?a=1&b=2").unwrap();
446 assert_eq!(a, b);
447 assert!(a.contains("a=1"));
448 assert_ne!(a, canonical_key("https://example.com/s?a=9").unwrap());
449 }
450
451 #[test]
452 fn agreement_between_providers_wins() {
453 let ranked = vec![
454 ("duckduckgo", vec![hit("https://a.dev/1"), hit("https://b.dev/2")]),
455 ("wikipedia", vec![hit("https://c.dev/3"), hit("https://b.dev/2?utm_source=q")]),
456 ];
457 let fused = fuse(&ranked, 10);
458 assert_eq!(fused[0].url, "https://b.dev/2", "fused: {fused:#?}");
459 assert_eq!(fused[0].providers, ["duckduckgo", "wikipedia"]);
460 assert_eq!(fused.len(), 3, "the duplicate should have collapsed");
461 }
462
463 #[test]
464 fn limit_is_respected_and_bad_urls_dropped() {
465 let ranked = vec![(
466 "duckduckgo",
467 vec![hit("https://a.dev/1"), hit("javascript:void"), hit("https://b.dev/2")],
468 )];
469 let fused = fuse(&ranked, 1);
470 assert_eq!(fused.len(), 1);
471 assert_eq!(fused[0].url, "https://a.dev/1");
472 }
473}