ratatoskr/
lib.rs

1//! Ratatoskr — the squirrel of Yggdrasil who runs messages between its roots
2//! and branches — carries questions out to the web and brings the
3//! answers back.
4//!
5//! [`Ratatoskr::search`] asks sources in order and returns the first useful
6//! answer: its own meta-search first (Wikipedia's and Grokipedia's best page,
7//! and Yahoo, DuckDuckGo, Bing and Yandex, healthiest first, two or more at a time, each
8//! search from a fresh browser, merged and ranked; see [`engine::meta`]),
9//! then an external search server if configured, Brave's and Google's
10//! official APIs when keyed,
11//! and Wikipedia last — an encyclopedia, not web search, kept only for plain
12//! facts when nothing else answers. [`Ratatoskr::fetch`] reads a page as text.
13//!
14//! Search is deliberately polite: a rate limit per engine, a cache so
15//! repeated questions never leave the machine, and a circuit breaker that
16//! stops asking an engine the moment it signals "no" (DuckDuckGo's 202
17//! anomaly page, 403, 429) and falls through to the next. A fetch reads one
18//! page a person asked for, as a browser would (see [`fetch`]), and can never
19//! reach loopback, private or link-local addresses.
20
21pub mod engine;
22pub mod fetch;
23pub mod format;
24pub mod http;
25pub mod mcp;
26pub mod polite;
27pub mod request;
28pub mod text;
29pub mod vertical;
30
31use std::sync::Arc;
32use std::time::Duration;
33
34use engine::{Engine, Hit};
35use polite::{Cache, CircuitBreaker, RateLimiter};
36pub use request::{RequestError, SafeSearch, SearchRequest, TimeLimit};
37
38/// The User-Agent of the API engines (Wikipedia, Grokipedia, Brave's and
39/// Google's APIs, a search server): who is asking, not a browser. The scraped
40/// engines and page fetches present as a browser instead ([`engine::browser`]).
41///
42/// An API operator who sees this traffic should be able to reach whoever runs
43/// it, so `RATATOSKR_CONTACT` (a URL or email of the deployment's owner) is
44/// appended when set. Read once; nothing about a particular install is baked in.
45pub fn user_agent() -> &'static str {
46    static AGENT: std::sync::OnceLock<String> = std::sync::OnceLock::new();
47    AGENT.get_or_init(|| agent_for(std::env::var("RATATOSKR_CONTACT").ok().as_deref()))
48}
49
50fn agent_for(contact: Option<&str>) -> String {
51    let base = concat!("Mozilla/5.0 (compatible; Ratatoskr/", env!("CARGO_PKG_VERSION"));
52    match contact.map(str::trim).filter(|c| !c.is_empty()) {
53        Some(contact) => format!("{base}; +{contact})"),
54        None => format!("{base})"),
55    }
56}
57
58/// What a search produced, and where it came from.
59#[derive(Clone, Debug, PartialEq)]
60pub struct SearchOutcome {
61    pub hits: Vec<Hit>,
62    /// The engine that answered.
63    pub engine: &'static str,
64    /// The engines whose answers made the results: several for the
65    /// meta-search, the engine itself otherwise.
66    pub sources: Vec<&'static str>,
67    /// How many results more than one engine returned, merged into one.
68    pub merged: usize,
69    pub from_cache: bool,
70}
71
72impl SearchOutcome {
73    /// Where the results came from, in a line: `meta (bing, yandex;
74    /// 3 duplicates merged; cached: false)`, or `wikipedia (cached: false)`
75    /// for one engine.
76    pub fn provenance(&self) -> String {
77        let mut parts = Vec::new();
78        if self.sources.as_slice() != [self.engine] && !self.sources.is_empty() {
79            parts.push(self.sources.join(", "));
80        }
81        match self.merged {
82            0 => {}
83            1 => parts.push("1 duplicate merged".into()),
84            n => parts.push(format!("{n} duplicates merged")),
85        }
86        parts.push(format!("cached: {}", self.from_cache));
87        format!("{} ({})", self.engine, parts.join("; "))
88    }
89}
90
91/// Why no engine could answer.
92#[derive(Clone, Debug, PartialEq)]
93pub struct SearchFailure {
94    /// Each engine tried (or skipped), with why it gave nothing.
95    pub attempts: Vec<(&'static str, String)>,
96}
97
98impl std::fmt::Display for SearchFailure {
99    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
100        let parts: Vec<String> = self.attempts.iter().map(|(e, why)| format!("{e}: {why}")).collect();
101        write!(f, "search unavailable ({})", parts.join("; "))
102    }
103}
104
105struct Slot {
106    engine: Arc<dyn Engine>,
107    limiter: RateLimiter,
108    breaker: CircuitBreaker,
109}
110
111/// Tunables; [`Settings::default`] is what production uses.
112#[derive(Clone, Debug)]
113pub struct Settings {
114    /// Minimum spacing between requests to one engine.
115    pub min_interval: Duration,
116    /// How long a tripped engine is skipped.
117    pub cooldown: Duration,
118    /// How long a search result is reused.
119    pub cache_ttl: Duration,
120    pub cache_entries: usize,
121    pub timeout: Duration,
122}
123
124impl Default for Settings {
125    fn default() -> Self {
126        Self {
127            min_interval: Duration::from_secs(2),
128            cooldown: Duration::from_secs(600),
129            cache_ttl: Duration::from_secs(900),
130            cache_entries: 256,
131            timeout: Duration::from_secs(15),
132        }
133    }
134}
135
136/// Which optional sources are configured. The meta-search (first) and
137/// Wikipedia (last) need nothing and are always present.
138#[derive(Clone, Debug, Default)]
139pub struct Sources {
140    /// An external search server ([`engine::search_api`]), e.g. `http://127.0.0.1:8765`.
141    pub search_api_url: Option<String>,
142    pub brave_key: Option<String>,
143    pub google_key: Option<String>,
144    /// Google's Programmable Search Engine id.
145    pub google_cx: Option<String>,
146}
147
148impl Sources {
149    /// From SEARCH_API_URL, BRAVE_SEARCH_API_KEY, GOOGLE_CSE_KEY, GOOGLE_CSE_ID.
150    pub fn from_env() -> Self {
151        let var = |key: &str| std::env::var(key).ok().filter(|v| !v.trim().is_empty());
152        Self {
153            search_api_url: var("SEARCH_API_URL"),
154            brave_key: var("BRAVE_SEARCH_API_KEY"),
155            google_key: var("GOOGLE_CSE_KEY"),
156            google_cx: var("GOOGLE_CSE_ID"),
157        }
158    }
159}
160
161/// The results of a news, images, videos or books search.
162#[derive(Clone, Debug, PartialEq, Eq)]
163pub struct Found<T> {
164    pub results: Vec<T>,
165    /// The sources that answered.
166    pub sources: Vec<&'static str>,
167    pub from_cache: bool,
168}
169
170/// One cache and one pool of sources for a kind of result.
171struct Kind<T> {
172    pool: vertical::Pool<T>,
173    cache: Cache<(Vec<T>, Vec<&'static str>)>,
174}
175
176impl<T: vertical::Keyed + Clone + Send + 'static> Kind<T> {
177    fn new(sources: Vec<Arc<dyn vertical::Source<T>>>, settings: &Settings) -> Self {
178        Self {
179            pool: vertical::Pool::new(sources, settings.cooldown, settings.min_interval),
180            cache: Cache::new(settings.cache_ttl, settings.cache_entries),
181        }
182    }
183
184    async fn search(&self, label: &'static str, request: &SearchRequest) -> Result<Found<T>, SearchFailure> {
185        let key = request.cache_key(label);
186        if let Some((_, (results, sources))) = self.cache.get(&key) {
187            return Ok(Found { results, sources, from_cache: true });
188        }
189        match self.pool.search(request).await {
190            Ok((results, sources)) if !results.is_empty() => {
191                self.cache.put(key, label, (results.clone(), sources.clone()));
192                Ok(Found { results, sources, from_cache: false })
193            }
194            Ok(_) => Err(SearchFailure { attempts: vec![(label, "no results".into())] }),
195            Err(error) => Err(SearchFailure { attempts: vec![(label, error.to_string())] }),
196        }
197    }
198}
199
200/// The sources of the news, images, videos and books searches.
201pub struct Verticals {
202    pub news: Vec<Arc<dyn vertical::Source<vertical::NewsResult>>>,
203    pub images: Vec<Arc<dyn vertical::Source<vertical::ImageResult>>>,
204    pub videos: Vec<Arc<dyn vertical::Source<vertical::VideoResult>>>,
205    pub books: Vec<Arc<dyn vertical::Source<vertical::BookResult>>>,
206}
207
208impl Verticals {
209    /// None: [`Ratatoskr::new`]'s default, for a web-search-only instance.
210    pub fn none() -> Self {
211        Self { news: Vec::new(), images: Vec::new(), videos: Vec::new(), books: Vec::new() }
212    }
213
214    /// The production sources:
215    /// - news: DuckDuckGo and Bing (Bing's index, so one of them a search) and
216    ///   Yahoo (its own);
217    /// - images: Bing, and DuckDuckGo when it does not ask its proof of work;
218    /// - videos: DuckDuckGo;
219    /// - books: Open Library's official API.
220    pub fn standard(settings: &Settings) -> Self {
221        use vertical::{bing, duckduckgo, openlibrary, yahoo};
222        let t = settings.timeout;
223        Self {
224            news: vec![
225                Arc::new(duckduckgo::DuckDuckGo::new(t).news()),
226                Arc::new(bing::Bing::new(t).news()),
227                Arc::new(yahoo::News(yahoo::Yahoo::new(t))),
228            ],
229            images: vec![Arc::new(bing::Bing::new(t).images()), Arc::new(duckduckgo::DuckDuckGo::new(t).images())],
230            videos: vec![Arc::new(duckduckgo::DuckDuckGo::new(t).videos())],
231            books: vec![Arc::new(openlibrary::OpenLibrary::new(engine::http_client(t)))],
232        }
233    }
234}
235
236pub struct Ratatoskr {
237    slots: Vec<Slot>,
238    /// Web answers with where they came from, so a cached one reports it too.
239    cache: Cache<engine::Report>,
240    fetcher: fetch::Fetcher,
241    news: Kind<vertical::NewsResult>,
242    images: Kind<vertical::ImageResult>,
243    videos: Kind<vertical::VideoResult>,
244    books: Kind<vertical::BookResult>,
245}
246
247impl Ratatoskr {
248    /// The given engines, tried in order.
249    pub fn new(engines: Vec<Arc<dyn Engine>>, settings: Settings) -> Self {
250        let slots = engines
251            .into_iter()
252            .map(|engine| Slot {
253                engine,
254                limiter: RateLimiter::new(settings.min_interval),
255                breaker: CircuitBreaker::new(settings.cooldown),
256            })
257            .collect();
258        let none = Verticals::none();
259        Self {
260            slots,
261            cache: Cache::new(settings.cache_ttl, settings.cache_entries),
262            fetcher: fetch::Fetcher::new(settings.timeout),
263            news: Kind::new(none.news, &settings),
264            images: Kind::new(none.images, &settings),
265            videos: Kind::new(none.videos, &settings),
266            books: Kind::new(none.books, &settings),
267        }
268    }
269
270    /// With these news, images, videos and books sources.
271    pub fn with_verticals(mut self, verticals: Verticals, settings: &Settings) -> Self {
272        self.news = Kind::new(verticals.news, settings);
273        self.images = Kind::new(verticals.images, settings);
274        self.videos = Kind::new(verticals.videos, settings);
275        self.books = Kind::new(verticals.books, settings);
276        self
277    }
278
279    /// News articles, with dates as ISO 8601 where the source gives one.
280    pub async fn news(&self, request: &SearchRequest) -> Result<Found<vertical::NewsResult>, SearchFailure> {
281        self.news.search("news", request).await
282    }
283
284    /// Images, with their page, thumbnail and size where known.
285    pub async fn images(&self, request: &SearchRequest) -> Result<Found<vertical::ImageResult>, SearchFailure> {
286        self.images.search("images", request).await
287    }
288
289    /// Videos, with duration, publisher and views where known.
290    pub async fn videos(&self, request: &SearchRequest) -> Result<Found<vertical::VideoResult>, SearchFailure> {
291        self.videos.search("videos", request).await
292    }
293
294    /// Books, with authors, year and cover where known.
295    pub async fn books(&self, request: &SearchRequest) -> Result<Found<vertical::BookResult>, SearchFailure> {
296        self.books.search("books", request).await
297    }
298
299    /// With a given fetcher (tests use [`fetch::Fetcher::allowing_private_for_tests`]).
300    pub fn with_fetcher(mut self, fetcher: fetch::Fetcher) -> Self {
301        self.fetcher = fetcher;
302        self
303    }
304
305    /// The production set, in order, from what is configured (see [`Sources`]).
306    pub fn standard(sources: &Sources) -> Self {
307        let settings = Settings::default();
308        let http = engine::http_client(settings.timeout);
309        // The engines that answer a browser. Brave's, Google's, Mojeek's and
310        // Startpage's result pages are left out: tried from two networks,
311        // Brave and Mojeek answered with a captcha, Google with its JavaScript
312        // wall and Startpage with Anubis's proof of work, and challenges are
313        // never solved here.
314        use engine::meta::Member;
315        let meta = vec![
316            Member::encyclopedia(Arc::new(engine::wikipedia::Wikipedia::best_page(http.clone(), "en"))),
317            Member::encyclopedia(Arc::new(engine::grokipedia::Grokipedia::new(http.clone()))),
318            Member::web(Arc::new(engine::yahoo::Yahoo::new(settings.timeout)), "bing"),
319            Member::web(Arc::new(engine::duckduckgo::DuckDuckGo::new(settings.timeout)), "bing"),
320            Member::web(Arc::new(engine::bing::Bing::new(settings.timeout)), "bing"),
321            Member::web(Arc::new(engine::yandex::Yandex::new(settings.timeout)), "yandex"),
322        ];
323        let mut engines: Vec<Arc<dyn Engine>> =
324            vec![Arc::new(engine::meta::Meta::new(meta, settings.cooldown, settings.min_interval))];
325        if let Some(base) = sources.search_api_url.as_deref().filter(|u| !u.trim().is_empty()) {
326            engines.push(Arc::new(engine::search_api::SearchApi::new(http.clone(), base.trim())));
327        }
328        if let Some(key) = sources.brave_key.as_deref().filter(|k| !k.trim().is_empty()) {
329            engines.push(Arc::new(engine::brave::Brave::new(http.clone(), key.trim())));
330        }
331        if let (Some(key), Some(cx)) = (sources.google_key.as_deref(), sources.google_cx.as_deref())
332            && !key.trim().is_empty()
333            && !cx.trim().is_empty()
334        {
335            engines.push(Arc::new(engine::google::Google::new(http.clone(), key.trim(), cx.trim())));
336        }
337        engines.push(Arc::new(engine::wikipedia::Wikipedia::new(http, "en")));
338        Self::new(engines, settings.clone()).with_verticals(Verticals::standard(&settings), &settings)
339    }
340
341    /// Search: the cache first, then each engine in order that is not cooling
342    /// down, waiting out its rate limit. An engine that refuses trips its
343    /// breaker; an empty answer falls through without tripping it.
344    pub async fn search(&self, query: &str, limit: usize) -> Result<SearchOutcome, SearchFailure> {
345        self.search_in(query, limit, None).await
346    }
347
348    /// [`Ratatoskr::search`] localized to `country` (ISO 3166-1 alpha-2).
349    pub async fn search_in(
350        &self,
351        query: &str,
352        limit: usize,
353        country: Option<&str>,
354    ) -> Result<SearchOutcome, SearchFailure> {
355        self.search_with(&SearchRequest::new(query, limit).country(country)).await
356    }
357
358    /// [`Ratatoskr::search`] with every option: country, time limit, safe
359    /// search, page ([`SearchRequest`]).
360    pub async fn search_with(&self, request: &SearchRequest) -> Result<SearchOutcome, SearchFailure> {
361        let key = request.cache_key("text");
362        if let Some((engine, report)) = self.cache.get(&key) {
363            let engine::Report { hits, sources, merged } = report;
364            return Ok(SearchOutcome { hits, engine, sources, merged, from_cache: true });
365        }
366        let mut attempts = Vec::new();
367        for slot in &self.slots {
368            let name = slot.engine.name();
369            if let Some(left) = slot.breaker.open_for() {
370                attempts.push((name, format!("cooling down for {}s", left.as_secs())));
371                continue;
372            }
373            if !slot.engine.paces_itself() {
374                slot.limiter.wait_turn().await;
375            }
376            match slot.engine.search_report(request).await {
377                Ok(report) if !report.hits.is_empty() => {
378                    slot.breaker.record_success();
379                    self.cache.put(key, name, report.clone());
380                    let engine::Report { hits, sources, merged } = report;
381                    return Ok(SearchOutcome { hits, engine: name, sources, merged, from_cache: false });
382                }
383                Ok(_) => attempts.push((name, "no results".into())),
384                Err(error) => {
385                    if error.is_refusal() {
386                        slot.breaker.trip();
387                        tracing::warn!("ratatoskr: {name} refused ({error}); skipping it for a while");
388                    }
389                    attempts.push((name, error.to_string()));
390                }
391            }
392        }
393        Err(SearchFailure { attempts })
394    }
395
396    /// Read `url` as text (see [`fetch::Fetcher::fetch`]).
397    pub async fn fetch(&self, url: &str) -> Result<fetch::Page, fetch::FetchError> {
398        self.fetcher.fetch(url).await
399    }
400
401    /// Per-engine health, for logs and /stats.
402    pub fn status(&self) -> Vec<(&'static str, Option<Duration>)> {
403        self.slots.iter().map(|s| (s.engine.name(), s.breaker.open_for())).collect()
404    }
405
406    /// Each news, images, videos and books source, by kind, with how long it
407    /// is still resting, if it is.
408    pub fn vertical_status(&self) -> Vec<(&'static str, &'static str, Option<Duration>)> {
409        let tag = |kind, list: Vec<(&'static str, Option<Duration>)>| list.into_iter().map(move |(n, r)| (kind, n, r));
410        tag("news", self.news.pool.status())
411            .chain(tag("images", self.images.pool.status()))
412            .chain(tag("videos", self.videos.pool.status()))
413            .chain(tag("books", self.books.pool.status()))
414            .collect()
415    }
416}
417
418#[cfg(test)]
419mod outcome_tests {
420    use super::SearchOutcome;
421
422    #[test]
423    fn provenance_names_the_engines_and_what_was_merged() {
424        let outcome = |engine, sources: &[&'static str], merged| SearchOutcome {
425            hits: Vec::new(),
426            engine,
427            sources: sources.to_vec(),
428            merged,
429            from_cache: false,
430        };
431        assert_eq!(
432            outcome("meta", &["bing", "yandex", "wikipedia"], 3).provenance(),
433            "meta (bing, yandex, wikipedia; 3 duplicates merged; cached: false)"
434        );
435        assert_eq!(
436            outcome("meta", &["bing", "yandex"], 1).provenance(),
437            "meta (bing, yandex; 1 duplicate merged; cached: false)"
438        );
439        assert_eq!(
440            outcome("wikipedia", &["wikipedia"], 0).provenance(),
441            "wikipedia (cached: false)",
442            "one engine: itself"
443        );
444    }
445}
446
447#[cfg(test)]
448mod agent_tests {
449    use super::agent_for;
450
451    #[test]
452    fn the_contact_is_optional_and_never_built_in() {
453        let bare = agent_for(None);
454        assert!(bare.starts_with("Mozilla/5.0 (compatible; Ratatoskr/") && bare.ends_with(')'));
455        assert!(!bare.contains("+"));
456        assert_eq!(agent_for(Some("  ")), bare);
457        assert!(agent_for(Some("https://example.org/bot")).ends_with("; +https://example.org/bot)"));
458    }
459}
ratatoskr/lib.rs

ratatoskr/
lib.rs
