Skip to main content

fetchium_core/search/
mod.rs

1//! Search backends and orchestrator (PRD ยง15).
2//!
3//! Phase 1: DuckDuckGo HTML scraper.
4//! Phase 2: 10 additional backends (HTTP + headless Chromium).
5//!
6//! ## HTTP backends (no feature flag required):
7//! - SearXNG (self-hosted localhost:4040 โ€” aggregates 9+ engines), DuckDuckGo, Wikipedia,
8//!   Brave, HackerNews, ArXiv, GitHub, Reddit, StackOverflow
9//!
10//! ## Headless backends (`--features headless`):
11//! - Google, Bing, Google Scholar
12//!
13//! ## SearXNG Self-Hosted (Primary โ€” free, unlimited, CAPTCHA-free)
14//! Run `cd ~/searxng-local && docker compose up -d` to start the local aggregator.
15//! Set `SEARXNG_URL=http://localhost:4040` to use exclusively.
16
17// HTTP backends (always available)
18pub mod arxiv;
19pub mod backend_selector;
20pub mod brave;
21pub mod dedup;
22pub mod duckduckgo;
23pub mod exa;
24pub mod fallback;
25pub mod firecrawl;
26pub mod github;
27pub mod hackernews;
28pub mod latency;
29pub mod orchestrator;
30pub mod reddit;
31pub mod searxng;
32pub mod serper;
33pub mod stackoverflow;
34pub mod tavily;
35pub mod wikipedia;
36
37pub use backend_selector::{AbsConfig, AdaptiveBackendSelector, BackendSelection};
38pub use latency::{LatencyConfig, LatencyPredictor, LatencyStats};
39
40// Headless backends (compiled unconditionally; return empty results without `headless` feature)
41pub mod bing;
42pub mod google;
43pub mod scholar;
44
45use crate::error::FetchiumResult;
46use crate::rank::fusion::QueryIntent;
47use crate::types::{BackendId, ResultItem};
48use async_trait::async_trait;
49
50/// Time range filter for date-restricted searches.
51#[derive(Debug, Clone, Copy, PartialEq, Eq)]
52pub enum TimeRange {
53    Day,
54    Week,
55    Month,
56    Year,
57}
58
59/// Context passed to backends for intent-aware, locale-targeted searches.
60#[derive(Debug, Clone)]
61pub struct SearchContext {
62    pub intent: QueryIntent,
63    pub time_range: Option<TimeRange>,
64    /// ISO 3166-1 alpha-2 country code for residential proxy targeting.
65    /// `None` = default/US routing. Set from query locale detection.
66    pub locale: Option<String>,
67    /// Query language hint for backends that support language targeting.
68    /// Uses lightweight query-language detection, not geo inference.
69    pub language: Option<String>,
70}
71
72/// Trait implemented by every search backend.
73///
74/// Backends are responsible for:
75/// 1. Querying their respective search endpoint
76/// 2. Parsing results into the unified `ResultItem` schema
77/// 3. Reporting their ID and headless requirements
78#[async_trait]
79pub trait SearchBackend: Send + Sync {
80    /// Unique identifier for this backend.
81    fn id(&self) -> BackendId;
82
83    /// Whether this backend requires a headless Chromium browser.
84    /// HTTP-only backends return `false` (default).
85    fn requires_headless(&self) -> bool {
86        false
87    }
88
89    /// Execute a search and return at most `max_results` results.
90    ///
91    /// Implementations should:
92    /// - Return partial results on soft failures (not full errors)
93    /// - Return `Err` only for hard failures (network down, auth broken, etc.)
94    /// - Never panic
95    async fn search(&self, query: &str, max_results: u32) -> FetchiumResult<Vec<ResultItem>>;
96
97    /// Execute a search with intent and time range context.
98    ///
99    /// Backends that support date filtering should override this method.
100    /// Default implementation delegates to `search()`, ignoring context.
101    async fn search_with_context(
102        &self,
103        query: &str,
104        max_results: u32,
105        _ctx: &SearchContext,
106    ) -> FetchiumResult<Vec<ResultItem>> {
107        self.search(query, max_results).await
108    }
109}