fetchium_core/search/mod.rs
1//! Search backends and orchestrator (PRD ยง15).
2//!
3//! Phase 1: DuckDuckGo HTML scraper.
4//! Phase 2: 10 additional backends (HTTP + headless Chromium).
5//!
6//! ## HTTP backends (no feature flag required):
7//! - SearXNG (self-hosted localhost:4040 โ aggregates 9+ engines), DuckDuckGo, Wikipedia,
8//! Brave, HackerNews, ArXiv, GitHub, Reddit, StackOverflow
9//!
10//! ## Headless backends (`--features headless`):
11//! - Google, Bing, Google Scholar
12//!
13//! ## SearXNG Self-Hosted (Primary โ free, unlimited, CAPTCHA-free)
14//! Run `cd ~/searxng-local && docker compose up -d` to start the local aggregator.
15//! Set `SEARXNG_URL=http://localhost:4040` to use exclusively.
16
17// HTTP backends (always available)
18pub mod arxiv;
19pub mod backend_selector;
20pub mod brave;
21pub mod dedup;
22pub mod duckduckgo;
23pub mod exa;
24pub mod fallback;
25pub mod firecrawl;
26pub mod github;
27pub mod hackernews;
28pub mod latency;
29pub mod orchestrator;
30pub mod reddit;
31pub mod searxng;
32pub mod serper;
33pub mod stackoverflow;
34pub mod tavily;
35pub mod wikipedia;
36
37pub use backend_selector::{AbsConfig, AdaptiveBackendSelector, BackendSelection};
38pub use latency::{LatencyConfig, LatencyPredictor, LatencyStats};
39
40// Headless backends (compiled unconditionally; return empty results without `headless` feature)
41pub mod bing;
42pub mod google;
43pub mod scholar;
44
45use crate::error::FetchiumResult;
46use crate::rank::fusion::QueryIntent;
47use crate::types::{BackendId, ResultItem};
48use async_trait::async_trait;
49
50/// Time range filter for date-restricted searches.
51#[derive(Debug, Clone, Copy, PartialEq, Eq)]
52pub enum TimeRange {
53 Day,
54 Week,
55 Month,
56 Year,
57}
58
59/// Context passed to backends for intent-aware, locale-targeted searches.
60#[derive(Debug, Clone)]
61pub struct SearchContext {
62 pub intent: QueryIntent,
63 pub time_range: Option<TimeRange>,
64 /// ISO 3166-1 alpha-2 country code for residential proxy targeting.
65 /// `None` = default/US routing. Set from query locale detection.
66 pub locale: Option<String>,
67 /// Query language hint for backends that support language targeting.
68 /// Uses lightweight query-language detection, not geo inference.
69 pub language: Option<String>,
70}
71
72/// Trait implemented by every search backend.
73///
74/// Backends are responsible for:
75/// 1. Querying their respective search endpoint
76/// 2. Parsing results into the unified `ResultItem` schema
77/// 3. Reporting their ID and headless requirements
78#[async_trait]
79pub trait SearchBackend: Send + Sync {
80 /// Unique identifier for this backend.
81 fn id(&self) -> BackendId;
82
83 /// Whether this backend requires a headless Chromium browser.
84 /// HTTP-only backends return `false` (default).
85 fn requires_headless(&self) -> bool {
86 false
87 }
88
89 /// Execute a search and return at most `max_results` results.
90 ///
91 /// Implementations should:
92 /// - Return partial results on soft failures (not full errors)
93 /// - Return `Err` only for hard failures (network down, auth broken, etc.)
94 /// - Never panic
95 async fn search(&self, query: &str, max_results: u32) -> FetchiumResult<Vec<ResultItem>>;
96
97 /// Execute a search with intent and time range context.
98 ///
99 /// Backends that support date filtering should override this method.
100 /// Default implementation delegates to `search()`, ignoring context.
101 async fn search_with_context(
102 &self,
103 query: &str,
104 max_results: u32,
105 _ctx: &SearchContext,
106 ) -> FetchiumResult<Vec<ResultItem>> {
107 self.search(query, max_results).await
108 }
109}