fetchium-core 1.0.0

Fetchium core library — search, extract, rank, validate, cache
Documentation
//! Search backends and orchestrator (PRD §15).
//!
//! Phase 1: DuckDuckGo HTML scraper.
//! Phase 2: 10 additional backends (HTTP + headless Chromium).
//!
//! ## HTTP backends (no feature flag required):
//! - SearXNG (self-hosted localhost:4040 — aggregates 9+ engines), DuckDuckGo, Wikipedia,
//!   Brave, HackerNews, ArXiv, GitHub, Reddit, StackOverflow
//!
//! ## Headless backends (`--features headless`):
//! - Google, Bing, Google Scholar
//!
//! ## SearXNG Self-Hosted (Primary — free, unlimited, CAPTCHA-free)
//! Run `cd ~/searxng-local && docker compose up -d` to start the local aggregator.
//! Set `SEARXNG_URL=http://localhost:4040` to use exclusively.

// HTTP backends (always available)
pub mod arxiv;
pub mod backend_selector;
pub mod brave;
pub mod dedup;
pub mod duckduckgo;
pub mod exa;
pub mod fallback;
pub mod firecrawl;
pub mod github;
pub mod hackernews;
pub mod latency;
pub mod orchestrator;
pub mod reddit;
pub mod searxng;
pub mod serper;
pub mod stackoverflow;
pub mod tavily;
pub mod wikipedia;

pub use backend_selector::{AbsConfig, AdaptiveBackendSelector, BackendSelection};
pub use latency::{LatencyConfig, LatencyPredictor, LatencyStats};

// Headless backends (compiled unconditionally; return empty results without `headless` feature)
pub mod bing;
pub mod google;
pub mod scholar;

use crate::error::FetchiumResult;
use crate::rank::fusion::QueryIntent;
use crate::types::{BackendId, ResultItem};
use async_trait::async_trait;

/// Time range filter for date-restricted searches.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum TimeRange {
    Day,
    Week,
    Month,
    Year,
}

/// Context passed to backends for intent-aware, locale-targeted searches.
#[derive(Debug, Clone)]
pub struct SearchContext {
    pub intent: QueryIntent,
    pub time_range: Option<TimeRange>,
    /// ISO 3166-1 alpha-2 country code for residential proxy targeting.
    /// `None` = default/US routing. Set from query locale detection.
    pub locale: Option<String>,
    /// Query language hint for backends that support language targeting.
    /// Uses lightweight query-language detection, not geo inference.
    pub language: Option<String>,
}

/// Trait implemented by every search backend.
///
/// Backends are responsible for:
/// 1. Querying their respective search endpoint
/// 2. Parsing results into the unified `ResultItem` schema
/// 3. Reporting their ID and headless requirements
#[async_trait]
pub trait SearchBackend: Send + Sync {
    /// Unique identifier for this backend.
    fn id(&self) -> BackendId;

    /// Whether this backend requires a headless Chromium browser.
    /// HTTP-only backends return `false` (default).
    fn requires_headless(&self) -> bool {
        false
    }

    /// Execute a search and return at most `max_results` results.
    ///
    /// Implementations should:
    /// - Return partial results on soft failures (not full errors)
    /// - Return `Err` only for hard failures (network down, auth broken, etc.)
    /// - Never panic
    async fn search(&self, query: &str, max_results: u32) -> FetchiumResult<Vec<ResultItem>>;

    /// Execute a search with intent and time range context.
    ///
    /// Backends that support date filtering should override this method.
    /// Default implementation delegates to `search()`, ignoring context.
    async fn search_with_context(
        &self,
        query: &str,
        max_results: u32,
        _ctx: &SearchContext,
    ) -> FetchiumResult<Vec<ResultItem>> {
        self.search(query, max_results).await
    }
}