opencrabs 0.3.73

//! Web Search Tool (Unified)
//!
//! Single search entry point that fans out to DuckDuckGo, Exa, and Brave
//! in parallel, merges results, and returns the best combined set. The
//! agent calls one tool and never thinks about which engine answered.
//!
//! Engines:
//! - **DuckDuckGo** (always available, free, captcha-prone)
//! - **Exa** (free MCP endpoint or direct API with `EXA_API_KEY`)
//! - **Brave** (requires `BRAVE_API_KEY`, registered only when configured)
//!
//! DDG captcha detection (HTTP 202 + structural form heuristic) silently
//! drops DDG from the result pool instead of surfacing an error.

use super::brave_search::BraveSearchTool;
use super::error::{Result, ToolError};
use super::exa_search::ExaSearchTool;
use super::r#trait::{Tool, ToolCapability, ToolExecutionContext, ToolResult};
use async_trait::async_trait;
use serde::{Deserialize, Serialize};
use serde_json::Value;
use std::sync::Arc;

/// Unified web search tool. Fans out to DDG + Exa (+ Brave if configured)
/// in parallel, merges results, dedupes by URL.
#[derive(Default)]
pub struct WebSearchTool {
    exa: Option<Arc<ExaSearchTool>>,
    brave: Option<Arc<BraveSearchTool>>,
}

impl WebSearchTool {
    /// Create a unified search tool with optional engine references.
    /// When both are `None`, behaves as DDG-only (backward compat).
    pub fn new(exa: Option<Arc<ExaSearchTool>>, brave: Option<Arc<BraveSearchTool>>) -> Self {
        Self { exa, brave }
    }

    /// DuckDuckGo search (extracted for parallel fan-out).
    async fn search_ddg(&self, query: &str, max_results: usize) -> Result<ToolResult> {
        let url = format!(
            "https://lite.duckduckgo.com/lite/?q={}",
            urlencoding::encode(query)
        );

        let mut last_err = String::new();
        for (attempt, ua) in DDG_USER_AGENTS.iter().enumerate() {
            let client = reqwest::Client::builder()
                .timeout(std::time::Duration::from_secs(10))
                .user_agent(*ua)
                .build()
                .map_err(|e| ToolError::Execution(format!("Failed to create HTTP client: {e}")))?;

            match client.get(&url).send().await {
                // DDG bot-detection challenge: HTTP 202 is their
                // protocol-level "you're blocked" signal. Stable across
                // deploys. Short-circuit instead of burning UA rotations.
                Ok(response) if response.status().as_u16() == 202 => {
                    return Ok(ToolResult::error(
                        "DuckDuckGo returned a bot-detection challenge (HTTP 202).".to_string(),
                    ));
                }
                Ok(response) if response.status().is_success() => {
                    let html = response.text().await.map_err(|e| {
                        ToolError::Execution(format!("Failed to read response: {e}"))
                    })?;
                    let results = parse_lite_results(&html, max_results);
                    // Zero results + form present = challenge page, not a
                    // genuine empty result. Structural check, no reliance on
                    // internal JS filenames DDG can rename at any time.
                    if results.is_empty() && html.contains("<form") {
                        return Ok(ToolResult::error(
                            "DuckDuckGo returned a captcha/challenge page instead of results."
                                .to_string(),
                        ));
                    }
                    return Ok(ToolResult::success(format_results(query, &results)));
                }
                Ok(response) => {
                    last_err = format!("HTTP {}", response.status());
                    tracing::warn!(
                        "web_search: DuckDuckGo returned {} (UA {}/{}) — rotating User-Agent",
                        response.status(),
                        attempt + 1,
                        DDG_USER_AGENTS.len(),
                    );
                }
                Err(e) => {
                    last_err = e.to_string();
                    tracing::warn!(
                        "web_search: request failed (UA {}/{}): {e}",
                        attempt + 1,
                        DDG_USER_AGENTS.len(),
                    );
                }
            }

            // Brief backoff before the next UA (skipped after the last attempt).
            if attempt + 1 < DDG_USER_AGENTS.len() {
                tokio::time::sleep(std::time::Duration::from_millis(250)).await;
            }
        }

        Ok(ToolResult::error(format!(
            "DuckDuckGo search failed after {} attempts (last error: {last_err}).",
            DDG_USER_AGENTS.len(),
        )))
    }
}

#[derive(Debug, Deserialize, Serialize)]
struct SearchInput {
    /// Search query
    query: String,

    /// Maximum number of results to return
    #[serde(default = "default_max_results")]
    max_results: usize,
}

fn default_max_results() -> usize {
    5
}

// DuckDuckGo HTML search result
#[derive(Debug, Deserialize)]
pub(crate) struct SearchResult {
    pub(crate) title: String,
    pub(crate) url: String,
}

#[async_trait]
impl Tool for WebSearchTool {
    fn name(&self) -> &str {
        "web_search"
    }

    fn description(&self) -> &str {
        "Search the internet for real-time information. Fans out to \
         DuckDuckGo, Exa, and Brave (when configured) in parallel and \
         returns merged, deduplicated results. \
         \n\nDEFAULT web-research tool — use this for any \"find me info \
         about X\" / \"what's the latest Y\" / \"check the docs for Z\" \
         request unless the user explicitly asks for browser interaction. \
         Always pick a search tool over `browser_navigate` for research. \
         \n\nFor GitHub content (issues, PRs, repos, code search) use the \
         `gh` CLI via `bash` instead — it returns structured JSON and is \
         authenticated."
    }

    fn input_schema(&self) -> Value {
        serde_json::json!({
            "type": "object",
            "properties": {
                "query": {
                    "type": "string",
                    "description": "Search query (e.g., 'latest Node.js LTS release', 'Rust async programming')"
                },
                "max_results": {
                    "type": "integer",
                    "description": "Maximum number of results to return (default: 5)",
                    "default": 5,
                    "minimum": 1,
                    "maximum": 10
                }
            },
            "required": ["query"]
        })
    }

    fn capabilities(&self) -> Vec<ToolCapability> {
        vec![ToolCapability::Network]
    }

    fn requires_approval(&self) -> bool {
        false // Web search is generally safe (read-only)
    }

    fn validate_input(&self, input: &Value) -> Result<()> {
        let input: SearchInput = serde_json::from_value(input.clone())
            .map_err(|e| ToolError::InvalidInput(format!("Invalid input: {}", e)))?;

        if input.query.trim().is_empty() {
            return Err(ToolError::InvalidInput("Query cannot be empty".to_string()));
        }

        if input.max_results == 0 || input.max_results > 10 {
            return Err(ToolError::InvalidInput(
                "max_results must be between 1 and 10".to_string(),
            ));
        }

        Ok(())
    }

    async fn execute(&self, input: Value, context: &ToolExecutionContext) -> Result<ToolResult> {
        let parsed: SearchInput = serde_json::from_value(input.clone())?;

        // No extra engines configured: DDG-only (backward compat with
        // tool_setup.rs registration before config is loaded).
        if self.exa.is_none() && self.brave.is_none() {
            return self.search_ddg(&parsed.query, parsed.max_results).await;
        }

        // Fan out to all engines in parallel.
        let ddg_query = parsed.query.clone();
        let ddg_max = parsed.max_results;
        let ddg_fut = self.search_ddg(&ddg_query, ddg_max);

        let exa_input = input.clone();
        let exa_fut = async {
            match &self.exa {
                Some(exa) => Some(exa.execute(exa_input, context).await),
                None => None,
            }
        };

        let brave_input = input.clone();
        let brave_fut = async {
            match &self.brave {
                Some(brave) => Some(brave.execute(brave_input, context).await),
                None => None,
            }
        };

        let (ddg_res, exa_res, brave_res) = tokio::join!(ddg_fut, exa_fut, brave_fut);

        // Collect successful outputs. DDG errors (captcha, rate limit)
        // are silently dropped; the other engines fill the gap.
        let mut sections: Vec<String> = Vec::new();

        match ddg_res {
            Ok(ref r) if r.success => sections.push(r.output.clone()),
            Ok(r) => {
                tracing::debug!("web_search: DDG dropped: {}", r.output);
            }
            Err(e) => {
                tracing::debug!("web_search: DDG failed: {e}");
            }
        }

        if let Some(Ok(r)) = exa_res
            && r.success
        {
            sections.push(r.output);
        }

        if let Some(Ok(r)) = brave_res
            && r.success
        {
            sections.push(r.output);
        }

        if sections.is_empty() {
            return Ok(ToolResult::error(
                "All search engines failed or returned no results. \
                 Try rephrasing the query or using http_request against a search API."
                    .to_string(),
            ));
        }

        Ok(ToolResult::success(sections.join("\n\n")))
    }
}

/// Rotated User-Agent pool for DuckDuckGo Lite (#525). DDG blocks repetitive UA
/// patterns from one IP, so trying a few realistic browser UAs recovers most
/// transient 403s.
pub(crate) const DDG_USER_AGENTS: &[&str] = &[
    "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36",
    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.3 Safari/605.1.15",
    "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:123.0) Gecko/20100101 Firefox/123.0",
    "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/121.0.0.0 Safari/537.36",
];

/// Format parsed search results into the tool's human-readable output.
pub(crate) fn format_results(query: &str, results: &[SearchResult]) -> String {
    let mut output = String::new();
    output.push_str(&format!("🔍 Search results for: \"{query}\"\n\n"));
    if results.is_empty() {
        output.push_str("ℹ️  No results found. Try:\n");
        output.push_str("  • Rephrasing your query\n");
        output.push_str("  • Using more specific keywords\n");
        output.push_str("  • Searching for a different topic\n");
    } else {
        for (i, result) in results.iter().enumerate() {
            output.push_str(&format!("{}. {}\n", i + 1, result.title));
            output.push_str(&format!("   🔗 {}\n\n", result.url));
        }
    }
    output
}

/// Parse DuckDuckGo Lite HTML response to extract search results
pub(crate) fn parse_lite_results(html: &str, max_results: usize) -> Vec<SearchResult> {
    let mut results = Vec::new();

    // Find result links in the HTML
    // DDG Lite uses <a> tags with class "result-link" for search results
    let link_regex =
        regex::Regex::new(r#"<a[^>]*class="result-link"[^>]*href="([^"]*)"[^>]*>([^<]*)</a>"#)
            .unwrap_or_else(|_| {
                regex::Regex::new(r#"<a[^>]*href="([^"]*)"[^>]*>([^<]*)</a>"#).unwrap()
            });

    for cap in link_regex.captures_iter(html) {
        if results.len() >= max_results {
            break;
        }

        let url = cap
            .get(1)
            .map(|m| m.as_str().to_string())
            .unwrap_or_default();
        let title = cap
            .get(2)
            .map(|m| m.as_str().to_string())
            .unwrap_or_default();

        // Skip non-http links and empty titles
        if url.starts_with("http") && !title.trim().is_empty() {
            results.push(SearchResult { title, url });
        }
    }

    // Fallback: if no results found with class="result-link", try generic link parsing
    if results.is_empty() {
        let generic_regex =
            regex::Regex::new(r#"<a[^>]*href="(https?://[^"]*)"[^>]*>([^<]{10,})</a>"#).unwrap();

        for cap in generic_regex.captures_iter(html) {
            if results.len() >= max_results {
                break;
            }

            let url = cap
                .get(1)
                .map(|m| m.as_str().to_string())
                .unwrap_or_default();
            let title = cap
                .get(2)
                .map(|m| m.as_str().to_string())
                .unwrap_or_default();

            // Skip duckduckgo own links
            if !url.contains("duckduckgo.com") && !title.trim().is_empty() {
                results.push(SearchResult { title, url });
            }
        }
    }

    results
}