bgustscraper 0.2.0

use anyhow::Result;
use scraper::{Html, Selector};
use reqwest::header::{USER_AGENT, HeaderMap};

pub async fn search_duckduckgo(prompt: &str) -> Result<Vec<String>> {
    let url = format!("https://html.duckduckgo.com/html/?q={}", urlencoding::encode(prompt));
    
    let mut headers = HeaderMap::new();
    headers.insert(USER_AGENT, "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36".parse()?);

    let client = reqwest::Client::builder()
        .default_headers(headers)
        .build()?;

    let response = client.get(url).send().await?.text().await?;
    let document = Html::parse_document(&response);
    let selector = Selector::parse("a.result__url").unwrap();

    let mut urls = Vec::new();
    for element in document.select(&selector) {
        if let Some(href) = element.value().attr("href") {
            let mut clean_url = href.trim().to_string();
            
            // 1. Manejar redirecciones de DuckDuckGo (e.g., //duckduckgo.com/l/?uddg=URL)
            if clean_url.contains("uddg=") {
                if let Some(pos) = clean_url.find("uddg=") {
                    let encoded_part = &clean_url[pos + 5..];
                    let end_pos = encoded_part.find('&').unwrap_or(encoded_part.len());
                    if let Ok(decoded) = urlencoding::decode(&encoded_part[..end_pos]) {
                        clean_url = decoded.into_owned();
                    }
                }
            }

            // 2. Asegurar esquema absoluto
            if clean_url.starts_with("//") {
                clean_url = format!("https:{}", clean_url);
            }

            if !clean_url.is_empty() && (clean_url.starts_with("http://") || clean_url.starts_with("https://")) {
                urls.push(clean_url);
            }
        }
    }

    Ok(urls)
}