webfind 0.5.0

use std::collections::HashSet;
use std::num::NonZeroU32;
use std::sync::{Arc, LazyLock};
use std::time::Instant;

use anyhow::{Context, Result};
use chrono::{DateTime, Utc};
use readability_rust::Readability;
use reqwest::Client;
use scraper::{Html, Selector};
use serde_json::Value;
use unicode_segmentation::UnicodeSegmentation;

use crate::engine::device_profile::{DeviceProfile, SessionManager};
use crate::engine::fingerprint::{FingerprintAuditLog, FingerprintGenerator};
use crate::engine::human_client::HumanClient;
use crate::engine::proxy_pool::ProxyPool;
use crate::schema::content::{
    CalloutEntity, CodeBlockEntity, DiagramEntity, Entities, ExampleEntity, FaqEntity,
    HeadingEntity, ImageInfo, MathEntity, OpenGraph, StepEntity, StructuredContent, TableEntity,
    TwitterCard,
};
use crate::schema::response::Keyword;

const USER_AGENT: &str = crate::engine::util::USER_AGENT;
const FETCH_TIMEOUT_SECS: u64 = 30;

/// Minimum word count for a page to be considered valid content.
///
/// Kept deliberately low: many legitimate pages (docs landing pages, error /
/// redirect stubs, single-purpose pages like example.com) carry only a few
/// dozen words yet are exactly what an agent wants scraped. A high floor
/// (previously 50) silently dropped these, so `webfind_fetch` returned
/// `is_valid_content: false` and research treated them as failed crawls —
/// which is why the LLM fell back to DuckDuckGo. 5 words still filters
/// empty / JS-shell pages while retaining real content.
pub(crate) const MIN_VALID_WORDS: u32 = 5;

/// Whether to rotate the User-Agent header per request.
#[derive(Clone, Copy)]
pub enum RotateUserAgent {
    Rotate,
    Fixed,
}

/// Fetcher: HTTP client + content extraction pipeline.
pub struct Fetcher {
    client: Client,
    human_client: Option<HumanClient>,
    profile: Option<DeviceProfile>,
    audit_log: Option<Arc<dyn FingerprintAuditLog>>,
    dynamic_fallback: bool,
    dynamic_wait_ms: u64,
    dynamic_deep: bool,
}

impl Fetcher {
    pub fn new() -> Result<Self> {
        let client = Client::builder()
            .user_agent(USER_AGENT)
            .timeout(std::time::Duration::from_secs(FETCH_TIMEOUT_SECS))
            .redirect(reqwest::redirect::Policy::limited(10))
            .gzip(true)
            .build()
            .context("failed to build HTTP client")?;
        Ok(Self {
            client,
            human_client: None,
            profile: None,
            audit_log: None,
            dynamic_fallback: false,
            dynamic_wait_ms: 2000,
            dynamic_deep: false,
        })
    }

    /// Build a Fetcher from an existing reqwest client.
    pub fn from_client(client: Client) -> Result<Self> {
        let mut fetcher = Self::new()?;
        fetcher.client = client;
        Ok(fetcher)
    }

    /// Attach a device profile so every request carries consistent headers.
    pub fn with_profile(mut self, profile: DeviceProfile) -> Self {
        self.profile = Some(profile);
        self
    }

    /// Build a fetcher with human-like request handling and privacy fingerprint rotation.
    pub fn new_human(
        proxy_pool: Option<ProxyPool>,
        session_manager: Option<SessionManager>,
        rotate_ua: RotateUserAgent,
        rps: u32,
    ) -> Result<Self> {
        let rps = NonZeroU32::new(rps.max(1)).expect("rps.max(1) is always >= 1");
        let ua_rotate = match rotate_ua {
            RotateUserAgent::Rotate => true,
            RotateUserAgent::Fixed => false,
        };
        let mut builder = HumanClient::builder()
            .requests_per_second(rps)
            .burst_size(rps.get() * 3)
            .rotate_ua(ua_rotate);
        if let Some(pool) = proxy_pool {
            builder = builder.proxy_pool(pool);
        }
        if let Some(mgr) = session_manager {
            builder = builder.session_manager(mgr);
        }
        let fingerprint_generator = Some(FingerprintGenerator::new());
        if let Some(generator) = fingerprint_generator {
            builder = builder.fingerprint_generator(generator);
        }
        let human_client = builder.build()?;

        let client = Client::builder()
            .user_agent(USER_AGENT)
            .timeout(std::time::Duration::from_secs(FETCH_TIMEOUT_SECS))
            .redirect(reqwest::redirect::Policy::limited(10))
            .gzip(true)
            .build()
            .context("failed to build fallback HTTP client")?;

        Ok(Self {
            client,
            human_client: Some(human_client),
            profile: None,
            audit_log: None,
            dynamic_fallback: false,
            dynamic_wait_ms: 2000,
            dynamic_deep: false,
        })
    }

    /// Attach an audit sink for fingerprint usage events.
    pub fn with_audit_log(mut self, audit_log: Arc<dyn FingerprintAuditLog>) -> Self {
        self.audit_log = Some(audit_log.clone());
        if let Some(ref mut hc) = self.human_client {
            hc.set_audit_log(audit_log);
        }
        self
    }

    /// Enable Chromium-based dynamic rendering fallback for JS-heavy pages.
    pub fn with_dynamic_fallback(mut self, wait_ms: u64) -> Self {
        self.dynamic_fallback = true;
        self.dynamic_wait_ms = wait_ms.max(500);
        self
    }

    /// Enable deep browser rendering: stealth mode + infinite-scroll so
    /// progressively rendered / bot-protected pages are fully captured.
    pub fn with_dynamic_deep(mut self, deep: bool) -> Self {
        self.dynamic_deep = deep;
        self
    }

    /// Fetch a URL and extract StructuredContent.
    pub async fn fetch_url(&self, url: &str) -> Result<StructuredContent> {
        // Reddit-specific path: try .json endpoint, fall back to old Reddit HTML.
        if super::reddit::is_reddit_url(url) {
            return self.fetch_reddit(url).await;
        }

        // Tier-0 Manifest Fast-Path: Only for domain roots or root /docs, never for specific subpages or blog articles.
        if let Ok(parsed_url) = url::Url::parse(url) {
            let path = parsed_url.path();
            let is_manifest_eligible = path.is_empty() || path == "/" || path == "/docs" || path == "/docs/";
            if is_manifest_eligible
                && parsed_url.query().is_none()
                && (parsed_url.scheme() == "http" || parsed_url.scheme() == "https")
                && let Some(manifest_content) = self.try_manifest_fast_path(&parsed_url).await
            {
                return Ok(manifest_content);
            }
        }

        let start = Instant::now();
        let response = if let Some(ref hc) = self.human_client {
            hc.get(url).await?
        } else {
            let mut req = self.client
                .get(url)
                .header("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,application/atom+xml,application/rss+json;q=0.8,*/*;q=0.7");
            if let Some(ref profile) = self.profile {
                req = profile.apply_headers(req);
            }
            req.send()
                .await
                .with_context(|| format!("HTTP request failed for {}", url))?
        };

        let status = response.status().as_u16();
        let final_url = response.url().to_string();
        let ssl_valid = final_url.starts_with("https://");
        let last_modified = response
            .headers()
            .get("last-modified")
            .and_then(|v| v.to_str().ok())
            .map(String::from);
        let content_type = response
            .headers()
            .get("content-type")
            .and_then(|v| v.to_str().ok())
            .map(String::from);

        let content_type_header = content_type.clone().unwrap_or_default();
        let (kind, charset) = parse_content_type_header(content_type.as_deref());

        let elapsed = start.elapsed().as_millis() as u64;

        // Binary resources are not indexed; return a minimal invalid record so
        // callers can log the URL but not waste tokens on garbage text.
        if kind == ContentKind::Binary {
            return Ok(make_invalid_content(
                url,
                &final_url,
                status,
                ssl_valid,
                elapsed,
                content_type,
            ));
        }

        // SIMD-accelerated zero-copy body read: validates UTF-8 at CPU hardware speed
        // (AVX2 / ARM NEON) and borrows the buffer directly without allocation.
        // Non-UTF-8 bodies decode lossily as fallback.
        let body_bytes = response
            .bytes()
            .await
            .context("failed to read response body")?;
        let body = match simdutf8::compat::from_utf8(&body_bytes) {
            Ok(valid_str) => std::borrow::Cow::Borrowed(valid_str),
            Err(_) => String::from_utf8_lossy(&body_bytes),
        };

        let mut content = match kind {
            ContentKind::Html => self.extract_from_html(
                &body,
                url,
                &final_url,
                status,
                ssl_valid,
                elapsed,
                last_modified.as_deref(),
            )?,
            ContentKind::Text | ContentKind::Json | ContentKind::Xml => extract_from_text(
                &body,
                url,
                &final_url,
                status,
                ssl_valid,
                elapsed,
                last_modified.as_deref(),
                kind,
            )?,
            ContentKind::Binary => unreachable!(),
        };

        content.encoding = charset;
        content.content_type_header = content_type_header.clone();
        content.content_type =
            super::content_classifier::classify_content_type(&content).to_string();

        // If static fetch yields no meaningful content and dynamic fallback is enabled,
        // render the page in Chromium and re-extract.
        if !content.is_valid_content && self.dynamic_fallback && kind == ContentKind::Html {
            #[cfg(feature = "dynamic")]
            {
                let dynamic = super::dynamic_fetcher::DynamicFetcher::new();
                let (rendered_html, dynamic_final_url) = dynamic
                    .render(url, self.dynamic_wait_ms, self.dynamic_deep)
                    .await?;
                let ssl = dynamic_final_url.starts_with("https://");
                let mut dyn_content = self.extract_from_html(
                    &rendered_html,
                    url,
                    &dynamic_final_url,
                    200,
                    ssl,
                    elapsed + self.dynamic_wait_ms,
                    None,
                )?;
                dyn_content.content_type_header = content_type_header.clone();
                dyn_content.content_type =
                    super::content_classifier::classify_content_type(&dyn_content).to_string();
                return Ok(dyn_content);
            }
            #[cfg(not(feature = "dynamic"))]
            {
                anyhow::bail!("dynamic fallback requested but 'dynamic' feature is not enabled");
            }
        }

        Ok(content)
    }

    /// Fetch a Reddit URL, preferring the `.json` endpoint and falling back
    /// to old.reddit.com HTML when JSON returns 403.
    async fn fetch_reddit(&self, url: &str) -> Result<StructuredContent> {
        let start = Instant::now();

        // Attempt 1: try the .json endpoint
        if let Some(json_url) = super::reddit::to_reddit_json_url(url) {
            tokio::time::sleep(super::reddit::REDDIT_RATE_LIMIT).await;
            let json_request = self.build_reddit_request(&json_url);
            match json_request.send().await {
                Ok(resp) if resp.status().is_success() => {
                    let status = resp.status().as_u16();
                    let final_url = resp.url().to_string();
                    let body = resp
                        .text()
                        .await
                        .context("failed to read Reddit JSON body")?;
                    let elapsed = start.elapsed().as_millis() as u64;

                    match super::reddit::parse_reddit_listing(&body) {
                        Ok(items) if !items.is_empty() => {
                            let text = super::reddit::listing_to_text(&items);
                            return self
                                .build_reddit_content(url, &final_url, status, elapsed, &text);
                        }
                        _ => {
                            // JSON parsed but no items — fall through to HTML
                        }
                    }
                }
                Ok(_) | Err(_) => {
                    // 403 or network error — fall through to HTML
                }
            }
        }

        // Attempt 2: fetch old.reddit.com HTML (cleaner than new Reddit)
        tokio::time::sleep(super::reddit::REDDIT_RATE_LIMIT).await;
        let old_url = super::reddit::to_old_reddit_url(url);
        let html_request = self.build_reddit_request(&old_url);
        let resp = html_request
            .send()
            .await
            .with_context(|| format!("HTTP request failed for Reddit {}", old_url))?;

        let status = resp.status().as_u16();
        let final_url = resp.url().to_string();
        let ssl_valid = final_url.starts_with("https://");
        let body_bytes = resp
            .bytes()
            .await
            .context("failed to read Reddit HTML body")?;
        let body = match simdutf8::compat::from_utf8(&body_bytes) {
            Ok(valid_str) => std::borrow::Cow::Borrowed(valid_str),
            Err(_) => String::from_utf8_lossy(&body_bytes),
        };
        let elapsed = start.elapsed().as_millis() as u64;

        self.extract_from_html(&body, url, &final_url, status, ssl_valid, elapsed, None)
    }

    /// Build a Reddit request with descriptive User-Agent without blocking thread sleep.
    fn build_reddit_request(&self, url: &str) -> reqwest::RequestBuilder {
        self.client
            .get(url)
            .header("User-Agent", super::reddit::REDDIT_USER_AGENT)
            .header(
                "Accept",
                "application/json, text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
            )
    }

    /// Build a StructuredContent from extracted Reddit text.
    fn build_reddit_content(
        &self,
        original_url: &str,
        final_url: &str,
        status_code: u16,
        fetch_duration_ms: u64,
        text: &str,
    ) -> Result<StructuredContent> {
        let collapsed = normalize_text(text);
        let title = collapsed
            .lines()
            .find(|l| !l.trim().is_empty())
            .map(|l| l.trim().chars().take(120).collect())
            .unwrap_or_else(|| title_from_url(final_url));

        let excerpt = collapsed.chars().take(300).collect::<String>();
        let word_count = count_words(&collapsed);
        let sentence_count = count_sentences(&collapsed);
        let reading_time_seconds = estimate_reading_time(word_count);
        let reading_ease = textstat::flesch_reading_ease(&collapsed);
        let grade_level = textstat::flesch_kincaid_grade(&collapsed);
        let keywords = extract_keywords(&collapsed, 15);
        let (language, language_confidence) = whatlang::detect(&collapsed)
            .map(|i| (i.lang().code().to_string(), i.confidence()))
            .unwrap_or_else(|| ("en".to_string(), 0.3));
        let content_markdown = collapsed.clone();

        Ok(StructuredContent {
            url: original_url.to_string(),
            final_url: final_url.to_string(),
            status_code,
            title,
            description: None,
            canonical_url: None,
            language,
            language_confidence,
            published_at: None,
            modified_at: None,
            author: None,
            site_name: Some("reddit.com".to_string()),
            content_text: collapsed.clone(),
            content_html: format!("<pre>{}</pre>", html_escape(&collapsed)),
            content_markdown,
            excerpt,
            word_count,
            char_count: collapsed.len() as u32,
            sentence_count,
            reading_time_seconds,
            reading_ease,
            grade_level,
            keywords,
            open_graph: None,
            twitter_card: None,
            json_ld: vec![],
            schema_type: Some("reddit".to_string()),
            images: vec![],
            internal_links: vec![],
            external_links: vec![],
            favicon: None,
            rss_url: None,
            normalized_text: collapsed,
            fetched_at: Utc::now(),
            fetch_duration_ms,
            html_size_bytes: 0,
            encoding: None,
            ssl_valid: true,
            redirect_count: 0,
            content_type: "reddit/json".to_string(),
            content_type_header: "application/json".to_string(),
            is_paywalled: false,
            is_valid_content: word_count >= MIN_VALID_WORDS,
            entities: extract_entities(text),
        })
    }

    /// Extract StructuredContent from raw HTML.
    #[allow(clippy::too_many_arguments)]
    pub fn extract_from_html(
        &self,
        html: &str,
        original_url: &str,
        final_url: &str,
        status_code: u16,
        ssl_valid: bool,
        fetch_duration_ms: u64,
        last_modified: Option<&str>,
    ) -> Result<StructuredContent> {
        let doc = Html::parse_document(html);

        // 1. Readability extraction
        let article = Readability::new(html, None)
            .ok()
            .and_then(|mut r| r.parse());

        // 1b. Documentation & official site semantic container extraction
        let doc_container = extract_doc_container(&doc);

        // Fallback body text when readability and doc container cannot isolate the article.
        let fallback_text = doc_container
            .as_ref()
            .map(|(_, text)| text.clone())
            .unwrap_or_else(|| fallback_body_text(&doc));

        // 2. Metadata from <head>
        let meta = extract_meta(&doc);

        // 3. OpenGraph
        let open_graph = extract_open_graph(&doc);

        // 4. Twitter Card
        let twitter_card = extract_twitter_card(&doc);

        // 5. JSON-LD
        let json_ld = extract_json_ld(&doc);

        // 6. Links
        let (internal_links, external_links) = extract_links(&doc, final_url);

        // 7. Images
        let images = extract_images(&doc);

        // 8. Title: readability > meta > og > h1 > URL path
        let h1_title = extract_h1(&doc);
        let title = article
            .as_ref()
            .and_then(|a| a.title.clone())
            .or_else(|| meta.title.clone())
            .or_else(|| open_graph.as_ref().and_then(|og| og.title.clone()))
            .or(h1_title)
            .unwrap_or_else(|| title_from_url(final_url))
            .trim()
            .to_string();

        // 9. Content text: Prefer doc container if it has substantial content, else readability article, else fallback
        let content_text = if let Some((_, ref dt)) = doc_container
            && dt.len() >= 100
        {
            dt.clone()
        } else {
            article
                .as_ref()
                .and_then(|a| a.text_content.clone())
                .filter(|t| !t.trim().is_empty())
                .unwrap_or(fallback_text)
        };

        // 10. Content HTML — cleaned for AI agent consumption
        let raw_html = if let Some((ref dh, _)) = doc_container
            && !dh.trim().is_empty()
        {
            dh.clone()
        } else {
            article
                .as_ref()
                .and_then(|a| a.content.clone())
                .unwrap_or_else(|| format!("<div>{}</div>", html_escape(&content_text)))
        };
        let content_html = clean_content_html(&raw_html);

        // 11. Excerpt
        let excerpt = article
            .as_ref()
            .and_then(|a| a.excerpt.clone())
            .or_else(|| meta.description.clone())
            .or_else(|| open_graph.as_ref().and_then(|og| og.description.clone()))
            .unwrap_or_default();

        // 12. Author
        let author = article
            .as_ref()
            .and_then(|a| a.byline.clone())
            .or_else(|| open_graph.as_ref().and_then(|og| og.article_author.clone()))
            .or_else(|| meta.author.clone());

        // 13. Site name
        let site_name = article
            .as_ref()
            .and_then(|a| a.site_name.clone())
            .or_else(|| open_graph.as_ref().and_then(|og| og.site_name.clone()))
            .or_else(|| extract_site_name(final_url));

        // 14. Language detection
        let (language, language_confidence) = detect_language(&content_text, html);

        // 15. Published date (priority: readability > JSON-LD > og > meta)
        let published_at = article
            .as_ref()
            .and_then(|a| a.published_time.clone())
            .or_else(|| {
                open_graph
                    .as_ref()
                    .and_then(|og| og.article_published_time.clone())
            })
            .or_else(|| meta.published_time.clone())
            .as_deref()
            .and_then(parse_date);

        // 16. Modified date
        let modified_at = open_graph
            .as_ref()
            .and_then(|og| og.article_modified_time.clone())
            .or_else(|| meta.modified_time.clone())
            .or_else(|| meta.updated_time.clone())
            .or_else(|| meta.last_modified.clone())
            .or_else(|| last_modified.map(String::from))
            .as_deref()
            .and_then(parse_date);

        // 17. Content metrics
        let word_count = count_words(&content_text);
        let char_count = content_text.len() as u32;
        let sentence_count = count_sentences(&content_text);
        let reading_time_seconds = estimate_reading_time(word_count);
        let reading_ease = textstat::flesch_reading_ease(&content_text);
        let grade_level = textstat::flesch_kincaid_grade(&content_text);

        // 18. Keywords
        let keywords = extract_keywords(&content_text, 15);

        // 19. Normalize text
        let normalized_text = normalize_text(&content_text);

        // 19b. Entity extraction — regex + DOM AST (code blocks, tables, headings).
        let mut entities = extract_entities(&content_text);
        let (code_blocks, tables, headings) = extract_dom_entities(&doc);
        entities.code_blocks = code_blocks;
        entities.tables = tables;
        entities.headings = headings;

        // 19c. Quality entities — heading-contextual examples/FAQs/steps/callouts
        // plus diagram reclassification. Relevance-sized, never char-capped.
        let (examples, faqs, steps, callouts) = extract_quality_entities(&doc);
        entities.examples = examples;
        entities.faqs = faqs;
        entities.steps = steps;
        entities.callouts = callouts;
        entities.diagrams = classify_diagrams(&entities.code_blocks, &images);

        // Pre-compute values before moves
        let content_markdown = html_to_markdown(&content_html);
        let is_valid = !content_text.is_empty() && word_count >= MIN_VALID_WORDS;

        // 20. Schema type from JSON-LD
        let schema_type = json_ld
            .iter()
            .find_map(|v| v.get("@type").and_then(|t| t.as_str()).map(String::from));

        // 21. Paywall detection
        let is_paywalled = detect_paywall(html);

        // 22. Favicon
        let favicon = extract_favicon(&doc);

        // 23. RSS
        let rss_url = extract_rss(&doc);

        // 24. Encoding from content type (if we had headers)
        let encoding: Option<String> = None;

        Ok(StructuredContent {
            url: original_url.to_string(),
            final_url: final_url.to_string(),
            status_code,
            title,
            description: meta.description,
            canonical_url: meta.canonical_url,
            language,
            language_confidence,
            published_at,
            modified_at,
            author,
            site_name,
            content_text,
            content_html,
            content_markdown,
            excerpt,
            word_count,
            char_count,
            sentence_count,
            reading_time_seconds,
            reading_ease,
            grade_level,
            keywords,
            open_graph,
            twitter_card,
            json_ld,
            schema_type,
            images,
            internal_links,
            external_links,
            favicon,
            rss_url,
            normalized_text,
            fetched_at: Utc::now(),
            fetch_duration_ms,
            html_size_bytes: html.len() as u64,
            encoding,
            ssl_valid,
            redirect_count: 0,
            content_type: String::new(),
            content_type_header: String::new(),
            is_paywalled,
            is_valid_content: is_valid,
            entities,
        })
    }

    /// Fast-path probe for machine-readable manifests (`/llms-full.txt`, `/llms.txt`, `/.well-known/ai-catalog.json`).
    async fn try_manifest_fast_path(&self, parsed_url: &url::Url) -> Option<StructuredContent> {
        let origin = match (parsed_url.scheme(), parsed_url.host_str(), parsed_url.port()) {
            (scheme, Some(host), Some(port)) => format!("{scheme}://{host}:{port}"),
            (scheme, Some(host), None) => format!("{scheme}://{host}"),
            _ => return None,
        };

        // Ordered priority: /llms-full.txt (complete corpus), /llms.txt (curated), /llm.txt, /.well-known/ai-catalog.json, /ai-catelog.json
        let candidates = [
            format!("{origin}/llms-full.txt"),
            format!("{origin}/llms.txt"),
            format!("{origin}/llm.txt"),
            format!("{origin}/.well-known/ai-catalog.json"),
            format!("{origin}/.well-known/ai-catelog.json"),
            format!("{origin}/ai-catalog.json"),
            format!("{origin}/ai-catelog.json"),
        ];

        for candidate_url in candidates {
            let start = Instant::now();
            let resp = self
                .client
                .get(&candidate_url)
                .timeout(std::time::Duration::from_millis(1800))
                .send()
                .await;

            if let Ok(response) = resp
                && response.status().is_success()
                && let Ok(body) = response.text().await
            {
                let trimmed = body.trim();
                    if !trimmed.is_empty()
                        && !trimmed.to_ascii_lowercase().starts_with("<!doctype html")
                        && !trimmed.to_ascii_lowercase().starts_with("<html")
                    {
                        let elapsed = start.elapsed().as_millis() as u64;
                        let kind = if candidate_url.ends_with(".json") {
                            ContentKind::Json
                        } else {
                            ContentKind::Text
                        };
                        let mut sc = extract_from_text(
                            &body,
                            parsed_url.as_str(),
                            &candidate_url,
                            200,
                            candidate_url.starts_with("https://"),
                            elapsed,
                            None,
                            kind,
                        )
                        .ok()?;
                        sc.content_type = "text/markdown; manifest=llms.txt".to_string();
                        sc.content_type_header = "text/plain".to_string();
                        sc.is_valid_content = true;
                        return Some(sc);
                    }
            }
        }
        None
    }
}

// ── Content-Type classification ───────────────────────────────────────────

#[derive(Debug, Clone, Copy, PartialEq, Eq)]
enum ContentKind {
    Html,
    Text,
    Json,
    Xml,
    Binary,
}

/// Classify a Content-Type header into a broad extraction strategy.
fn parse_content_type_header(ct: Option<&str>) -> (ContentKind, Option<String>) {
    let raw = ct.unwrap_or("text/html").to_lowercase();
    let mut parts = raw.split(';');
    let mime = parts.next().unwrap_or("text/html").trim();
    let charset = parts.find_map(|p| {
        let p = p.trim();
        p.strip_prefix("charset=")
            .or_else(|| p.strip_prefix("charset ="))
            .map(|s| s.trim_matches(['"', '\''].as_slice()).to_string())
    });

    let kind = if mime == "text/html" || mime == "application/xhtml+xml" {
        ContentKind::Html
    } else if mime.starts_with("text/") {
        ContentKind::Text
    } else if mime == "application/json" || mime == "application/ld+json" || mime.ends_with("+json")
    {
        ContentKind::Json
    } else if mime == "application/xml"
        || mime == "application/atom+xml"
        || mime == "application/rss+xml"
        || mime == "application/rdf+xml"
        || mime == "text/xml"
    {
        ContentKind::Xml
    } else if mime.starts_with("image/")
        || mime.starts_with("audio/")
        || mime.starts_with("video/")
        || mime == "application/pdf"
        || mime == "application/zip"
        || mime == "application/gzip"
        || mime == "application/x-gzip"
        || mime == "application/x-tar"
        || mime == "application/x-bzip2"
        || mime == "application/x-7z-compressed"
        || mime == "application/x-rar-compressed"
        || mime == "application/msword"
        || mime == "application/vnd.openxmlformats-officedocument"
        || mime == "application/octet-stream"
    {
        ContentKind::Binary
    } else {
        // Unknown types are treated as HTML; servers frequently mislabel.
        ContentKind::Html
    };

    (kind, charset)
}

/// Extract useful text from plain text, JSON, or XML/feed responses.
#[allow(clippy::too_many_arguments)]
fn extract_from_text(
    body: &str,
    original_url: &str,
    final_url: &str,
    status_code: u16,
    ssl_valid: bool,
    fetch_duration_ms: u64,
    last_modified: Option<&str>,
    kind: ContentKind,
) -> Result<StructuredContent> {
    let collapsed = normalize_text(body);
    let title = if kind == ContentKind::Json {
        // Try to pull a top-level "title" field without fully parsing.
        serde_json::from_str::<Value>(body)
            .ok()
            .and_then(|v| v.get("title").and_then(|t| t.as_str()).map(String::from))
            .or_else(|| {
                collapsed
                    .lines()
                    .find(|l| !l.trim().is_empty())
                    .map(|l| l.trim().chars().take(120).collect())
            })
    } else {
        collapsed
            .lines()
            .find(|l| !l.trim().is_empty())
            .map(|l| l.trim().chars().take(120).collect())
    }
    .unwrap_or_else(|| title_from_url(final_url));

    let excerpt = collapsed.chars().take(300).collect::<String>();
    let word_count = count_words(&collapsed);
    let sentence_count = count_sentences(&collapsed);
    let reading_time_seconds = estimate_reading_time(word_count);
    let reading_ease = textstat::flesch_reading_ease(&collapsed);
    let grade_level = textstat::flesch_kincaid_grade(&collapsed);
    let keywords = extract_keywords(&collapsed, 15);
    let (language, language_confidence) = detect_language(&collapsed, body);

    let content_markdown = if kind == ContentKind::Json {
        format!("```json\n{}\n```", body.trim())
    } else if kind == ContentKind::Xml {
        format!("```xml\n{}\n```", body.trim())
    } else {
        collapsed.clone()
    };

    let modified_at = last_modified.and_then(parse_date);

    Ok(StructuredContent {
        url: original_url.to_string(),
        final_url: final_url.to_string(),
        status_code,
        title,
        description: None,
        canonical_url: None,
        language,
        language_confidence,
        published_at: None,
        modified_at,
        author: None,
        site_name: extract_site_name(final_url),
        content_text: collapsed.clone(),
        content_html: format!("<pre>{}</pre>", html_escape(&collapsed)),
        content_markdown,
        excerpt,
        word_count,
        char_count: collapsed.len() as u32,
        sentence_count,
        reading_time_seconds,
        reading_ease,
        grade_level,
        keywords,
        open_graph: None,
        twitter_card: None,
        json_ld: vec![],
        schema_type: if kind == ContentKind::Json {
            Some("json".to_string())
        } else if kind == ContentKind::Xml {
            Some("xml".to_string())
        } else {
            None
        },
        images: vec![],
        internal_links: vec![],
        external_links: vec![],
        favicon: None,
        rss_url: None,
        normalized_text: collapsed,
        fetched_at: Utc::now(),
        fetch_duration_ms,
        html_size_bytes: body.len() as u64,
        encoding: None,
        ssl_valid,
        redirect_count: 0,
        content_type: String::new(),
        content_type_header: String::new(),
        is_paywalled: false,
        is_valid_content: word_count >= MIN_VALID_WORDS,
        entities: extract_entities(body),
    })
}

/// Build a minimal invalid record for binary or otherwise un-indexable URLs.
fn make_invalid_content(
    original_url: &str,
    final_url: &str,
    status_code: u16,
    ssl_valid: bool,
    fetch_duration_ms: u64,
    ct_header: Option<String>,
) -> StructuredContent {
    StructuredContent {
        url: original_url.to_string(),
        final_url: final_url.to_string(),
        status_code,
        title: title_from_url(final_url),
        description: ct_header.clone(),
        canonical_url: None,
        language: "en".to_string(),
        language_confidence: 0.0,
        published_at: None,
        modified_at: None,
        author: None,
        site_name: extract_site_name(final_url),
        content_text: String::new(),
        content_html: String::new(),
        content_markdown: String::new(),
        excerpt: ct_header
            .clone()
            .unwrap_or_else(|| "Binary or unsupported content type".to_string()),
        word_count: 0,
        char_count: 0,
        sentence_count: 0,
        reading_time_seconds: 0,
        reading_ease: 0.0,
        grade_level: 0.0,
        keywords: vec![],
        open_graph: None,
        twitter_card: None,
        json_ld: vec![],
        schema_type: Some("binary".to_string()),
        images: vec![],
        internal_links: vec![],
        external_links: vec![],
        favicon: None,
        rss_url: None,
        normalized_text: String::new(),
        fetched_at: Utc::now(),
        fetch_duration_ms,
        html_size_bytes: 0,
        encoding: None,
        ssl_valid,
        redirect_count: 0,
        content_type: String::new(),
        content_type_header: ct_header.unwrap_or_default(),
        is_paywalled: false,
        is_valid_content: false,
        entities: Entities::default(),
    }
}

// ── Meta extraction ─────────────────────────────────────────────────────────

struct PageMeta {
    title: Option<String>,
    description: Option<String>,
    canonical_url: Option<String>,
    published_time: Option<String>,
    modified_time: Option<String>,
    updated_time: Option<String>,
    last_modified: Option<String>,
    author: Option<String>,
}

fn extract_meta(doc: &Html) -> PageMeta {
    let mut title = None;
    let mut description = None;
    let mut canonical_url = None;
    let mut published_time = None;
    let mut modified_time = None;
    let mut updated_time = None;
    let mut last_modified = None;
    let mut author = None;

    // <title>
    if let Ok(sel) = Selector::parse("title")
        && let Some(el) = doc.select(&sel).next() {
            let t: String = el.text().collect::<Vec<_>>().join("").trim().to_string();
            if !t.is_empty() {
                title = Some(t);
            }
        }

    // <meta> tags
    if let Ok(sel) = Selector::parse("meta[name],meta[property]") {
        for el in doc.select(&sel) {
            let name = el
                .value()
                .attr("name")
                .or_else(|| el.value().attr("property"));
            let content = el.value().attr("content");
            if let (Some(name), Some(content)) = (name, content) {
                match name {
                    "description" => {
                        if description.is_none() {
                            description = Some(content.to_string());
                        }
                    }
                    "author" => author = Some(content.to_string()),
                    "article:published_time"
                    | "datePublished"
                    | "publish_date"
                    | "date"
                    | "publish-date" => {
                        if published_time.is_none() {
                            published_time = Some(content.to_string());
                        }
                    }
                    "article:modified_time"
                    | "dateModified"
                    | "modified_time"
                    | "modified-date" => {
                        if modified_time.is_none() {
                            modified_time = Some(content.to_string());
                        }
                    }
                    "updated_time" | "dateUpdated" | "updated-date" => {
                        if updated_time.is_none() {
                            updated_time = Some(content.to_string());
                        }
                    }
                    "last-modified" | "last_modified"
                        if last_modified.is_none() => {
                            last_modified = Some(content.to_string());
                        }
                    _ => {}
                }
            }
        }
    }

    // <link rel="canonical">
    if let Ok(sel) = Selector::parse("link[rel=canonical]")
        && let Some(el) = doc.select(&sel).next()
            && let Some(href) = el.value().attr("href") {
                canonical_url = Some(href.to_string());
            }

    PageMeta {
        title,
        description,
        canonical_url,
        published_time,
        modified_time,
        updated_time,
        last_modified,
        author,
    }
}

// ── OpenGraph ───────────────────────────────────────────────────────────────

fn extract_open_graph(doc: &Html) -> Option<OpenGraph> {
    let mut og = OpenGraph {
        title: None,
        r#type: None,
        image: None,
        url: None,
        description: None,
        site_name: None,
        locale: None,
        article_author: None,
        article_published_time: None,
        article_modified_time: None,
        article_section: None,
        article_tags: vec![],
    };

    if let Ok(sel) = Selector::parse("meta[property^=\"og:\"]") {
        for el in doc.select(&sel) {
            let prop = el.value().attr("property")?;
            let content = el.value().attr("content")?;
            match prop {
                "og:title" => og.title = Some(content.to_string()),
                "og:type" => og.r#type = Some(content.to_string()),
                "og:image" => og.image = Some(content.to_string()),
                "og:url" => og.url = Some(content.to_string()),
                "og:description" => og.description = Some(content.to_string()),
                "og:site_name" => og.site_name = Some(content.to_string()),
                "og:locale" => og.locale = Some(content.to_string()),
                "article:author" => og.article_author = Some(content.to_string()),
                "article:published_time" => og.article_published_time = Some(content.to_string()),
                "article:modified_time" => og.article_modified_time = Some(content.to_string()),
                "article:section" => og.article_section = Some(content.to_string()),
                "article:tag" => og.article_tags.push(content.to_string()),
                _ => {}
            }
        }
    }

    if og.title.is_some() || og.description.is_some() {
        Some(og)
    } else {
        None
    }
}

// ── Twitter Card ────────────────────────────────────────────────────────────

fn extract_twitter_card(doc: &Html) -> Option<TwitterCard> {
    let mut tc = TwitterCard {
        card: None,
        site: None,
        creator: None,
        title: None,
        description: None,
        image: None,
    };

    if let Ok(sel) = Selector::parse("meta[name^=\"twitter:\"]") {
        for el in doc.select(&sel) {
            let name = el.value().attr("name")?;
            let content = el.value().attr("content")?;
            match name {
                "twitter:card" => tc.card = Some(content.to_string()),
                "twitter:site" => tc.site = Some(content.to_string()),
                "twitter:creator" => tc.creator = Some(content.to_string()),
                "twitter:title" => tc.title = Some(content.to_string()),
                "twitter:description" => tc.description = Some(content.to_string()),
                "twitter:image" => tc.image = Some(content.to_string()),
                _ => {}
            }
        }
    }

    if tc.card.is_some() || tc.title.is_some() {
        Some(tc)
    } else {
        None
    }
}

// ── JSON-LD ─────────────────────────────────────────────────────────────────

fn extract_json_ld(doc: &Html) -> Vec<serde_json::Value> {
    let mut results = vec![];
    if let Ok(sel) = Selector::parse("script[type=\"application/ld+json\"]") {
        for el in doc.select(&sel) {
            let text: String = el.text().collect();
            if let Ok(val) = serde_json::from_str::<serde_json::Value>(&text) {
                results.push(val);
            }
        }
    }
    results
}

// ── Links ───────────────────────────────────────────────────────────────────

fn extract_links(doc: &Html, base_url: &str) -> (Vec<String>, Vec<String>) {
    let base_domain = extract_domain(base_url);
    let mut internal = vec![];
    let mut external = vec![];
    let mut seen = HashSet::new();

    if let Ok(sel) = Selector::parse("a[href]") {
        for el in doc.select(&sel) {
            if let Some(href) = el.value().attr("href") {
                let href = href.trim();
                if href.is_empty() || href.starts_with('#') || href.starts_with("javascript:") {
                    continue;
                }
                if !seen.insert(href) {
                    continue;
                }

                let full_url = if href.starts_with("http") {
                    href.to_string()
                } else if href.starts_with("//") {
                    format!("https:{}", href)
                } else if href.starts_with('/') {
                    if let Some(pos) = base_url.find("://") {
                        let scheme_end = pos + 3;
                        if let Some(slash_pos) = base_url[scheme_end..].find('/') {
                            format!("{}{}", &base_url[..scheme_end + slash_pos], href)
                        } else {
                            format!("{}{}", base_url, href)
                        }
                    } else {
                        continue;
                    }
                } else {
                    continue;
                };

                let link_domain = extract_domain(&full_url);
                if link_domain == base_domain || link_domain.is_empty() {
                    internal.push(full_url);
                } else {
                    external.push(full_url);
                }
            }
        }
    }

    (internal, external)
}

// ── Images ──────────────────────────────────────────────────────────────────

fn extract_images(doc: &Html) -> Vec<ImageInfo> {
    let mut images = vec![];
    if let Ok(sel) = Selector::parse("img[src]") {
        for el in doc.select(&sel) {
            if let Some(src) = el.value().attr("src") {
                let alt = el.value().attr("alt").map(String::from);
                let width = el.value().attr("width").and_then(|w| w.parse::<u32>().ok());
                let height = el
                    .value()
                    .attr("height")
                    .and_then(|h| h.parse::<u32>().ok());
                images.push(ImageInfo {
                    url: src.to_string(),
                    alt,
                    width,
                    height,
                });
            }
        }
    }
    images
}

// ── Favicon ─────────────────────────────────────────────────────────────────

fn extract_favicon(doc: &Html) -> Option<String> {
    if let Ok(sel) = Selector::parse("link[rel~=\"icon\"]")
        && let Some(el) = doc.select(&sel).next()
            && let Some(href) = el.value().attr("href") {
                return Some(href.to_string());
            }
    Some("/favicon.ico".to_string())
}

// ── RSS ─────────────────────────────────────────────────────────────────────

fn extract_rss(doc: &Html) -> Option<String> {
    if let Ok(sel) = Selector::parse("link[type=\"application/rss+xml\"]")
        && let Some(el) = doc.select(&sel).next()
            && let Some(href) = el.value().attr("href") {
                return Some(href.to_string());
            }
    None
}

// ── Language Detection ──────────────────────────────────────────────────────

fn detect_language(text: &str, html: &str) -> (String, f64) {
    if let Some(info) = whatlang::detect(text) {
        let lang = info.lang().code().to_string();
        let conf = info.confidence();
        return (lang, conf);
    }

    // Fallback: check html lang attribute
    let doc = Html::parse_document(html);
    if let Ok(sel) = Selector::parse("html[lang]")
        && let Some(el) = doc.select(&sel).next()
            && let Some(lang) = el.value().attr("lang") {
                let code = lang.split('-').next().unwrap_or("en").to_string();
                return (code, 0.5);
            }

    ("en".to_string(), 0.3)
}

// ── Date Parsing ────────────────────────────────────────────────────────────

fn parse_date(s: &str) -> Option<DateTime<Utc>> {
    dateparser::parse(s).ok()
}

// ── Keyword Extraction ──────────────────────────────────────────────────────

pub(crate) fn extract_keywords(text: &str, max: usize) -> Vec<Keyword> {
    if text.len() < 100 {
        return vec![];
    }

    let stop_words: Vec<String> = stop_words::get(stop_words::LANGUAGE::English)
        .into_iter()
        .collect();

    let rake = keyword_extraction::rake::Rake::new(
        keyword_extraction::rake::RakeParams::WithDefaults(text, &stop_words),
    );

    let ranked = rake.get_ranked_keyword_scores(max);

    ranked
        .into_iter()
        .enumerate()
        .map(|(i, (word, score))| Keyword {
            text: word,
            tfidf_score: score as f64,
            rank: (i + 1) as u32,
        })
        .collect()
}

// ── Word / Sentence Counting ───────────────────────────────────────────────

pub(crate) fn count_words(text: &str) -> u32 {
    text.unicode_words().count() as u32
}

pub(crate) fn count_sentences(text: &str) -> u32 {
    text.unicode_sentences().count() as u32
}

pub fn estimate_reading_time(word_count: u32) -> u32 {
    (word_count as f64 / 238.0 * 60.0).ceil() as u32
}

// ── Text Normalization ──────────────────────────────────────────────────────

pub(crate) fn normalize_text(text: &str) -> String {
    use unicode_normalization::UnicodeNormalization;
    text.nfc()
        .collect::<String>()
        .split_whitespace()
        .collect::<Vec<_>>()
        .join(" ")
}

fn html_to_markdown(html: &str) -> String {
    // Large width prevents artificial line breaks across code blocks and long links
    let raw_md = html2text::from_read(html.as_bytes(), 10_000).unwrap_or_default();
    purge_doc_boilerplate(&raw_md)
}

// ── Regex entity extraction ───────────────────────────────────────────────
//
// Deterministic, zero-cost entity extraction (emails, phones, addresses, ...)
// applied to the *clean* page text inside the fetch pipeline. This gives the
// LLM machine-checkable facts instead of raw text and avoids paying tokens or
// risking nondeterminism by asking a model to extract them. Regexes are
// compiled once and reused via std LazyLock.

/// Emails: standard local@domain[.tld], tolerant of a trailing period.
static RE_EMAIL: LazyLock<regex::Regex> = LazyLock::new(|| {
    regex::Regex::new(r"(?i)\b[a-z0-9._%+-]+@[a-z0-9.-]+\.[a-z]{2,}\b").expect("valid email regex")
});

/// Phone numbers: E.164/US-centric — optional +country, then 7-15 digits
/// separated by space/dash/dot. Requires at least 7 digits total so a date
/// like `2017-05-13` (fewer digits) is not misclassified as a phone.
static RE_PHONE: LazyLock<regex::Regex> = LazyLock::new(|| {
    regex::Regex::new(r"(?:\+?\d{1,3}[\s\-\.]?)?\(?\d{3}\)?[\s\-\.]?\d{3}[\s\-\.]\d{4}\b")
        .expect("valid phone regex")
});

/// Addresses: a leading number + street words, then a street suffix, then an
/// optional city / state / ZIP tail (e.g. "Austin, TX 78701").
static RE_ADDRESS: LazyLock<regex::Regex> = LazyLock::new(|| {
    regex::Regex::new(
        r"(?i)\b\d{1,6}\s+(?:[a-z0-9]+\.?[\s\-']+){1,6}(?:street|st|avenue|ave|road|rd|boulevard|blvd|lane|ln|drive|dr|court|ct|way|place|pl|highway|hwy|parkway|pkwy|square|sq|suite|ste)\b[,\s]*(?:[a-z][a-z\s.]*?)?(?:[,]\s*[a-z]{2})?\s*\d{5}(?:-\d{4})?",
    )
    .expect("valid address regex")
});

/// URLs/URIs: http(s), ftp, mailto, www, or a bare domain with a path.
/// Uses `[^\s<>]` (no quote chars in the class) so it stays a valid raw string.
static RE_URL: LazyLock<regex::Regex> = LazyLock::new(|| {
    regex::Regex::new(
        r"(?i)\b(?:https?://|ftp://|mailto:|www\.)[^\s<>]+|(?:[a-z0-9](?:[a-z0-9-]*[a-z0-9])?\.)+(?:com|org|net|io|ai|dev|app|co|edu|gov|info|me|us|uk|ca|de|fr|in)(?:/[^\s<>]*)?",
    )
    .expect("valid url regex")
});

/// Prices: currency symbol then digits (with optional cents), or digits then symbol.
static RE_PRICE: LazyLock<regex::Regex> = LazyLock::new(|| {
    regex::Regex::new(
        r"(?:\$\s?\d{1,3}(?:,\d{3})*(?:\.\d{2})?|(?:\d{1,3}(?:,\d{3})*(?:\.\d{2})?)\s?(?:USD|EUR|GBP|€|£))",
    )
    .expect("valid price regex")
});

/// Dates: ISO 8601, and common human dates (Month D, YYYY; D Month YYYY; MM/DD/YYYY).
static RE_DATE: LazyLock<regex::Regex> = LazyLock::new(|| {
    regex::Regex::new(
        r"(?ix)\b\d{4}-\d{1,2}-\d{1,2}\b|\b(?:jan|feb|mar|apr|may|jun|jul|aug|sep|oct|nov|dec)[a-z]*[\s.,]+\d{1,2}[\s.,]*(?:,?\s*\d{2,4})?\b|\b\d{1,2}[/-]\d{1,2}[/-]\d{2,4}\b",
    )
    .expect("valid date regex")
});

/// IPv4 / IPv6 addresses.
static RE_IP: LazyLock<regex::Regex> = LazyLock::new(|| {
    regex::Regex::new(
        r"(?:\b(?:25[0-5]|2[0-4][0-9]|1?[0-9][0-9]?)(?:\.(?:25[0-5]|2[0-4][0-9]|1?[0-9][0-9]?)){3}\b|(?i)\b(?:[a-f0-9]{1,4}:){2,7}[a-f0-9]{1,4}\b)",
    )
    .expect("valid ip regex")
});

/// Social handles: @username for Twitter/X, GitHub, etc. Must be preceded by
/// whitespace or line start so an email local-part (`name@example.com`) is not
/// mistaken for a handle.
static RE_SOCIAL: LazyLock<regex::Regex> = LazyLock::new(|| {
    regex::Regex::new(r"(?:\A|\s)(@[a-zA-Z0-9_]{3,30})\b").expect("valid social regex")
});

/// Extract structured entities from cleaned page text.
/// Version strings like v1.2.3, 0.3.4, 2.0.0-beta.
static RE_VERSION: LazyLock<regex::Regex> = LazyLock::new(|| {
    regex::Regex::new(r"\b(?:v\d+\.\d+(?:\.\d+)?|\b\d+\.\d+\.\d+(?:-[a-zA-Z0-9.]+)?)\b")
        .expect("valid version regex")
});

/// Software licenses like MIT, Apache-2.0, BSD-3-Clause, GPLv3, MPL-2.0.
static RE_LICENSE: LazyLock<regex::Regex> = LazyLock::new(|| {
    regex::Regex::new(
        r"(?i)\b(?:MIT|Apache[\s\-]2(?:\.0)?|BSD[\s\-]?(?:2|3)[\s\-]Clause|GPLv[23]|AGPLv[23]|LGPLv[23]|MPL[\s\-]2(?:\.0)?|ISC|Unlicense|CC0)\b",
    )
    .expect("valid license regex")
});

/// Math formulas in TeX delimiters: `$$...$$`, `\(...\)`, `\[...\]`.
/// Single `$...$` is deliberately excluded: it collides with prices ($19.99).
static RE_MATH: LazyLock<regex::Regex> = LazyLock::new(|| {
    regex::Regex::new(r"\$\$.+?\$\$|\\\([^\)]+?\\\)|\\\[[^\]]+?\\\]")
        .expect("valid math regex")
});

/// Extract structured entities from cleaned page text.
pub(crate) fn extract_entities(text: &str) -> Entities {
    fn unique(v: Vec<String>) -> Vec<String> {
        let mut seen = std::collections::HashSet::new();
        v.into_iter()
            .filter(|s| !s.trim().is_empty())
            .filter(|s| seen.insert(s.to_lowercase()))
            .collect()
    }

    let emails = unique(
        RE_EMAIL
            .find_iter(text)
            .map(|m| m.as_str().trim_end_matches('.').to_string())
            .collect(),
    );
    let phones = unique(
        RE_PHONE
            .find_iter(text)
            .map(|m| m.as_str().trim().to_string())
            .collect(),
    );
    let addresses = unique(
        RE_ADDRESS
            .find_iter(text)
            .map(|m| m.as_str().trim().to_string())
            .collect(),
    );
    let urls = unique(
        RE_URL
            .find_iter(text)
            .map(|m| {
                m.as_str()
                    .trim_end_matches(['.', ')', ';', ',', '"', '\'', '>', ']'].as_slice())
                    .to_string()
            })
            .collect(),
    );
    let repository_urls = urls
        .iter()
        .filter(|u| {
            u.contains("github.com/")
                || u.contains("gitlab.com/")
                || u.contains("crates.io/crates/")
                || u.contains("npmjs.com/package/")
        })
        .cloned()
        .collect();

    let package_versions = unique(
        RE_VERSION
            .find_iter(text)
            .map(|m| m.as_str().trim().to_string())
            .collect(),
    );
    let licenses = unique(
        RE_LICENSE
            .find_iter(text)
            .map(|m| m.as_str().trim().to_string())
            .collect(),
    );

    let prices = unique(
        RE_PRICE
            .find_iter(text)
            .map(|m| m.as_str().trim().to_string())
            .collect(),
    );
    let dates = unique(
        RE_DATE
            .find_iter(text)
            .map(|m| m.as_str().trim().to_string())
            .collect(),
    );
    let ip_addresses = unique(
        RE_IP
            .find_iter(text)
            .map(|m| m.as_str().trim().to_string())
            .collect(),
    );
    let social_handles = unique(
        RE_SOCIAL
            .captures_iter(text)
            .filter_map(|c| c.get(1).map(|m| m.as_str().trim().to_string()))
            .collect(),
    );
    let maths = {
        let mut seen = std::collections::HashSet::new();
        RE_MATH
            .find_iter(text)
            .filter_map(|m| {
                let raw = m.as_str().trim().to_string();
                if raw.len() < 6 || !seen.insert(raw.to_lowercase()) {
                    return None;
                }
                let display = raw.starts_with("$$") || raw.starts_with("\\[");
                // Parenthesized `\(...\)` needs an operator, otherwise it is
                // prose in brackets (false-positive guard). Display forms are
                // unambiguous by delimiter alone.
                if !display
                    && !raw.contains(['=', '^', '_', '+', '*', '/', '\\'])
                {
                    return None;
                }
                Some(MathEntity { tex: raw, display })
            })
            .take(200)
            .collect()
    };

    let parsed_examples = crate::engine::ranker::Ranker::extract_examples(text);
    let code_blocks = parsed_examples
        .iter()
        .map(|e| CodeBlockEntity {
            language: e.language.clone(),
            code: e.snippet.clone(),
            line_count: e.snippet.lines().count(),
        })
        .collect();
    let examples = parsed_examples
        .into_iter()
        .map(|e| ExampleEntity {
            context_heading: e.context,
            language: e.language,
            code: e.snippet,
        })
        .collect();

    Entities {
        code_blocks,
        tables: Vec::new(),
        headings: Vec::new(),
        diagrams: Vec::new(),
        examples,
        faqs: Vec::new(),
        steps: Vec::new(),
        maths,
        callouts: Vec::new(),
        package_versions,
        licenses,
        repository_urls,
        emails,
        phones,
        addresses,
        urls,
        prices,
        dates,
        ip_addresses,
        social_handles,
    }
}

/// Syntax language from `language-*` / `lang-*` classes on a code element.
fn block_language(elem: scraper::ElementRef<'_>, fallback: scraper::ElementRef<'_>) -> Option<String> {
    elem.value()
        .attr("class")
        .or_else(|| fallback.value().attr("class"))
        .and_then(|classes| {
            classes.split_whitespace().find_map(|c| {
                c.strip_prefix("language-")
                    .or_else(|| c.strip_prefix("lang-"))
                    .map(String::from)
            })
        })
}

/// Extract structured technical entities from the DOM AST (code blocks, tables, headings).
pub(crate) fn extract_dom_entities(doc: &scraper::Html) -> (Vec<CodeBlockEntity>, Vec<TableEntity>, Vec<HeadingEntity>) {
    let mut code_blocks = Vec::new();
    let mut tables = Vec::new();
    let mut headings = Vec::new();

    // 1. Extract code blocks with syntax languages
    if let Ok(pre_sel) = Selector::parse("pre") {
        let code_child_sel = Selector::parse("code").ok();
        for elem in doc.select(&pre_sel) {
            let code_child = code_child_sel.as_ref().and_then(|sel| elem.select(sel).next());
            let target_elem = code_child.unwrap_or(elem);
            let code_text = target_elem.text().collect::<Vec<_>>().join("");
            let trimmed = code_text.trim();
            if trimmed.is_empty() || trimmed.len() < 10 {
                continue;
            }
            let language = block_language(target_elem, elem);
            let line_count = trimmed.lines().count();
            code_blocks.push(CodeBlockEntity {
                language,
                code: trimmed.to_string(),
                line_count,
            });
            if code_blocks.len() >= 50 {
                break;
            }
        }
    }

    // 2. Extract tables
    if let (Ok(table_sel), Ok(th_sel), Ok(tr_sel), Ok(td_sel), Ok(caption_sel)) = (
        Selector::parse("table"),
        Selector::parse("th"),
        Selector::parse("tr"),
        Selector::parse("td"),
        Selector::parse("caption"),
    ) {
        for table in doc.select(&table_sel) {
            let caption = table
                .select(&caption_sel)
                .next()
                .map(|c| c.text().collect::<Vec<_>>().join(" ").trim().to_string());
            let headers: Vec<String> = table
                .select(&th_sel)
                .map(|th| th.text().collect::<Vec<_>>().join(" ").trim().to_string())
                .filter(|h| !h.is_empty())
                .collect();
            let mut rows = Vec::new();
            for row in table.select(&tr_sel) {
                let cells: Vec<String> = row
                    .select(&td_sel)
                    .map(|td| td.text().collect::<Vec<_>>().join(" ").trim().to_string())
                    .collect();
                if !cells.is_empty() {
                    rows.push(cells);
                }
                if rows.len() >= 100 {
                    break;
                }
            }
            if !headers.is_empty() || !rows.is_empty() {
                tables.push(TableEntity {
                    caption,
                    headers,
                    rows,
                });
            }
            if tables.len() >= 20 {
                break;
            }
        }
    }

    // 3. Extract document outline hierarchy
    if let Ok(heading_sel) = Selector::parse("h1, h2, h3, h4, h5, h6") {
        for h in doc.select(&heading_sel) {
            let tag_name = h.value().name();
            let level = match tag_name {
                "h1" => 1,
                "h2" => 2,
                "h3" => 3,
                "h4" => 4,
                "h5" => 5,
                "h6" => 6,
                _ => 2,
            };
            let text = h.text().collect::<Vec<_>>().join(" ").trim().to_string();
            if !text.is_empty() {
                let anchor = h.value().attr("id").or_else(|| h.value().attr("name")).map(String::from);
                headings.push(HeadingEntity {
                    level,
                    text,
                    anchor,
                });
            }
            if headings.len() >= 100 {
                break;
            }
        }
    }

    (code_blocks, tables, headings)
}

/// Diagram languages that reclassify a code block as a diagram source.
const DIAGRAM_LANGS: &[&str] = &["mermaid", "dot", "graphviz", "plantuml", "d2", "nomnoml"];

/// Alt-text / URL hints that mark an image as a diagram.
const DIAGRAM_HINTS: &[&str] = &[
    "diagram", "chart", "graph", "figure", "schema", "architect", "flowchart", "plot", "erd",
    "uml",
];

/// Substrings that match a hint by accident (`photograph` contains `graph`).
/// Denylisted so photos never inflate diagram counts (false-positive guard).
const DIAGRAM_FP_DENY: &[&str] = &[
    "photograph",
    "photography",
    "photographic",
    "telegraph",
    "choreograph",
];

/// True when alt text or URL signals a diagram, minus known false positives.
fn is_diagram_image(alt: &str, url: &str) -> bool {
    let hay = format!("{alt} {url}").to_lowercase();
    if DIAGRAM_FP_DENY.iter().any(|d| hay.contains(d)) {
        return false;
    }
    DIAGRAM_HINTS.iter().any(|h| hay.contains(h))
}

/// Reclassify diagram code blocks and diagram images into diagrams.
/// Pure post-process over already-extracted vectors: no DOM walk, no char budget.
/// Per-vector caps are DoS guards, not content budgets.
pub(crate) fn classify_diagrams(
    code_blocks: &[CodeBlockEntity],
    images: &[ImageInfo],
) -> Vec<DiagramEntity> {
    let mut out = Vec::new();
    for cb in code_blocks {
        let is_diagram = cb
            .language
            .as_deref()
            .map(|l| l.to_lowercase())
            .is_some_and(|l| DIAGRAM_LANGS.contains(&l.as_str()));
        if is_diagram {
            out.push(DiagramEntity {
                kind: cb.language.clone().unwrap_or_default(),
                caption: None,
                source: cb.code.clone(),
            });
        }
        if out.len() >= 50 {
            break;
        }
    }
    for img in images {
        if is_diagram_image(img.alt.as_deref().unwrap_or(""), &img.url) {
            out.push(DiagramEntity {
                kind: "image".to_string(),
                caption: img.alt.clone(),
                source: img.url.clone(),
            });
        }
        if out.len() >= 50 {
            break;
        }
    }
    out
}

/// Headings that mark procedural content for step extraction.
fn is_howto_heading(text: &str) -> bool {
    const HINTS: &[&str] = &["how", "step", "guide", "tutorial", "instruction"];
    let lowered = text.to_lowercase();
    HINTS.iter().any(|h| lowered.contains(h))
}

/// Callout kind from a blockquote's leading word; defaults to `quote`.
/// Handles GitHub-style admonitions (`[!WARNING] ...`) as well as plain
/// leading words (`Warning: ...`). Unknown shapes stay `quote` so pull-quotes
/// never inflate alert counts (false-positive guard).
fn callout_kind(text: &str) -> &str {
    const KINDS: &[&str] = &["note", "warning", "tip", "important", "caution", "danger"];
    let trimmed = text.trim_start();
    let first = if let Some(bracketed) = trimmed
        .strip_prefix("[!")
        .and_then(|rest| rest.split(']').next())
    {
        bracketed.to_lowercase()
    } else {
        trimmed
            .split_whitespace()
            .next()
            .unwrap_or("")
            .trim_matches([':', '!', '[', ']', '*'])
            .to_lowercase()
    };
    KINDS
        .iter()
        .find(|k| first == **k)
        .copied()
        .unwrap_or("quote")
}

/// Extract heading-contextual quality entities in a single document-order pass:
/// examples (code + nearest preceding heading), FAQs (question heading + first
/// following paragraph), procedural steps (ordered lists under how-to headings),
/// callouts (blockquotes). Caps are DoS guards, not content budgets.
pub(crate) fn extract_quality_entities(
    doc: &scraper::Html,
) -> (
    Vec<ExampleEntity>,
    Vec<FaqEntity>,
    Vec<StepEntity>,
    Vec<CalloutEntity>,
) {
    let mut examples = Vec::new();
    let mut faqs = Vec::new();
    let mut steps = Vec::new();
    let mut callouts = Vec::new();

    let Ok(sel) = Selector::parse("h1, h2, h3, h4, h5, h6, pre, blockquote, ol, p") else {
        return (examples, faqs, steps, callouts);
    };
    let code_sel = Selector::parse("code").ok();

    let mut last_heading: Option<String> = None;
    let mut pending_faq: Option<String> = None;

    for elem in doc.select(&sel) {
        match elem.value().name() {
            "h1" | "h2" | "h3" | "h4" | "h5" | "h6" => {
                let text = elem.text().collect::<Vec<_>>().join(" ").trim().to_string();
                if text.is_empty() {
                    continue;
                }
                if text.ends_with('?') {
                    pending_faq = Some(text.clone());
                } else {
                    pending_faq = None;
                }
                last_heading = Some(text);
            }
            "pre" => {
                let code_child = code_sel
                    .as_ref()
                    .and_then(|sel| elem.select(sel).next());
                let target = code_child.unwrap_or(elem);
                let code = target.text().collect::<Vec<_>>().join("");
                let trimmed = code.trim();
                if trimmed.is_empty() || trimmed.len() < 10 {
                    continue;
                }
                if examples.len() >= 50 {
                    continue;
                }
                examples.push(ExampleEntity {
                    context_heading: last_heading.clone(),
                    language: block_language(target, elem),
                    code: trimmed.to_string(),
                });
            }
            "blockquote" => {
                let text = elem.text().collect::<Vec<_>>().join(" ").trim().to_string();
                if text.is_empty() || callouts.len() >= 50 {
                    continue;
                }
                callouts.push(CalloutEntity {
                    kind: callout_kind(&text).to_string(),
                    text,
                });
            }
            "ol" => {
                if steps.len() >= 100 {
                    continue;
                }
                let howto = last_heading.as_deref().is_some_and(is_howto_heading);
                if !howto {
                    continue;
                }
                if let Ok(li_sel) = Selector::parse("li") {
                    for li in elem.select(&li_sel) {
                        let title =
                            li.text().collect::<Vec<_>>().join(" ").trim().to_string();
                        if title.is_empty() || steps.len() >= 100 {
                            continue;
                        }
                        steps.push(StepEntity {
                            position: (steps.len() + 1) as u32,
                            context_heading: last_heading.clone(),
                            title,
                        });
                    }
                }
            }
            "p" => {
                if let Some(question) = pending_faq.take() {
                    // Length floors filter rhetorical fragments and empty
                    // answers (false-positive guard); multi-paragraph answers
                    // keep their first paragraph by design (recall trade-off).
                    if question.len() < 8 {
                        continue;
                    }
                    let answer =
                        elem.text().collect::<Vec<_>>().join(" ").trim().to_string();
                    if answer.len() >= 10 && faqs.len() < 50 {
                        faqs.push(FaqEntity { question, answer });
                    }
                }
            }
            _ => {}
        }
    }

    (examples, faqs, steps, callouts)
}

/// Strip web noise from HTML content using proper DOM traversal, retaining only
/// semantic structural elements suitable for AI agent consumption.
///
/// Pipeline:
///   1. Strip HTML comments (`<!-- ... -->`)
///   2. Parse with HTML5 parser (handles malformed input gracefully)
///   3. DOM tree traversal: only keep structural/semantic tags
///   4. Normalize whitespace
///
/// Retains: h1-h6, p, div, span, a, ul, ol, li, table, pre, code, blockquote, etc.
/// Removes: script, style, nav, header, footer, aside, form, input, iframe, etc.
fn clean_content_html(html: &str) -> String {
    // Step 1: strip HTML comments
    let no_comments = strip_html_comments(html);
    // Step 2: parse fragment via HTML5 spec parser
    let doc = Html::parse_fragment(&no_comments);

    // Step 3: traverse the body and rebuild clean HTML.
    // If <body ...> is not present (e.g. standalone container fragments like <main> or <article>),
    // traverse from the root elements of the fragment directly.
    let mut out = String::with_capacity(no_comments.len());
    if let Ok(body_sel) = Selector::parse("body")
        && let Some(body) = doc.select(&body_sel).next()
    {
        traverse_clean(body, &mut out);
    } else {
        for child in doc.root_element().children() {
            if let Some(child_el) = scraper::ElementRef::wrap(child) {
                traverse_clean(child_el, &mut out);
            }
        }
    }

    // Step 4: normalize whitespace
    normalize_html_whitespace(&out)
}

/// Tags whose entire subtree is kept (structural + semantic content).
/// Matches PLAN.md spec: h1-h6, p, div, span, standard inline formatting.
const CLEAN_KEEP: &[&str] = &[
    "h1",
    "h2",
    "h3",
    "h4",
    "h5",
    "h6",
    "p",
    "div",
    "span",
    "a",
    "ul",
    "ol",
    "li",
    "table",
    "tr",
    "td",
    "th",
    "pre",
    "code",
    "blockquote",
    "em",
    "strong",
    "i",
    "b",
    "u",
    "br",
    "hr",
    "img",
    "figure",
    "figcaption",
    "dl",
    "dt",
    "dd",
    "abbr",
    "cite",
    "q",
    "sub",
    "sup",
    "section",
    "article",
    "main",
];

/// Tags that are completely removed including their content.
/// Matches PLAN.md spec: script, style, nav, header, footer, aside, form, button, head, noscript.
const CLEAN_REMOVE: &[&str] = &[
    "script", "style", "noscript", "iframe", "canvas", "svg", "nav", "header", "footer", "aside",
    "form", "button", "input", "select", "textarea", "label", "option", "head", "link", "meta",
    "dialog",
];

/// Helper to identify noisy classes/IDs for cookies, anchors, sidebar menus, and copy buttons
fn is_noise_element(el: &scraper::ElementRef) -> bool {
    let val = el.value();
    
    // 1. Aria hidden decorative elements
    if val.attr("aria-hidden") == Some("true") {
        return true;
    }

    // 2. Class-based noise check
    if let Some(classes) = val.attr("class") {
        let cls = classes.to_ascii_lowercase();
        // Rustdoc / Sphinx / GitHub / Wikipedia / SO / Substack / Medium / Discourse noise elements
        if cls.contains("anchor")
            || cls.contains("headerlink")
            || cls.contains("srclink")
            || cls.contains("src-link")
            || cls.contains("view-source")
            || cls.split_whitespace().any(|c| c == "src")
            || cls.contains("copy-button")
            || cls.contains("copy-icon")
            || cls.contains("collapse-toggle")
            || cls.contains("cookie")
            || cls.contains("consent")
            || cls.contains("onetrust")
            || cls.contains("didomi")
            || cls.contains("sidebar")
            || cls.contains("extra-services")
            || cls.contains("submission-history")
            || cls.contains("katex-html")
            || cls.contains("bc-table")
            || cls.contains("baseline-indicator")
            || cls.contains("breadcrumb")
            || cls.contains("mw-editsection")
            || cls.contains("reference")
            || cls.contains("post-signature")
            || cls.contains("user-info")
            || cls.contains("js-voting-container")
            || cls.contains("js-vote-count")
            || cls.contains("comments-link")
            || cls.contains("subline")
            || cls.contains("file-navigation")
            || cls.contains("repository-content-header")
            || cls.contains("blob-num")
            || cls.contains("line-numbers")
            || cls.contains("subscribe")
            || cls.contains("newsletter")
            || cls.contains("social-share")
            || cls.contains("share-bar")
            || cls.contains("reaction")
            || cls.contains("clap")
            || cls.contains("author-card")
            || cls.contains("author-bio")
            || cls.contains("related-posts")
            || cls.contains("recommended")
            || cls.contains("topic-timeline")
            || cls.contains("topic-map")
            || cls.contains("page-rating")
            || cls.contains("feedback-prompt")
            // Cross-domain: Marketing, Law, Real Estate, Healthcare, Finance
            || cls.contains("affiliate")
            || cls.contains("ad-disclosure")
            || cls.contains("sponsored")
            || cls.contains("disclaimer")
            || cls.contains("modal")
            || cls.contains("popup")
            || cls.contains("overlay")
            || cls.contains("signup-wall")
            || cls.contains("lead-form")
            || cls.contains("quote-widget")
            || cls.contains("mortgage-calculator")
            || cls.contains("rate-quote")
            || cls.contains("contact-agent")
            || cls.contains("enrollment")
            || cls.contains("cta-banner")
            || cls.contains("no-print")
            || cls.contains("print-only")
        {
            return true;
        }
    }

    // 3. ID-based noise check
    if let Some(id) = val.attr("id") {
        let id_lower = id.to_ascii_lowercase();
        if id_lower.contains("cookie")
            || id_lower.contains("consent")
            || id_lower.contains("onetrust")
            || id_lower.contains("sidebar")
            || id_lower.contains("mw-navigation")
            || id_lower.contains("subscribe")
            || id_lower.contains("newsletter")
            || id_lower.contains("modal")
            || id_lower.contains("popup")
            || id_lower.contains("disclaimer")
            || id_lower == "toc"
        {
            return true;
        }
    }

    false
}

/// Recursive DOM traversal: rebuilds clean HTML from the scraper tree.
fn traverse_clean(el: scraper::ElementRef, out: &mut String) {
    let tag = el.value().name();
    let tag_lower = tag.to_ascii_lowercase();

    if CLEAN_REMOVE.contains(&tag_lower.as_str()) || is_noise_element(&el) {
        return;
    }

    // Void / self-closing elements — emit inline
    if tag_lower == "br" {
        out.push_str("<br>");
        return;
    }
    if tag_lower == "hr" {
        out.push_str("<hr>");
        return;
    }
    if tag_lower == "img" {
        if let Some(src) = el.value().attr("src") {
            let alt = el.value().attr("alt").unwrap_or("");
            out.push_str(&format!(
                "<img src=\"{}\" alt=\"{}\">",
                html_escape(src),
                html_escape(alt)
            ));
        }
        return;
    }

    // Unknown / custom elements — strip entirely (e.g. <div class="ad">, <tracking-widget>)
    if !CLEAN_KEEP.contains(&tag_lower.as_str()) && !CLEAN_REMOVE.contains(&tag_lower.as_str()) {
        // Unknown tag: skip it + its content
        return;
    }

    // Open the tag with preserved attributes for critical elements
    out.push('<');
    out.push_str(tag_lower.as_str());

    // Preserve href on links (traceability / citation)
    if tag_lower == "a"
        && let Some(href) = el.value().attr("href") {
            out.push_str(&format!(" href=\"{}\"", html_escape(href)));
        }

    // Preserve code language classes (e.g. class="language-rust" or class="highlight-python")
    if (tag_lower == "code" || tag_lower == "pre")
        && let Some(cls) = el.value().attr("class") {
            let lang_tokens: Vec<&str> = cls.split_whitespace()
                .filter(|c| c.starts_with("language-") || c.starts_with("lang-") || c.starts_with("highlight-"))
                .collect();
            if !lang_tokens.is_empty() {
                out.push_str(&format!(" class=\"{}\"", html_escape(&lang_tokens.join(" "))));
            }
        }

    out.push('>');

    // Recursively process children
    for child in el.children() {
        if let Some(child_el) = scraper::ElementRef::wrap(child) {
            traverse_clean(child_el, out);
        } else if let Some(text) = child.value().as_text() {
            out.push_str(text);
        }
    }

    out.push_str("</");
    out.push_str(tag_lower.as_str());
    out.push('>');
    out.push('\n');
}

/// Strip HTML comments (`<!-- ... -->`) from a string.
fn strip_html_comments(html: &str) -> String {
    let mut result = String::with_capacity(html.len());
    let bytes = html.as_bytes();
    let len = bytes.len();
    let mut i = 0;

    while i < len {
        // Check for <!--
        if i + 3 < len
            && bytes[i] == b'<'
            && bytes[i + 1] == b'!'
            && bytes[i + 2] == b'-'
            && bytes[i + 3] == b'-'
        {
            // Skip until -->
            i += 4;
            while i + 2 < len && !(bytes[i] == b'-' && bytes[i + 1] == b'-' && bytes[i + 2] == b'>')
            {
                i += 1;
            }
            i += 3; // skip -->
        } else {
            result.push(bytes[i] as char);
            i += 1;
        }
    }
    result
}

/// Collapse consecutive whitespace (including newlines) into single spaces,
/// then trim leading/trailing whitespace.
fn normalize_html_whitespace(s: &str) -> String {
    let mut out = String::with_capacity(s.len());
    let mut prev_space = false;
    for ch in s.chars() {
        if ch.is_whitespace() {
            if !prev_space {
                out.push(' ');
                prev_space = true;
            }
        } else {
            out.push(ch);
            prev_space = false;
        }
    }
    out.trim().to_string()
}

// ── Paywall Detection ───────────────────────────────────────────────────────

fn detect_paywall(html: &str) -> bool {
    let indicators = [
        "paywall",
        "subscribe to continue",
        "membership required",
        "premium content",
        "sign in to read",
        "members-only",
        "subscription required",
        "behind a paywall",
    ];
    let lower = html.to_lowercase();
    indicators.iter().any(|i| lower.contains(i))
}

// ── Helpers ─────────────────────────────────────────────────────────────────

fn extract_domain(url: &str) -> String {
    super::util::extract_domain(url).unwrap_or_default()
}

/// Extract the first <h1> text as a title fallback.
fn extract_h1(doc: &Html) -> Option<String> {
    Selector::parse("h1")
        .ok()
        .and_then(|sel| doc.select(&sel).next())
        .map(|el| el.text().collect::<Vec<_>>().join("").trim().to_string())
        .filter(|t| !t.is_empty())
}

/// Extract semantic documentation container (<article>, <main>, role="main", docs containers).
/// Returns (clean_inner_html, clean_text).
fn extract_doc_container(doc: &Html) -> Option<(String, String)> {
    const DOC_SELECTORS: &[&str] = &[
        "article",
        "main",
        "[role=\"main\"]",
        ".markdown-body",
        ".mw-parser-output",
        "#answers",
        "blockquote.abstract",
        ".article-content",
        ".docs-content",
        ".markdown-section",
        ".documentation",
        ".document",
        ".rfcMarkup",
        "#content",
        "#main-content",
        "#page-content",
    ];

    for sel_str in DOC_SELECTORS {
        if let Ok(sel) = Selector::parse(sel_str)
            && let Some(el) = doc.select(&sel).next() {
                let mut text_parts = Vec::new();
                let skip_sel = Selector::parse(
                    "script,style,noscript,iframe,canvas,svg,nav,header,footer,aside,form,button",
                )
                .ok();
                collect_text_nodes(el, &skip_sel, &mut text_parts);
                let raw_text = normalize_text(&text_parts.join(" "));
                let purged_text = purge_doc_boilerplate(&raw_text);

                if !purged_text.is_empty() && purged_text.len() >= 40 {
                    let inner_html = el.html();
                    return Some((inner_html, purged_text));
                }
            }
    }
    None
}

/// Strip documentation boilerplate, feedback questions, and community widgets.
fn purge_doc_boilerplate(text: &str) -> String {
    let boilerplate_markers = [
        "was this helpful?",
        "give feedback about this article",
        "need more help? try these next steps",
        "was this article helpful?",
        "send feedback about our help center",
        "post to the help community",
        "thanks for reading! subscribe for free",
        "leave a comment",
        "sign up to continue reading",
        "ready for more? subscribe",
        // Cross-domain boilerplate delimiters
        "the information on this website is for informational purposes only and is not intended as medical advice",
        "this information is not intended to be legal advice",
        "this content is for educational purposes only and does not constitute financial advice",
        "we may receive a commission if you click on a link",
        "advertiser disclosure: the offers that appear",
    ];

    let lower = text.to_lowercase();
    let mut cutoff = text.len();

    for marker in boilerplate_markers {
        if let Some(pos) = lower.find(marker)
            && pos < cutoff && pos > 0 {
                cutoff = pos;
            }
    }

    text[..cutoff].trim().to_string()
}

/// Strip scripts, styles, and noscript tags, then collect visible body text.
fn fallback_body_text(doc: &Html) -> String {
    let mut text_parts = Vec::new();
    if let Ok(body_sel) = Selector::parse("body")
        && let Some(body) = doc.select(&body_sel).next() {
            let skip_sel = Selector::parse(
                "script,style,noscript,iframe,canvas,svg,nav,header,footer,aside,form,button",
            )
            .ok();
            collect_text_nodes(body, &skip_sel, &mut text_parts);
        }
    let raw_text = normalize_text(&text_parts.join(" "));
    purge_doc_boilerplate(&raw_text)
}

fn collect_text_nodes(
    node: scraper::ElementRef,
    skip_sel: &Option<Selector>,
    out: &mut Vec<String>,
) {
    for child in node.children() {
        if let Some(child_el) = scraper::ElementRef::wrap(child) {
            if is_noise_element(&child_el) {
                continue;
            }
            if skip_sel
                .as_ref()
                .map(|_sel| {
                    child_el.value().name() == "script"
                        || child_el.value().name() == "style"
                        || child_el.value().name() == "noscript"
                        || child_el.value().name() == "iframe"
                        || child_el.value().name() == "canvas"
                        || child_el.value().name() == "svg"
                })
                .unwrap_or(false)
            {
                continue;
            }
            collect_text_nodes(child_el, skip_sel, out);
        } else if let Some(text) = child.value().as_text() {
            let trimmed = text.trim();
            if !trimmed.is_empty() {
                out.push(trimmed.to_string());
            }
        }
    }
}

/// Build a human-readable title from the last path segment of a URL.
pub(crate) fn title_from_url(url: &str) -> String {
    let path = url
        .trim_start_matches("https://")
        .trim_start_matches("http://")
        .split('?')
        .next()
        .unwrap_or(url);
    path.rsplit('/')
        .find(|s| !s.is_empty())
        .map(percent_decode)
        .unwrap_or_else(|| extract_domain(url))
        .trim()
        .to_string()
}

/// Decode common percent-encoded characters in a URL fragment.
fn percent_decode(s: &str) -> String {
    let mut out = String::with_capacity(s.len());
    let mut chars = s.chars();
    while let Some(c) = chars.next() {
        if c == '%' {
            let a = chars.next();
            let b = chars.next();
            if let (Some(a), Some(b)) = (a, b)
                && let Ok(byte) = u8::from_str_radix(&format!("{}{}", a, b), 16) {
                    out.push(byte as char);
                    continue;
                }
        }
        out.push(c);
    }
    out
}

/// Derive a site name from the registered domain, or fall back to the full host.
pub(crate) fn extract_site_name(url: &str) -> Option<String> {
    let host = extract_domain(url);
    if host.is_empty() {
        return None;
    }
    // Strip a leading "www." again and return the domain without port.
    host.split(':')
        .next()
        .map(|h| h.trim_start_matches("www.").to_string())
        .filter(|h| !h.is_empty())
}

/// Minimal HTML-escape for fallback plain text blocks.
pub(crate) fn html_escape(text: &str) -> String {
    text.replace('&', "&amp;")
        .replace('<', "&lt;")
        .replace('>', "&gt;")
        .replace('"', "&quot;")
}

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn test_extract_meta() {
        let html = "<html><head>\
            <title>My Page Title</title>\
            <meta name=\"description\" content=\"A great page about Rust\">\
            <meta name=\"author\" content=\"Gaurav\">\
            <meta property=\"og:title\" content=\"OG Title\">\
            <link rel=\"canonical\" href=\"https://example.com/canonical\">\
            </head><body><p>Hello world</p></body></html>";
        let doc = Html::parse_document(html);
        let meta = extract_meta(&doc);
        assert_eq!(meta.title.as_deref(), Some("My Page Title"));
        assert_eq!(meta.description.as_deref(), Some("A great page about Rust"));
        assert_eq!(meta.author.as_deref(), Some("Gaurav"));
        assert_eq!(
            meta.canonical_url.as_deref(),
            Some("https://example.com/canonical")
        );
    }

    #[test]
    fn test_extract_open_graph() {
        let html = "<html><head>\
            <meta property=\"og:title\" content=\"OG Title\">\
            <meta property=\"og:description\" content=\"OG Desc\">\
            <meta property=\"og:image\" content=\"https://example.com/img.jpg\">\
            <meta property=\"og:type\" content=\"article\">\
            </head><body></body></html>";
        let doc = Html::parse_document(html);
        let og = extract_open_graph(&doc).unwrap();
        assert_eq!(og.title.as_deref(), Some("OG Title"));
        assert_eq!(og.description.as_deref(), Some("OG Desc"));
        assert_eq!(og.r#type.as_deref(), Some("article"));
    }

    #[test]
    fn test_extract_links() {
        let html = "<html><body>\
            <a href=\"/page1\">Internal</a>\
            <a href=\"https://external.com/page\">External</a>\
            <a href=\"#anchor\">Anchor</a>\
            <a href=\"javascript:void(0)\">JS</a>\
            </body></html>";
        let doc = Html::parse_document(html);
        let (internal, external) = extract_links(&doc, "https://example.com/home");
        assert!(internal.iter().any(|u| u.contains("/page1")));
        assert!(external.iter().any(|u| u.contains("external.com")));
        assert!(!internal.iter().any(|u| u.contains("#anchor")));
    }

    #[test]
    fn test_domain_extraction() {
        assert_eq!(
            extract_domain("https://www.example.com/path"),
            "example.com"
        );
        assert_eq!(
            extract_domain("http://blog.example.co.uk/post"),
            "blog.example.co.uk"
        );
    }

    #[test]
    fn test_word_count() {
        assert_eq!(count_words("Hello world"), 2);
        assert_eq!(count_words("one two three four five"), 5);
    }

    #[test]
    fn test_short_page_is_valid_content() {
        // A short-but-real page (e.g. example.com, ~19 words) must be treated
        // as valid content so webfind_fetch does not mark it invalid and drop
        // it. Regression for the >50 word threshold that pushed the LLM to
        // DuckDuckGo.
        let html = r#"<!DOCTYPE html><html><head><title>Example Domain</title></head>
<body><h1>Example Domain</h1>
<p>This domain is for use in documentation examples without needing permission. Avoid use in operations. Learn more.</p>
</body></html>"#;
        let fetcher = Fetcher::new().unwrap();
        let content = fetcher
            .extract_from_html(
                html,
                "https://example.com/",
                "https://example.com/",
                200,
                true,
                5,
                None,
            )
            .unwrap();
        assert!(content.word_count > 0);
        assert!(
            content.is_valid_content,
            "short page marked invalid (word_count={})",
            content.word_count
        );
    }

    #[test]
    fn test_empty_page_is_invalid_content() {
        let html = "<!DOCTYPE html><html><head><title>Empty</title></head><body></body></html>";
        let fetcher = Fetcher::new().unwrap();
        let content = fetcher
            .extract_from_html(html, "https://x.com/", "https://x.com/", 200, true, 5, None)
            .unwrap();
        assert!(!content.is_valid_content, "empty page must stay invalid");
    }

    #[test]
    fn test_reading_time() {
        assert_eq!(estimate_reading_time(238), 60);
        assert_eq!(estimate_reading_time(100), 26);
    }

    #[test]
    fn test_paywall_detection() {
        assert!(detect_paywall("Please subscribe to continue reading"));
        assert!(detect_paywall("This is premium content behind a paywall"));
        assert!(!detect_paywall("Normal article content"));
    }

    #[test]
    fn test_extract_entities() {
        let text = concat!(
            "Contact support@example.com or sales@corp.io. ",
            "Call +1 555-123-4567. ",
            "Visit https://example.com/pricing or www.example.org. ",
            "The product costs $1,299.00 (or 999 USD). ",
            "Founded on 2024-03-15. Server at 192.168.0.1. ",
            "Follow us @OpenCodeHQ. Office at 123 Main Street, Austin, TX 78701.",
        );
        let e = extract_entities(text);
        assert!(e.emails.iter().any(|m| m == "support@example.com"));
        assert!(e.emails.iter().any(|m| m == "sales@corp.io"));
        assert!(e.phones.iter().any(|m| m.contains("555-123-4567")));
        assert!(e.urls.iter().any(|m| m == "https://example.com/pricing"));
        assert!(e.prices.iter().any(|m| m.contains("1,299")));
        assert!(e.prices.iter().any(|m| m.contains("999")));
        assert!(e.dates.iter().any(|m| m == "2024-03-15"));
        assert!(e.ip_addresses.iter().any(|m| m == "192.168.0.1"));
        assert!(e.social_handles.iter().any(|m| m == "@OpenCodeHQ"));
        assert!(e.addresses.iter().any(|m| m.contains("123 Main Street")));
        assert!(!e.is_empty());
    }

    #[test]
    fn test_extract_entities_avoids_false_positives() {
        // A date must not be classified as a phone; an email local-part must
        // not be classified as a social handle.
        let e = extract_entities("Released 2017-05-13. Reach sales@corp.io directly.");
        assert!(
            e.phones.is_empty(),
            "date misclassified as phone: {:?}",
            e.phones
        );
        assert!(
            !e.social_handles.iter().any(|m| m.contains("corp")),
            "email local-part misclassified as handle: {:?}",
            e.social_handles
        );
        assert!(e.dates.iter().any(|m| m == "2017-05-13"));
        assert!(e.emails.iter().any(|m| m == "sales@corp.io"));
    }

    #[test]
    fn test_extract_entities_empty() {
        let e = extract_entities("Just some plain text with no structured data here.");
        assert!(e.is_empty());
        assert!(e.emails.is_empty() && e.phones.is_empty());
    }

    #[test]
    fn test_extract_technical_entities() {
        let text = "WebFind v0.3.4 is licensed under MIT. Visit https://github.com/Gaurav-Wankhede/webfind for details.";
        let e = extract_entities(text);
        assert!(e.package_versions.iter().any(|v| v == "v0.3.4"));
        assert!(e.licenses.iter().any(|l| l == "MIT"));
        assert!(e.repository_urls.iter().any(|u| u.contains("github.com/Gaurav-Wankhede/webfind")));
        assert!(!e.is_empty());
    }

    #[test]
    fn test_extract_dom_entities() {
        let html = r#"<!DOCTYPE html><html><body>
            <h1 id="intro">Introduction</h1>
            <pre><code class="language-rust">fn main() { println!("hello world"); }</code></pre>
            <table>
                <caption>Features</caption>
                <thead><tr><th>Feature</th><th>Latency</th></tr></thead>
                <tbody><tr><td>Search</td><td>12ms</td></tr></tbody>
            </table>
        </body></html>"#;
        let doc = Html::parse_document(html);
        let (code_blocks, tables, headings) = extract_dom_entities(&doc);
        assert_eq!(headings.len(), 1);
        assert_eq!(headings[0].level, 1);
        assert_eq!(headings[0].text, "Introduction");
        assert_eq!(headings[0].anchor.as_deref(), Some("intro"));

        assert_eq!(code_blocks.len(), 1);
        assert_eq!(code_blocks[0].language.as_deref(), Some("rust"));
        assert!(code_blocks[0].code.contains("fn main()"));

        assert_eq!(tables.len(), 1);
        assert_eq!(tables[0].caption.as_deref(), Some("Features"));
        assert_eq!(tables[0].headers, vec!["Feature", "Latency"]);
        assert_eq!(tables[0].rows, vec![vec!["Search", "12ms"]]);
    }

    #[test]
    fn test_doc_container_and_boilerplate_purging() {
        let html = r#"<!DOCTYPE html><html><head><title>API Docs</title></head>
<body>
<nav><a href="/home">Home</a></nav>
<article>
  <h1>API Overview</h1>
  <p>The API provides programmatic access to your resources.</p>
  <div class="feedback">Was this helpful? Submit feedback to our help community.</div>
</article>
<footer>Footer links and copyright</footer>
</body></html>"#;
        let doc = Html::parse_document(html);
        let extracted = extract_doc_container(&doc);
        assert!(extracted.is_some());
        let (_, text) = extracted.unwrap();
        assert!(text.contains("API Overview"));
        assert!(text.contains("programmatic access to your resources"));
        assert!(!text.to_lowercase().contains("was this helpful"));
        assert!(!text.to_lowercase().contains("submit feedback"));
    }

    #[test]
    fn test_programming_doc_and_research_paper_noise_removal() {
        // Simulates Rustdoc anchor §, copy button, cookie banner, and arXiv abstract
        let html = r##"<!DOCTYPE html><html><head><title>Vec in std::vec</title></head>
<body>
<div id="onetrust-consent-sdk" class="cookie-banner"><p>Accept all tracking cookies</p></div>
<nav class="sidebar"><ul><li>Methods</li></ul></nav>
<main>
  <h1>Struct Vec<a class="anchor" href="#vec">§</a></h1>
  <div class="code-block">
    <button class="copy-button">Copy code</button>
    <pre><code>pub struct Vec&lt;T&gt; { /* fields */ }</code></pre>
  </div>
  <blockquote class="abstract">
    <span class="descriptor">Abstract:</span> We present a new deep learning architecture.
  </blockquote>
  <div class="extra-services">
    <a href="/pdf/123">Download PDF</a>
    <button>Export BibTeX</button>
  </div>
</main>
</body></html>"##;
        let doc = Html::parse_document(html);
        let extracted = extract_doc_container(&doc);
        assert!(extracted.is_some());
        let (_, text) = extracted.unwrap();

        // Must keep core content
        assert!(text.contains("Struct Vec"));
        assert!(text.contains("pub struct Vec"));
        assert!(text.contains("We present a new deep learning architecture"));

        // Must purge all noise elements
        assert!(!text.contains("§"), "Anchor symbol § must be stripped");
        assert!(!text.contains("Copy code"), "Copy button must be stripped");
        assert!(!text.contains("Accept all tracking cookies"), "Cookie banner must be stripped");
        assert!(!text.contains("Download PDF"), "arXiv extra services must be stripped");
        assert!(!text.contains("Export BibTeX"), "BibTeX export buttons must be stripped");
    }

    #[test]
    fn test_wikipedia_and_stackoverflow_noise_removal() {
        let html = r##"<!DOCTYPE html><html><head><title>Transformer Architecture</title></head>
<body>
<div id="mw-navigation">Navigation sidebar and portals</div>
<div class="mw-parser-output">
  <h2>Overview<span class="mw-editsection"><a href="#">[edit]</a></span></h2>
  <p>Transformers use multi-head self-attention.<sup class="reference"><a href="#cite">[1]</a></sup></p>
  <div class="post-signature user-info">answered 2 hours ago by user123 (15.2k reputation)</div>
  <div class="js-voting-container"><button class="js-vote-count">142 votes</button></div>
</div>
</body></html>"##;
        let doc = Html::parse_document(html);
        let extracted = extract_doc_container(&doc);
        assert!(extracted.is_some());
        let (_, text) = extracted.unwrap();

        // Must retain authentic core knowledge
        assert!(text.contains("Overview"));
        assert!(text.contains("Transformers use multi-head self-attention"));

        // Must strip Wikipedia and SO UI clutter
        assert!(!text.contains("[edit]"), "Wikipedia [edit] links must be stripped");
        assert!(!text.contains("[1]"), "Wikipedia reference superscripts must be stripped");
        assert!(!text.contains("user123"), "User reputation flair must be stripped");
        assert!(!text.contains("142 votes"), "Vote counters must be stripped");
        assert!(!text.contains("Navigation sidebar"), "Wikipedia navigation must be stripped");
    }

    #[test]
    fn test_blog_and_forum_noise_removal() {
        let html = r##"<!DOCTYPE html><html><head><title>System Architecture Guide</title></head>
<body>
<article class="post">
  <h1>Distributed Raft Consensus</h1>
  <div class="social-share-bar"><button>Share on Twitter</button></div>
  <p>Raft guarantees strong consistency across replicas using leader heartbeats.</p>
  <div class="reactions-widget"><button class="clap-button">500 claps</button></div>
  <div class="author-card">Written by Alex. Passionate about cloud infra.</div>
  <div class="newsletter-subscribe">Ready for more? Subscribe for free!</div>
</article>
</body></html>"##;
        let doc = Html::parse_document(html);
        let extracted = extract_doc_container(&doc);
        assert!(extracted.is_some());
        let (_, text) = extracted.unwrap();

        // Must retain core knowledge
        assert!(text.contains("Distributed Raft Consensus"));
        assert!(text.contains("Raft guarantees strong consistency across replicas"));

        // Must strip social, claps, author promo, and newsletter prompts
        assert!(!text.contains("Share on Twitter"));
        assert!(!text.contains("500 claps"));
        assert!(!text.contains("Passionate about cloud infra"));
        assert!(!text.contains("Subscribe for free"));
    }

    #[test]
    fn test_cross_domain_noise_removal() {
        let html = r##"<!DOCTYPE html><html><head><title>Property Deed and Mortgage Overview</title></head>
<body>
<div class="affiliate-disclosure"><p>Advertiser disclosure: We may receive compensation from partners.</p></div>
<article>
  <h1>Understanding Title Deeds</h1>
  <div class="lead-modal popup"><p>Get pre-approved for a mortgage today!</p></div>
  <p>A deed is a signed legal document that transfers ownership of an asset to a new owner.</p>
  <div class="mortgage-calculator widget">Calculate your monthly rate</div>
  <div class="disclaimer-footer">This information is not intended to be legal advice.</div>
</article>
</body></html>"##;
        let doc = Html::parse_document(html);
        let extracted = extract_doc_container(&doc);
        assert!(extracted.is_some());
        let (_, text) = extracted.unwrap();

        // Must retain authentic core knowledge
        assert!(text.contains("Understanding Title Deeds"));
        assert!(text.contains("A deed is a signed legal document that transfers ownership"));

        // Must strip affiliate, lead modal, mortgage widget, and legal advice disclaimers
        assert!(!text.contains("Advertiser disclosure"));
        assert!(!text.contains("Get pre-approved for a mortgage"));
        assert!(!text.contains("Calculate your monthly rate"));
        assert!(!text.contains("legal advice"));
    }

    #[test]
    fn test_quality_entities_from_doc() {
        let html = r##"<!DOCTYPE html><html><head><title>Guide</title></head>
<body><article>
<h2>Quickstart Example</h2>
<pre><code class="language-rust">fn main() { println!("hi"); }</code></pre>
<h2>What is Raft?</h2>
<p>Raft is a consensus algorithm.</p>
<h2>How to install</h2>
<ol><li>Download the binary</li><li>Run it</li></ol>
<blockquote>Warning: do not run as root.</blockquote>
<pre><code class="language-mermaid">graph TD; A-->B;</code></pre>
</article></body></html>"##;
        let doc = Html::parse_document(html);
        let (examples, faqs, steps, callouts) = extract_quality_entities(&doc);
        assert_eq!(examples.len(), 2);
        assert_eq!(
            examples[0].context_heading.as_deref(),
            Some("Quickstart Example")
        );
        assert_eq!(examples[0].language.as_deref(), Some("rust"));
        assert_eq!(faqs.len(), 1);
        assert_eq!(faqs[0].question, "What is Raft?");
        assert_eq!(faqs[0].answer, "Raft is a consensus algorithm.");
        assert_eq!(steps.len(), 2);
        assert_eq!(steps[0].position, 1);
        assert_eq!(steps[0].title, "Download the binary");
        assert_eq!(callouts.len(), 1);
        assert_eq!(callouts[0].kind, "warning");
    }

    #[test]
    fn test_classify_diagrams() {
        let blocks = vec![
            CodeBlockEntity {
                language: Some("mermaid".into()),
                code: "graph TD; A-->B;".into(),
                line_count: 1,
            },
            CodeBlockEntity {
                language: Some("rust".into()),
                code: "fn main() {}".into(),
                line_count: 1,
            },
        ];
        let images = vec![ImageInfo {
            url: "https://x.com/architecture-diagram.png".into(),
            alt: Some("System architecture".into()),
            width: None,
            height: None,
        }];
        let diagrams = classify_diagrams(&blocks, &images);
        assert_eq!(diagrams.len(), 2);
        assert_eq!(diagrams[0].kind, "mermaid");
        assert_eq!(diagrams[1].kind, "image");
    }

    #[test]
    fn test_math_entities_skip_prices() {
        let entities = extract_entities("Einstein wrote $$E=mc^2$$ in 1905. It costs $19.99 today.");
        assert_eq!(entities.maths.len(), 1);
        assert!(entities.maths[0].display);
        assert!(entities.maths[0].tex.contains("E=mc^2"));
    }

    #[test]
    fn test_quality_entities_avoid_false_positives() {
        // photograph (contains "graph"), rhetorical fragment, nav list,
        // pull-quote, and prose brackets must not inflate entity counts.
        let html = r##"<!DOCTYPE html><html><head><title>FP</title></head>
<body><article>
<img src="team-photograph.jpg" alt="Team photograph retreat" />
<h2>What now?</h2>
<p>Ok.</p>
<h2>Latest News</h2>
<ol><li>Launch</li><li>Update</li></ol>
<blockquote>Great product, five stars.</blockquote>
<pre><code>plain snippet</code></pre>
</article></body></html>"##;
        let doc = Html::parse_document(html);
        let diagrams = classify_diagrams(
            &extract_dom_entities(&doc).0,
            &extract_images(&doc),
        );
        assert!(diagrams.is_empty(), "photo misclassified: {diagrams:?}");
        let (examples, faqs, steps, callouts) = extract_quality_entities(&doc);
        assert!(faqs.is_empty(), "rhetorical fragment misclassified: {faqs:?}");
        assert!(steps.is_empty(), "nav list misclassified: {steps:?}");
        assert_eq!(callouts.len(), 1);
        assert_eq!(callouts[0].kind, "quote");
        assert_eq!(examples.len(), 1, "code without heading context lost");
        let maths = extract_entities("see (figure 3) for details");
        assert!(maths.maths.is_empty());
    }

    #[test]
    fn test_quality_entities_golden_counts() {
        // WCXB-style miniature: exact TP counts per entity on one fixture.
        let html = r##"<!DOCTYPE html><html><head><title>Golden</title></head>
<body><article>
<h2>Install Example</h2>
<pre><code class="language-bash">cargo install webfind</code></pre>
<h2>How do I configure it?</h2>
<p>Pass flags on the command line.</p>
<h2>How to deploy</h2>
<ol><li>Build the binary</li><li>Ship it</li></ol>
<blockquote>[!WARNING] Back up first.</blockquote>
<pre><code class="language-mermaid">graph TD; A-->B;</code></pre>
</article></body></html>"##;
        let doc = Html::parse_document(html);
        let (code_blocks, _, _) = extract_dom_entities(&doc);
        let diagrams = classify_diagrams(&code_blocks, &[]);
        let (examples, faqs, steps, callouts) = extract_quality_entities(&doc);
        assert_eq!(examples.len(), 2);
        assert_eq!(diagrams.len(), 1);
        assert_eq!(diagrams[0].kind, "mermaid");
        assert_eq!(faqs.len(), 1);
        assert_eq!(steps.len(), 2);
        assert_eq!(callouts.len(), 1);
        assert_eq!(callouts[0].kind, "warning");
    }
}