auditmysite 1.2.0

//! robots.txt audit — informational, no score impact.
//!
//! Fetches and parses the robots.txt of the audited domain. Classifies known
//! bots by category (search engines, verified AI crawlers, unverified AI
//! crawlers, generic crawlers) and surfaces key findings in the report.

use reqwest::Client;
use serde::{Deserialize, Serialize};
use std::time::Duration;

/// robots.txt audit result. Never affects the SEO score — informational only.
#[derive(Debug, Clone, Serialize, Deserialize, Default)]
pub struct RobotsAudit {
    /// Whether robots.txt was fetched successfully
    pub fetched: bool,
    /// HTTP or parse error, if any
    pub error: Option<String>,
    /// Parsed rule groups (one per User-agent block)
    pub groups: Vec<RobotsGroup>,
    /// Sitemap directives found in robots.txt
    pub sitemaps: Vec<String>,
    /// (user-agent, delay-seconds) pairs
    pub crawl_delays: Vec<(String, u32)>,
    /// True when the audited page has noindex AND appears in the sitemap.xml
    #[serde(default, skip_serializing_if = "Option::is_none")]
    pub noindex_in_sitemap: Option<bool>,
    /// True when `User-agent: *` has `Disallow: /` — blocks everything
    pub has_wildcard_disallow_all: bool,
    /// True when any AI crawler (any sub-type) is blocked — legacy alias
    pub blocks_ai_crawlers: bool,
    /// True when training-only bots (GPTBot, Google-Extended, …) are blocked
    pub blocks_ai_training: bool,
    /// True when citation/search AI bots (PerplexityBot, Amazonbot, …) are blocked
    pub blocks_ai_citation: bool,
    /// Human-readable policy label inferred from the rules
    pub inferred_policy: String,
}

/// A single rule group in robots.txt (one or more User-agents sharing Allow/Disallow rules)
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct RobotsGroup {
    pub user_agent: String,
    pub bot_class: BotClass,
    pub allows: Vec<String>,
    pub disallows: Vec<String>,
}

/// Bot category, used for display grouping
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum BotClass {
    /// Wildcard (*) rule
    Wildcard,
    /// Verified search engine crawler (Googlebot, Bingbot, …)
    SearchEngine,
    /// AI training bot — blocking is standard practice (GPTBot, Google-Extended, CCBot, …)
    AiTraining,
    /// AI citation / search bot — blocking is unusual (PerplexityBot, Amazonbot, …)
    AiCitation,
    /// AI bot with mixed purpose: both training and citation (ClaudeBot, meta-externalagent, …)
    AiMixed,
    /// Unverified scrapers, SEO bots (dotbot, semrushbot, …)
    UnknownAi,
    /// General / known-good (archive.org, etc.)
    General,
    /// Not in any known list
    Unknown,
}

impl BotClass {
    /// Human-readable, report-visible label for this bot class.
    pub fn label(&self, en: bool) -> &'static str {
        match self {
            BotClass::Wildcard => {
                if en {
                    "All crawlers (*)"
                } else {
                    "Alle Crawler (*)"
                }
            }
            BotClass::SearchEngine => {
                if en {
                    "Search engine"
                } else {
                    "Suchmaschine"
                }
            }
            BotClass::AiTraining => {
                if en {
                    "AI training"
                } else {
                    "KI-Training"
                }
            }
            BotClass::AiCitation => {
                if en {
                    "AI search / citation"
                } else {
                    "KI-Suche / Zitation"
                }
            }
            BotClass::AiMixed => {
                if en {
                    "AI training & search"
                } else {
                    "KI-Training & Suche"
                }
            }
            BotClass::UnknownAi => {
                if en {
                    "Unknown scraper"
                } else {
                    "Unbekannter Scraper"
                }
            }
            BotClass::General => {
                if en {
                    "General"
                } else {
                    "Allgemein"
                }
            }
            BotClass::Unknown => {
                if en {
                    "Unclassified"
                } else {
                    "Nicht klassifiziert"
                }
            }
        }
    }
}

impl std::fmt::Display for BotClass {
    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
        write!(f, "{}", self.label(false))
    }
}

// ─── Bot Registry ────────────────────────────────────────────────────────────

const SEARCH_BOTS: &[&str] = &[
    "googlebot",
    "bingbot",
    "slurp", // Yahoo
    "duckduckbot",
    "baiduspider",
    "yandexbot",
    "sogou",
    "exabot",
    "ia_archiver", // Wayback Machine
    "msnbot",
    "teoma",
    "ask jeeves",
];

/// Training-only bots — blocking is standard practice (citationFriendly default)
const AI_TRAINING_BOTS: &[&str] = &[
    "gptbot",            // OpenAI — model training
    "google-extended",   // Google — AI training
    "applebot-extended", // Apple — AI training
    "bytespider",        // ByteDance / TikTok — training
    "ccbot",             // Common Crawl — training datasets
    "omgili",            // Webz.io — training
];

/// Citation / AI-search bots — blocking is unusual and may reduce AI visibility
const AI_CITATION_BOTS: &[&str] = &[
    "perplexitybot", // Perplexity AI search
    "youbot",        // You.com AI search
    "amazonbot",     // Amazon AI / Alexa
    "oai-searchbot", // OpenAI search (separate from training GPTBot)
    "claude-web",    // Anthropic web browsing (citation only)
];

/// Mixed-purpose bots: both training data collection and AI responses/search
const AI_MIXED_BOTS: &[&str] = &[
    "claudebot",          // Anthropic — training + citation
    "anthropic-ai",       // Anthropic general
    "chatgpt-user",       // OpenAI — user browsing + training
    "meta-externalagent", // Meta — training + social AI
    "facebookbot",        // Meta
    "cohere-ai",          // Cohere
    "diffbot",            // Diffbot — structured data + training
];

/// Unverified scrapers / SEO bots — often used for training but not primary AI crawlers
const UNKNOWN_AI_BOTS: &[&str] = &[
    "dotbot",
    "petalbot", // Huawei search
    "wpbot",
    "semrushbot",
    "ahrefsbot",
    "mj12bot",
    "rogerbot",
];

fn classify_bot(ua: &str) -> BotClass {
    if ua == "*" {
        return BotClass::Wildcard;
    }
    let lower = ua.to_lowercase();
    for pat in SEARCH_BOTS {
        if lower.contains(pat) {
            return BotClass::SearchEngine;
        }
    }
    for pat in AI_TRAINING_BOTS {
        if lower.contains(pat) {
            return BotClass::AiTraining;
        }
    }
    for pat in AI_CITATION_BOTS {
        if lower.contains(pat) {
            return BotClass::AiCitation;
        }
    }
    for pat in AI_MIXED_BOTS {
        if lower.contains(pat) {
            return BotClass::AiMixed;
        }
    }
    for pat in UNKNOWN_AI_BOTS {
        if lower.contains(pat) {
            return BotClass::UnknownAi;
        }
    }
    BotClass::Unknown
}

// ─── Fetch ───────────────────────────────────────────────────────────────────

/// Fetch and parse the robots.txt for the given URL's domain.
/// If `canonical_url` is provided and `is_noindex` is true, also checks
/// whether the page appears in any declared sitemap.xml.
/// Never returns an error — failures are recorded in `RobotsAudit.error`.
pub async fn audit_robots_txt(
    url: &str,
    canonical_url: Option<&str>,
    is_noindex: bool,
    locale: &str,
) -> RobotsAudit {
    let base = extract_base(url);
    let robots_url = format!("{}/robots.txt", base.trim_end_matches('/'));

    let client = match Client::builder()
        .timeout(Duration::from_secs(10))
        .user_agent("auditmysite/1.0")
        .build()
    {
        Ok(c) => c,
        Err(e) => {
            return RobotsAudit {
                error: Some(format!("HTTP-Client konnte nicht erstellt werden: {e}")),
                ..Default::default()
            }
        }
    };

    let text = match client.get(&robots_url).send().await {
        Ok(resp) if resp.status().is_success() => match resp.text().await {
            Ok(t) => t,
            Err(e) => {
                return RobotsAudit {
                    error: Some(format!("robots.txt konnte nicht gelesen werden: {e}")),
                    ..Default::default()
                }
            }
        },
        Ok(resp) => {
            return RobotsAudit {
                error: Some(format!("HTTP {}", resp.status())),
                ..Default::default()
            }
        }
        Err(e) => {
            return RobotsAudit {
                error: Some(format!("Netzwerkfehler: {e}")),
                ..Default::default()
            }
        }
    };

    let mut audit = parse_robots_txt(&text, locale);
    audit.fetched = true;

    // noindex-in-sitemap check: only run when page is noindex and we have a canonical URL
    if is_noindex {
        if let Some(canon) = canonical_url {
            audit.noindex_in_sitemap =
                Some(check_noindex_in_sitemap(&audit.sitemaps, canon, &client).await);
        }
    }

    audit
}

/// Fetch the first declared sitemap and check whether `canonical_url` appears as a `<loc>` entry.
async fn check_noindex_in_sitemap(
    sitemaps: &[String],
    canonical_url: &str,
    client: &Client,
) -> bool {
    let Some(sitemap_url) = sitemaps.first() else {
        return false;
    };

    let Ok(resp) = client
        .get(sitemap_url)
        .timeout(Duration::from_secs(8))
        .send()
        .await
    else {
        return false;
    };
    if !resp.status().is_success() {
        return false;
    }
    let Ok(body) = resp.text().await else {
        return false;
    };

    // Normalise URL for comparison: strip trailing slash
    let norm = |s: &str| s.trim_end_matches('/').to_lowercase();
    let target = norm(canonical_url);

    // Simple <loc>…</loc> scan — avoids an XML parser dependency
    let mut pos = 0;
    while let Some(start) = body[pos..].find("<loc>") {
        let abs = pos + start + 5;
        if let Some(end) = body[abs..].find("</loc>") {
            let loc = norm(&body[abs..abs + end]);
            if loc == target {
                return true;
            }
            pos = abs + end + 6;
        } else {
            break;
        }
    }
    false
}

fn extract_base(url: &str) -> String {
    if let Some(rest) = url.strip_prefix("https://") {
        let host = rest.split('/').next().unwrap_or(rest);
        return format!("https://{}", host);
    }
    if let Some(rest) = url.strip_prefix("http://") {
        let host = rest.split('/').next().unwrap_or(rest);
        return format!("http://{}", host);
    }
    url.to_string()
}

// ─── Parser ──────────────────────────────────────────────────────────────────

fn parse_robots_txt(text: &str, locale: &str) -> RobotsAudit {
    let en = locale == "en";
    let mut groups: Vec<RobotsGroup> = Vec::new();
    let mut sitemaps: Vec<String> = Vec::new();
    let mut crawl_delays: Vec<(String, u32)> = Vec::new();

    // Accumulate current group
    let mut cur_agents: Vec<String> = Vec::new();
    let mut cur_allows: Vec<String> = Vec::new();
    let mut cur_disallows: Vec<String> = Vec::new();

    for line in text.lines() {
        let line = line.trim();
        // Skip empty lines and comments
        if line.is_empty() || line.starts_with('#') {
            // Empty line separates groups — flush if we have rules
            if line.is_empty()
                && !cur_agents.is_empty()
                && (!cur_allows.is_empty() || !cur_disallows.is_empty())
            {
                flush_group(
                    &mut cur_agents,
                    &mut cur_allows,
                    &mut cur_disallows,
                    &mut groups,
                );
            }
            continue;
        }

        let Some(colon_pos) = line.find(':') else {
            continue;
        };
        let key = line[..colon_pos].trim().to_lowercase();
        let value = line[colon_pos + 1..].trim().to_string();

        match key.as_str() {
            "user-agent" => {
                // New user-agent: if we have pending rules flush, else just accumulate agents
                if !cur_allows.is_empty() || !cur_disallows.is_empty() {
                    flush_group(
                        &mut cur_agents,
                        &mut cur_allows,
                        &mut cur_disallows,
                        &mut groups,
                    );
                }
                cur_agents.push(value);
            }
            "allow" if !value.is_empty() => cur_allows.push(value),
            "disallow" => cur_disallows.push(value),
            "sitemap" if !value.is_empty() => sitemaps.push(value),
            "crawl-delay" => {
                if let Ok(delay) = value.parse::<u32>() {
                    for agent in &cur_agents {
                        crawl_delays.push((agent.clone(), delay));
                    }
                }
            }
            _ => {}
        }
    }

    // Flush remaining group
    if !cur_agents.is_empty() {
        flush_group(
            &mut cur_agents,
            &mut cur_allows,
            &mut cur_disallows,
            &mut groups,
        );
    }

    // Derived signals
    let has_wildcard_disallow_all = groups
        .iter()
        .any(|g| g.bot_class == BotClass::Wildcard && g.disallows.iter().any(|d| d == "/"));

    let is_fully_blocked = |g: &RobotsGroup| g.disallows.iter().any(|d| d == "/");

    let blocks_ai_training = groups
        .iter()
        .any(|g| g.bot_class == BotClass::AiTraining && is_fully_blocked(g));

    let blocks_ai_citation = groups
        .iter()
        .any(|g| g.bot_class == BotClass::AiCitation && is_fully_blocked(g));

    let blocks_ai_mixed = groups
        .iter()
        .any(|g| g.bot_class == BotClass::AiMixed && is_fully_blocked(g));

    let blocks_ai_crawlers = blocks_ai_training
        || blocks_ai_citation
        || blocks_ai_mixed
        || groups
            .iter()
            .any(|g| g.bot_class == BotClass::UnknownAi && is_fully_blocked(g));

    let mut audit = RobotsAudit {
        fetched: false, // set by caller after network step
        error: None,
        groups,
        sitemaps,
        crawl_delays,
        has_wildcard_disallow_all,
        blocks_ai_crawlers,
        blocks_ai_training,
        blocks_ai_citation,
        inferred_policy: String::new(),
        noindex_in_sitemap: None,
    };

    // Bake the policy label in the requested language. The PDF re-derives a
    // localized label from the same pure function at presentation time (#406).
    audit.inferred_policy = infer_robots_policy(&audit, en);
    audit
}

/// Re-derive the human-readable robots policy label in the requested language.
///
/// Pure function of the stored audit fields — used both at analysis time
/// (canonical English, for JSON) and by the PDF to localize without re-fetching
/// robots.txt (#406).
pub fn infer_robots_policy(audit: &RobotsAudit, en: bool) -> String {
    let is_fully_blocked = |g: &RobotsGroup| g.disallows.iter().any(|d| d == "/");

    let blocks_ai_mixed = audit
        .groups
        .iter()
        .any(|g| g.bot_class == BotClass::AiMixed && is_fully_blocked(g));

    let has_any_ai_rule = audit.groups.iter().any(|g| {
        matches!(
            g.bot_class,
            BotClass::AiTraining | BotClass::AiCitation | BotClass::AiMixed
        )
    });

    if audit.has_wildcard_disallow_all {
        if en {
            "Everything blocked (Disallow: *)".to_string()
        } else {
            "Alles gesperrt (Disallow: *)".to_string()
        }
    } else if audit.blocks_ai_citation {
        if en {
            "AI fully blocked".to_string()
        } else {
            "KI vollständig blockiert".to_string()
        }
    } else if blocks_ai_mixed && audit.blocks_ai_training {
        if en {
            "AI training blocked (conservative)".to_string()
        } else {
            "KI-Training blockiert (konservativ)".to_string()
        }
    } else if audit.blocks_ai_training {
        if en {
            "AI training blocked, AI search allowed".to_string()
        } else {
            "KI-Training blockiert, KI-Suche erlaubt".to_string()
        }
    } else if has_any_ai_rule {
        if en {
            "AI access open".to_string()
        } else {
            "KI-Zugang offen".to_string()
        }
    } else if en {
        "No explicit AI rule".to_string()
    } else {
        "Keine explizite KI-Regel".to_string()
    }
}

fn flush_group(
    agents: &mut Vec<String>,
    allows: &mut Vec<String>,
    disallows: &mut Vec<String>,
    groups: &mut Vec<RobotsGroup>,
) {
    for agent in agents.drain(..) {
        let class = classify_bot(&agent);
        groups.push(RobotsGroup {
            user_agent: agent,
            bot_class: class,
            allows: allows.clone(),
            disallows: disallows.clone(),
        });
    }
    allows.clear();
    disallows.clear();
}

// ─── Tests ───────────────────────────────────────────────────────────────────

#[cfg(test)]
mod tests {
    use super::*;

    const SAMPLE: &str = r#"
User-agent: *
Disallow: /admin/
Allow: /

User-agent: GPTBot
Disallow: /

User-agent: Googlebot
Allow: /

Sitemap: https://example.com/sitemap.xml
Crawl-delay: 10
"#;

    #[test]
    fn test_parse_basic() {
        let audit = parse_robots_txt(SAMPLE, "de");
        assert!(!audit.has_wildcard_disallow_all);
        assert!(audit.blocks_ai_crawlers);
        assert!(audit
            .sitemaps
            .contains(&"https://example.com/sitemap.xml".to_string()));
    }

    #[test]
    fn test_wildcard_disallow_all() {
        let text = "User-agent: *\nDisallow: /\n";
        let audit = parse_robots_txt(text, "de");
        assert!(audit.has_wildcard_disallow_all);
    }

    #[test]
    fn test_bot_classification() {
        assert_eq!(classify_bot("GPTBot"), BotClass::AiTraining);
        assert_eq!(classify_bot("Googlebot"), BotClass::SearchEngine);
        assert_eq!(classify_bot("CCBot"), BotClass::AiTraining);
        assert_eq!(classify_bot("*"), BotClass::Wildcard);
        assert_eq!(classify_bot("SomeRandomBot"), BotClass::Unknown);
        assert_eq!(classify_bot("claudebot"), BotClass::AiMixed);
        assert_eq!(classify_bot("PerplexityBot"), BotClass::AiCitation);
        assert_eq!(classify_bot("semrushbot"), BotClass::UnknownAi);
    }

    #[test]
    fn test_extract_base() {
        assert_eq!(
            extract_base("https://example.com/path?q=1"),
            "https://example.com"
        );
        assert_eq!(
            extract_base("http://sub.example.com/"),
            "http://sub.example.com"
        );
    }
}