auditmysite 1.4.0

//! Technical SEO analysis
//!
//! Checks HTTPS, canonical URLs, language, sitemap, robots.txt.

use chromiumoxide::cdp::js_protocol::runtime::EvaluateParams;
use chromiumoxide::Page;
use futures::stream::{self, StreamExt};
use reqwest::{redirect::Policy, Client, StatusCode};
use serde::{Deserialize, Serialize};
use std::collections::BTreeSet;
use tracing::{info, warn};

use crate::error::{AuditError, Result};
use crate::taxonomy::Severity;

/// Technical SEO analysis results
#[derive(Debug, Clone, Default, Serialize, Deserialize)]
pub struct TechnicalSeo {
    /// URL uses HTTPS
    pub https: bool,
    /// Has canonical URL
    pub has_canonical: bool,
    /// Canonical URL value
    pub canonical_url: Option<String>,
    /// Has lang attribute
    pub has_lang: bool,
    /// Language value
    pub lang: Option<String>,
    /// Has robots meta
    pub has_robots_meta: bool,
    /// Robots meta content
    pub robots_meta: Option<String>,
    /// Has hreflang tags
    pub has_hreflang: bool,
    /// Hreflang values
    pub hreflang: Vec<HreflangTag>,
    /// x-default hreflang entry is present (only meaningful when has_hreflang is true)
    pub hreflang_has_x_default: bool,
    /// Page URL is not listed in its own hreflang set (only meaningful when has_hreflang is true)
    pub hreflang_missing_self_reference: bool,
    /// Page appears to be part of a paginated archive/listing.
    pub pagination_detected: bool,
    /// `<link rel="prev">` target, if present.
    pub pagination_prev: Option<String>,
    /// `<link rel="next">` target, if present.
    pub pagination_next: Option<String>,
    /// Internal links that contain query parameters (crawl budget dilution risk)
    pub internal_links_with_query_params: u32,
    /// Word count on page
    pub word_count: u32,
    /// Internal links count
    pub internal_links: u32,
    /// External links count
    pub external_links: u32,
    /// Dofollow links (no nofollow/ugc/sponsored rel)
    pub dofollow_links: u32,
    /// Nofollow links (rel=nofollow/ugc/sponsored)
    pub nofollow_links: u32,
    /// Resolved paths of internal links (for inbound link computation in batch mode, capped at 500)
    #[serde(skip_serializing_if = "Vec::is_empty", default)]
    pub internal_link_targets: Vec<String>,
    /// Broken links found
    pub broken_links: Vec<String>,
    /// HTTP subresources referenced from an HTTPS page.
    #[serde(skip_serializing_if = "Vec::is_empty", default)]
    pub mixed_content: Vec<MixedContentResource>,
    /// Progressive Web App manifest and service-worker signals.
    #[serde(default)]
    pub pwa: PwaAnalysis,
    /// Basic AMP conformance signals when AMP markup is present.
    #[serde(default)]
    pub amp: AmpAnalysis,
    /// Sensitive form transport, method, and credential-autocomplete checks.
    #[serde(default)]
    pub form_security: FormSecurityAnalysis,
    /// Visible text excerpt for topic analysis and redundancy checks
    pub text_excerpt: String,
    /// Uses externally hosted Google Fonts assets
    pub uses_remote_google_fonts: bool,
    /// Matching Google Fonts assets or stylesheets
    pub google_fonts_sources: Vec<String>,
    /// Tracking cookies detected on the page
    pub tracking_cookies: Vec<TrackingCookie>,
    /// Cookie metadata from the browser cookie jar. Values are never collected.
    #[serde(skip_serializing_if = "Vec::is_empty", default)]
    pub cookie_inventory: Vec<CookieInventoryItem>,
    /// localStorage/sessionStorage keys detected on the page (values are never collected).
    #[serde(skip_serializing_if = "Vec::is_empty", default)]
    pub storage_items: Vec<StorageItem>,
    /// Tracking providers or signals detected from scripts/resources
    pub tracking_signals: Vec<String>,
    /// Cloudflare Zaraz detected
    pub zaraz: ZarazDetection,
    /// Favicon detected (<link rel="icon"> or apple-touch-icon)
    pub has_favicon: bool,
    /// www / non-www redirect check result
    #[serde(skip_serializing_if = "Option::is_none")]
    pub www_redirect: Option<WwwRedirectCheck>,
    /// og:image is present without an og:image:alt description.
    pub og_image_missing_alt: bool,
    /// twitter:image is present without a twitter:image:alt description.
    pub twitter_image_missing_alt: bool,
    /// Issues found
    pub issues: Vec<TechnicalIssue>,
}

/// Result of the www ↔ non-www redirect check
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct WwwRedirectCheck {
    /// The URL variant that was audited (primary)
    pub primary: String,
    /// The alternative variant (www ↔ non-www)
    pub alternative: String,
    /// HTTP status code returned by the alternative
    pub alternative_status: u16,
    /// Does the alternative redirect (301/302/308) to the primary?
    pub redirects_to_primary: bool,
    /// Does the page canonical point to the primary domain? None if no canonical.
    pub canonical_matches_primary: Option<bool>,
}

/// Tracking cookie detected on the page.
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct TrackingCookie {
    pub name: String,
    pub scope: String,
    pub provider: String,
}

/// Browser cookie metadata. Values are intentionally omitted.
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct CookieInventoryItem {
    pub name: String,
    pub domain: String,
    pub path: String,
    pub secure: bool,
    pub http_only: bool,
    pub same_site: Option<String>,
    pub session: bool,
    pub third_party: bool,
    #[serde(skip_serializing_if = "Option::is_none")]
    pub provider: Option<String>,
    #[serde(skip_serializing_if = "Option::is_none")]
    pub category: Option<String>,
}

/// Browser storage key detected on the page. Values are intentionally omitted.
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct StorageItem {
    pub area: String,
    pub key: String,
    #[serde(skip_serializing_if = "Option::is_none")]
    pub provider: Option<String>,
    #[serde(skip_serializing_if = "Option::is_none")]
    pub category: Option<String>,
}

/// Zaraz detection summary.
#[derive(Debug, Clone, Default, Serialize, Deserialize)]
pub struct ZarazDetection {
    pub detected: bool,
    pub signals: Vec<String>,
}

/// Hreflang tag information
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct HreflangTag {
    pub lang: String,
    pub url: String,
}

/// HTTP subresource referenced by an HTTPS page.
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
pub struct MixedContentResource {
    pub url: String,
    pub resource_type: String,
    pub blockable: bool,
}

/// Basic AMP markup signal analysis.
#[derive(Debug, Clone, Default, Serialize, Deserialize)]
pub struct AmpAnalysis {
    pub detected: bool,
    pub canonical_present: bool,
    pub runtime_script_present: bool,
    pub boilerplate_present: bool,
    #[serde(skip_serializing_if = "Vec::is_empty", default)]
    pub issues: Vec<String>,
}

/// Basic Progressive Web App signal analysis.
#[derive(Debug, Clone, Default, Serialize, Deserialize)]
pub struct PwaAnalysis {
    pub manifest_url: Option<String>,
    pub manifest_valid: bool,
    pub name: Option<String>,
    pub short_name: Option<String>,
    pub start_url: Option<String>,
    pub display: Option<String>,
    pub theme_color: Option<String>,
    pub icons_count: u32,
    pub has_maskable_icon: bool,
    pub service_worker_supported: bool,
    pub service_worker_controlled: bool,
    pub service_worker_registrations: u32,
}

/// Sensitive form security analysis.
#[derive(Debug, Clone, Default, Serialize, Deserialize)]
pub struct FormSecurityAnalysis {
    pub sensitive_forms: u32,
    pub insecure_transport_forms: u32,
    pub get_sensitive_forms: u32,
    pub password_autocomplete_issues: u32,
}

/// Technical SEO issue
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct TechnicalIssue {
    pub issue_type: String,
    pub message: String,
    pub severity: Severity,
}

/// Analyze technical SEO aspects
pub async fn analyze_technical_seo(page: &Page, url: &str) -> Result<TechnicalSeo> {
    info!("Analyzing technical SEO...");

    let https = url.starts_with("https://");

    let js_code = r#"
    (() => {
        const result = {};

        // Canonical
        const canonical = document.querySelector('link[rel="canonical"]');
        result.canonical = canonical ? canonical.getAttribute('href') : null;

        // Language
        result.lang = document.documentElement.getAttribute('lang');

        // Robots meta
        const robots = document.querySelector('meta[name="robots"]');
        result.robots = robots ? robots.getAttribute('content') : null;

        // Hreflang
        result.hreflang = [];
        document.querySelectorAll('link[rel="alternate"][hreflang]').forEach(el => {
            result.hreflang.push({
                lang: el.getAttribute('hreflang'),
                url: el.getAttribute('href')
            });
        });
        result.hreflangHasXDefault = result.hreflang.some(h => h.lang === 'x-default');
        result.hreflangHasSelfReference = result.hreflang.some(h => {
            try {
                const norm = u => new URL(u, window.location.href).href.replace(/\/$/, '');
                return norm(h.url) === norm(window.location.href);
            } catch(e) { return false; }
        });

        // Pagination
        const relHref = rel => {
            const el = Array.from(document.querySelectorAll('link[rel][href]')).find(link => {
                const rels = (link.getAttribute('rel') || '').toLowerCase().split(/\s+/);
                return rels.includes(rel);
            });
            return el ? el.href : null;
        };
        result.paginationPrev = relHref('prev');
        result.paginationNext = relHref('next');
        result.paginationLinkCount = document.querySelectorAll(
            'nav[class*="pag" i], [class*="pagination" i], [aria-label*="pagination" i], [aria-label*="seiten" i]'
        ).length;
        result.paginationAnchorCount = Array.from(document.querySelectorAll('a[href]')).filter(a => {
            const text = (a.textContent || '').trim().toLowerCase();
            const label = (a.getAttribute('aria-label') || '').toLowerCase();
            const rel = (a.getAttribute('rel') || '').toLowerCase();
            const href = a.getAttribute('href') || '';
            return rel.includes('next') || rel.includes('prev') ||
                /^(next|prev|previous|weiter|zurück|vorherige|nächste|[0-9]+)$/.test(text) ||
                label.includes('next') || label.includes('previous') ||
                label.includes('weiter') || label.includes('zurück') ||
                /([?&](page|paged|p)=\d+|\/page\/\d+\/?)/i.test(href);
        }).length;

        // Word count (approximate)
        const text = document.body ? document.body.innerText : '';
        result.wordCount = text.split(/\s+/).filter(w => w.length > 0).length;
        result.textExcerpt = text.slice(0, 4000);

        // Links
        const links = document.querySelectorAll('a[href]');
        let internal = 0, external = 0;
        const currentHost = window.location.host;
        let dofollow = 0, nofollow = 0;

        const internalTargets = [];
        const internalLinkCheckTargets = [];
        links.forEach(a => {
            try {
                const rel = (a.getAttribute('rel') || '').toLowerCase().split(/\s+/);
                const isNofollow = rel.includes('nofollow') || rel.includes('ugc') || rel.includes('sponsored');
                if (isNofollow) { nofollow++; } else { dofollow++; }

                const href = a.getAttribute('href');
                if (!href || /^(mailto|tel|javascript):/i.test(href)) return;
                if (href.startsWith('#')) {
                    internal++;
                    return;
                }
                const linkUrl = new URL(href, window.location.href);
                if (linkUrl.protocol !== 'http:' && linkUrl.protocol !== 'https:') return;
                const isInternal = linkUrl.host === currentHost;
                if (isInternal) {
                    internal++;
                    linkUrl.hash = '';
                    internalLinkCheckTargets.push(linkUrl.href);
                    internalTargets.push(linkUrl.pathname);
                } else {
                    external++;
                }
            } catch (e) {}
        });

        result.internalLinks = internal;
        result.externalLinks = external;
        result.dofollowLinks = dofollow;
        result.nofollowLinks = nofollow;
        result.internalLinkTargets = internalTargets.slice(0, 500);
        result.internalLinkCheckTargets = Array.from(new Set(internalLinkCheckTargets)).slice(0, 50);

        // Crawl budget: internal links with query parameters
        let internalWithQP = 0;
        links.forEach(a => {
            try {
                const href = a.getAttribute('href');
                if (!href || !href.includes('?')) return;
                if (href.startsWith('/')) { internalWithQP++; return; }
                if (href.startsWith('http') && new URL(href).host === currentHost) internalWithQP++;
            } catch(e) {}
        });
        result.internalLinksWithQueryParams = internalWithQP;

        result.stylesheetUrls = Array.from(
            document.querySelectorAll('link[rel=\"stylesheet\"][href]'),
            el => el.href
        );
        result.scriptUrls = Array.from(document.querySelectorAll('script[src]'), el => el.src);
        result.resourceUrls = performance
            .getEntriesByType('resource')
            .map(entry => entry.name)
            .filter(Boolean);
        result.mixedContentResources = [];
        if (window.location.protocol === 'https:') {
            const pushMixed = (url, type, blockable) => {
                try {
                    const resolved = new URL(url, window.location.href);
                    if (resolved.protocol === 'http:') {
                        result.mixedContentResources.push({
                            url: resolved.href,
                            type,
                            blockable
                        });
                    }
                } catch(e) {}
            };
            document.querySelectorAll('script[src]').forEach(el => pushMixed(el.getAttribute('src'), 'script', true));
            document.querySelectorAll('link[rel="stylesheet"][href]').forEach(el => pushMixed(el.getAttribute('href'), 'stylesheet', true));
            document.querySelectorAll('iframe[src], object[data], embed[src]').forEach(el => pushMixed(el.getAttribute('src') || el.getAttribute('data'), 'frame', true));
            document.querySelectorAll('img[src], source[src], video[src], audio[src]').forEach(el => pushMixed(el.getAttribute('src'), 'media', false));
            performance.getEntriesByType('resource').forEach(entry => {
                if (entry.name && entry.name.startsWith('http://')) {
                    const type = entry.initiatorType || 'resource';
                    pushMixed(entry.name, type, ['script', 'link', 'css', 'xmlhttprequest', 'fetch', 'iframe'].includes(type));
                }
            });
        }
        result.cookieNames = document.cookie
            .split(';')
            .map(part => part.trim().split('=')[0])
            .filter(Boolean);
        result.storageItems = [];
        const readStorageKeys = area => {
            try {
                const storage = window[area];
                if (!storage) return;
                for (let i = 0; i < storage.length; i++) {
                    const key = storage.key(i);
                    if (key) result.storageItems.push({ area, key });
                }
            } catch(e) {}
        };
        readStorageKeys('localStorage');
        readStorageKeys('sessionStorage');
        result.hasZarazGlobal = typeof window.zaraz !== 'undefined';

        const htmlEl = document.documentElement;
        const hasAmpAttr = htmlEl.hasAttribute('amp') ||
            Array.from(htmlEl.attributes).some(attr => attr.name === '\u26a1');
        const ampRuntime = !!document.querySelector('script[src^="https://cdn.ampproject.org/v0.js"], script[src="//cdn.ampproject.org/v0.js"]');
        const ampBoilerplate = !!document.querySelector('style[amp-boilerplate]');
        const ampCustomScripts = Array.from(document.querySelectorAll('script[custom-element], script[custom-template]'));
        const ampDetected = hasAmpAttr || ampRuntime || ampBoilerplate || ampCustomScripts.length > 0;
        const ampIssues = [];
        if (ampDetected) {
            if (!hasAmpAttr) ampIssues.push('missing_html_amp_attribute');
            if (!result.canonical) ampIssues.push('missing_canonical');
            if (!ampRuntime) ampIssues.push('missing_amp_runtime');
            if (!ampBoilerplate) ampIssues.push('missing_amp_boilerplate');
            document.querySelectorAll('script:not([type="application/ld+json"]):not([src^="https://cdn.ampproject.org/"]):not([src^="//cdn.ampproject.org/"])')
                .forEach(script => {
                    if (!script.hasAttribute('custom-element') && !script.hasAttribute('custom-template')) {
                        ampIssues.push('custom_script_not_allowed');
                    }
                });
        }
        result.amp = {
            detected: ampDetected,
            canonicalPresent: !!result.canonical,
            runtimeScriptPresent: ampRuntime,
            boilerplatePresent: ampBoilerplate,
            issues: Array.from(new Set(ampIssues))
        };

        // Social-preview image alt text (og:image:alt / twitter:image:alt)
        const ogImageEl = document.querySelector('meta[property="og:image"]');
        const ogImageAltEl = document.querySelector('meta[property="og:image:alt"]');
        result.hasOgImage = !!ogImageEl;
        result.hasOgImageAlt = !!(ogImageAltEl && (ogImageAltEl.getAttribute('content') || '').trim());
        const twitterImageEl = document.querySelector('meta[name="twitter:image"]');
        const twitterImageAltEl = document.querySelector('meta[name="twitter:image:alt"]');
        result.hasTwitterImage = !!twitterImageEl;
        result.hasTwitterImageAlt = !!(twitterImageAltEl && (twitterImageAltEl.getAttribute('content') || '').trim());

        // Favicon
        const faviconEl = document.querySelector(
            'link[rel="icon"], link[rel="shortcut icon"], link[rel="apple-touch-icon"]'
        );
        result.hasFavicon = !!faviconEl;

        const manifestEl = document.querySelector('link[rel~="manifest"][href]');
        result.webManifestUrl = manifestEl ? manifestEl.href : null;
        const themeColorEl = document.querySelector('meta[name="theme-color"]');
        result.themeColor = themeColorEl ? themeColorEl.getAttribute('content') : null;
        result.serviceWorkerSupported = 'serviceWorker' in navigator;
        result.serviceWorkerControlled = !!(navigator.serviceWorker && navigator.serviceWorker.controller);

        const sensitiveNamePattern = /(pass|pwd|credit|card|cc-|cc_|cvc|cvv|iban|account|konto|login|email|mail|tel|phone|token|secret)/i;
        const isSensitiveInput = input => {
            const type = (input.getAttribute('type') || 'text').toLowerCase();
            const name = [
                input.getAttribute('name'),
                input.getAttribute('id'),
                input.getAttribute('autocomplete'),
                input.getAttribute('placeholder'),
                input.getAttribute('aria-label')
            ].filter(Boolean).join(' ');
            return ['password', 'email', 'tel'].includes(type) ||
                ['cc-number', 'cc-csc', 'cc-exp', 'one-time-code'].includes((input.getAttribute('autocomplete') || '').toLowerCase()) ||
                sensitiveNamePattern.test(name);
        };
        result.formSecurity = {
            sensitiveForms: 0,
            insecureTransportForms: 0,
            getSensitiveForms: 0,
            passwordAutocompleteIssues: 0
        };
        document.querySelectorAll('form').forEach(form => {
            const inputs = Array.from(form.querySelectorAll('input, textarea, select'));
            const sensitiveInputs = inputs.filter(isSensitiveInput);
            const passwordInputs = inputs.filter(input => (input.getAttribute('type') || '').toLowerCase() === 'password');
            if (sensitiveInputs.length === 0 && passwordInputs.length === 0) return;

            result.formSecurity.sensitiveForms++;

            const method = (form.getAttribute('method') || 'get').toLowerCase();
            if (method === 'get') {
                result.formSecurity.getSensitiveForms++;
            }

            try {
                const action = form.getAttribute('action') || window.location.href;
                const target = new URL(action, window.location.href);
                if (target.protocol === 'http:') {
                    result.formSecurity.insecureTransportForms++;
                }
            } catch(e) {
                if (window.location.protocol === 'http:') {
                    result.formSecurity.insecureTransportForms++;
                }
            }

            passwordInputs.forEach(input => {
                const autocomplete = (input.getAttribute('autocomplete') || '').trim().toLowerCase();
                if (autocomplete === 'off' || (autocomplete && !['current-password', 'new-password', 'one-time-code'].includes(autocomplete))) {
                    result.formSecurity.passwordAutocompleteIssues++;
                }
            });
        });

        return JSON.stringify(result);
    })()
    "#;

    let js_result = page
        .evaluate(js_code)
        .await
        .map_err(|e| AuditError::CdpError(format!("Technical SEO analysis failed: {}", e)))?;

    let json_str = js_result.value().and_then(|v| v.as_str()).unwrap_or("{}");

    let parsed: serde_json::Value = serde_json::from_str(json_str).unwrap_or_default();

    let canonical_url = parsed["canonical"].as_str().map(String::from);
    let lang = parsed["lang"].as_str().map(String::from);
    let robots_meta = parsed["robots"].as_str().map(String::from);

    let hreflang: Vec<HreflangTag> = parsed["hreflang"]
        .as_array()
        .map(|arr| {
            arr.iter()
                .filter_map(|v| {
                    Some(HreflangTag {
                        lang: v["lang"].as_str()?.to_string(),
                        url: v["url"].as_str()?.to_string(),
                    })
                })
                .collect()
        })
        .unwrap_or_default();

    let hreflang_has_x_default = parsed["hreflangHasXDefault"].as_bool().unwrap_or(false);
    let hreflang_missing_self_reference = !hreflang.is_empty()
        && !parsed["hreflangHasSelfReference"]
            .as_bool()
            .unwrap_or(false);
    let pagination_prev = parsed["paginationPrev"].as_str().map(String::from);
    let pagination_next = parsed["paginationNext"].as_str().map(String::from);
    let pagination_link_count = parsed["paginationLinkCount"].as_u64().unwrap_or(0) as u32;
    let pagination_anchor_count = parsed["paginationAnchorCount"].as_u64().unwrap_or(0) as u32;
    let pagination_detected =
        detect_pagination(url, pagination_link_count, pagination_anchor_count);
    let internal_links_with_query_params =
        parsed["internalLinksWithQueryParams"].as_u64().unwrap_or(0) as u32;

    let word_count = parsed["wordCount"].as_u64().unwrap_or(0) as u32;
    let internal_links = parsed["internalLinks"].as_u64().unwrap_or(0) as u32;
    let external_links = parsed["externalLinks"].as_u64().unwrap_or(0) as u32;
    let dofollow_links = parsed["dofollowLinks"].as_u64().unwrap_or(0) as u32;
    let nofollow_links = parsed["nofollowLinks"].as_u64().unwrap_or(0) as u32;
    let internal_link_targets = parse_string_array(&parsed["internalLinkTargets"]);
    let internal_link_check_targets = parse_string_array(&parsed["internalLinkCheckTargets"]);
    let text_excerpt = parsed["textExcerpt"].as_str().unwrap_or("").to_string();
    let stylesheet_urls = parse_string_array(&parsed["stylesheetUrls"]);
    let script_urls = parse_string_array(&parsed["scriptUrls"]);
    let resource_urls = parse_string_array(&parsed["resourceUrls"]);
    let mixed_content = parse_mixed_content_resources(&parsed["mixedContentResources"]);
    let cookie_names = parse_string_array(&parsed["cookieNames"]);
    let storage_items = parse_storage_items(&parsed["storageItems"]);
    let has_zaraz_global = parsed["hasZarazGlobal"].as_bool().unwrap_or(false);
    let has_favicon = parsed["hasFavicon"].as_bool().unwrap_or(false);
    let has_og_image = parsed["hasOgImage"].as_bool().unwrap_or(false);
    let has_og_image_alt = parsed["hasOgImageAlt"].as_bool().unwrap_or(false);
    let has_twitter_image = parsed["hasTwitterImage"].as_bool().unwrap_or(false);
    let has_twitter_image_alt = parsed["hasTwitterImageAlt"].as_bool().unwrap_or(false);
    let web_manifest_url = parsed["webManifestUrl"].as_str().map(String::from);
    let theme_color = parsed["themeColor"].as_str().map(String::from);
    let service_worker_supported = parsed["serviceWorkerSupported"].as_bool().unwrap_or(false);
    let service_worker_controlled = parsed["serviceWorkerControlled"].as_bool().unwrap_or(false);
    let form_security = parse_form_security_analysis(&parsed["formSecurity"]);
    let amp = parse_amp_analysis(&parsed["amp"]);

    let google_fonts_sources =
        collect_google_fonts_sources(&stylesheet_urls, &resource_urls, &script_urls);
    let tracking_cookies: Vec<TrackingCookie> = cookie_names
        .iter()
        .filter_map(|name| classify_tracking_cookie(name))
        .collect();
    let cookie_inventory = collect_cookie_inventory(page, url).await;
    let tracking_signals = collect_tracking_signals(
        &script_urls,
        &resource_urls,
        &tracking_cookies,
        &cookie_inventory,
        &storage_items,
    );
    let zaraz = detect_zaraz(
        &script_urls,
        &resource_urls,
        &tracking_cookies,
        has_zaraz_global,
    );
    let broken_links = check_broken_internal_links(url, &internal_link_check_targets).await;
    let service_worker_registrations = read_service_worker_registration_count(page).await;
    let pwa = analyze_pwa(
        web_manifest_url,
        theme_color,
        service_worker_supported,
        service_worker_controlled,
        service_worker_registrations,
    )
    .await;

    info!(
        "Technical SEO: HTTPS={}, canonical={}, lang={}, words={}, favicon={}, google_fonts={}, tracking_cookies={}, zaraz={}, pagination={}",
        https,
        canonical_url.is_some(),
        lang.is_some(),
        word_count,
        has_favicon,
        !google_fonts_sources.is_empty(),
        tracking_cookies.len(),
        zaraz.detected,
        pagination_detected
    );

    // www / non-www redirect check
    let www_redirect = check_www_redirect(url, canonical_url.as_deref()).await;

    let mut technical = TechnicalSeo {
        https,
        has_canonical: canonical_url.is_some(),
        canonical_url,
        has_lang: lang.is_some(),
        lang,
        has_robots_meta: robots_meta.is_some(),
        robots_meta,
        has_hreflang: !hreflang.is_empty(),
        hreflang,
        hreflang_has_x_default,
        hreflang_missing_self_reference,
        pagination_detected,
        pagination_prev,
        pagination_next,
        internal_links_with_query_params,
        word_count,
        internal_links,
        external_links,
        dofollow_links,
        nofollow_links,
        internal_link_targets,
        broken_links,
        mixed_content,
        pwa,
        amp,
        form_security,
        text_excerpt,
        uses_remote_google_fonts: !google_fonts_sources.is_empty(),
        google_fonts_sources,
        tracking_cookies,
        cookie_inventory,
        storage_items,
        tracking_signals,
        zaraz,
        has_favicon,
        www_redirect,
        og_image_missing_alt: has_og_image && !has_og_image_alt,
        twitter_image_missing_alt: has_twitter_image && !has_twitter_image_alt,
        issues: Vec::new(),
    };

    // Issues are baked in canonical English (for JSON); the PDF re-derives
    // localized issues via collect_technical_issues at presentation time (#406).
    technical.issues = collect_technical_issues(&technical, true);

    Ok(technical)
}

/// Build the technical-SEO issue list in the requested language.
///
/// Pure function of the analysis struct — called with English at analysis time
/// (canonical, for JSON) and re-called with the report locale by the PDF
/// presentation builder (#406).
pub fn collect_technical_issues(t: &TechnicalSeo, en: bool) -> Vec<TechnicalIssue> {
    let mut issues = Vec::new();

    if !t.https {
        issues.push(TechnicalIssue {
            issue_type: "no_https".to_string(),
            message: if en {
                "Page is not served over HTTPS".to_string()
            } else {
                "Seite wird nicht über HTTPS ausgeliefert".to_string()
            },
            severity: Severity::High,
        });
    }

    if t.canonical_url.is_none() {
        issues.push(TechnicalIssue {
            issue_type: "no_canonical".to_string(),
            message: if en {
                "Missing canonical URL".to_string()
            } else {
                "Fehlende Canonical-URL".to_string()
            },
            severity: Severity::Medium,
        });
    }

    if t.lang.is_none() {
        issues.push(TechnicalIssue {
            issue_type: "no_lang".to_string(),
            message: if en {
                "Missing lang attribute on html element".to_string()
            } else {
                "Fehlendes lang-Attribut am html-Element".to_string()
            },
            severity: Severity::Medium,
        });
    }

    if t.word_count < 300 {
        issues.push(TechnicalIssue {
            issue_type: "thin_content".to_string(),
            message: if en {
                format!(
                    "Page has thin content ({} words, recommended: 300+)",
                    t.word_count
                )
            } else {
                format!(
                    "Seite hat dünnen Inhalt ({} Wörter, empfohlen: 300+)",
                    t.word_count
                )
            },
            severity: Severity::Medium,
        });
    }

    if t.internal_links == 0 {
        issues.push(TechnicalIssue {
            issue_type: "no_internal_links".to_string(),
            message: if en {
                "Page has no internal links".to_string()
            } else {
                "Seite hat keine internen Links".to_string()
            },
            severity: Severity::Medium,
        });
    }

    if !t.google_fonts_sources.is_empty() {
        issues.push(TechnicalIssue {
            issue_type: "remote_google_fonts".to_string(),
            message: if en {
                "Externally hosted Google Fonts detected".to_string()
            } else {
                "Extern gehostete Google Fonts erkannt".to_string()
            },
            severity: Severity::Low,
        });
    }

    if let Some(ref robots) = t.robots_meta {
        if robots.to_lowercase().contains("noindex") {
            issues.push(TechnicalIssue {
                issue_type: "noindex".to_string(),
                message: if en {
                    "Page has a noindex directive — it is not indexed by search engines".to_string()
                } else {
                    "Seite hat noindex-Direktive — wird von Suchmaschinen nicht indexiert"
                        .to_string()
                },
                severity: Severity::High,
            });
        }
    }

    if !t.hreflang.is_empty() && !t.hreflang_has_x_default {
        issues.push(TechnicalIssue {
            issue_type: "hreflang_no_x_default".to_string(),
            message: if en {
                "hreflang annotations without an x-default entry".to_string()
            } else {
                "hreflang-Annotierungen ohne x-default-Eintrag".to_string()
            },
            severity: Severity::Low,
        });
    }

    if t.hreflang_missing_self_reference {
        issues.push(TechnicalIssue {
            issue_type: "hreflang_no_self_reference".to_string(),
            message: if en {
                "hreflang set contains no self-reference for the current page".to_string()
            } else {
                "hreflang-Set enthält keine Self-Reference für die aktuelle Seite".to_string()
            },
            severity: Severity::Low,
        });
    }

    if t.internal_links_with_query_params > 0 {
        issues.push(TechnicalIssue {
            issue_type: "internal_links_with_query_params".to_string(),
            message: if en {
                format!(
                    "{} internal {} with query parameters — can dilute crawl budget",
                    t.internal_links_with_query_params,
                    if t.internal_links_with_query_params == 1 {
                        "link"
                    } else {
                        "links"
                    }
                )
            } else {
                format!(
                    "{} interne {} mit Query-Parametern — können Crawl-Budget verwässern",
                    t.internal_links_with_query_params,
                    if t.internal_links_with_query_params == 1 {
                        "Link"
                    } else {
                        "Links"
                    }
                )
            },
            severity: Severity::Low,
        });
    }

    collect_pagination_issues(
        t.pagination_detected,
        t.pagination_prev.as_deref(),
        t.pagination_next.as_deref(),
        t.www_redirect.as_ref().map(|c| c.primary.as_str()),
        en,
        &mut issues,
    );

    if !t.broken_links.is_empty() {
        issues.push(TechnicalIssue {
            issue_type: "broken_internal_links".to_string(),
            message: if en {
                format!(
                    "{} broken internal {} detected: {}",
                    t.broken_links.len(),
                    if t.broken_links.len() == 1 {
                        "link"
                    } else {
                        "links"
                    },
                    t.broken_links
                        .iter()
                        .take(3)
                        .cloned()
                        .collect::<Vec<_>>()
                        .join("; ")
                )
            } else {
                format!(
                    "{} defekte interne {} erkannt: {}",
                    t.broken_links.len(),
                    if t.broken_links.len() == 1 {
                        "Link"
                    } else {
                        "Links"
                    },
                    t.broken_links
                        .iter()
                        .take(3)
                        .cloned()
                        .collect::<Vec<_>>()
                        .join("; ")
                )
            },
            severity: Severity::High,
        });
    }

    if !t.mixed_content.is_empty() {
        let blockable_count = t
            .mixed_content
            .iter()
            .filter(|resource| resource.blockable)
            .count();
        let samples = t
            .mixed_content
            .iter()
            .take(3)
            .map(|resource| resource.url.clone())
            .collect::<Vec<_>>()
            .join("; ");
        issues.push(TechnicalIssue {
            issue_type: "mixed_content".to_string(),
            message: if en {
                format!(
                    "{} HTTP {} referenced from HTTPS page ({} blockable): {}",
                    t.mixed_content.len(),
                    if t.mixed_content.len() == 1 {
                        "subresource"
                    } else {
                        "subresources"
                    },
                    blockable_count,
                    samples
                )
            } else {
                format!(
                    "{} {} auf HTTPS-Seite referenziert ({} blockierbar): {}",
                    t.mixed_content.len(),
                    if t.mixed_content.len() == 1 {
                        "HTTP-Subressource"
                    } else {
                        "HTTP-Subressourcen"
                    },
                    blockable_count,
                    samples
                )
            },
            severity: if blockable_count > 0 {
                Severity::High
            } else {
                Severity::Medium
            },
        });
    }

    if t.og_image_missing_alt {
        issues.push(TechnicalIssue {
            issue_type: "og_image_missing_alt".to_string(),
            message: if en {
                "og:image is set without an og:image:alt description for the preview image"
                    .to_string()
            } else {
                "og:image ist gesetzt, aber ohne og:image:alt-Beschreibung für das Vorschaubild"
                    .to_string()
            },
            severity: Severity::Low,
        });
    }

    if t.twitter_image_missing_alt {
        issues.push(TechnicalIssue {
            issue_type: "twitter_image_missing_alt".to_string(),
            message: if en {
                "twitter:image is set without a twitter:image:alt description for the preview image"
                    .to_string()
            } else {
                "twitter:image ist gesetzt, aber ohne twitter:image:alt-Beschreibung für das Vorschaubild"
                    .to_string()
            },
            severity: Severity::Low,
        });
    }

    collect_pwa_issues(&t.pwa, en, &mut issues);
    collect_amp_issues(&t.amp, en, &mut issues);
    collect_form_security_issues(&t.form_security, en, &mut issues);

    if let Some(ref check) = t.www_redirect {
        if !check.redirects_to_primary && check.alternative_status == 200 {
            let canonical_note = match (check.canonical_matches_primary, en) {
                (Some(true), true) => " Canonical tag is set correctly, but a 301 is still recommended — canonicals are hints, not directives.",
                (Some(true), false) => " Canonical-Tag ist korrekt gesetzt, ein 301 wird dennoch empfohlen — Canonicals sind Hinweise, keine Direktiven.",
                (Some(false), true) => " Additionally, canonical does not point to the primary domain.",
                (Some(false), false) => " Zusätzlich zeigt das Canonical nicht auf die primäre Domain.",
                (None, true) => " No canonical tag found.",
                (None, false) => " Kein Canonical-Tag gefunden.",
            };
            issues.push(TechnicalIssue {
                issue_type: "www_no_redirect".to_string(),
                message: if en {
                    format!(
                        "Both {} and {} serve content (HTTP 200). Add a 301 redirect from the alternative to the primary to prevent duplicate content.{}",
                        check.primary, check.alternative, canonical_note
                    )
                } else {
                    format!(
                        "Sowohl {} als auch {} liefern Inhalt (HTTP 200). Richte eine 301-Weiterleitung von der Alternative auf die primäre Variante ein, um Duplicate Content zu vermeiden.{}",
                        check.primary, check.alternative, canonical_note
                    )
                },
                severity: Severity::Medium,
            });
        }
        if let Some(false) = check.canonical_matches_primary {
            issues.push(TechnicalIssue {
                issue_type: "canonical_domain_mismatch".to_string(),
                message: if en {
                    format!(
                        "Canonical URL points to a different domain variant than the audited URL ({} vs canonical).",
                        check.primary
                    )
                } else {
                    format!(
                        "Canonical-URL zeigt auf eine andere Domain-Variante als die geprüfte URL ({} vs. Canonical).",
                        check.primary
                    )
                },
                severity: Severity::Medium,
            });
        }
    }

    issues
}

/// Check whether the www ↔ non-www counterpart of `url` redirects to `url`
/// or serves its own content (duplicate).
async fn check_www_redirect(url: &str, canonical: Option<&str>) -> Option<WwwRedirectCheck> {
    // Build the alternative URL variant
    let parsed = url::Url::parse(url).ok()?;
    let host = parsed.host_str()?;

    let (primary_host, alt_host) = if let Some(stripped) = host.strip_prefix("www.") {
        (host.to_string(), stripped.to_string())
    } else {
        (host.to_string(), format!("www.{}", host))
    };

    // Only makes sense for plain domains; skip IPs, localhost, and subdomains
    if primary_host.parse::<std::net::IpAddr>().is_ok() || primary_host == "localhost" {
        return None;
    }
    // Determine the bare host (without www prefix) and skip if it's a subdomain
    let bare_host = host.strip_prefix("www.").unwrap_or(host);
    if bare_host.split('.').count() > 2 {
        return None;
    }

    let mut alt_url = parsed.clone();
    alt_url.set_host(Some(&alt_host)).ok()?;
    let alternative = alt_url.as_str().to_string();
    let primary = url.to_string();

    let client = reqwest::Client::builder()
        .timeout(std::time::Duration::from_secs(8))
        .redirect(reqwest::redirect::Policy::none())
        .user_agent("Mozilla/5.0 (compatible; AuditMySite/1.0)")
        .build()
        .ok()?;

    let response = match client.head(&alternative).send().await {
        Ok(r) => r,
        Err(e) => {
            tracing::debug!("www-redirect check failed for {}: {}", alternative, e);
            return Some(WwwRedirectCheck {
                primary,
                alternative,
                alternative_status: 0,
                redirects_to_primary: false,
                canonical_matches_primary: canonical_matches(&primary_host, canonical),
            });
        }
    };

    let status = response.status().as_u16();
    let redirects_to_primary = if matches!(status, 301 | 302 | 307 | 308) {
        response
            .headers()
            .get(reqwest::header::LOCATION)
            .and_then(|v| v.to_str().ok())
            .map(|loc| {
                // Location may be absolute or relative; check if it resolves to primary host
                if let Ok(loc_url) = url::Url::parse(loc) {
                    loc_url.host_str() == Some(&primary_host)
                } else {
                    // relative redirect — stays on same host
                    false
                }
            })
            .unwrap_or(false)
    } else {
        false
    };

    Some(WwwRedirectCheck {
        primary,
        alternative,
        alternative_status: status,
        redirects_to_primary,
        canonical_matches_primary: canonical_matches(&primary_host, canonical),
    })
}

/// Returns Some(true) if canonical points to primary_host, Some(false) if it points elsewhere,
/// None if no canonical.
fn canonical_matches(primary_host: &str, canonical: Option<&str>) -> Option<bool> {
    let canon = canonical?;
    let parsed = url::Url::parse(canon).ok()?;
    Some(parsed.host_str() == Some(primary_host))
}

fn parse_string_array(value: &serde_json::Value) -> Vec<String> {
    value
        .as_array()
        .map(|items| {
            items
                .iter()
                .filter_map(|v| v.as_str())
                .map(str::trim)
                .filter(|s| !s.is_empty())
                .map(ToOwned::to_owned)
                .collect()
        })
        .unwrap_or_default()
}

fn parse_storage_items(value: &serde_json::Value) -> Vec<StorageItem> {
    let mut seen = BTreeSet::new();
    value
        .as_array()
        .map(|items| {
            items
                .iter()
                .filter_map(|item| {
                    let area = item["area"].as_str()?.trim();
                    let key = item["key"].as_str()?.trim();
                    if area.is_empty() || key.is_empty() {
                        return None;
                    }
                    let dedupe_key = format!("{area}:{key}");
                    if !seen.insert(dedupe_key) {
                        return None;
                    }
                    let classified = classify_tracking_cookie(key);
                    let provider = classified.as_ref().map(|cookie| cookie.provider.clone());
                    Some(StorageItem {
                        area: area.to_string(),
                        key: key.to_string(),
                        category: provider
                            .as_deref()
                            .and_then(classify_tracking_provider_category),
                        provider,
                    })
                })
                .collect()
        })
        .unwrap_or_default()
}

fn parse_amp_analysis(value: &serde_json::Value) -> AmpAnalysis {
    let issues = value
        .get("issues")
        .and_then(|items| items.as_array())
        .map(|items| {
            items
                .iter()
                .filter_map(|item| item.as_str())
                .map(str::trim)
                .filter(|item| !item.is_empty())
                .map(ToOwned::to_owned)
                .collect()
        })
        .unwrap_or_default();

    AmpAnalysis {
        detected: value
            .get("detected")
            .and_then(|v| v.as_bool())
            .unwrap_or(false),
        canonical_present: value
            .get("canonicalPresent")
            .and_then(|v| v.as_bool())
            .unwrap_or(false),
        runtime_script_present: value
            .get("runtimeScriptPresent")
            .and_then(|v| v.as_bool())
            .unwrap_or(false),
        boilerplate_present: value
            .get("boilerplatePresent")
            .and_then(|v| v.as_bool())
            .unwrap_or(false),
        issues,
    }
}

async fn collect_cookie_inventory(page: &Page, page_url: &str) -> Vec<CookieInventoryItem> {
    let page_host = url::Url::parse(page_url)
        .ok()
        .and_then(|url| url.host_str().map(str::to_ascii_lowercase))
        .unwrap_or_default();

    let cookies = match page.get_cookies().await {
        Ok(cookies) => cookies,
        Err(e) => {
            warn!("cookie inventory collection failed: {}", e);
            return Vec::new();
        }
    };

    cookies
        .into_iter()
        .map(|cookie| {
            let domain = cookie.domain.trim_start_matches('.').to_ascii_lowercase();
            let third_party = !page_host.is_empty()
                && domain != page_host
                && !page_host.ends_with(&format!(".{domain}"));
            let classified = classify_tracking_cookie(&cookie.name);
            let provider = classified.as_ref().map(|cookie| cookie.provider.clone());
            CookieInventoryItem {
                category: provider
                    .as_deref()
                    .and_then(classify_tracking_provider_category),
                provider,
                name: cookie.name,
                domain: cookie.domain,
                path: cookie.path,
                secure: cookie.secure,
                http_only: cookie.http_only,
                same_site: cookie
                    .same_site
                    .as_ref()
                    .map(|same_site| same_site.as_ref().to_string()),
                session: cookie.session,
                third_party,
            }
        })
        .collect()
}

fn parse_mixed_content_resources(value: &serde_json::Value) -> Vec<MixedContentResource> {
    let mut seen = BTreeSet::new();
    value
        .as_array()
        .map(|items| {
            items
                .iter()
                .filter_map(|item| {
                    let url = item["url"].as_str()?.trim();
                    if url.is_empty() || !seen.insert(url.to_string()) {
                        return None;
                    }
                    Some(MixedContentResource {
                        url: url.to_string(),
                        resource_type: item["type"]
                            .as_str()
                            .unwrap_or("resource")
                            .trim()
                            .to_string(),
                        blockable: item["blockable"].as_bool().unwrap_or(false),
                    })
                })
                .collect()
        })
        .unwrap_or_default()
}

fn parse_form_security_analysis(value: &serde_json::Value) -> FormSecurityAnalysis {
    FormSecurityAnalysis {
        sensitive_forms: value["sensitiveForms"].as_u64().unwrap_or(0) as u32,
        insecure_transport_forms: value["insecureTransportForms"].as_u64().unwrap_or(0) as u32,
        get_sensitive_forms: value["getSensitiveForms"].as_u64().unwrap_or(0) as u32,
        password_autocomplete_issues: value["passwordAutocompleteIssues"].as_u64().unwrap_or(0)
            as u32,
    }
}

async fn analyze_pwa(
    manifest_url: Option<String>,
    theme_color: Option<String>,
    service_worker_supported: bool,
    service_worker_controlled: bool,
    service_worker_registrations: u32,
) -> PwaAnalysis {
    let Some(ref manifest_url) = manifest_url else {
        return PwaAnalysis {
            manifest_url,
            theme_color,
            service_worker_supported,
            service_worker_controlled,
            service_worker_registrations,
            ..PwaAnalysis::default()
        };
    };

    let manifest = fetch_web_manifest(manifest_url).await;
    let Some(manifest) = manifest else {
        return PwaAnalysis {
            manifest_url: Some(manifest_url.clone()),
            theme_color,
            service_worker_supported,
            service_worker_controlled,
            service_worker_registrations,
            ..PwaAnalysis::default()
        };
    };

    let icons = manifest["icons"].as_array().cloned().unwrap_or_default();
    let has_maskable_icon = icons.iter().any(|icon| {
        icon["purpose"]
            .as_str()
            .map(|purpose| purpose.split_whitespace().any(|part| part == "maskable"))
            .unwrap_or(false)
    });
    let manifest_theme_color = manifest["theme_color"].as_str().map(String::from);
    let display = manifest["display"].as_str().map(String::from);
    let start_url = manifest["start_url"].as_str().map(String::from);
    let name = manifest["name"].as_str().map(String::from);
    let short_name = manifest["short_name"].as_str().map(String::from);
    let manifest_valid = (name.is_some() || short_name.is_some())
        && start_url.is_some()
        && display.is_some()
        && !icons.is_empty()
        && (manifest_theme_color.is_some() || theme_color.is_some());

    PwaAnalysis {
        manifest_url: Some(manifest_url.clone()),
        manifest_valid,
        name,
        short_name,
        start_url,
        display,
        theme_color: manifest_theme_color.or(theme_color),
        icons_count: icons.len() as u32,
        has_maskable_icon,
        service_worker_supported,
        service_worker_controlled,
        service_worker_registrations,
    }
}

async fn fetch_web_manifest(manifest_url: &str) -> Option<serde_json::Value> {
    let client = Client::builder()
        .timeout(std::time::Duration::from_secs(5))
        .redirect(Policy::limited(4))
        .build()
        .ok()?;
    let response = client
        .get(manifest_url)
        .header("User-Agent", "auditmysite-pwa-checker/1.0")
        .send()
        .await
        .ok()?;
    if !response.status().is_success() {
        return None;
    }
    response.json::<serde_json::Value>().await.ok()
}

async fn read_service_worker_registration_count(page: &Page) -> u32 {
    let expression = r#"
        (async () => {
            if (!('serviceWorker' in navigator) || !navigator.serviceWorker.getRegistrations) {
                return 0;
            }
            try {
                const registrations = await navigator.serviceWorker.getRegistrations();
                return registrations.length || 0;
            } catch (e) {
                return 0;
            }
        })()
    "#;
    let Ok(params) = EvaluateParams::builder()
        .expression(expression.to_string())
        .await_promise(true)
        .build()
    else {
        return 0;
    };
    page.execute(params)
        .await
        .ok()
        .and_then(|result| result.result.result.value)
        .and_then(|value| value.as_u64())
        .unwrap_or(0) as u32
}

fn collect_pwa_issues(pwa: &PwaAnalysis, en: bool, issues: &mut Vec<TechnicalIssue>) {
    if pwa.manifest_url.is_none() {
        issues.push(TechnicalIssue {
            issue_type: "pwa_missing_manifest".to_string(),
            message: if en {
                "Missing web app manifest".to_string()
            } else {
                "Fehlendes Web-App-Manifest".to_string()
            },
            severity: Severity::Low,
        });
        return;
    }

    if !pwa.manifest_valid {
        issues.push(TechnicalIssue {
            issue_type: "pwa_invalid_manifest".to_string(),
            message: if en {
                "Web app manifest is missing required PWA fields".to_string()
            } else {
                "Web-App-Manifest fehlen erforderliche PWA-Felder".to_string()
            },
            severity: Severity::Medium,
        });
    }

    // Deliberately no "missing service worker" issue: a web app manifest is
    // routinely shipped for icons and theme color alone, so its presence does
    // not establish that the site intends to be an installable, offline-capable
    // PWA. `PwaAnalysis.service_worker_*` stays in the JSON as information.
}

fn collect_amp_issues(amp: &AmpAnalysis, en: bool, issues: &mut Vec<TechnicalIssue>) {
    if !amp.detected || amp.issues.is_empty() {
        return;
    }

    for issue in &amp.issues {
        let (message_en, message_de, severity) = match issue.as_str() {
            "missing_html_amp_attribute" => (
                "AMP markup detected, but the html element has no amp attribute",
                "AMP-Markup erkannt, aber das html-Element hat kein amp-Attribut",
                Severity::Medium,
            ),
            "missing_canonical" => (
                "AMP page is missing a canonical link",
                "AMP-Seite hat keinen Canonical-Link",
                Severity::Medium,
            ),
            "missing_amp_runtime" => (
                "AMP page is missing the AMP runtime script",
                "AMP-Seite bindet das AMP-Runtime-Script nicht ein",
                Severity::Medium,
            ),
            "missing_amp_boilerplate" => (
                "AMP page is missing amp-boilerplate CSS",
                "AMP-Seite enthält kein amp-boilerplate-CSS",
                Severity::Low,
            ),
            "custom_script_not_allowed" => (
                "AMP page contains a custom script outside the AMP runtime",
                "AMP-Seite enthält ein eigenes Script außerhalb der AMP-Runtime",
                Severity::Medium,
            ),
            _ => continue,
        };

        issues.push(TechnicalIssue {
            issue_type: format!("amp_{issue}"),
            message: if en { message_en } else { message_de }.to_string(),
            severity,
        });
    }
}

fn collect_form_security_issues(
    form_security: &FormSecurityAnalysis,
    en: bool,
    issues: &mut Vec<TechnicalIssue>,
) {
    if form_security.insecure_transport_forms > 0 {
        issues.push(TechnicalIssue {
            issue_type: "form_insecure_transport".to_string(),
            message: if en {
                format!(
                    "{} sensitive {} submit to HTTP or are hosted on HTTP",
                    form_security.insecure_transport_forms,
                    if form_security.insecure_transport_forms == 1 {
                        "form"
                    } else {
                        "forms"
                    }
                )
            } else {
                format!(
                    "{} {} senden an HTTP oder werden über HTTP ausgeliefert",
                    form_security.insecure_transport_forms,
                    if form_security.insecure_transport_forms == 1 {
                        "sensibles Formular"
                    } else {
                        "sensible Formulare"
                    }
                )
            },
            severity: Severity::High,
        });
    }

    if form_security.get_sensitive_forms > 0 {
        issues.push(TechnicalIssue {
            issue_type: "form_sensitive_get".to_string(),
            message: if en {
                format!(
                    "{} sensitive {} use GET and can leak values into URLs and logs",
                    form_security.get_sensitive_forms,
                    if form_security.get_sensitive_forms == 1 {
                        "form"
                    } else {
                        "forms"
                    }
                )
            } else {
                format!(
                    "{} {} verwenden GET und können Werte in URLs und Logs preisgeben",
                    form_security.get_sensitive_forms,
                    if form_security.get_sensitive_forms == 1 {
                        "sensibles Formular"
                    } else {
                        "sensible Formulare"
                    }
                )
            },
            severity: Severity::High,
        });
    }

    if form_security.password_autocomplete_issues > 0 {
        issues.push(TechnicalIssue {
            issue_type: "form_password_autocomplete".to_string(),
            message: if en {
                format!(
                    "{} password {} have autocomplete disabled or misconfigured",
                    form_security.password_autocomplete_issues,
                    if form_security.password_autocomplete_issues == 1 {
                        "field"
                    } else {
                        "fields"
                    }
                )
            } else {
                format!(
                    "{} {} haben autocomplete deaktiviert oder fehlkonfiguriert",
                    form_security.password_autocomplete_issues,
                    if form_security.password_autocomplete_issues == 1 {
                        "Passwortfeld"
                    } else {
                        "Passwortfelder"
                    }
                )
            },
            severity: Severity::Medium,
        });
    }
}

async fn check_broken_internal_links(source_url: &str, targets: &[String]) -> Vec<String> {
    let Some(source_origin) = url_origin(source_url) else {
        return Vec::new();
    };
    let client = Client::builder()
        .redirect(Policy::limited(6))
        .timeout(std::time::Duration::from_secs(5))
        .build()
        .unwrap_or_else(|_| Client::new());

    let mut unique = BTreeSet::new();
    for target in targets {
        if url_origin(target).as_deref() == Some(source_origin.as_str()) {
            unique.insert(target.clone());
        }
    }

    stream::iter(unique.into_iter().take(50))
        .map(|target| {
            let client = client.clone();
            async move {
                check_link_status(&client, &target)
                    .await
                    .map(|reason| format!("{target} ({reason})"))
            }
        })
        .buffer_unordered(8)
        .filter_map(|result| async move { result })
        .collect()
        .await
}

async fn check_link_status(client: &Client, target: &str) -> Option<String> {
    match client
        .head(target)
        .header("User-Agent", "auditmysite-link-checker/1.0")
        .send()
        .await
    {
        Ok(response)
            if response.status() == StatusCode::METHOD_NOT_ALLOWED
                || response.status() == StatusCode::NOT_IMPLEMENTED =>
        {
            check_link_status_with_get(client, target).await
        }
        Ok(response) if response.status().as_u16() >= 400 => {
            Some(format!("HTTP {}", response.status().as_u16()))
        }
        Ok(_) => None,
        Err(head_error) => match check_link_status_with_get(client, target).await {
            Some(reason) => Some(reason),
            None => Some(head_error.to_string()),
        },
    }
}

async fn check_link_status_with_get(client: &Client, target: &str) -> Option<String> {
    match client
        .get(target)
        .header("User-Agent", "auditmysite-link-checker/1.0")
        .send()
        .await
    {
        Ok(response) if response.status().as_u16() >= 400 => {
            Some(format!("HTTP {}", response.status().as_u16()))
        }
        Ok(_) => None,
        Err(error) => Some(error.to_string()),
    }
}

fn url_origin(url: &str) -> Option<String> {
    let parsed = url::Url::parse(url).ok()?;
    let host = parsed.host_str()?;
    let port = parsed
        .port()
        .map(|port| format!(":{port}"))
        .unwrap_or_default();
    Some(format!("{}://{}{}", parsed.scheme(), host, port))
}

fn detect_pagination(
    url: &str,
    pagination_container_count: u32,
    pagination_anchor_count: u32,
) -> bool {
    if pagination_container_count > 0 || pagination_anchor_count >= 2 {
        return true;
    }

    let Ok(parsed) = url::Url::parse(url) else {
        return false;
    };

    let has_page_query = parsed.query_pairs().any(|(key, value)| {
        matches!(key.as_ref(), "page" | "paged" | "p")
            && value.parse::<u32>().is_ok_and(|page| page > 1)
    });
    let has_page_path = parsed
        .path_segments()
        .map(|segments| {
            let items: Vec<_> = segments.collect();
            items
                .windows(2)
                .any(|pair| pair[0].eq_ignore_ascii_case("page") && pair[1].parse::<u32>().is_ok())
        })
        .unwrap_or(false);

    has_page_query || has_page_path
}

fn collect_pagination_issues(
    pagination_detected: bool,
    prev: Option<&str>,
    next: Option<&str>,
    url: Option<&str>,
    en: bool,
    issues: &mut Vec<TechnicalIssue>,
) {
    if !pagination_detected {
        return;
    }

    // Deliberately no "missing rel=prev/next" issue: Google dropped the pair
    // as an indexing signal in 2019, so reporting its absence as an SEO problem
    // is misleading. The checks below stay — they only fire when rel links are
    // actually present and wrong, which is broken markup either way.
    if prev.is_none() && next.is_none() {
        return;
    }

    if let Some(url) = url {
        if prev.is_some_and(|href| same_url(href, url))
            || next.is_some_and(|href| same_url(href, url))
        {
            issues.push(TechnicalIssue {
                issue_type: "pagination_self_referential_rel_link".to_string(),
                message: if en {
                    "Pagination rel link points to the current page instead of an adjacent page"
                        .to_string()
                } else {
                    "Pagination-rel-Link zeigt auf die aktuelle Seite statt auf eine Nachbarseite"
                        .to_string()
                },
                severity: Severity::Low,
            });
        }
    }

    if let (Some(prev), Some(next)) = (prev, next) {
        if same_url(prev, next) {
            issues.push(TechnicalIssue {
                issue_type: "pagination_duplicate_prev_next".to_string(),
                message: if en {
                    "Pagination rel=\"prev\" and rel=\"next\" point to the same URL".to_string()
                } else {
                    "Pagination rel=\"prev\" und rel=\"next\" zeigen auf dieselbe URL".to_string()
                },
                severity: Severity::Low,
            });
        }
    }
}

fn same_url(a: &str, b: &str) -> bool {
    let normalize = |input: &str| {
        url::Url::parse(input)
            .map(|mut parsed| {
                parsed.set_fragment(None);
                parsed.to_string().trim_end_matches('/').to_string()
            })
            .unwrap_or_else(|_| input.trim_end_matches('/').to_string())
    };
    normalize(a) == normalize(b)
}

fn collect_google_fonts_sources(
    stylesheet_urls: &[String],
    resource_urls: &[String],
    script_urls: &[String],
) -> Vec<String> {
    let mut urls = BTreeSet::new();
    for url in stylesheet_urls
        .iter()
        .chain(resource_urls.iter())
        .chain(script_urls.iter())
    {
        if is_google_fonts_url(url) {
            urls.insert(url.to_string());
        }
    }
    urls.into_iter().collect()
}

fn is_google_fonts_url(url: &str) -> bool {
    let lower = url.to_ascii_lowercase();
    lower.contains("fonts.googleapis.com") || lower.contains("fonts.gstatic.com")
}

pub(crate) fn classify_tracking_cookie(name: &str) -> Option<TrackingCookie> {
    let lower = name.trim().to_ascii_lowercase();
    let (scope, provider) = if lower.starts_with("_ga")
        || lower == "_gid"
        || lower == "_gat"
        || lower.starts_with("_gcl")
    {
        ("extern", "Google")
    } else if lower == "_fbp" || lower == "_fbc" || lower == "fr" {
        ("extern", "Meta")
    } else if lower.starts_with("_hj") {
        ("extern", "Hotjar")
    } else if lower == "hubspotutk" {
        ("extern", "HubSpot")
    } else if lower.starts_with("_pk_") || lower == "mtm_cookie_consent" {
        ("lokal", "Matomo")
    } else if lower.contains("zaraz") {
        ("lokal", "Cloudflare Zaraz")
    } else {
        return None;
    };

    Some(TrackingCookie {
        name: name.to_string(),
        scope: scope.to_string(),
        provider: provider.to_string(),
    })
}

pub(crate) fn classify_tracking_provider_category(provider: &str) -> Option<String> {
    let category = match provider {
        "Google" | "Matomo" | "Cloudflare Zaraz" => "analytics",
        "Meta" => "social",
        "Hotjar" | "HubSpot" => "marketing",
        _ => return None,
    };
    Some(category.to_string())
}

fn collect_tracking_signals(
    script_urls: &[String],
    resource_urls: &[String],
    tracking_cookies: &[TrackingCookie],
    cookie_inventory: &[CookieInventoryItem],
    storage_items: &[StorageItem],
) -> Vec<String> {
    let mut signals = BTreeSet::new();
    for url in script_urls.iter().chain(resource_urls.iter()) {
        if let Some(signal) = classify_tracking_url(url) {
            signals.insert(signal.to_string());
        }
    }
    for cookie in tracking_cookies {
        signals.insert(format!("{}-Cookie: {}", cookie.provider, cookie.name));
    }
    for cookie in cookie_inventory {
        if let Some(provider) = &cookie.provider {
            signals.insert(format_tracking_signal(
                provider,
                cookie.category.as_deref(),
                "Cookie",
                &cookie.name,
            ));
        }
    }
    for item in storage_items {
        if let Some(provider) = &item.provider {
            signals.insert(format!(
                "{} ({})",
                format_tracking_signal(provider, item.category.as_deref(), "Storage", &item.key),
                item.area
            ));
        }
    }
    signals.into_iter().collect()
}

fn format_tracking_signal(
    provider: &str,
    category: Option<&str>,
    signal_type: &str,
    name: &str,
) -> String {
    match category {
        Some(category) => format!("{provider} [{category}]-{signal_type}: {name}"),
        None => format!("{provider}-{signal_type}: {name}"),
    }
}

fn classify_tracking_url(url: &str) -> Option<&'static str> {
    let lower = url.to_ascii_lowercase();
    if lower.contains("/cdn-cgi/zaraz/") {
        Some("Cloudflare Zaraz")
    } else if lower.contains("googletagmanager.com")
        || lower.contains("google-analytics.com")
        || lower.contains("analytics.google.com")
    {
        Some("Google Analytics / Tag Manager")
    } else if lower.contains("connect.facebook.net") || lower.contains("facebook.com/tr") {
        Some("Meta Pixel")
    } else if lower.contains("static.cloudflareinsights.com") {
        Some("Cloudflare Web Analytics")
    } else if lower.contains("plausible.io") {
        Some("Plausible")
    } else if lower.contains("matomo") || lower.contains("/matomo.") || lower.contains("piwik") {
        Some("Matomo")
    } else {
        None
    }
}

fn detect_zaraz(
    script_urls: &[String],
    resource_urls: &[String],
    tracking_cookies: &[TrackingCookie],
    has_zaraz_global: bool,
) -> ZarazDetection {
    let mut signals = BTreeSet::new();
    if has_zaraz_global {
        signals.insert("window.zaraz".to_string());
    }
    for url in script_urls.iter().chain(resource_urls.iter()) {
        if url.to_ascii_lowercase().contains("/cdn-cgi/zaraz/") {
            signals.insert(url.clone());
        }
    }
    for cookie in tracking_cookies {
        if cookie.provider == "Cloudflare Zaraz" {
            signals.insert(format!("Cookie: {}", cookie.name));
        }
    }
    ZarazDetection {
        detected: !signals.is_empty(),
        signals: signals.into_iter().collect(),
    }
}

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn test_technical_seo_default() {
        let tech = TechnicalSeo::default();
        assert!(!tech.https);
        assert!(!tech.has_canonical);
    }

    #[test]
    fn test_google_fonts_detection() {
        assert!(is_google_fonts_url(
            "https://fonts.googleapis.com/css2?family=Inter:wght@400;700&display=swap"
        ));
        assert!(is_google_fonts_url(
            "https://fonts.gstatic.com/s/inter/v20/font.woff2"
        ));
        assert!(!is_google_fonts_url("https://example.com/app.css"));
    }

    #[test]
    fn test_tracking_cookie_classification() {
        let google = classify_tracking_cookie("_ga").unwrap();
        assert_eq!(google.scope, "extern");
        assert_eq!(google.provider, "Google");

        let zaraz = classify_tracking_cookie("zarazExample").unwrap();
        assert_eq!(zaraz.scope, "lokal");
        assert_eq!(zaraz.provider, "Cloudflare Zaraz");

        assert!(classify_tracking_cookie("sessionid").is_none());
    }

    #[test]
    fn test_storage_items_are_inventory_only_and_feed_tracking_signals() {
        let value = serde_json::json!([
            {"area": "localStorage", "key": "_ga"},
            {"area": "localStorage", "key": "_ga"},
            {"area": "sessionStorage", "key": "app_state"}
        ]);

        let storage_items = parse_storage_items(&value);
        assert_eq!(storage_items.len(), 2);
        assert_eq!(storage_items[0].provider.as_deref(), Some("Google"));
        assert_eq!(storage_items[0].category.as_deref(), Some("analytics"));
        assert_eq!(storage_items[1].provider, None);

        let signals = collect_tracking_signals(&[], &[], &[], &[], &storage_items);
        assert!(signals
            .iter()
            .any(|signal| signal == "Google [analytics]-Storage: _ga (localStorage)"));
    }

    #[test]
    fn test_cookie_inventory_feeds_tracking_signals() {
        let cookie_inventory = vec![CookieInventoryItem {
            name: "_fbp".to_string(),
            domain: ".example.com".to_string(),
            path: "/".to_string(),
            secure: true,
            http_only: true,
            same_site: Some("Lax".to_string()),
            session: false,
            third_party: false,
            provider: Some("Meta".to_string()),
            category: Some("social".to_string()),
        }];

        let signals = collect_tracking_signals(&[], &[], &[], &cookie_inventory, &[]);
        assert!(signals
            .iter()
            .any(|signal| signal == "Meta [social]-Cookie: _fbp"));
    }

    #[test]
    fn test_zaraz_detection() {
        let zaraz = detect_zaraz(
            &["https://www.example.com/cdn-cgi/zaraz/s.js".to_string()],
            &[],
            &[],
            false,
        );
        assert!(zaraz.detected);
        assert_eq!(zaraz.signals.len(), 1);
    }

    #[test]
    fn test_zaraz_detection_does_not_false_positive_on_polish_word() {
        // Regression: "zaraz" ("right away" in Polish) must not be treated as
        // a Cloudflare Zaraz signal unless it appears on the actual
        // /cdn-cgi/zaraz/ path.
        let zaraz = detect_zaraz(
            &["https://example.pl/blog/zaraz-wracam-do-pracy".to_string()],
            &[],
            &[],
            false,
        );
        assert!(!zaraz.detected);
        assert!(zaraz.signals.is_empty());
    }

    #[test]
    fn test_classify_tracking_url_does_not_false_positive_on_polish_word() {
        assert_eq!(
            classify_tracking_url("https://example.pl/blog/zaraz-wracam-do-pracy"),
            None
        );
        assert_eq!(
            classify_tracking_url("https://example.com/cdn-cgi/zaraz/s.js"),
            Some("Cloudflare Zaraz")
        );
    }

    #[test]
    fn test_url_origin_preserves_scheme_host_and_port() {
        assert_eq!(
            url_origin("https://example.com:8443/path?x=1"),
            Some("https://example.com:8443".to_string())
        );
        assert_eq!(
            url_origin("http://localhost:3000/about"),
            Some("http://localhost:3000".to_string())
        );
        assert_eq!(url_origin("/relative"), None);
    }

    #[test]
    fn test_parse_mixed_content_resources_deduplicates() {
        let value = serde_json::json!([
            {"url": "http://cdn.example.com/app.js", "type": "script", "blockable": true},
            {"url": "http://cdn.example.com/app.js", "type": "script", "blockable": true},
            {"url": "http://cdn.example.com/photo.jpg", "type": "media", "blockable": false}
        ]);

        let resources = parse_mixed_content_resources(&value);

        assert_eq!(resources.len(), 2);
        assert_eq!(resources[0].resource_type, "script");
        assert!(resources[0].blockable);
        assert!(!resources[1].blockable);
    }

    #[test]
    fn test_collect_pwa_issues_missing_manifest() {
        let mut issues = Vec::new();
        collect_pwa_issues(&PwaAnalysis::default(), true, &mut issues);

        assert_eq!(issues.len(), 1);
        assert_eq!(issues[0].issue_type, "pwa_missing_manifest");
        assert_eq!(issues[0].severity, Severity::Low);
    }

    #[test]
    fn test_collect_pwa_issues_invalid_manifest() {
        let pwa = PwaAnalysis {
            manifest_url: Some("https://example.com/manifest.webmanifest".to_string()),
            manifest_valid: false,
            service_worker_supported: true,
            service_worker_registrations: 0,
            ..PwaAnalysis::default()
        };
        let mut issues = Vec::new();
        collect_pwa_issues(&pwa, true, &mut issues);

        assert!(issues
            .iter()
            .any(|issue| issue.issue_type == "pwa_invalid_manifest"));
    }

    /// A manifest without a service worker is not reported: shipping a
    /// manifest for icons/theme color is ordinary, and does not make a site a
    /// PWA that owes anyone offline support.
    #[test]
    fn test_manifest_without_service_worker_is_not_an_issue() {
        let pwa = PwaAnalysis {
            manifest_url: Some("https://example.com/manifest.webmanifest".to_string()),
            manifest_valid: true,
            service_worker_supported: true,
            service_worker_registrations: 0,
            ..PwaAnalysis::default()
        };
        let mut issues = Vec::new();
        collect_pwa_issues(&pwa, true, &mut issues);

        assert!(issues.is_empty(), "{issues:#?}");
    }

    #[test]
    fn test_parse_amp_analysis_and_collect_issues() {
        let value = serde_json::json!({
            "detected": true,
            "canonicalPresent": false,
            "runtimeScriptPresent": false,
            "boilerplatePresent": true,
            "issues": ["missing_canonical", "missing_amp_runtime"]
        });

        let amp = parse_amp_analysis(&value);
        assert!(amp.detected);
        assert!(!amp.canonical_present);
        assert_eq!(amp.issues.len(), 2);

        let mut issues = Vec::new();
        collect_amp_issues(&amp, true, &mut issues);
        assert!(issues
            .iter()
            .any(|issue| issue.issue_type == "amp_missing_canonical"));
        assert!(issues
            .iter()
            .any(|issue| issue.issue_type == "amp_missing_amp_runtime"));
    }

    #[test]
    fn test_collect_form_security_issues_flags_sensitive_get_and_http() {
        let form_security = FormSecurityAnalysis {
            sensitive_forms: 2,
            insecure_transport_forms: 1,
            get_sensitive_forms: 1,
            password_autocomplete_issues: 0,
        };
        let mut issues = Vec::new();

        collect_form_security_issues(&form_security, true, &mut issues);

        assert!(issues
            .iter()
            .any(|issue| issue.issue_type == "form_insecure_transport"
                && issue.severity == Severity::High));
        assert!(issues
            .iter()
            .any(|issue| issue.issue_type == "form_sensitive_get"
                && issue.severity == Severity::High));
    }

    #[test]
    fn test_collect_form_security_issues_flags_password_autocomplete() {
        let form_security = FormSecurityAnalysis {
            password_autocomplete_issues: 2,
            ..FormSecurityAnalysis::default()
        };
        let mut issues = Vec::new();

        collect_form_security_issues(&form_security, true, &mut issues);

        assert_eq!(issues.len(), 1);
        assert_eq!(issues[0].issue_type, "form_password_autocomplete");
        assert_eq!(issues[0].severity, Severity::Medium);
    }

    #[test]
    fn test_detect_pagination_from_url_patterns() {
        assert!(detect_pagination("https://example.com/blog/page/2/", 0, 0));
        assert!(detect_pagination("https://example.com/blog?page=3", 0, 0));
        assert!(detect_pagination("https://example.com/blog", 1, 0));
        assert!(detect_pagination("https://example.com/blog", 0, 2));
        assert!(!detect_pagination("https://example.com/blog", 0, 1));
        assert!(!detect_pagination("https://example.com/blog?page=1", 0, 0));
    }

    /// Absent rel=prev/next is not reported: Google stopped using the pair for
    /// indexing in 2019, so its absence is no longer an SEO defect.
    #[test]
    fn test_missing_pagination_rel_links_is_not_an_issue() {
        let mut issues = Vec::new();
        collect_pagination_issues(
            true,
            None,
            None,
            Some("https://example.com/blog/page/2/"),
            true,
            &mut issues,
        );

        assert!(issues.is_empty(), "{issues:#?}");
    }

    #[test]
    fn test_collect_pagination_issues_self_reference_and_duplicate() {
        let mut issues = Vec::new();
        collect_pagination_issues(
            true,
            Some("https://example.com/blog/page/2/#top"),
            Some("https://example.com/blog/page/2/"),
            Some("https://example.com/blog/page/2/"),
            true,
            &mut issues,
        );

        assert!(issues
            .iter()
            .any(|issue| issue.issue_type == "pagination_self_referential_rel_link"));
        assert!(issues
            .iter()
            .any(|issue| issue.issue_type == "pagination_duplicate_prev_next"));
    }

    #[test]
    fn test_collect_technical_issues_flags_missing_social_image_alt() {
        let technical = TechnicalSeo {
            og_image_missing_alt: true,
            twitter_image_missing_alt: true,
            https: true,
            has_lang: true,
            canonical_url: Some("https://example.com/".to_string()),
            word_count: 500,
            internal_links: 1,
            ..TechnicalSeo::default()
        };

        let issues = collect_technical_issues(&technical, true);

        assert!(issues
            .iter()
            .any(|issue| issue.issue_type == "og_image_missing_alt"
                && issue.severity == Severity::Low));
        assert!(issues
            .iter()
            .any(|issue| issue.issue_type == "twitter_image_missing_alt"
                && issue.severity == Severity::Low));
    }

    #[test]
    fn test_collect_technical_issues_no_finding_when_social_image_absent_or_alt_present() {
        let technical = TechnicalSeo {
            og_image_missing_alt: false,
            twitter_image_missing_alt: false,
            https: true,
            has_lang: true,
            canonical_url: Some("https://example.com/".to_string()),
            word_count: 500,
            internal_links: 1,
            ..TechnicalSeo::default()
        };

        let issues = collect_technical_issues(&technical, true);

        assert!(!issues
            .iter()
            .any(|issue| issue.issue_type == "og_image_missing_alt"));
        assert!(!issues
            .iter()
            .any(|issue| issue.issue_type == "twitter_image_missing_alt"));
    }
}