domain-status 0.2.1

Concurrent URL status checker that captures comprehensive metadata in SQLite.
Documentation
// Parse module tests.

use super::*;
use crate::error_handling::ProcessingStats;
use scraper::Html;

fn test_error_stats() -> ProcessingStats {
    ProcessingStats::new()
}

#[test]
fn test_extract_title_basic() {
    let html = r#"<html><head><title>Test Page</title></head><body></body></html>"#;
    let document = Html::parse_document(html);
    let stats = test_error_stats();
    assert_eq!(extract_title(&document, &stats), "Test Page");
}

#[test]
fn test_extract_title_mixed_case_tags() {
    let stats = test_error_stats();
    for html in [
        r#"<html><head><TITLE>Test</TITLE></head></html>"#,
        r#"<html><head><Title>Test</Title></head></html>"#,
        r#"<html><head><tItLe>Test</tItLe></head></html>"#,
    ] {
        let document = Html::parse_document(html);
        assert_eq!(
            extract_title(&document, &stats),
            "Test",
            "parser must extract title from mixed-case tags: {html}"
        );
    }
}

#[test]
fn test_extract_title_with_whitespace() {
    // Common gotcha: titles with extra whitespace/newlines
    let html = r#"<html><head><title>
        Test Page
    </title></head></html>"#;
    let document = Html::parse_document(html);
    let stats = test_error_stats();
    assert_eq!(extract_title(&document, &stats), "Test Page");
}

#[test]
fn test_extract_title_with_html_entities() {
    // HTML entities should be decoded
    let html = r#"<html><head><title>Test &amp; Page &lt;Title&gt;</title></head></html>"#;
    let document = Html::parse_document(html);
    let stats = test_error_stats();
    assert_eq!(extract_title(&document, &stats), "Test & Page <Title>");
}

#[test]
fn test_extract_title_empty() {
    let html = r#"<html><head><title></title></head></html>"#;
    let document = Html::parse_document(html);
    let stats = test_error_stats();
    assert_eq!(extract_title(&document, &stats), "");
}

#[test]
fn test_extract_title_missing() {
    // Missing title is now tracked as a warning, not an error
    let html = r#"<html><head></head><body></body></html>"#;
    let document = Html::parse_document(html);
    let stats = test_error_stats();
    assert_eq!(extract_title(&document, &stats), "");
    assert_eq!(
        stats.get_warning_count(crate::error_handling::WarningType::MissingTitle),
        1
    );
}

#[test]
fn test_extract_title_multiple_tags() {
    // Edge case: multiple title tags (should get first)
    let html = r#"<html><head><title>First</title><title>Second</title></head></html>"#;
    let document = Html::parse_document(html);
    let stats = test_error_stats();
    assert_eq!(extract_title(&document, &stats), "First");
}

#[test]
fn test_extract_title_rawtext_content_model_preserves_literal_angle_brackets() {
    // <title> is a RAWTEXT element per the HTML5 spec: html5ever never parses its
    // contents as child elements/comments, so anything that looks like a tag or
    // comment inside <title>...</title> is literal text, not markup. text()
    // (the primary extraction path) faithfully returns that literal content —
    // this is correct HTML5 parsing behavior, not a bug, and is why title
    // extraction doesn't need general tag-stripping on the common path (only the
    // rarely-hit inner_html fallback below needs it; see `strip_html_tags`).
    let html = r#"<html><head><title><!-- literally text, not a comment --></title></head></html>"#;
    let document = Html::parse_document(html);
    let stats = test_error_stats();
    let title = extract_title(&document, &stats);
    assert_eq!(title, "<!-- literally text, not a comment -->");
}

#[test]
fn test_extract_meta_description_basic() {
    let html =
        r#"<html><head><meta name="description" content="A test description"></head></html>"#;
    let document = Html::parse_document(html);
    let stats = test_error_stats();
    assert_eq!(
        extract_meta_description(&document, &stats),
        Some("A test description".to_string())
    );
}

#[test]
fn test_extract_meta_description_with_whitespace() {
    let html = r#"<html><head><meta name="description" content="  A test description  "></head></html>"#;
    let document = Html::parse_document(html);
    let stats = test_error_stats();
    assert_eq!(
        extract_meta_description(&document, &stats),
        Some("A test description".to_string())
    );
}

#[test]
fn test_extract_meta_description_missing() {
    // Missing meta description is tracked as a warning, not an error
    let html = r#"<html><head></head></html>"#;
    let document = Html::parse_document(html);
    let stats = test_error_stats();
    assert!(extract_meta_description(&document, &stats).is_none());
    // Missing meta description is tracked as a warning, not an error
}

// Analytics ID extraction tests
#[test]
fn test_extract_analytics_ids_gtm_data_layer_format() {
    // Test GTM in dataLayer format
    let html = r#"
        <script>
            (function(w,d,s,l,i){w[l]=w[l]||[];w[l].push({'gtm.start':
            new Date().getTime(),event:'gtm.js'});var f=d.getElementsByTagName(s)[0],
            j=d.createElement(s),dl=l!='dataLayer'?'&l='+l:'';j.async=true;j.src=
            'https://www.googletagmanager.com/gtm.js?id='+i+dl;f.parentNode.insertBefore(j,f);
            })(window, document, 'script', 'dataLayer','GTM-TEST001');
        </script>
    "#;
    let ids = extract_analytics_ids(html);
    let gtm_ids: Vec<&str> = ids
        .iter()
        .filter(|id| id.provider == AnalyticsProvider::GoogleTagManager)
        .map(|id| id.id.as_str())
        .collect();
    assert!(
        gtm_ids.contains(&"GTM-TEST001"),
        "Should find GTM-TEST001: {:?}",
        gtm_ids
    );
}

#[test]
fn test_extract_analytics_ids_gtm_json_format() {
    // Test GTM in JSON format
    let html = r#"
        <script type="application/json">{"gtm":{"tagIds":["GTM-TEST002"]}}</script>
    "#;
    let ids = extract_analytics_ids(html);
    let gtm_ids: Vec<&str> = ids
        .iter()
        .filter(|id| id.provider == AnalyticsProvider::GoogleTagManager)
        .map(|id| id.id.as_str())
        .collect();
    assert!(
        gtm_ids.contains(&"GTM-TEST002"),
        "Should find GTM-TEST002: {:?}",
        gtm_ids
    );
}

#[test]
fn test_extract_analytics_ids_gtm_url_format() {
    // Test GTM in URL format (iframe/script src)
    let html = r#"
        <iframe src="https://www.googletagmanager.com/ns.html?id=GTM-XXXXX"></iframe>
        <script src="https://www.googletagmanager.com/gtm.js?id=GTM-YYYYY"></script>
    "#;
    let ids = extract_analytics_ids(html);
    let gtm_ids: Vec<&str> = ids
        .iter()
        .filter(|id| id.provider == AnalyticsProvider::GoogleTagManager)
        .map(|id| id.id.as_str())
        .collect();
    assert!(
        gtm_ids.contains(&"GTM-XXXXX"),
        "Should find GTM-XXXXX: {:?}",
        gtm_ids
    );
    assert!(
        gtm_ids.contains(&"GTM-YYYYY"),
        "Should find GTM-YYYYY: {:?}",
        gtm_ids
    );
}

#[test]
fn test_extract_analytics_ids_gtm_rejects_false_positives() {
    // Test that data layer variables and other false positives are NOT matched
    let html = r#"
        <script>
            dataLayer.push({
                'gtm-company': 'Acme Corp',
                'gtm-industry': 'Technology',
                'gtm-province': 'California',
                'gtm-message': 'Hello',
                'gtm-subproperty': 'sub',
                'gtm-script': 'script.js',
                'gtm-purhcase': 'purchase'
            });
        </script>
    "#;
    let ids = extract_analytics_ids(html);
    let gtm_ids: Vec<&str> = ids
        .iter()
        .filter(|id| id.provider == AnalyticsProvider::GoogleTagManager)
        .map(|id| id.id.as_str())
        .collect();
    // Should NOT match any of these false positives
    assert!(
        !gtm_ids.contains(&"gtm-company"),
        "Should NOT match gtm-company: {:?}",
        gtm_ids
    );
    assert!(
        !gtm_ids.contains(&"gtm-industry"),
        "Should NOT match gtm-industry: {:?}",
        gtm_ids
    );
    assert!(
        !gtm_ids.contains(&"gtm-province"),
        "Should NOT match gtm-province: {:?}",
        gtm_ids
    );
    assert!(
        !gtm_ids.contains(&"gtm-message"),
        "Should NOT match gtm-message: {:?}",
        gtm_ids
    );
}