spectre_pdf 1.0.0

/// scorer.rs — Text quality scoring.
///
/// Ported from spectre v0.1/v0.2 RustValidator.
/// Exposed as module-level `score_text()` in v0.3.
use regex::Regex;
use std::sync::OnceLock;

static JS_PATTERN: OnceLock<Regex> = OnceLock::new();
static SEPARATOR_PATTERN: OnceLock<Regex> = OnceLock::new();

fn js_regex() -> &'static Regex {
    JS_PATTERN.get_or_init(|| {
        Regex::new(
            r"(?i)(function\s*\(|eval\(|document\.|window\.|\.innerHTML|<script|javascript:)",
        )
        .expect("scorer JS regex literal compiles — change requires updating this expect")
    })
}

fn separator_regex() -> &'static Regex {
    SEPARATOR_PATTERN.get_or_init(|| {
        Regex::new(r"(?m)^[-=_]{5,}$").expect(
            "scorer separator regex literal compiles — change requires updating this expect",
        )
    })
}

/// Score a single text string. Returns 0.0 (garbage) to 1.0 (clean).
///
/// The heuristic measures fraction of *Unicode-replacement* and
/// *control-character* codepoints, NOT raw `byte > 127` ratio. The earlier
/// pre-0.4.0 implementation penalized any text with ≥30% high bytes, which
/// rated clean Chinese/Japanese/Korean documents (whose chars are ~3 UTF-8
/// bytes apiece, all > 127) the same as binary noise. This version rates
/// non-Latin text fairly while still flagging the failure modes that
/// motivated the heuristic: CID dumps (replacement chars), corrupt streams
/// (control bytes), and embedded JS payloads (regex match below).
pub fn score_text_impl(text: &str) -> f64 {
    if text.is_empty() || text.len() < 10 {
        return 0.0;
    }

    let length = text.len() as f64;
    let total_chars = text.chars().count().max(1) as f64;
    let mut penalties: f64 = 0.0;

    // 1. Garbage-character ratio. Counts the actual signals of broken
    //    extraction:
    //      - U+FFFD (replacement char — UTF-8 decode failure)
    //      - private-use area (often emitted by CID-font dumps)
    //      - format / surrogate / unassigned codepoints.
    //    Genuine non-Latin printable text scores zero here.
    let garbage = text
        .chars()
        .filter(|c| {
            matches!(*c, '\u{FFFD}')
                || ('\u{E000}'..='\u{F8FF}').contains(c)        // BMP private-use
                || ('\u{F0000}'..='\u{FFFFD}').contains(c)      // SPUA-A
                || ('\u{100000}'..='\u{10FFFD}').contains(c) // SPUA-B
        })
        .count() as f64;
    let garbage_ratio = garbage / total_chars;
    if garbage_ratio > 0.30 {
        penalties += 0.5;
    } else if garbage_ratio > 0.10 {
        penalties += garbage_ratio * 1.5;
    }

    // 2. Control character ratio (char-based, not byte-based, so multi-byte
    //    UTF-8 letters never accidentally trip this).
    let control_count = text
        .chars()
        .filter(|c| (*c as u32) < 32 && !matches!(*c, '\n' | '\r' | '\t'))
        .count() as f64;
    let control_ratio = control_count / total_chars;
    if control_ratio > 0.05 {
        penalties += 0.4;
    } else if control_ratio > 0.01 {
        penalties += control_ratio * 8.0;
    }

    // 3. JS pattern detection. PDFs embedding JS payloads (or returning
    //    JS-as-content from broken extraction) score below clean prose with
    //    enough headroom for callers to filter on a single threshold.
    let js_matches = js_regex().find_iter(text).count() as f64;
    let js_ratio = js_matches / (length / 100.0).max(1.0);
    if js_ratio > 0.10 {
        penalties += 0.5;
    } else if js_ratio > 0.03 {
        penalties += js_ratio * 4.0;
    }

    // 4. Average line length heuristic
    let lines: Vec<&str> = text.split('\n').filter(|l| !l.trim().is_empty()).collect();
    if !lines.is_empty() {
        let avg_line_len = lines.iter().map(|l| l.len()).sum::<usize>() as f64 / lines.len() as f64;
        if avg_line_len < 5.0 {
            penalties += 0.4;
        } else if avg_line_len < 15.0 {
            penalties += 0.15;
        }

        // 5. Separator line ratio
        let separators = separator_regex().find_iter(text).count() as f64;
        let sep_ratio = separators / lines.len() as f64;
        if sep_ratio > 0.3 {
            penalties += 0.2;
        }
    } else {
        penalties += 0.3;
    }

    // 6. Word density
    let word_count = text.split_whitespace().count() as f64;
    if word_count < 5.0 {
        penalties += 0.3;
    } else if word_count / length < 0.05 {
        penalties += 0.1;
    }

    (1.0 - penalties).clamp(0.0, 1.0)
}