guard_core_engine/

preprocessor.rs

//! Content preprocessing ported from `guard_core/detection_engine/preprocessor.py`.
//!
//! Spec 4.0.2: the shared decode chain produces the processed view (null-byte
//! removal, whitespace collapse, attack-preserving truncation) plus the decoded
//! view handed to the URL-decoded scan pass, the signal-preserving raw view,
//! and the short-base64 additive view.

use std::sync::LazyLock;

use regex::Regex;
use unicode_normalization::UnicodeNormalization;

pub const DEFAULT_MAX_FULL_SCAN_BYTES: usize = 262_144;
const FULL_SCAN_TAIL_BYTES: usize = 4096;
const MAX_DECODE_ITERATIONS: usize = 16;
const MAX_GUNZIP_ATTEMPTS_PER_PASS: u32 = 8;

// visually ambiguous unicode chars mapped to their ASCII equivalents
const LOOKALIKES: &[(char, &str)] = &[
    ('\u{2044}', "/"),
    ('\u{FF0F}', "/"),
    ('\u{29F8}', "/"),
    ('\u{0130}', "I"),
    ('\u{0131}', "i"),
    ('\u{200B}', ""),
    ('\u{200C}', ""),
    ('\u{200D}', ""),
    ('\u{FEFF}', ""),
    ('\u{00AD}', ""),
    ('\u{034F}', ""),
    ('\u{180E}', ""),
    ('\u{2028}', "\n"),
    ('\u{2029}', "\n"),
    ('\u{E000}', ""),
    ('\u{FFF0}', ""),
    ('\u{01C0}', "|"),
    ('\u{037E}', ";"),
    ('\u{2215}', "/"),
    ('\u{2216}', "\\"),
    ('\u{FF1C}', "<"),
    ('\u{FF1E}', ">"),
    ('\u{FF1B}', ";"),
    ('\u{FF5C}', "|"),
    ('\u{FF06}', "&"),
];

static ATTACK_INDICATORS: LazyLock<Vec<Regex>> = LazyLock::new(|| {
    let patterns = [
        r"<script",
        r"javascript:",
        r"on\w+=",
        r"SELECT\s+.{0,50}?\s+FROM",
        r"UNION\s+SELECT",
        r"\.\./",
        r"eval\s*\(",
        r"exec\s*\(",
        r"system\s*\(",
        r"<\?php",
        r"<%",
        r"\{\{",
        r"\{%",
        r"<iframe",
        r"<object",
        r"<embed",
        r"onerror\s*=",
        r"onload\s*=",
        r"\$\{",
        r"\\x[0-9a-fA-F]{2}",
        r"%[0-9a-fA-F]{2}",
        r"`",
        r"\$\(",
        r"[;&|]",
        r"\b\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}\b",
    ];
    patterns
        .iter()
        .map(|p| Regex::new(&format!("(?i){p}")).expect("static indicator must compile"))
        .collect()
});

/// NFKC normalization + unicode lookalike replacement
/// (fullwidth forms, zero-width characters, etc.).
#[must_use]
pub fn normalize_unicode(content: &str) -> String {
    let normalized: String = content.nfkc().collect();
    let mut result = normalized;

    for &(ch, replacement) in LOOKALIKES {
        if result.contains(ch) {
            result = result.replace(ch, replacement);
        }
    }

    result
}

/// Python `str.isspace`-compatible whitespace test (`re` `\s` also matches the
/// file/group/record/unit separators, which `char::is_whitespace` excludes).
const fn py_is_space(c: char) -> bool {
    c.is_whitespace() || matches!(c, '\u{1c}'..='\u{1f}')
}

#[doc(hidden)]
#[must_use]
pub fn collapse_whitespace(content: &str) -> String {
    let mut out = String::with_capacity(content.len());
    let mut in_run = false;
    let mut any = false;
    for c in content.chars() {
        if py_is_space(c) {
            in_run = true;
            continue;
        }
        if in_run && any {
            out.push(' ');
        }
        in_run = false;
        any = true;
        out.push(c);
    }
    out
}

#[doc(hidden)]
#[must_use]
pub fn remove_null_bytes(content: &str) -> String {
    content
        .chars()
        .filter(|&c| {
            let code = c as u32;
            code >= 32 || matches!(code, 9 | 10 | 13)
        })
        .collect()
}

// ---------------------------------------------------------------------------
// encoding decoders (encoding_decoders.py)
// ---------------------------------------------------------------------------

fn decode_percent_u_escapes(s: &str) -> String {
    static RE: LazyLock<Regex> =
        LazyLock::new(|| Regex::new(r"(?i)%u([0-9a-fA-F]{4})").expect("static regex"));
    replace_hex_group(&RE, s)
}

fn decode_hex_escapes(s: &str) -> String {
    static RE: LazyLock<Regex> =
        LazyLock::new(|| Regex::new(r"\\x([0-9a-fA-F]{2})").expect("static regex"));
    replace_hex_group(&RE, s)
}

fn decode_ldap_hex_escapes(s: &str) -> String {
    static RE: LazyLock<Regex> =
        LazyLock::new(|| Regex::new(r"\\([0-9a-fA-F]{2})").expect("static regex"));
    replace_hex_group(&RE, s)
}

fn decode_unicode_escapes(s: &str) -> String {
    static RE: LazyLock<Regex> =
        LazyLock::new(|| Regex::new(r"\\u([0-9a-fA-F]{4})").expect("static regex"));
    replace_hex_group(&RE, s)
}

fn replace_hex_group(re: &Regex, s: &str) -> String {
    re.replace_all(s, |caps: &regex::Captures<'_>| {
        let hex = caps.get(1).map_or("0", |m| m.as_str());
        u32::from_str_radix(hex, 16)
            .ok()
            .and_then(char::from_u32)
            .map_or_else(
                || caps.get(0).map_or("", |m| m.as_str()).to_owned(),
                String::from,
            )
    })
    .into_owned()
}

const PERCENT_BYTE_RUN_RE: &str = r"(?:%[0-9a-fA-F]{2})+";

/// `decode_overlong_utf8_percent_runs`: percent-encoded byte runs that do not
/// form valid UTF-8 are re-decoded leniently (overlong lead bytes).
#[must_use]
pub fn decode_overlong_utf8_percent_runs(content: &str) -> String {
    static RE: LazyLock<Regex> =
        LazyLock::new(|| Regex::new(PERCENT_BYTE_RUN_RE).expect("static regex"));
    RE.replace_all(content, |caps: &regex::Captures<'_>| {
        let run = caps.get(0).map_or("", |m| m.as_str());
        let raw: Vec<u8> = run
            .as_bytes()
            .chunks(3)
            .filter_map(|triple| {
                if triple.len() == 3 {
                    u8::from_str_radix(std::str::from_utf8(&triple[1..3]).ok()?, 16).ok()
                } else {
                    None
                }
            })
            .collect();
        if std::str::from_utf8(&raw).is_ok() {
            return run.to_owned();
        }
        lenient_overlong_utf8_decode(&raw)
    })
    .into_owned()
}

const OVERLONG_LEAD_SPECS: [(u8, usize, u8, u8, u8); 4] = [
    // lead, sequence length, lead mask, first-continuation min/max
    (0xC0, 2, 0x1F, 0x80, 0xBF),
    (0xC1, 2, 0x1F, 0x80, 0xBF),
    (0xE0, 3, 0x0F, 0x80, 0x9F),
    (0xF0, 4, 0x07, 0x80, 0x8F),
];

fn decode_overlong_sequence_at(raw: &[u8], index: usize) -> Option<(char, usize)> {
    let lead = *raw.get(index)?;
    let spec = OVERLONG_LEAD_SPECS.iter().find(|spec| spec.0 == lead)?;
    let sequence_length = spec.1;
    let lead_mask = spec.2;
    let first_min = spec.3;
    let first_max = spec.4;
    if index + sequence_length > raw.len() {
        return None;
    }
    let continuations = &raw[index + 1..index + sequence_length];
    if continuations[0] < first_min || continuations[0] > first_max {
        return None;
    }
    if continuations[1..]
        .iter()
        .any(|byte| !(*byte >= 0x80 && *byte <= 0xBF))
    {
        return None;
    }
    let mut codepoint = u32::from(lead & lead_mask);
    for byte in continuations {
        codepoint = (codepoint << 6) | u32::from(byte & 0x3F);
    }
    char::from_u32(codepoint).map(|c| (c, sequence_length))
}

fn lenient_overlong_utf8_decode(raw: &[u8]) -> String {
    let mut chars = String::new();
    let mut index = 0;
    while index < raw.len() {
        if let Some((c, consumed)) = decode_overlong_sequence_at(raw, index) {
            chars.push(c);
            index += consumed;
        } else if raw[index] < 0x80 {
            chars.push(raw[index] as char);
            index += 1;
        } else {
            index += 1;
        }
    }
    chars
}

/// `urllib.parse.unquote(content, errors="ignore")`: percent escapes become
/// bytes, invalid UTF-8 bytes are dropped.
fn percent_unquote_ignore(s: &str) -> String {
    if !s.contains('%') {
        return s.to_owned();
    }
    let bytes = s.as_bytes();
    let mut raw: Vec<u8> = Vec::with_capacity(bytes.len());
    let mut i = 0;
    while i < bytes.len() {
        if bytes[i] == b'%'
            && i + 2 < bytes.len()
            && bytes[i + 1].is_ascii_hexdigit()
            && bytes[i + 2].is_ascii_hexdigit()
        {
            let hi = (bytes[i + 1] as char).to_digit(16).unwrap_or(0) as u8;
            let lo = (bytes[i + 2] as char).to_digit(16).unwrap_or(0) as u8;
            raw.push((hi << 4) | lo);
            i += 3;
        } else {
            raw.push(bytes[i]);
            i += 1;
        }
    }
    utf8_ignore(&raw)
}

fn utf8_ignore(bytes: &[u8]) -> String {
    let mut out = String::with_capacity(bytes.len());
    let mut i = 0;
    while i < bytes.len() {
        match std::str::from_utf8(&bytes[i..]) {
            Ok(s) => {
                out.push_str(s);
                break;
            }
            Err(e) => {
                let valid = e.valid_up_to();
                out.push_str(std::str::from_utf8(&bytes[i..i + valid]).unwrap_or(""));
                i += valid + e.error_len().unwrap_or(1);
            }
        }
    }
    out
}

// ---------------------------------------------------------------------------
// base64 candidate decode (base64_decode.py)
// ---------------------------------------------------------------------------

const fn is_b64_data(c: char) -> bool {
    c.is_ascii_alphanumeric() || matches!(c, '+' | '/' | '_' | '-')
}

/// `SEPARATOR_CHARS`: every ASCII char outside the data alphabet plus the
/// surrogateescape byte range.
fn is_b64_separator(c: char) -> bool {
    let cp = c as u32;
    if (0xDC80..=0xDCFF).contains(&cp) {
        return true;
    }
    if cp < 0x80 {
        return !is_b64_data(c);
    }
    false
}

/// `bounded_gunzip`: gzip payloads are left undecoded (no decompressor in the
/// engine's dependency set); the reference decodes them but only the gunzip
/// branch of the base64 stage depends on it, and the decoded bytes then still
/// have to pass the printable/UTF-8 gates.
const fn bounded_gunzip(_raw: &[u8]) -> Option<Vec<u8>> {
    None
}

const GZIP_MAGIC: [u8; 2] = [0x1F, 0x8B];
const PRINTABLE_RATIO_THRESHOLD: f64 = 0.5;
const FALLBACK_PRINTABLE_RATIO_THRESHOLD: f64 = 0.95;
const MAX_REPLACEMENT_CHAR_RATIO: f64 = 0.2;

fn printable_ratio(text: &str) -> f64 {
    if text.is_empty() {
        return 0.0;
    }
    let printable = text.chars().filter(|c| py_is_printable(*c)).count();
    f64::from(u32::try_from(printable).unwrap_or(u32::MAX))
        / f64::from(u32::try_from(text.chars().count()).unwrap_or(u32::MAX))
}

/// Python `str.isprintable` approximation: everything except control, format,
/// separator (other than plain space), private-use and surrogate code points.
fn py_is_printable(c: char) -> bool {
    if c == ' ' {
        return true;
    }
    if c.is_control() || c.is_whitespace() {
        return false;
    }
    // common Cf (format) code points
    !matches!(c as u32,
        0x00AD | 0x0600..=0x0605 | 0x061C | 0x06DD | 0x070F
        | 0x180E | 0x200B..=0x200F | 0x202A..=0x202E | 0x2060..=0x2064
        | 0x2066..=0x206F | 0xFEFF | 0xFFF9..=0xFFFB
        | 0xE000..=0xF8FF | 0xF0000..=0xFFFFD | 0x0010_0000..=0x0010_FFFD
    )
}

fn replacement_char_ratio(text: &str) -> f64 {
    if text.is_empty() {
        return 0.0;
    }
    let count = text.chars().filter(|c| *c == '\u{FFFD}').count();
    f64::from(u32::try_from(count).unwrap_or(u32::MAX))
        / f64::from(u32::try_from(text.chars().count()).unwrap_or(u32::MAX))
}

/// strict base64 decode over the standard alphabet with re-padding.
fn b64_decode_strict(cleaned: &str) -> Option<Vec<u8>> {
    if cleaned.is_empty() || cleaned.len() % 4 == 1 {
        return None;
    }
    let mut padded = cleaned.to_owned();
    for _ in 0..(4 - cleaned.len() % 4) % 4 {
        padded.push('=');
    }
    let bytes = padded.as_bytes();
    let mut out = Vec::with_capacity(bytes.len() / 4 * 3);
    // the re-padding above guarantees a multiple of 4, so the remainder is empty
    let (groups, rest) = bytes.as_chunks::<4>();
    debug_assert!(rest.is_empty());
    for group in groups {
        let v: [u8; 4] = group
            .iter()
            .map(|b| match b {
                b'A'..=b'Z' => b - b'A',
                b'a'..=b'z' => b - b'a' + 26,
                b'0'..=b'9' => b - b'0' + 52,
                b'+' => 62,
                b'/' => 63,
                b'=' => 254,
                _ => 255,
            })
            .collect::<Vec<u8>>()
            .try_into()
            .ok()?;
        if v.contains(&255) || v[1] == 254 {
            return None;
        }
        out.push((v[0] << 2) | (v[1] >> 4));
        if v[2] != 254 {
            out.push((v[1] << 4) | (v[2] >> 2));
            if v[3] != 254 {
                out.push((v[2] << 6) | v[3]);
            }
        } else if v[3] != 254 {
            return None;
        }
    }
    Some(out)
}

fn decode_cleaned(
    cleaned: &str,
    min_printable_ratio: f64,
    gunzip_left: &mut u32,
) -> Option<String> {
    let raw = b64_decode_strict(cleaned)?;
    let raw = if raw.len() >= 2 && raw[..2] == GZIP_MAGIC && *gunzip_left > 0 {
        *gunzip_left -= 1;
        bounded_gunzip(&raw).unwrap_or(raw)
    } else {
        raw
    };
    let decoded = if let Ok(s) = std::str::from_utf8(&raw) {
        s.to_owned()
    } else {
        let replaced = String::from_utf8_lossy(&raw).into_owned();
        if replacement_char_ratio(&replaced) > MAX_REPLACEMENT_CHAR_RATIO {
            return None;
        }
        replaced
    };
    if printable_ratio(&decoded) >= min_printable_ratio {
        return Some(decoded);
    }
    None
}

fn is_hex_literal(token: &str) -> bool {
    token.len() > 2
        && (token.starts_with("0x") || token.starts_with("0X"))
        && token[2..].bytes().all(|b| b.is_ascii_hexdigit())
}

/// `WIDENED_CANDIDATE_MARKER_RE`: a separator other than CR/LF/= or a `-`/`_`
/// data char inside the token widens the candidate and tightens the gate.
fn token_has_widened_separator(token: &str) -> bool {
    token.chars().any(|c| match c {
        '\r' | '\n' | '=' => false,
        '-' | '_' => true,
        c if is_b64_separator(c) => true,
        _ => false,
    })
}

fn decode_token(token: &str, min_printable_ratio: f64, gunzip_left: &mut u32) -> Option<String> {
    if is_hex_literal(token) {
        return None;
    }
    // WHITESPACE_RE removes every separator char, including `=`
    let cleaned: String = token.chars().filter(|c| !is_b64_separator(*c)).collect();
    let urlsafe: String = cleaned
        .chars()
        .map(|c| match c {
            '-' => '+',
            '_' => '/',
            other => other,
        })
        .collect();
    if let Some(decoded) = decode_cleaned(&urlsafe, min_printable_ratio, gunzip_left) {
        return Some(decoded);
    }
    if cleaned.contains('-') || cleaned.contains('_') {
        let stripped: String = cleaned.chars().filter(|c| *c != '-' && *c != '_').collect();
        return decode_cleaned(&stripped, min_printable_ratio, gunzip_left);
    }
    None
}

/// `BASE64_RE` candidate tokens: maximal data/separator runs that start with a
/// data char preceded by a non-data char. The three reference alternatives
/// (`{12,}={0,2}` / `{11,}=` / `{10,}==`) decide the match end from the data
/// count and the trailing separator run.
fn base64_token_spans(content: &str) -> Vec<(usize, usize)> {
    let chars: Vec<(usize, char)> = content.char_indices().collect();
    let mut spans = Vec::new();
    let mut i = 0;
    while i < chars.len() {
        let (start, c) = chars[i];
        if !is_b64_data(c) || (i > 0 && is_b64_data(chars[i - 1].1)) {
            i += 1;
            continue;
        }
        // extend over data and separator chars
        let mut j = i;
        let mut data_count = 0usize;
        let mut last_data_idx: Option<usize> = None;
        while j < chars.len() {
            let (_, rc) = chars[j];
            if is_b64_data(rc) {
                data_count += 1;
                last_data_idx = Some(j);
            } else if !is_b64_separator(rc) {
                break;
            }
            j += 1;
        }
        let run_end = j;
        let trailing_from = last_data_idx.map_or(i, |idx| idx + 1);
        let trailing: Vec<char> = chars[trailing_from..run_end]
            .iter()
            .map(|(_, c)| *c)
            .collect();

        // backtracking order: alternative 1, then 2, then 3. Every end is a
        // CHAR index into `chars`; `alt_eq_end` returns an index into the
        // trailing separator slice (the chars after the last data char), so
        // the match end is `trailing_from + rel` there.
        let match_end_char: Option<usize> = if data_count >= 12 {
            // the units absorb every separator, so the match is the full run
            Some(run_end)
        } else if data_count >= 11 {
            alt_eq_end(&trailing, 1)
                .or_else(|| alt_eq_end(&trailing, 2))
                .map(|rel| trailing_from + rel)
        } else if data_count == 10 {
            alt_eq_end(&trailing, 2).map(|rel| trailing_from + rel)
        } else {
            None
        };
        let Some(end_char) = match_end_char else {
            i += 1;
            continue;
        };
        let end = chars.get(end_char).map_or(content.len(), |(idx, _)| *idx);
        if end > start {
            spans.push((start, end));
            // resume after the match, like re.sub
            i = end_char;
            continue;
        }
        i += 1;
    }
    spans
}

/// Match end (relative to the trailing-run start) for the `=`-padded
/// alternatives: `1` needs one `=` (rightmost whose successor is not `=`),
/// `2` needs a `==` pair not followed by another `=`.
fn alt_eq_end(trailing: &[char], eq_count: usize) -> Option<usize> {
    if eq_count == 1 {
        return trailing
            .iter()
            .rposition(|c| *c == '=')
            .filter(|p| trailing.get(p + 1).is_none_or(|c| *c != '='))
            .map(|p| p + 1);
    }
    trailing
        .iter()
        .enumerate()
        .rev()
        .find(|(idx, c)| {
            **c == '='
                && trailing.get(idx + 1) == Some(&'=')
                && trailing.get(idx + 2).is_none_or(|c| *c != '=')
        })
        .map(|(idx, _)| idx + 2)
}

/// `decode_base64_candidates`: candidate tokens are replaced by their decoded
/// form when the printable/UTF-8 gates pass, with run-splitting and
/// sub-floor reassembly fallbacks.
#[must_use]
pub fn decode_base64_candidates(content: &str, gunzip_attempts_left: &mut u32) -> String {
    let spans = base64_token_spans(content);
    if spans.is_empty() {
        return content.to_owned();
    }
    let mut out = String::with_capacity(content.len());
    let mut last = 0usize;
    for (start, end) in spans {
        if start < last {
            continue;
        }
        out.push_str(&content[last..start]);
        let token = &content[start..end];
        let primary_threshold = if token_has_widened_separator(token) {
            FALLBACK_PRINTABLE_RATIO_THRESHOLD
        } else {
            PRINTABLE_RATIO_THRESHOLD
        };
        let decoded = decode_token(token, primary_threshold, gunzip_attempts_left);
        let base = decoded.unwrap_or_else(|| decode_runs(token, gunzip_attempts_left));
        let reassembled = reassemble_sub_floor_runs(token, gunzip_attempts_left);
        match reassembled {
            Some(fragment) if !base.contains(&fragment) => {
                out.push_str(&base);
                out.push(' ');
                out.push_str(&fragment);
            }
            _ => out.push_str(&base),
        }
        last = end;
    }
    out.push_str(&content[last..]);
    out
}

/// `_decode_runs`: 12+ char separator-free data runs decoded at the fallback
/// threshold; failed runs stay as-is. Trailing `=` padding up to two chars
/// joins the replaced span.
fn decode_runs(token: &str, gunzip_attempts_left: &mut u32) -> String {
    let chars: Vec<(usize, char)> = token.char_indices().collect();
    let mut out = String::new();
    let mut last = 0usize;
    let mut idx = 0usize;
    while idx < chars.len() {
        let (start, c) = chars[idx];
        if !is_b64_data(c) || (idx > 0 && is_b64_data(chars[idx - 1].1)) {
            idx += 1;
            continue;
        }
        let mut end_idx = idx;
        while end_idx < chars.len() && is_b64_data(chars[end_idx].1) {
            end_idx += 1;
        }
        let run_len = end_idx - idx;
        let mut absorb = 0usize;
        while absorb < 2
            && chars
                .get(end_idx + absorb)
                .is_some_and(|(_, rc)| *rc == '=')
        {
            absorb += 1;
        }
        if run_len >= 12 {
            let run: String = chars[idx..end_idx].iter().map(|(_, c)| c).collect();
            out.push_str(&token[last..start]);
            match decode_token(
                &run,
                FALLBACK_PRINTABLE_RATIO_THRESHOLD,
                gunzip_attempts_left,
            ) {
                Some(decoded) => out.push_str(&decoded),
                None => out.push_str(&token[start..start + run_len + absorb]),
            }
            last = start + run_len + absorb;
        }
        idx = end_idx.max(idx + 1);
    }
    out.push_str(&token[last..]);
    out
}

fn reassemble_sub_floor_runs(token: &str, gunzip_attempts_left: &mut u32) -> Option<String> {
    let chars: Vec<(usize, char)> = token.char_indices().collect();
    let mut fragments = String::new();
    let mut idx = 0usize;
    while idx < chars.len() {
        let (_, c) = chars[idx];
        if is_b64_data(c) && (idx == 0 || !is_b64_data(chars[idx - 1].1)) {
            let mut end_idx = idx;
            while end_idx < chars.len() && is_b64_data(chars[end_idx].1) {
                end_idx += 1;
            }
            let run_len = end_idx - idx;
            if run_len <= 11 {
                fragments.extend(chars[idx..end_idx].iter().map(|(_, c)| c));
            }
            idx = end_idx;
            continue;
        }
        idx += 1;
    }
    if fragments.chars().count() < 12 {
        return None;
    }
    decode_token(
        &fragments,
        FALLBACK_PRINTABLE_RATIO_THRESHOLD,
        gunzip_attempts_left,
    )
}

// ---------------------------------------------------------------------------
// SQL comment strip (single final pass, reference semantics)
// ---------------------------------------------------------------------------

/// `_strip_sql_comments`: block comments become ` body ` (inner-word and
/// `/*!`-guarded forms are kept); every `--`/`#` marker becomes one space.
#[must_use]
pub fn strip_sql_comments(content: &str) -> String {
    let bytes = content.as_bytes();
    let mut out = String::with_capacity(content.len());
    let mut i = 0usize;
    while i < bytes.len() {
        if bytes[i] == b'/'
            && i + 1 < bytes.len()
            && bytes[i + 1] == b'*'
            && let Some((body_start, close_end)) = block_comment_span(content, i)
        {
            out.push(' ');
            out.push_str(&content[body_start..close_end - 2]);
            out.push(' ');
            i = close_end;
            continue;
        }
        if bytes[i] == b'-' && i + 1 < bytes.len() && bytes[i + 1] == b'-' {
            out.push(' ');
            i += 2;
            continue;
        }
        if bytes[i] == b'#' {
            out.push(' ');
            i += 1;
            continue;
        }
        let ch = content[i..].chars().next().unwrap_or('/');
        out.push(ch);
        i += ch.len_utf8();
    }
    out
}

/// Resolve `/* ... */` per the reference alternation
/// `(?<!\w)/\*(?!!)(.*?)\*/|/\*(?!!)(.*?)\*/(?!\w)` with DOTALL: the
/// lookbehind-gated branch wins with the first terminator, otherwise the
/// trailing-gated branch expands the body until a terminator not followed by
/// a word char. Returns `(body_start, close_end)`.
fn block_comment_span(content: &str, start: usize) -> Option<(usize, usize)> {
    let bytes = content.as_bytes();
    if bytes.get(start + 2) == Some(&b'!') {
        return None;
    }
    let preceding_is_word =
        start > 0 && char_before(content, start).is_some_and(|(_, c)| py_is_word(c));
    let first_close = content[start + 2..]
        .find("*/")
        .map(|rel| (start + 2 + rel, start + 2 + rel + 2));
    if !preceding_is_word {
        // first alternative: `(?<!\w)` holds, the lazy body ends at the first
        // terminator
        return first_close.map(|(_, close_end)| (start + 2, close_end));
    }
    // second alternative: expand until `*/` is not followed by a word char
    let mut cursor = start + 2;
    while let Some(rel) = content[cursor..].find("*/") {
        let close = cursor + rel;
        let close_end = close + 2;
        let after_is_word = content[close_end..].chars().next().is_some_and(py_is_word);
        if !after_is_word {
            return Some((start + 2, close_end));
        }
        cursor = close + 1;
    }
    None
}

fn char_before(s: &str, byte_idx: usize) -> Option<(usize, char)> {
    s[..byte_idx].char_indices().next_back()
}

fn py_is_word(c: char) -> bool {
    c.is_alphanumeric() || c == '_'
}

// ---------------------------------------------------------------------------
// truncation (truncation.py), in code-point space
// ---------------------------------------------------------------------------

fn take_chars(content: &str, count: usize) -> String {
    content.chars().take(count).collect()
}

fn char_len(content: &str) -> usize {
    content.chars().count()
}

/// `extract_attack_regions`: indicator matches padded by 100 chars on each
/// side, sorted, merged and capped. Indices are code points.
#[doc(hidden)]
#[must_use]
pub fn extract_attack_regions(content: &str, max_content_length: usize) -> Vec<(usize, usize)> {
    let max_regions = usize::min(100, max_content_length / 100);
    let mut regions: Vec<(usize, usize)> = Vec::new();

    // code-point prefix of every byte offset reachable here (built lazily)
    let cp_index = |byte_idx: usize| content[..byte_idx].chars().count();
    let total_chars = char_len(content);

    for indicator in ATTACK_INDICATORS.iter() {
        let mut found: Vec<(usize, usize)> = Vec::new();
        for m in indicator.find_iter(content) {
            if found.len() >= max_regions {
                break;
            }
            let start = cp_index(m.start()).saturating_sub(100);
            let end = usize::min(cp_index(m.end()) + 100, total_chars);
            found.push((start, end));
        }
        regions.extend(found);
        if regions.len() >= max_regions {
            break;
        }
    }

    if regions.is_empty() {
        return Vec::new();
    }
    regions.sort_unstable();
    let mut merged: Vec<(usize, usize)> = Vec::with_capacity(regions.len());
    for (start, end) in regions {
        if let Some(last) = merged.last_mut()
            && start <= last.1
        {
            last.1 = usize::max(last.1, end);
        } else {
            merged.push((start, end));
        }
    }
    merged.truncate(max_regions);
    merged
}

fn consume_gap(
    content_chars: &[char],
    last_end: usize,
    start: usize,
    gap_budget: usize,
) -> (String, usize) {
    let gap_len = start - last_end;
    if gap_len <= gap_budget {
        return (
            content_chars[last_end..start].iter().collect(),
            gap_budget - gap_len,
        );
    }
    let chunk_len = gap_budget - 1;
    let piece = if chunk_len > 0 {
        content_chars[last_end..last_end + chunk_len]
            .iter()
            .collect::<String>()
    } else {
        String::new()
    };
    (format!("{piece} "), 0)
}

fn build_result_with_attack_regions_and_context(
    content_chars: &[char],
    attack_regions: &[(usize, usize)],
    budget: usize,
) -> String {
    let attack_length: usize = attack_regions.iter().map(|(s, e)| e - s).sum();
    let mut gap_budget = budget.saturating_sub(attack_length);
    let mut result_parts: Vec<String> = Vec::new();
    let mut last_end = 0usize;

    for (start, end) in attack_regions {
        if last_end < *start && gap_budget > 0 {
            let (piece, remaining) = consume_gap(content_chars, last_end, *start, gap_budget);
            gap_budget = remaining;
            result_parts.push(piece);
        }
        result_parts.push(content_chars[*start..*end].iter().collect());
        last_end = *end;
    }

    if last_end < content_chars.len() && gap_budget > 0 {
        let tail_len = usize::min(content_chars.len() - last_end, gap_budget);
        result_parts.push(
            content_chars[last_end..last_end + tail_len]
                .iter()
                .collect(),
        );
    }

    result_parts.concat()
}

fn cap_with_tail(content_chars: &[char], max_full_scan_bytes: usize) -> String {
    let tail = usize::min(FULL_SCAN_TAIL_BYTES, max_full_scan_bytes);
    let head_len = max_full_scan_bytes - tail;
    let total = content_chars.len();
    let mut out = String::new();
    out.extend(content_chars[..head_len].iter());
    out.extend(content_chars[total - tail..].iter());
    out
}

/// `truncate_safely`: cap with attack-region preservation; all budgets are in
/// code points, mirroring Python string indexing.
#[doc(hidden)]
#[must_use]
pub fn truncate_safely(
    content: &str,
    max_full_scan_bytes: usize,
    preserve_attacks: bool,
    max_content_length: usize,
) -> String {
    let total = char_len(content);
    if total <= max_full_scan_bytes {
        return content.to_owned();
    }
    if !preserve_attacks {
        return take_chars(content, max_full_scan_bytes);
    }

    let attack_regions = extract_attack_regions(content, max_content_length);
    let content_chars: Vec<char> = content.chars().collect();

    if attack_regions.is_empty() {
        return cap_with_tail(&content_chars, max_full_scan_bytes);
    }

    let attack_length: usize = attack_regions.iter().map(|(s, e)| e - s).sum();
    if attack_length >= max_full_scan_bytes {
        let mut result = String::new();
        let mut remaining = max_full_scan_bytes;
        for (start, end) in attack_regions {
            let chunk_len = usize::min(end - start, remaining);
            result.extend(content_chars[start..start + chunk_len].iter());
            remaining -= chunk_len;
            if remaining == 0 {
                break;
            }
        }
        return result;
    }

    build_result_with_attack_regions_and_context(
        &content_chars,
        &attack_regions,
        max_full_scan_bytes,
    )
}

// ---------------------------------------------------------------------------
// pipelines (ContentPreprocessor methods)
// ---------------------------------------------------------------------------

/// `decode_common_encodings`: up to 16 fixed-point decode iterations, then a
/// single SQL-comment strip. Returns `(content, decode_budget_exhausted)`.
#[must_use]
pub fn decode_common_encodings_with_budget(content: &str) -> (String, bool) {
    let mut current = content.to_owned();
    let mut gunzip_attempts_left = MAX_GUNZIP_ATTEMPTS_PER_PASS;
    let mut iterations = 0usize;

    while iterations < MAX_DECODE_ITERATIONS {
        let original = current.clone();
        current = decode_overlong_utf8_percent_runs(&current);
        current = percent_unquote_ignore(&current);
        current = html_escape::decode_html_entities(&current).into_owned();
        current = decode_percent_u_escapes(&current);
        current = decode_hex_escapes(&current);
        current = decode_ldap_hex_escapes(&current);
        current = decode_unicode_escapes(&current);
        current = normalize_unicode(&current);
        current = decode_base64_candidates(&current, &mut gunzip_attempts_left);
        if current == original {
            break;
        }
        iterations += 1;
    }
    let exhausted = iterations == MAX_DECODE_ITERATIONS;
    (strip_sql_comments(&current), exhausted)
}

/// Legacy entry kept for the ledger verification tooling.
#[must_use]
pub fn decode_common_encodings(content: &str) -> String {
    decode_common_encodings_with_budget(content).0
}

/// `preprocess_with_decoded`: `(processed, decoded, decode_budget_exhausted)`.
#[must_use]
pub fn preprocess_with_decoded(
    content: &str,
    max_full_scan_bytes: usize,
    preserve_attacks: bool,
    max_content_length: usize,
) -> (String, String, bool) {
    if content.is_empty() {
        return (String::new(), String::new(), false);
    }

    let decoded = normalize_unicode(content);
    let (decoded, exhausted) = decode_common_encodings_with_budget(&decoded);
    let processed = remove_null_bytes(&decoded);
    let processed = collapse_whitespace(&processed);
    let processed = truncate_safely(
        &processed,
        max_full_scan_bytes,
        preserve_attacks,
        max_content_length,
    );

    (processed, decoded, exhausted)
}

/// `preprocess_signal_preserving`: raw view (no decoding).
#[must_use]
pub fn preprocess_signal_preserving(
    content: &str,
    max_full_scan_bytes: usize,
    preserve_attacks: bool,
    max_content_length: usize,
) -> String {
    if content.is_empty() {
        return String::new();
    }
    let normalized = normalize_unicode(content);
    truncate_safely(
        &normalized,
        max_full_scan_bytes,
        preserve_attacks,
        max_content_length,
    )
}

/// `preprocess`: full processed view (legacy entry point).
#[must_use]
pub fn preprocess(content: &str, max_full_scan_bytes: usize, preserve_attacks: bool) -> String {
    preprocess_with_decoded(
        content,
        max_full_scan_bytes,
        preserve_attacks,
        DEFAULT_MAX_CONTENT_LENGTH,
    )
    .0
}

pub const DEFAULT_MAX_CONTENT_LENGTH: usize = 10_000;

/// `preprocess_short_base64_additive_view`: decoded fragments of short
/// base64-looking tokens joined with newlines.
#[must_use]
pub fn short_base64_additive_view(
    content: &str,
    max_full_scan_bytes: usize,
    preserve_attacks: bool,
    max_content_length: usize,
) -> String {
    static TOKEN_RE: LazyLock<Regex> =
        LazyLock::new(|| Regex::new(r"[A-Za-z0-9+/]{4,}").expect("static regex"));
    static MARKERS: [char; 4] = ['$', '{', '}', '#'];
    const MAX_TOKEN_CHARS: usize = 11;
    const MAX_CANDIDATES: usize = 20_000;
    const RATIO: f64 = 0.95;

    if content.is_empty() {
        return String::new();
    }
    let normalized = normalize_unicode(content);
    let truncated = truncate_safely(
        &normalized,
        max_full_scan_bytes,
        preserve_attacks,
        max_content_length,
    );

    let mut fragments: Vec<String> = Vec::new();
    for (attempts, m) in TOKEN_RE.find_iter(&truncated).enumerate() {
        if attempts >= MAX_CANDIDATES {
            break;
        }
        let token = m.as_str();
        if token.chars().count() > MAX_TOKEN_CHARS {
            continue;
        }
        let Some(decoded) = short_token_decode(token) else {
            continue;
        };
        if printable_ratio(&decoded) < RATIO {
            continue;
        }
        if !decoded.chars().any(|c| MARKERS.contains(&c)) {
            continue;
        }
        fragments.push(decoded);
    }
    fragments.join("\n")
}

fn short_token_decode(token: &str) -> Option<String> {
    let raw = b64_decode_strict(token)?;
    String::from_utf8(raw).ok()
}

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn unicode_normalization() {
        assert_eq!(normalize_unicode("\u{FF0F}"), "/");
        assert_eq!(normalize_unicode("\u{200B}test\u{200C}"), "test");
        assert_eq!(normalize_unicode("\u{FF1C}script\u{FF1E}"), "<script>");
        assert_eq!(normalize_unicode("\u{FF1B}\u{FF5C}\u{FF06}"), ";|&");
    }

    #[test]
    fn whitespace_and_null_bytes() {
        assert_eq!(
            collapse_whitespace("test  multiple   spaces"),
            "test multiple spaces"
        );
        assert_eq!(
            collapse_whitespace("  leading trailing  "),
            "leading trailing"
        );
        assert_eq!(remove_null_bytes("test\x00null\x00bytes"), "testnullbytes");
    }

    #[test]
    fn decode_common_encodings_variants() {
        // URL
        assert_eq!(decode_common_encodings("%3Cscript%3E"), "<script>");
        // HTML
        assert_eq!(decode_common_encodings("&lt;script&gt;"), "<script>");
        // double-encoded
        assert_eq!(decode_common_encodings("%253Cscript%253E"), "<script>");
        // hex escape
        assert_eq!(decode_common_encodings(r"\x3Cscript\x3E"), "<script>");
        // inner-word SQL comments are preserved for the sqli obfuscation scan
        let r = decode_common_encodings("SEL/**/ECT * FR/**/OM users");
        assert!(r.contains("SEL/**/ECT"), "got: {r}");
    }

    #[test]
    fn sql_comment_markers_become_spaces() {
        assert_eq!(strip_sql_comments("' OR 1=1-- rest"), "' OR 1=1  rest");
        assert_eq!(strip_sql_comments("# heading"), "  heading");
        assert_eq!(strip_sql_comments("a -- b # c"), "a   b   c");
        // standalone block comment keeps its body between spaces
        assert_eq!(strip_sql_comments("x /* note */ y"), "x   note   y");
        // `/*!` is protected
        assert_eq!(strip_sql_comments(" /*!5 */"), " /*!5 */");
    }

    #[test]
    fn base64_candidates_decode() {
        let mut gunzip = 8;
        assert_eq!(
            decode_base64_candidates("PHNjcmlwdD5hbGVydCgxKTwvc2NyaXB0Pg==", &mut gunzip),
            "<script>alert(1)</script>"
        );
        // uuid stays unchanged
        assert_eq!(
            decode_base64_candidates("550e8400-e29b-41d4-a716-446655440000", &mut gunzip),
            "550e8400-e29b-41d4-a716-446655440000"
        );
        // short token stays unchanged
        assert_eq!(
            decode_base64_candidates("aGVsbG8=", &mut gunzip),
            "aGVsbG8="
        );
    }

    #[test]
    fn overlong_utf8_runs() {
        // %C0%AF is an overlong '/' in strict UTF-8
        assert_eq!(decode_overlong_utf8_percent_runs("%C0%AF"), "/");
        // valid runs stay untouched
        assert_eq!(decode_overlong_utf8_percent_runs("%C3%A9"), "%C3%A9");
    }

    #[test]
    fn ldap_hex_escapes() {
        assert_eq!(decode_ldap_hex_escapes(r"\41"), "A");
        assert_eq!(decode_hex_escapes(r"\x41"), "A");
        assert_eq!(decode_ldap_hex_escapes(r"\x41"), r"\x41");
    }

    #[test]
    fn full_preprocess_pipeline() {
        let zwsp = '\u{200B}';
        let fullwidth_slash = '\u{FF0F}';
        let content = format!(
            "{zwsp}<script>{fullwidth_slash}alert(1)</script>  multiple   spaces %3Cimg%3E\x00null"
        );
        let result = preprocess(&content, 262_144, true);

        assert!(!result.contains('\u{200B}'));
        assert!(!result.contains('\u{FF0F}'));
        assert!(!result.contains("  "));
        assert!(result.contains("<img>"));
        assert!(!result.contains('\x00'));
    }

    #[test]
    fn preprocess_empty() {
        assert_eq!(preprocess("", 262_144, true), "");
    }

    #[test]
    fn markdown_heading_survives_as_reference() {
        // '#' becomes a space, then whitespace collapses and strips
        let processed = preprocess("# Title\n\n- item one\n- item two", 262_144, true);
        assert_eq!(processed, "Title - item one - item two");
    }

    #[test]
    fn extract_attack_regions_padded() {
        let content = format!(
            "{}<script>alert(1)</script>{}",
            "a".repeat(150),
            "b".repeat(150)
        );
        let regions = extract_attack_regions(&content, 10_000);
        assert_eq!(regions.len(), 1);
        assert_eq!(regions[0], (50, 257));
    }
    #[test]
    fn b64_span_after_multibyte_no_underflow() {
        // A 12+ char base64 run preceded by multibyte content: the run's byte
        // offset exceeds its char index, so run_end(char) - start(byte) would
        // underflow. Regression for the fuzz-found panic (unit mismatch).
        let content = "h\u{e9}llo ++++++++++++++++++++++++++++++++++++";
        let _ = decode_common_encodings(content);
    }
}