differential_engine/lang/
generic.rs

1//! The generic line normaliser — deliberately identical to the validated
2//! prototype (same regexes, same order), so shape-class populations stay
3//! byte-comparable with its recorded outputs. Do not "improve" this in place:
4//! behaviour changes belong in a language plugin with its own id (ADR 0015).
5//!
6//! Normalisation ONLY. Symbol extraction used to live here too; it is a
7//! separate use case with its own port (`artefact::symbols`) and its readers
8//! live in an adapter crate. Sharing a module made a symbol change look like a
9//! normalisation change, which is the one thing this file must never allow.
10
11use std::sync::LazyLock;
12
13use regex::bytes::Regex;
14
15// (?-u): byte-level ASCII classes, matching Python bytes-pattern semantics.
16static STR_RE: LazyLock<Regex> =
17    LazyLock::new(|| Regex::new(r#"(?-u)"[^"]*"|'[^']*'|`[^`]*`"#).unwrap());
18static NUM_RE: LazyLock<Regex> = LazyLock::new(|| Regex::new(r"(?-u)\b\d+\b").unwrap());
19static IDENT_RE: LazyLock<Regex> =
20    LazyLock::new(|| Regex::new(r"(?-u)[A-Za-z_][A-Za-z0-9_\-]{3,}").unwrap());
21static WS_RE: LazyLock<Regex> = LazyLock::new(|| Regex::new(r"(?-u)\s+").unwrap());
22
23/// Strings → `"S"`, numbers → `N`, identifiers (length ≥ 4) → `I`, whitespace
24/// collapsed to single spaces, trimmed.
25pub fn normalize_line(line: &[u8]) -> Vec<u8> {
26    let s = STR_RE.replace_all(line, b"\"S\"".as_slice());
27    let s = NUM_RE.replace_all(&s, b"N".as_slice());
28    let s = IDENT_RE.replace_all(&s, b"I".as_slice());
29    let s = WS_RE.replace_all(&s, b" ".as_slice());
30    trim_ascii(&s).to_vec()
31}
32
33fn trim_ascii(s: &[u8]) -> &[u8] {
34    let start = s
35        .iter()
36        .position(|b| !b.is_ascii_whitespace())
37        .unwrap_or(s.len());
38    let end = s
39        .iter()
40        .rposition(|b| !b.is_ascii_whitespace())
41        .map_or(start, |e| e + 1);
42    &s[start..end]
43}
differential_engine/lang/generic.rs

differential_engine/lang/
generic.rs
