characterminator 0.1.2

find and eliminate characters outside an allowed set, per file type and per file, so text costs fewer tokens
Documentation
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
//! Hazard detection (V34): which lint, if any, a character fires.
//!
//! The CODE POINTS are not here. They are the generated data file the
//! charset node compiles in (`src/charset` `hazard.ctrm-sets`), read
//! through its public surface (`src:V39`). What lives here is the part
//! that is this node's: which lint each class is reported under, the
//! order the classes are tried in, and the one cut a set cannot make --
//! a byte order mark at byte 0 is an encoding signature, not a hazard.
//!
//! A joiner inside an RGI ZWJ sequence, and a tag inside an RGI tag
//! sequence, are no hazard (V63). That cut is POSITIONAL, so it needs the
//! text: a caller asks [`Hazards::exempt`] once per text and hands the
//! offsets to [`Hazards::lint_at`]; a library caller asks
//! [`Hazards::lints_in`], which does both. The list lives in `sequence`.

use super::sequence::Sequences;
use crate::charset::{CharSet, SetCatalog, builtin};
use crate::lint::Lint;
use crate::scan::{Hit, scan_str};
use std::sync::LazyLock;

/// The hazard classes in the order they are TRIED, each with the lint it
/// is reported under. The first class holding a character names its lint.
///
/// The classes overlap -- every bidi control, tag and the BOM is also
/// default-ignorable -- so the order is the meaning: the narrow classes
/// come first, and `invisible` is what is left when nothing narrower
/// claims a character. A Trojan Source override therefore reports as
/// `bidi-control`, which is what a reader needs to know about it.
const CLASSES: [(&str, &str); 5] = [
    ("hazard-bidi", "bidi-control"),
    ("hazard-tag", "tag-character"),
    ("hazard-bom", "stray-bom"),
    ("hazard-control", "control-character"),
    ("hazard-invisible", "invisible"),
];

/// The lint whose ONE exemption is positional.
const STRAY_BOM: &str = "stray-bom";

/// The one hazard class that may carry MEANING: a C0 or C1 control (a
/// form feed, an escape sequence in a captured log). Every other class
/// holds characters that draw nothing -- they hide or reorder text and
/// add none -- so `fix` may delete one; this one it rewrites only through
/// a map entry (`src/fix:V104`).
const CONTROL: &str = "control-character";

/// Whether a hazard lint's characters carry no visible text, so `fix`
/// deletes one no map entry covers (`src/fix:V104`): every class but
/// `control-character`.
pub(crate) fn carries_no_text(lint: Lint) -> bool {
    lint.group == crate::lint::Group::Hazard && lint.name != CONTROL
}

/// The fidelity the classes are resolved at. They carry no labelled
/// member (`src/charset:V41`), so any family answers the same; this is
/// the default one, named rather than imported because a family is an
/// opaque name outside the fix node.
const FAMILY: &str = "text";

/// The compiled-in hazards, parsed on first use.
static BUILTIN: LazyLock<Hazards> =
    LazyLock::new(|| Hazards::compiled().unwrap_or_else(|_| Hazards::none()));

/// Every hazard class, resolved once per run, paired with its lint.
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct Hazards {
    classes: Vec<(Lint, CharSet)>,
    /// Each compiled-in preset that excuses joiners, and which (V57).
    excusing: Vec<Excuse>,
    /// The RGI sequences whose joiners and tags are no hazard (V63).
    sequences: Sequences,
}

impl Hazards {
    /// The classes as this build ships them.
    ///
    /// Read from the COMPILED-IN file, never from the run's catalog: a hazard
    /// is the one finding no configuration may switch off (`src/lint:V36`), and
    /// a catalog is configuration.
    ///
    /// INFALLIBLE because the data is compiled in, and parsed ONCE per
    /// process. A defect in it -- a file that fails to parse, a class it
    /// does not declare, a lint the registry lacks -- is this crate's, and
    /// the test `the_compiled_data_parses` keeps it from shipping.
    ///
    /// ```
    /// use characterminator::Hazards;
    ///
    /// let found = Hazards::builtin().lints_in("a\u{202E}b");
    /// let names: Vec<&str> = found.iter().map(|(_, l)| l.name).collect();
    /// assert_eq!(names, ["bidi-control"]);
    /// ```
    #[must_use]
    pub fn builtin() -> Self {
        BUILTIN.clone()
    }

    /// No class at all: the fallback a passing `the_compiled_data_parses`
    /// proves is never taken.
    fn none() -> Self {
        Self {
            classes: Vec::new(),
            excusing: Vec::new(),
            sequences: Sequences::default(),
        }
    }

    /// The compiled-in data, parsed: what [`Hazards::builtin`] keeps.
    fn compiled() -> Result<Self, String> {
        let classes = classes()?;
        let excusing = excusing(&classes)?;
        let sequences = Sequences::deferred();
        Ok(Self {
            classes,
            excusing,
            sequences,
        })
    }

    /// The byte offsets in `text` of every joiner and tag character that
    /// sits inside an RGI emoji sequence (V63), ascending.
    pub(crate) fn exempt(&self, text: &str) -> Vec<usize> {
        self.sequences.exempt(text)
    }

    /// The byte offsets in `text` of every VS15 or VS16 right after a
    /// character `granted` accepts that has a standardized variation
    /// sequence, ascending: a presentation choice on a granted character,
    /// neither a hazard nor outside the set.
    pub(crate) fn selectors(
        &self,
        text: &str,
        granted: impl Fn(char) -> bool,
    ) -> Vec<usize> {
        super::sequence::selectors(text, granted)
    }

    /// [`Hazards::lint_for`], less a hit at an `exempt` offset.
    pub(crate) fn lint_at(&self, hit: Hit, exempt: &[usize]) -> Option<Lint> {
        let inside = exempt.binary_search(&hit.position.byte).is_ok();
        self.lint_for(hit).filter(|_| !inside)
    }

    /// Whether a file granted `granted` (a `+`-joined set name, as `check`
    /// names a union) is excused `character` (V57). Only a joiner can be,
    /// and only by a COMPILED-IN preset that grants it and no other hazard:
    /// `persian` excuses ZWNJ, `any` excuses nothing.
    pub(crate) fn excuses(&self, granted: &str, character: char) -> bool {
        granted.split('+').any(|name| {
            self.excusing.iter().any(|(set, joiners, _)| {
                set == name && joiners.contains(&character)
            })
        })
    }

    /// These hazards, excusing only through the presets `catalog` -- the
    /// run's -- resolves EXACTLY as the build ships them (V57). A NAME is
    /// not provenance: a `.ctrm-sets` line, or `--no-builtin-sets` with a
    /// `--set`, can call anything `hindi`, and such a set excuses nothing.
    #[must_use]
    pub(crate) fn vouched_by(mut self, catalog: &SetCatalog) -> Self {
        self.excusing.retain(|(name, _, shipped)| {
            catalog
                .resolve(name, FAMILY)
                .is_ok_and(|run| run == *shipped)
        });
        self
    }

    /// Whether a character is in ANY class, wherever it sits.
    ///
    /// A scan asks this before it knows a position, so a byte order mark
    /// answers yes here and is let off by [`Hazards::lints_in`].
    pub fn contains(&self, character: char) -> bool {
        self.classes.iter().any(|(_, set)| set.contains(character))
    }

    /// Every hazard in `text`, each with the lint it fires, in text order:
    /// what `check` reports for a file no rule grants anything (V113).
    ///
    /// CONTEXT-AWARE, as `check` is: a BOM at byte 0 is an encoding
    /// signature (V34), and a joiner, tag or VS16 inside an RGI emoji
    /// sequence is no hazard (V63). The script joiner excuse (V57) is
    /// not applied: it belongs to a file's rule, and a text has none.
    ///
    /// ```
    /// use characterminator::Hazards;
    ///
    /// let hazards = Hazards::builtin();
    /// // A family emoji: its joiners are part of the sequence.
    /// let family = "\u{1F468}\u{200D}\u{1F469}\u{200D}\u{1F467}";
    /// assert!(hazards.lints_in(family).is_empty());
    /// // A signature at byte 0 is no hazard; a joiner on its own is.
    /// let found = hazards.lints_in("\u{FEFF}a\u{200D}b");
    /// let at: Vec<(usize, &str)> =
    ///     found.iter().map(|(h, l)| (h.position.byte, l.name)).collect();
    /// assert_eq!(at, [(4, "invisible")]);
    /// ```
    #[must_use]
    pub fn lints_in(&self, text: &str) -> Vec<(Hit, Lint)> {
        let hits = scan_str(text, |c| !self.contains(c));
        if hits.is_empty() {
            return Vec::new();
        }
        let exempt = self.exempt(text);
        hits.into_iter()
            .filter_map(|hit| Some((hit, self.lint_at(hit, &exempt)?)))
            .collect()
    }

    /// The hazard lint one hit fires, or `None` when it is no hazard.
    ///
    /// CONTEXT-FREE: one character and its byte offset, nothing around
    /// it, so it cannot see an emoji sequence (V63). Crate-private for
    /// that reason (V113, B66); a library caller asks
    /// [`Hazards::lints_in`].
    ///
    /// A BOM at byte 0 is `None`, and stops there: it is not passed on
    /// to `invisible`, which holds it too, because the exemption is for
    /// the CHARACTER at that place rather than for one class's name.
    pub(crate) fn lint_for(&self, hit: Hit) -> Option<Lint> {
        let lint = self.class_of(hit.character)?;
        let signature = lint.name == STRAY_BOM && hit.position.byte == 0;
        (!signature).then_some(lint)
    }

    /// [`Hazards::lint_at`] for text with NO file start: tool output
    /// (`src/cli/guard:V53`). Each string of it begins at byte 0 of its own,
    /// and none of them is a file an encoder signed, so a BOM there is a
    /// stray like any other.
    pub(crate) fn lint_unsigned(
        &self,
        hit: Hit,
        exempt: &[usize],
    ) -> Option<Lint> {
        let inside = exempt.binary_search(&hit.position.byte).is_ok();
        self.class_of(hit.character).filter(|_| !inside)
    }

    /// The lint of the first class holding `character`, wherever it sits.
    pub(crate) fn class_of(&self, character: char) -> Option<Lint> {
        let (lint, _) = self
            .classes
            .iter()
            .find(|(_, set)| set.contains(character))?;
        Some(*lint)
    }
}

/// One hazard class and the lint it fires.
type Class = (Lint, CharSet);

/// Each class of the compiled-in hazard file, paired with its lint.
fn classes() -> Result<Vec<Class>, String> {
    let catalog = builtin::hazard_catalog().map_err(|e| e.to_string())?;
    CLASSES
        .iter()
        .map(|(set, lint)| {
            let lint = Lint::named(lint)
                .ok_or_else(|| format!("no `{lint}` lint registered"))?;
            let set = catalog.resolve(set, FAMILY);
            Ok((lint, set.map_err(|e| e.to_string())?))
        })
        .collect()
}

/// ZERO WIDTH NON-JOINER and ZERO WIDTH JOINER: the only hazards a grant
/// may excuse (V57). Persian needs ZWNJ inside words and Devanagari needs
/// both, so in those scripts they are SPELLING; a bidi control, a tag
/// character or a C0 control is never anybody's spelling.
const JOINERS: [char; 2] = ['\u{200C}', '\u{200D}'];

/// A preset name, the joiners it excuses, and the set as it ships.
type Excuse = (String, Vec<char>, CharSet);

/// The compiled-in presets that grant a joiner and NO other hazard, with
/// the joiners each grants. Read from the builtin catalog, never the
/// run's: a `.ctrm-sets` redeclaring `persian` cannot widen this.
fn excusing(classes: &[(Lint, CharSet)]) -> Result<Vec<Excuse>, String> {
    let catalog = builtin::catalog().map_err(|e| e.to_string())?;
    let mut found = Vec::new();
    for name in catalog.names() {
        let set = catalog.resolve(name, FAMILY).map_err(|e| e.to_string())?;
        let joiners: Vec<char> =
            JOINERS.into_iter().filter(|j| set.contains(*j)).collect();
        if !joiners.is_empty() && !grants_other_hazard(&set, classes) {
            found.push((name.to_owned(), joiners, set));
        }
    }
    Ok(found)
}

/// Whether `set` grants any hazard that is not a joiner. `any` does, so
/// granting everything never excuses anything.
fn grants_other_hazard(set: &CharSet, classes: &[(Lint, CharSet)]) -> bool {
    classes
        .iter()
        .flat_map(|(_, class)| &class.ranges)
        .any(|range| {
            (range.start..=range.end)
                .any(|c| !JOINERS.contains(&c) && set.contains(c))
        })
}

#[cfg(test)]
mod tests {
    use super::Hazards;
    use crate::lint::{Group, Level, Levels, Lint, Target};
    use crate::scan::{Hit, Position};

    fn hazards() -> Hazards {
        Hazards::builtin()
    }

    /// `Hazards::builtin` is infallible only because this holds: the
    /// compiled-in data parses, so its empty fallback is never taken.
    #[test]
    fn the_compiled_data_parses() {
        let compiled = Hazards::compiled();
        assert!(compiled.is_ok(), "{compiled:?}");
        assert_eq!(compiled.ok(), Some(Hazards::builtin()));
        assert_ne!(Hazards::builtin(), Hazards::none());
    }

    fn at(byte: usize, character: char) -> Hit {
        let position = Position {
            line: 1,
            column: 1,
            byte,
        };
        Hit {
            position,
            character,
        }
    }

    /// The lint NAME a character fires at byte 5, or `""` for none.
    fn fired(character: char) -> &'static str {
        hazards().lint_for(at(5, character)).map_or("", |l| l.name)
    }

    /// Trojan Source (CVE-2021-42574): the overrides and isolates that
    /// make code display in one order and compile in another.
    #[test]
    fn a_bidi_override_or_isolate_is_a_bidi_control() {
        for point in ['\u{202A}', '\u{202E}', '\u{2066}', '\u{2069}'] {
            assert_eq!(fired(point), "bidi-control", "{point:?}");
        }
        assert_eq!(fired('\u{200F}'), "bidi-control");
    }

    /// ASCII smuggling: U+E0041 is TAG LATIN CAPITAL LETTER A, an `A`
    /// nobody sees and a model reads.
    #[test]
    fn a_tag_letter_is_a_tag_character() {
        assert_eq!(fired('\u{E0041}'), "tag-character");
        assert_eq!(fired('\u{E0001}'), "tag-character");
        assert_eq!(fired('\u{E007F}'), "tag-character");
    }

    #[test]
    fn a_control_other_than_the_layout_three_is_a_control_character() {
        for point in ['\u{0007}', '\u{000B}', '\u{001B}', '\u{007F}'] {
            assert_eq!(fired(point), "control-character", "{point:?}");
        }
        assert_eq!(fired('\u{0085}'), "control-character");
    }

    #[test]
    fn tab_newline_and_carriage_return_are_no_hazard() {
        assert_eq!(fired('\t'), "");
        assert_eq!(fired('\n'), "");
        assert_eq!(fired('\r'), "");
    }

    #[test]
    fn the_rest_of_the_ignorables_are_invisible() {
        for point in ['\u{200B}', '\u{200D}', '\u{00AD}', '\u{FE0F}'] {
            assert_eq!(fired(point), "invisible", "{point:?}");
        }
        assert_eq!(fired('\u{3164}'), "invisible");
        assert_eq!(fired('\u{E0100}'), "invisible");
    }

    #[test]
    fn ordinary_text_is_no_hazard() {
        for point in ['A', ' ', '\u{00E9}', '\u{2014}', '\u{1F600}'] {
            assert_eq!(fired(point), "", "{point:?}");
        }
        assert!(!hazards().contains('A'));
    }

    /// The positional cut: a signature at byte 0, a stray anywhere else
    /// -- and the one at byte 0 is not handed on to `invisible` either.
    #[test]
    fn a_bom_is_a_hazard_everywhere_but_byte_zero() {
        let found = hazards();
        assert_eq!(found.lint_for(at(0, '\u{FEFF}')), None);
        let stray = found.lint_for(at(3, '\u{FEFF}')).map(|l| l.name);
        assert_eq!(stray, Some("stray-bom"));
        assert!(found.contains('\u{FEFF}'));
    }

    /// B19: the excuse follows the SET, not its name. The shipped catalog
    /// keeps `hindi`'s; a redeclared `hindi`, or none at all, loses it.
    #[test]
    fn only_the_shipped_preset_vouches_for_a_joiner() {
        use crate::charset::{SetCatalog, builtin, parse_line};
        let zwnj = '\u{200C}';
        let shipped = builtin::catalog().unwrap_or_else(|_| SetCatalog::new());
        assert!(hazards().vouched_by(&shipped).excuses("hindi", zwnj));
        let mut redeclared = shipped;
        let line = parse_line("hindi ascii U+200C U+200D");
        if let Ok(Some(definition)) = line {
            redeclared.insert(definition);
        }
        assert!(!hazards().vouched_by(&redeclared).excuses("hindi", zwnj));
        let none = SetCatalog::new();
        assert!(!hazards().vouched_by(&none).excuses("hindi", zwnj));
    }

    /// Tool output has no file start, so byte 0 of it signs nothing.
    #[test]
    fn unsigned_text_lets_no_bom_off() {
        let lint = hazards().lint_unsigned(at(0, '\u{FEFF}'), &[]);
        assert_eq!(lint.map(|l| l.name), Some("stray-bom"));
    }

    /// Only byte 0 is exempt, and only for the BOM: another hazard at the
    /// start of a file is as much a hazard as anywhere.
    #[test]
    fn byte_zero_lets_off_the_bom_and_nothing_else() {
        let first = hazards().lint_for(at(0, '\u{202E}')).map(|l| l.name);
        assert_eq!(first, Some("bidi-control"));
    }

    /// Every lint a class names is a registered hazard, so it forbids.
    #[test]
    fn every_class_fires_a_hazard_lint_at_forbid() {
        for (lint, _) in &hazards().classes {
            assert_eq!(lint.group, Group::Hazard, "{}", lint.name);
            assert_eq!(lint.default_level(), Level::Forbid, "{}", lint.name);
        }
    }

    /// `src/lint:V36`: `!hazard=allow`, and the lint named on its own, move
    /// nothing.
    #[test]
    fn no_rule_lowers_a_real_hazard_lint() {
        let lint = hazards().lint_for(at(5, '\u{202E}'));
        let lint = lint.unwrap_or(Lint::new("bidi-control", Group::Hazard));
        let mut levels = Levels::new();
        levels.set(Target::Group(Group::Hazard), Level::Allow);
        levels.set(Target::Lint(lint), Level::Warn);
        levels.set_charset(Level::Allow);
        assert_eq!(levels.level_of(lint), Level::Forbid);
    }
}