characterminator 0.1.2

find and eliminate characters outside an allowed set, per file type and per file, so text costs fewer tokens
Documentation
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
//! The tests of `builtin.rs`, in a file of their own so the module
//! reads as code (sherd V50). Still its child: `super` is `builtin`.

use super::{
    ASCII, CharSet, HAZARD, SETS, SetDefinition, SetMember, ascii,
    ascii_definition, catalog, definitions, hazard_catalog, intrinsic_catalog,
};

#[test]
fn ascii_grants_printable_text() {
    let set = ascii();
    assert!(set.contains('\u{0020}'));
    assert!(set.contains('\u{0041}'));
    assert!(set.contains('\u{007E}'));
}

#[test]
fn ascii_grants_tab_and_newline() {
    let set = ascii();
    assert!(set.contains('\u{0009}'));
    assert!(set.contains('\u{000A}'));
}

#[test]
fn ascii_withholds_carriage_return_and_other_controls() {
    let set = ascii();
    assert!(!set.contains('\u{000D}'));
    assert!(!set.contains('\u{0000}'));
    assert!(!set.contains('\u{001F}'));
    assert!(!set.contains('\u{007F}'));
}

#[test]
fn ascii_withholds_everything_above_ascii() {
    let set = ascii();
    assert!(!set.contains('\u{00A0}'));
    assert!(!set.contains('\u{2014}'));
    assert!(!set.contains('\u{1F600}'));
}

#[test]
fn the_definition_resolves_to_the_same_set() {
    assert_eq!(ascii_definition().name, ASCII);
    assert_eq!(ascii().ranges.len(), 2);
}

#[test]
fn the_intrinsic_catalog_composes_the_same_set() {
    assert_eq!(intrinsic_catalog().resolve(ASCII, TEXT), Ok(ascii()));
}

#[test]
fn the_intrinsic_catalog_declares_only_ascii() {
    let catalog = intrinsic_catalog();
    let listed = catalog.names().collect::<Vec<_>>();
    assert_eq!(listed, vec![ASCII]);
}

/// The default fidelity (`src/rules:V29`), spelled rather than
/// imported: a family is an opaque name to this node (V41).
const TEXT: &str = "text";

/// The presets `src/charset/SPEC.md` INTERFACES names, which this
/// file ships.
///
/// Written out rather than read back from the catalog: a test that
/// asked the data file what it declares would pass just as happily
/// after a preset was deleted from it.
const DECLARED: [&str; 16] = [
    "any",
    "arabic",
    "box",
    "caveman",
    "cr",
    "cyrillic",
    "emoji",
    "greek",
    "hazard",
    "hindi",
    "latin-ext",
    "latin1",
    "legal",
    "marks",
    "math",
    "persian",
    // `typography` is tested by name below rather than listed here:
    // the array length is the count this file promises, and a preset
    // added without a test is what that count is for.
];

fn declared() -> Vec<SetDefinition> {
    let parsed = definitions();
    assert!(parsed.is_ok(), "the compiled-in data file must parse");
    parsed.unwrap_or_default()
}

fn preset(name: &str) -> CharSet {
    preset_at(name, TEXT)
}

/// The same, at a fidelity the caller names (V41).
fn preset_at(name: &str, family: &str) -> CharSet {
    let resolved = catalog()
        .ok()
        .and_then(|sets| sets.resolve(name, family).ok());
    assert!(resolved.is_some(), "{name} must resolve");
    resolved.unwrap_or_else(|| CharSet::new(name.to_owned(), Vec::new()))
}

#[test]
fn the_data_file_parses() {
    assert!(definitions().is_ok());
}

#[test]
fn the_data_file_is_pure_ascii() {
    assert!(SETS.is_ascii());
}

/// V22: every member is `U+XXXX`, which is what keeps the file ASCII.
/// A fidelity label is allowed in front of one (V41) and carries no
/// character of its own, so it is followed rather than rejected.
///
/// So is a member NAMING another builtin set (V25), which is how the
/// generated `hazard` is the union of its classes rather than a second
/// copy of their ranges. Only a name the builtin text itself declares:
/// one reaching outside it would make a preset depend on whatever a
/// user happened to declare.
fn is_code_point(member: &SetMember, builtin: &[String]) -> bool {
    match member {
        SetMember::Range(_) => true,
        SetMember::Labelled { member, .. } => is_code_point(member, builtin),
        SetMember::Named(name) => builtin.contains(name),
        SetMember::Literal(_) => false,
    }
}

#[test]
fn every_member_is_written_as_a_code_point() {
    let names: Vec<String> = declared().into_iter().map(|d| d.name).collect();
    for definition in declared() {
        for member in &definition.members {
            assert!(
                is_code_point(member, &names),
                "{} holds a member that is not U+XXXX",
                definition.name
            );
        }
    }
}

#[test]
fn every_preset_the_spec_names_is_declared_and_grants_something() {
    for name in DECLARED {
        assert!(!preset(name).is_empty(), "{name} grants nothing");
    }
    assert!(!preset("typography").is_empty());
}

#[test]
fn the_data_file_declares_no_set_twice() {
    let mut names: Vec<String> =
        declared().into_iter().map(|set| set.name).collect();
    let total = names.len();
    names.sort_unstable();
    names.dedup();
    assert_eq!(names.len(), total);
}

#[test]
fn the_data_file_does_not_declare_ascii() {
    let ascii_declared = declared().iter().any(|set| set.name == ASCII);
    assert!(!ascii_declared);
}

#[test]
fn the_catalog_carries_ascii_as_well_as_the_presets() {
    assert_eq!(preset(ASCII), ascii());
    assert!(catalog().is_ok_and(|sets| sets.get("caveman").is_some()));
}

#[test]
fn caveman_grants_the_format_symbols() {
    let set = preset("caveman");
    assert!(set.contains('\u{2192}'));
    assert!(set.contains('\u{22A5}'));
    assert!(set.contains('\u{00A7}'));
}

/// R3: stopping at FORMAT.md's own list misses 32% of the files.
#[test]
fn caveman_grants_the_measured_symbols_too() {
    let set = preset("caveman");
    assert!(set.contains('\u{21D2}'));
    assert!(set.contains('\u{2235}'));
    assert!(set.contains('\u{00B7}'));
    assert!(!set.contains('\u{1F600}'));
}

/// R19: the four that 4-6 of 13 Rust repos used and no preset held.
#[test]
fn caveman_grants_the_wave_one_gap() {
    let set = preset("caveman");
    assert!(set.contains('\u{2261}'));
    assert!(set.contains('\u{03A3}'));
    assert!(set.contains('\u{27FA}'));
    assert!(set.contains('\u{2287}'));
    assert!(!set.contains('\u{226A}'));
}

#[test]
fn box_grants_both_drawing_blocks() {
    let set = preset("box");
    assert!(set.contains('\u{250C}'));
    assert!(set.contains('\u{257F}'));
    assert!(set.contains('\u{25A0}'));
    assert!(!set.contains('\u{2580}'));
}

#[test]
fn typography_grants_what_the_builtin_map_targets() {
    let set = preset("typography");
    assert!(set.contains('\u{2014}'));
    assert!(set.contains('\u{2019}'));
    assert!(set.contains('\u{201D}'));
    assert!(set.contains('\u{2026}'));
    assert!(set.contains('\u{00A0}'));
    assert!(set.contains('\u{2212}'));
}

/// B73: granting typography keeps a Polish quotation whole, the opening
/// low-9 quote included, rather than rewriting that one alone.
#[test]
fn typography_grants_the_low_9_quotes_the_map_targets() {
    let set = preset("typography");
    let quotes = ['\u{201A}', '\u{201B}', '\u{201E}', '\u{201F}'];
    assert!(quotes.iter().all(|quote| set.contains(*quote)));
}

#[test]
fn emoji_grants_single_code_points_and_withholds_the_joiners() {
    let set = preset("emoji");
    assert!(set.contains('\u{1F600}'));
    assert!(set.contains('\u{2600}'));
    assert!(!set.contains('\u{200D}'));
    assert!(!set.contains('\u{FE0F}'));
    assert!(!set.contains('\u{1F3FB}'));
    assert!(!set.contains('\u{20E3}'));
}

#[test]
fn cr_grants_only_the_carriage_return() {
    let set = preset("cr");
    assert!(set.contains('\u{000D}'));
    assert_eq!(set.ranges.len(), 1);
}

#[test]
fn a_coarse_block_grants_its_whole_block() {
    assert!(preset("latin1").contains('\u{00E9}'));
    assert!(preset("latin-ext").contains('\u{0142}'));
    assert!(preset("cyrillic").contains('\u{0416}'));
    assert!(preset("greek").contains('\u{03B1}'));
    assert!(preset("arabic").contains('\u{0645}'));
}

#[test]
fn any_grants_every_code_point_there_is() {
    let set = preset("any");
    assert!(set.contains('\u{0000}'));
    assert!(set.contains('\u{E000}'));
    assert!(set.contains('\u{10FFFF}'));
}

#[test]
fn a_preset_is_a_handful_of_ranges() {
    for name in DECLARED {
        let ranges = preset(name).ranges.len();
        assert!(ranges <= 32, "{name} holds {ranges} ranges");
    }
}
/// The preset that needed V41: one name, two spellings, and only the
/// fidelity in force is granted.
#[test]
fn marks_grants_the_text_spelling_at_text_fidelity() {
    let set = preset("marks");
    // CHECK MARK and BALLOT X, the `text` members.
    assert!(set.contains('\u{2713}'));
    assert!(set.contains('\u{2717}'));
    // WHITE HEAVY CHECK MARK and CROSS MARK, which are `emoji`.
    assert!(!set.contains('\u{2705}'));
    assert!(!set.contains('\u{274C}'));
}

#[test]
fn marks_grants_the_emoji_spelling_at_emoji_fidelity() {
    let set = preset_at("marks", "emoji");
    assert!(set.contains('\u{2705}'));
    assert!(set.contains('\u{274C}'));
    assert!(!set.contains('\u{2713}'));
    assert!(!set.contains('\u{2717}'));
}

/// An unlabelled member belongs to every fidelity: WARNING SIGN is the
/// same code point in both spellings, so labelling it twice would say
/// there were two of it.
#[test]
fn an_unlabelled_member_survives_every_fidelity() {
    assert!(preset("marks").contains('\u{26A0}'));
    assert!(preset_at("marks", "emoji").contains('\u{26A0}'));
    assert!(preset_at("marks", "nerd").contains('\u{26A0}'));
}

/// An UNKNOWN family grants the unlabelled members and nothing else.
/// No walk up the family tree happens here (V41): that tree lives in
/// the map, and a set that followed it would make this node depend on
/// `src/fix`.
#[test]
fn an_unknown_family_grants_only_what_carries_no_label() {
    let set = preset_at("marks", "nerd");
    assert_eq!(set.ranges.len(), 1);
    assert!(!set.contains('\u{2713}'));
    assert!(!set.contains('\u{2705}'));
}

/// The classes the generated hazard file declares, in the order the
/// lint node reads them. Spelled out for the reason [`DECLARED`] is.
const HAZARD_CLASSES: [&str; 5] = [
    "hazard-bidi",
    "hazard-tag",
    "hazard-bom",
    "hazard-control",
    "hazard-invisible",
];

fn hazard_class(name: &str) -> CharSet {
    let resolved = hazard_catalog()
        .ok()
        .and_then(|sets| sets.resolve(name, TEXT).ok());
    assert!(resolved.is_some(), "{name} must resolve on its own");
    resolved.unwrap_or_else(|| CharSet::new(name.to_owned(), Vec::new()))
}

/// How many code points a set holds, counted rather than computed so
/// no arithmetic is needed to say it.
fn size(set: &CharSet) -> usize {
    set.ranges.iter().map(|r| (r.start..=r.end).count()).sum()
}

#[test]
fn the_hazard_file_is_ascii_and_reads_without_the_presets() {
    assert!(HAZARD.is_ascii());
    assert!(SETS.ends_with(HAZARD));
    for name in HAZARD_CLASSES {
        assert!(!hazard_class(name).is_empty(), "{name} is empty");
    }
    let listed = hazard_catalog().map(|c| c.names().count());
    assert_eq!(listed, Ok(6));
}

/// The generated line is checked against the figure upstream prints
/// beneath it ("Total code points: 4174" in DerivedCoreProperties
/// 18.0.0), so a member lost or mistyped in a regeneration shows here.
#[test]
fn the_ignorable_class_holds_exactly_what_unicode_counts() {
    assert_eq!(size(&hazard_class("hazard-invisible")), 4174);
}

/// V34's members, at least one per class it names: ZWSP, ZWJ, soft
/// hyphen, VS16, a Hangul filler, an invisible math operator; RLO and
/// LRI; TAG A; BEL, ESC, DELETE and a C1 control; the BOM.
const V34_MEMBERS: &str = "\u{200B}\u{200D}\u{00AD}\u{FE0F}\u{3164}\
        \u{2061}\u{202E}\u{2066}\u{E0041}\u{0007}\u{001B}\u{007F}\
        \u{009B}\u{FEFF}";

/// ... and the three controls it keeps.
#[test]
fn hazard_holds_every_class_v34_names_and_keeps_the_layout_controls() {
    let set = preset("hazard");
    for point in V34_MEMBERS.chars() {
        assert!(set.contains(point), "U+{:04X}", u32::from(point));
    }
    for point in ['\t', '\n', '\r', ' ', 'A', '\u{00A0}', '\u{1F3FD}'] {
        assert!(!set.contains(point), "U+{:04X}", u32::from(point));
    }
}

/// The catalog's `hazard` and the one the lint node reads are the same
/// bytes, so `ctrm sets` lists what actually fires.
#[test]
fn the_catalog_hazard_is_the_hazard_the_lint_reads() {
    assert_eq!(preset("hazard"), hazard_class("hazard"));
}

/// A bidi override is classed as bidi and not only as invisible: the
/// narrow classes are what name the lint, so each must hold its own.
#[test]
fn each_hazard_class_holds_its_own_members() {
    assert!(hazard_class("hazard-bidi").contains('\u{202E}'));
    assert!(hazard_class("hazard-bidi").contains('\u{200F}'));
    assert!(!hazard_class("hazard-bidi").contains('\u{200B}'));
    assert_eq!(size(&hazard_class("hazard-tag")), 128);
    assert_eq!(size(&hazard_class("hazard-bom")), 1);
    assert_eq!(size(&hazard_class("hazard-control")), 62);
}

/// `SetCatalog::builtin` is infallible only because both halves of it
/// parse: the presets and every locale. Its fallback is never taken.
#[test]
fn the_library_catalog_holds_presets_and_locales() {
    let mut built = catalog().unwrap_or_default();
    assert!(super::super::adopt_all(&mut built).is_ok());
    let library = super::SetCatalog::builtin();
    assert_eq!(library, built);
    assert!(library.resolve("caveman", "text").is_ok());
    assert!(library.resolve("pl", "text").is_ok());
}

/// Every vendored sequence line decodes, by kind: none is skipped by the
/// one `U+` decoder the map and the set grammar share.
#[test]
fn every_vendored_emoji_sequence_decodes() {
    use super::SequenceKind::{Flag, Keycap, Presentation, Tag, Zwj};
    let all: Vec<_> = super::emoji_sequences().collect();
    let count = |kind| all.iter().filter(|s| s.kind == kind).count();
    let kinds = [Zwj, Tag, Keycap, Presentation, Flag].map(count);
    assert_eq!(kinds, [1614, 3, 12, 207, 259]);
    let heart = all.iter().find(|s| s.text == "\u{2764}\u{FE0F}");
    assert_eq!(heart.map(|s| s.kind), Some(Presentation));
}

/// V77, B36: each generator's `fetch` makes its directory before it
/// writes there, and a failed download says why instead of exiting
/// with curl's bare code.
const GENERATORS: [&str; 2] = [
    include_str!("locale/cldr-letters.sh"),
    include_str!("emoji-sequences.sh"),
];

#[test]
fn every_generator_fetch_makes_its_dir_and_reports_failure() {
    for script in GENERATORS {
        let body = script.split_once("\nfetch() {\n").map(|(_, b)| b);
        assert!(body.is_some_and(|b| b.starts_with("  mkdir -p \"$1\"\n")));
        assert!(script.contains("curl -sSfL") && !script.contains("curl -sfL"));
    }
}

/// `src/cli/explain:V127`: every builtin set but the generated locales is
/// curated -- it has a `#:` description -- and every description names a
/// set the build declares, so `ctrm sets` cannot list an unexplained
/// preset or explain one that is gone.
#[test]
fn every_builtin_preset_has_exactly_one_description() {
    let described: Vec<&str> = super::described().map(|(n, _)| n).collect();
    let built = catalog().unwrap_or_default();
    let mut declared: Vec<&str> = built.names().collect();
    let mut listed = described.clone();
    declared.sort_unstable();
    listed.sort_unstable();
    assert_eq!(listed, declared);
    assert!(described.first() == Some(&"ascii"));
    assert!(super::described().all(|(_, said)| !said.trim().is_empty()));
}