characterminator 0.1.2

find and eliminate characters outside an allowed set, per file type and per file, so text costs fewer tokens
Documentation
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
//! The JSON form: the STABLE CONTRACT (V11).
//!
//! What that means in practice: a key does not change its name, its type or
//! its meaning under a caller, and the tests below assert whole documents
//! EXACTLY rather than probing field by field. A contract that only promises
//! "contains what you need" is not one, because nothing then catches a key
//! quietly renamed.
//!
//! One compact document per run, and pure ASCII: see `value` and `escape`.
//!
//! A document carries what the invariants NAME and no more. A violation
//! carries its position in all three units and its code point
//! (`src/scan:V12`), the set that did not contain it (the root spec's
//! interface section), and the lint name and level (`src/lint:V36`). The
//! lint's GROUP is deliberately absent: it follows from the name, and a key
//! added to a stable contract is a key that can never be taken back.

use crate::charset::{CharRange, CharSet};
use crate::lint::{Level, Lint};
#[cfg(test)]
use crate::render::Violation;
use crate::render::escape::string;
use crate::render::group::Fold;
use crate::render::line::{Line, Spelled};
use crate::render::name::{Codepoint, codepoint, level_name, method_name};
use crate::render::order;
use crate::render::total::{self, Total};
use crate::render::value::{
    Fields, array, field, list, number, object, optional,
};
use crate::render::{Change, Explanation, FileStats, InForce, Skipped};
use crate::rules::{LevelChoice, Origin, Rule, Sourced};
use crate::scan::{Hit, Unreadable};
use crate::tokens::Count;
use std::borrow::Cow;
use std::path::Path;

/// About what one violation object costs, so a report of millions is
/// sized once rather than grown by doubling (R17).
const VIOLATION_BYTES: usize = 200;

/// The contract's version, the first key of every document (V122). It
/// rises only when a key is renamed, retyped or dropped; a key ADDED
/// leaves it alone, so a consumer ignores keys it does not know.
pub(super) const SCHEMA: u32 = 1;

/// A run that never reached a report, as a document (V122): `--format
/// json` promises json on stdout, and a script parsing it should read the
/// failure rather than choke on an empty stream.
pub(super) fn error(verb: &str, message: &str) -> String {
    let mut out = String::new();
    let mut doc = Fields::open(&mut out);
    doc.number("schema", SCHEMA);
    doc.text("verb", verb);
    doc.text("error", message);
    doc.close();
    out
}

/// `check`: every violation, and every file that could not be read.
#[cfg(test)]
pub(super) fn check(
    items: &[Violation<'_>],
    skipped: &[Skipped<'_>],
) -> String {
    check_lines(order::lines(items), skipped)
}

/// [`check`] over violations ALREADY in report order.
pub(super) fn check_lines<'a>(
    lines: impl IntoIterator<Item = Line<'a>>,
    skipped: &[Skipped<'_>],
) -> String {
    let mut out = String::new();
    let mut doc = Fields::open(&mut out);
    doc.number("schema", SCHEMA);
    doc.text("verb", "check");
    violations(doc.key("violations"), lines);
    doc.raw("skipped", &unread(skipped));
    doc.close();
    out
}

/// The array of violation objects, each path escaped once per file.
fn violations<'a>(out: &mut String, lines: impl IntoIterator<Item = Line<'a>>) {
    let lines = lines.into_iter();
    out.reserve(lines.size_hint().0.saturating_mul(VIOLATION_BYTES));
    let mut path = Spelled::new(literal);
    list(out, lines, |out, item| {
        violation(out, path.of(item.path), item)
    });
}

/// A path as a json literal, for [`Spelled`].
fn literal(path: &str) -> Cow<'_, str> {
    Cow::Owned(string(path))
}

/// `path` is the json literal, already escaped.
fn violation(out: &mut String, path: &str, item: Line<'_>) {
    let mut fields = Fields::open(out);
    violation_fields(&mut fields, path, item);
    fields.close();
}

/// Every key of a violation, into an object a caller may add to.
fn violation_fields(fields: &mut Fields<'_>, path: &str, item: Line<'_>) {
    fields.raw("path", path);
    hit_fields(fields, item.finding.hit);
    let (lint, level) = (item.finding.lint, item.finding.level);
    verdict_fields(fields, item.set, lint, level);
    let to = item.remedy.and_then(|remedy| remedy.to.as_deref());
    let fixable = item.remedy.is_some_and(|remedy| remedy.fixable);
    remedy_fields(fields, to, fixable);
}

/// `check --summary` (V124): one object per (file, code point), its first
/// occurrence's violation keys plus `count`.
pub(super) fn summary(groups: &[Fold<'_>], skipped: &[Skipped<'_>]) -> String {
    let mut out = String::new();
    let mut doc = Fields::open(&mut out);
    doc.number("schema", SCHEMA);
    doc.text("verb", "check");
    let mut path = Spelled::new(literal);
    list(doc.key("groups"), groups, |out, group| {
        let mut fields = Fields::open(out);
        violation_fields(&mut fields, path.of(group.first.path), group.first);
        fields.number("count", group.count);
        fields.close();
    });
    doc.raw("skipped", &unread(skipped));
    doc.close();
    out
}

/// The set it was judged against, the lint, and the level: the same three
/// keys on a violation and a rewrite (V95).
fn verdict_fields(
    fields: &mut Fields<'_>,
    set: &str,
    lint: Lint,
    level: Level,
) {
    fields.text("set", set);
    fields.text("lint", lint.name);
    fields.text("level", level_name(level));
}

/// What `fix` would write here, `null` when no rewrite starts here, and
/// whether a `fix` run clears it (V122).
fn remedy_fields(fields: &mut Fields<'_>, to: Option<&str>, fixable: bool) {
    fields.raw("replacement", &optional(to));
    fields.raw("fixable", if fixable { "true" } else { "false" });
}

/// The position in all three units, plus the character itself: a consumer
/// that wants to show it should not have to parse `U+XXXX` back into a code
/// point, and the escaper keeps it ASCII on the way out.
fn hit_fields(fields: &mut Fields<'_>, hit: Hit) {
    fields.number("line", hit.position.line);
    fields.number("column", hit.position.column);
    fields.number("byte", hit.position.byte);
    fields.number(
        "codepoint",
        format_args!("\"{}\"", Codepoint(hit.character)),
    );
    let mut buffer = [0_u8; 4];
    fields.text("character", hit.character.encode_utf8(&mut buffer));
}

fn unread(items: &[Skipped<'_>]) -> String {
    let rendered: Vec<String> =
        order::skipped(items).into_iter().map(unread_item).collect();
    array(&rendered)
}

fn unread_item(item: &Skipped<'_>) -> String {
    let mut fields = vec![field("path", &string(item.path))];
    fields.extend(reason_fields(item.reason));
    object(&fields)
}

fn reason_fields(reason: Unreadable) -> Vec<String> {
    match reason {
        Unreadable::NotUtf8 { byte, line, column } => vec![
            field("reason", &string("not-utf8")),
            field("byte", &number(byte)),
            field("line", &number(line)),
            field("column", &number(column)),
        ],
        Unreadable::Binary => vec![field("reason", &string("binary"))],
    }
}

/// `fix`: every rewrite, every character no map entry covers (`src/fix:V4`)
/// in `check`'s violation shape, and every file that could not be read.
pub(super) fn fix<'a>(
    items: &[Change<'_>],
    unmapped: impl IntoIterator<Item = Line<'a>>,
    skipped: &[Skipped<'_>],
) -> String {
    let mut out = String::new();
    let mut doc = Fields::open(&mut out);
    doc.number("schema", SCHEMA);
    doc.text("verb", "fix");
    let mut path = Spelled::new(literal);
    let rewrites = order::changes(items);
    list(doc.key("rewrites"), rewrites, |out, item| {
        change(out, path.of(item.path), item);
    });
    violations(doc.key("unmapped"), unmapped);
    doc.raw("skipped", &unread(skipped));
    doc.close();
    out
}

fn change(out: &mut String, path: &str, item: &Change<'_>) {
    let mut fields = Fields::open(out);
    fields.raw("path", path);
    hit_fields(&mut fields, item.rewrite.hit);
    fields.text("to", &item.rewrite.to);
    verdict_fields(&mut fields, item.set, item.lint, item.level);
    remedy_fields(&mut fields, Some(&item.rewrite.to), true);
    fields.close();
}

/// `stats`: one row per file, and every file that is not text.
pub(super) fn stats(
    files: &[FileStats<'_>],
    skipped: &[Skipped<'_>],
) -> String {
    let rows: Vec<String> = order::stats(files).into_iter().map(row).collect();
    object(&[
        field("schema", &number(SCHEMA)),
        field("verb", &string("stats")),
        field("files", &array(&rows)),
        field("total", &total::of(files).map_or_else(null, total_row)),
        field("skipped", &unread(skipped)),
    ])
}

/// Every counted file summed (V96), `null` when none was counted.
fn total_row(sum: Total) -> String {
    object(&[
        field("files", &number(sum.files)),
        field("outside", &number(sum.outside)),
        field("bytes", &number(sum.bytes)),
        field("tokens_now", &count(sum.now)),
        field("tokens_after", &count(sum.after)),
    ])
}

fn row(item: &FileStats<'_>) -> String {
    object(&[
        field("path", &string(item.path)),
        field("outside", &number(item.outside)),
        field("bytes", &number(item.bytes)),
        field("tokens_now", &count(item.now)),
        field("tokens_after", &count(item.after)),
    ])
}

/// The method travels WITH the number (`src/tokens:V10`), as an object
/// rather than as a suffix on a string: a consumer should not have to strip
/// a tilde to get at the figure.
fn count(value: Count) -> String {
    object(&[
        field("tokens", &number(value.tokens)),
        field("method", &string(method_name(value.method))),
    ])
}

/// `explain`: the effective set, and the rule that won.
pub(super) fn explain(item: &Explanation<'_>) -> String {
    object(&[
        field("schema", &number(SCHEMA)),
        field("verb", &string("explain")),
        field("path", &optional(item.path)),
        field("set", &char_set(item.set)),
        field("rule", &item.rule.map_or_else(null, rule)),
        field("effective", &in_force(&item.in_force)),
    ])
}

/// What is in force, each setting with its line (`src/rules:V20`, B28).
/// ADDED after `rule`, so every key a consumer already read keeps its
/// place and meaning (V11). A null origin is the default family.
fn in_force(item: &InForce<'_>) -> String {
    let family = object(&[
        field("name", &string(item.family)),
        field("origin", &item.family_origin.map_or_else(null, origin)),
    ]);
    let levels: Vec<String> = item.levels.iter().map(sourced_level).collect();
    object(&[
        field("family", &family),
        field(
            "default_level",
            &item.level.as_ref().map_or_else(null, bare),
        ),
        field("levels", &array(&levels)),
    ])
}

fn bare(item: &Sourced<'_, Level>) -> String {
    object(&[
        field("level", &string(level_name(item.value))),
        field("origin", &origin(item.origin)),
    ])
}

fn sourced_level(item: &Sourced<'_, LevelChoice>) -> String {
    object(&[
        field("target", &string(&item.value.target)),
        field("level", &string(level_name(item.value.level))),
        field("origin", &origin(item.origin)),
    ])
}

/// `sets`: the builtin sets and what they hold.
pub(super) fn sets(items: &[CharSet]) -> String {
    let rendered: Vec<String> = items.iter().map(char_set).collect();
    object(&[
        field("schema", &number(SCHEMA)),
        field("verb", &string("sets")),
        field("sets", &array(&rendered)),
    ])
}

fn char_set(item: &CharSet) -> String {
    let ranges: Vec<String> = item.ranges.iter().map(range).collect();
    object(&[
        field("name", &string(&item.name)),
        field("ranges", &array(&ranges)),
    ])
}

/// Endpoints as `U+XXXX` rather than as bare numbers: the same spelling the
/// human form uses, and one a person can read in a raw document.
fn range(item: &CharRange) -> String {
    object(&[
        field("start", &string(&codepoint(item.start))),
        field("end", &string(&codepoint(item.end))),
    ])
}

/// The absent rule: nothing matched, so `src/rules:V1` decided.
fn null() -> String {
    String::from("null")
}

/// `default_level` is the bare `!<level>` a rule line wrote, or `null` when
/// it wrote none (T43). It sits beside `levels` rather than inside it as a
/// choice with a magic target, for the reason the `Rule` field does: the
/// bare form names no target. `null` rather than the group default a run
/// falls back to, because the document reports what the RULE says, and a
/// synthesised level would put words in a line that never wrote them.
fn rule(item: &Rule) -> String {
    let sets: Vec<String> = item.sets.iter().map(|set| string(set)).collect();
    let levels: Vec<String> = item.levels.iter().map(level_choice).collect();
    let bare = item.default_level;
    object(&[
        field("pattern", &string(&item.pattern)),
        field("sets", &array(&sets)),
        field("family", &optional(item.family.as_deref())),
        field("levels", &array(&levels)),
        field("default_level", &optional(bare.map(level_name))),
        field("origin", &origin(&item.origin)),
    ])
}

fn level_choice(item: &LevelChoice) -> String {
    object(&[
        field("target", &string(&item.target)),
        field("level", &string(level_name(item.level))),
    ])
}

/// A tagged union: `kind` says which case, and the remaining keys follow
/// from it.
fn origin(item: &Origin) -> String {
    match item {
        Origin::File { path, line } => file_origin(path, *line),
        Origin::Argument { index } => object(&[
            field("kind", &string("argument")),
            field("index", &number(*index)),
        ]),
        Origin::Builtin { line } => object(&[
            field("kind", &string("builtin")),
            field("line", &number(*line)),
        ]),
    }
}

/// The path is rendered LOSSILY. What a non-UTF-8 path should print is a
/// rendering question and this is the honest answer: the replacement
/// character, which the escaper then writes as `\ufffd` -- visibly wrong in
/// the output rather than silently dropped from it.
fn file_origin(path: &Path, line: usize) -> String {
    object(&[
        field("kind", &string("file")),
        field("path", &string(&path.to_string_lossy())),
        field("line", &number(line)),
    ])
}

#[cfg(test)]
#[path = "json_test.rs"]
mod tests;