characterminator 0.1.2

find and eliminate characters outside an allowed set, per file type and per file, so text costs fewer tokens
Documentation
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
//! The SARIF form: `check`'s findings as a SARIF 2.1.0 log (V50), the
//! format GitHub code scanning reads.
//!
//! Written with the same primitives as the json contract (`value`,
//! `escape`), so it is compact, pure ASCII and needs no dependency. It is
//! DETERMINISTIC: results go out in the one report order (`order`), and
//! the log carries no timestamp, no absolute root and nothing else that
//! differs between two runs over one tree.
//!
//! The `rules` array is the lint REGISTRY (`crate::lint::LINTS`) read as
//! data, each row's group name and default level taken from the lint
//! vocabulary's own accessors. A lint registered there -- a hazard lint,
//! say -- reaches code scanning with no edit here, which a hand-written
//! list could not promise.
//!
//! COLUMNS. SARIF reads `startColumn` as UTF-16 code units unless the run
//! says otherwise, and `src/scan` counts CHARACTERS (Unicode scalar
//! values). The two disagree on every line holding an astral character
//! before the hit, so the run declares `columnKind: unicodeCodePoints`,
//! which is what the scanner actually counts.
//!
//! LINES. The scanner ends a line at LF only. A consumer that also breaks
//! at a lone CR places a hit in such a file on a different line; a CRLF
//! file reads the same either way.

use crate::lint::{LINTS, Level, Lint};
use crate::render::Skipped;
#[cfg(test)]
use crate::render::Violation;
use crate::render::escape::{inner, string};
use crate::render::line::{Line, Spelled};
use crate::render::name::Codepoint;
use crate::render::order;
use crate::render::value::{Fields, array, field, list, number, object};
use crate::scan::{Position, Unreadable};
use std::borrow::Cow;
use std::fmt::Write;

/// The published schema for the version below, so a validator can find
/// it without being told.
const SCHEMA: &str = "https://json.schemastore.org/sarif-2.1.0.json";

/// The one SARIF version this writes.
const VERSION: &str = "2.1.0";

/// The tool as a reader invokes it: the binary, not the crate.
const NAME: &str = "ctrm";

/// `check`: one run, every violation a result, every unread file a
/// notification.
#[cfg(test)]
pub(super) fn check(
    items: &[Violation<'_>],
    skipped: &[Skipped<'_>],
) -> String {
    check_lines(order::lines(items), skipped)
}

/// [`check`] over violations ALREADY in report order.
pub(super) fn check_lines<'a>(
    lines: impl IntoIterator<Item = Line<'a>>,
    skipped: &[Skipped<'_>],
) -> String {
    let mut out = String::new();
    let mut log = Fields::open(&mut out);
    log.text("$schema", SCHEMA);
    log.text("version", VERSION);
    let runs = log.key("runs");
    runs.push('[');
    run(runs, lines, skipped);
    runs.push(']');
    log.close();
    out
}

/// The one run, written into `out`.
fn run<'a>(
    out: &mut String,
    lines: impl IntoIterator<Item = Line<'a>>,
    skipped: &[Skipped<'_>],
) {
    let mut fields = Fields::open(out);
    fields.raw("tool", &object(&[field("driver", &driver())]));
    fields.text("columnKind", "unicodeCodePoints");
    results(fields.key("results"), lines);
    fields.raw("invocations", &array(&[invocation(skipped)]));
    fields.close();
}

/// About what one result object costs, so a log of millions is sized
/// once rather than grown by doubling (R17).
const RESULT_BYTES: usize = 240;

/// Every result, each path made a URI literal once per file.
fn results<'a>(out: &mut String, lines: impl IntoIterator<Item = Line<'a>>) {
    let lines = lines.into_iter();
    out.reserve(lines.size_hint().0.saturating_mul(RESULT_BYTES));
    let mut path = Spelled::new(uri_literal);
    list(out, lines, |out, item| {
        result(out, path.of(item.path), item)
    });
}

/// A path as the json literal of its URI, for [`Spelled`].
fn uri_literal(path: &str) -> Cow<'_, str> {
    Cow::Owned(string(&uri(path)))
}

/// The tool, from Cargo's own metadata rather than restated here, so a
/// version bump cannot leave the log claiming the old one.
fn driver() -> String {
    let mut fields = vec![
        field("name", &string(NAME)),
        field("version", &string(env!("CARGO_PKG_VERSION"))),
    ];
    if let Some(uri) = information_uri(env!("CARGO_PKG_REPOSITORY")) {
        fields.push(field("informationUri", &string(uri)));
    }
    let rules: Vec<String> = LINTS.iter().copied().map(rule).collect();
    fields.push(field("rules", &array(&rules)));
    object(&fields)
}

/// Cargo hands an absent `repository` over as the empty string. That is
/// OMITTED rather than written: an empty URI is not a link, and inventing
/// one would be worse.
fn information_uri(repository: &str) -> Option<&str> {
    (!repository.is_empty()).then_some(repository)
}

/// One registered lint. The group travels as a TAG, which is how code
/// scanning filters, so `hazard` findings can be told apart at a glance.
fn rule(lint: Lint) -> String {
    let level = field("level", &string(level_word(lint.default_level())));
    let tags = field("tags", &array(&[string(lint.group.name())]));
    object(&[
        field("id", &string(lint.name)),
        field("defaultConfiguration", &object(&[level])),
        field("properties", &object(&[tags])),
    ])
}

/// SARIF's level words. `forbid` and `deny` both fail a run, so both are
/// `error`; `allow` is `none`, which only a rule's default can carry,
/// because an allowed finding is never reported.
const fn level_word(level: Level) -> &'static str {
    match level {
        Level::Allow => "none",
        Level::Warn => "warning",
        Level::Deny | Level::Forbid => "error",
    }
}

/// One result. `uri` is the json literal of the path's URI, already
/// escaped; the bytes are those [`location`] writes for a region.
fn result(out: &mut String, uri: &str, item: Line<'_>) {
    let mut fields = Fields::open(out);
    fields.text("ruleId", item.finding.lint.name);
    fields.text("level", level_word(item.finding.level));
    message(fields.key("message"), item);
    locations(fields.key("locations"), uri, item.finding.hit.position);
    fields.close();
}

/// The one-element `locations` array [`location`] writes, with a region.
fn locations(out: &mut String, uri: &str, at: Position) {
    out.push_str(r#"[{"physicalLocation":{"artifactLocation":{"uri":"#);
    out.push_str(uri);
    out.push_str(r#"},"region":"#);
    region(out, at);
    out.push_str("}}]");
}

/// The code point and the set the file was judged against: the two facts
/// the human line carries besides the position. Phrased so it stays true
/// for a lint that fires INSIDE the set, as a hazard does under `any`.
///
/// Written as the `{"text":...}` object directly; the code point needs no
/// escaping, so only the set's name is passed through the escaper.
fn message(out: &mut String, item: Line<'_>) {
    let found = Codepoint(item.finding.hit.character);
    let _infallible = write!(out, "{{\"text\":\"{found} (set in force: ");
    inner(out, item.set);
    out.push_str(")\"}");
}

fn text(words: &str) -> String {
    object(&[field("text", &string(words))])
}

/// One character, exactly. `endColumn` is EXCLUSIVE and defaults to the
/// end of the line when absent, so it is written: a region that silently
/// covered the rest of the line would underline text that did nothing.
fn region(out: &mut String, at: Position) {
    let mut fields = Fields::open(out);
    fields.number("startLine", at.line);
    fields.number("startColumn", at.column);
    fields.number("endColumn", at.column.saturating_add(1));
    fields.close();
}

fn location(path: &str, region: Option<String>) -> String {
    let artifact = object(&[field("uri", &string(&uri(path)))]);
    let mut fields = vec![field("artifactLocation", &artifact)];
    if let Some(at) = region {
        fields.push(field("region", &at));
    }
    object(&[field("physicalLocation", &object(&fields))])
}

/// A path as a URI reference (RFC 3986). Every byte outside the unreserved
/// set and `/` is percent-encoded, so a space, a `%`, a `:` that would read
/// as a scheme, or a non-ASCII name all survive.
///
/// A relative path stays relative, to the directory `ctrm` ran in, which
/// code scanning resolves against the checkout. An absolute one -- a file
/// named outside that directory -- becomes a `file://` URI rather than a
/// relative reference that would resolve somewhere it is not.
fn uri(path: &str) -> String {
    let encoded: String = path.bytes().map(uri_byte).collect();
    if path.starts_with('/') {
        return format!("file://{encoded}");
    }
    encoded
}

fn uri_byte(byte: u8) -> String {
    if byte.is_ascii_alphanumeric() || b"-._~/".contains(&byte) {
        return String::from(char::from(byte));
    }
    format!("%{byte:02X}")
}

/// The run itself. It SUCCEEDED whatever it found: a violation is a
/// result, and a run that could not start emits no log at all.
fn invocation(skipped: &[Skipped<'_>]) -> String {
    let notes: Vec<String> = order::skipped(skipped)
        .into_iter()
        .map(notification)
        .collect();
    object(&[
        field("executionSuccessful", "true"),
        field("toolExecutionNotifications", &array(&notes)),
    ])
}

/// An unread file is NAMED (`src/scan:V8`), never dropped: a log that
/// left it out would read exactly like a clean file. Invalid UTF-8 is a
/// `warning`, because text went unchecked; a binary skip is a `note`,
/// because that is the scanner working as intended.
fn notification(item: &Skipped<'_>) -> String {
    let (level, words, at) = match item.reason {
        Unreadable::NotUtf8 { byte, line, column } => (
            "warning",
            format!("not read: invalid UTF-8 at {line}:{column}"),
            Some(undecoded(line, column, byte)),
        ),
        Unreadable::Binary => ("note", String::from("not read: binary"), None),
    };
    object(&[
        field("level", &string(level)),
        field("message", &text(&words)),
        field("locations", &array(&[location(item.path, at)])),
    ])
}

/// Where decoding failed, as a SARIF region: the line and column a viewer
/// jumps to, and the byte offset beside them.
fn undecoded(line: usize, column: usize, byte: usize) -> String {
    object(&[
        field("startLine", &number(line)),
        field("startColumn", &number(column)),
        field("byteOffset", &number(byte)),
    ])
}

#[cfg(test)]
mod tests {
    use super::{check, information_uri, level_word, uri};
    use crate::lint::{Finding, Group, LINTS, Level, Lint};
    use crate::render::{Skipped, Violation};
    use crate::scan::{Hit, Position, Unreadable};

    /// The registered lint, so the result's `ruleId` names a row of the
    /// `rules` array the way a real run's does.
    const fn outside_set() -> Lint {
        crate::lint::OUTSIDE_SET
    }

    /// U+2014 EM DASH, written as an escape because the source is ASCII.
    fn hit() -> Hit {
        let position = Position {
            line: 2,
            column: 5,
            byte: 7,
        };
        Hit {
            position,
            character: '\u{2014}',
        }
    }

    fn em_dash(path: &str, level: Level) -> Violation<'_> {
        let (hit, lint) = (hit(), outside_set());
        let finding = Finding { hit, lint, level };
        Violation {
            path,
            finding,
            set: "ascii",
        }
    }

    fn skip(path: &str, reason: Unreadable) -> Skipped<'_> {
        Skipped { path, reason }
    }

    /// The fixed head of every log: the schema, the version, and the
    /// driver with one rule per REGISTERED lint, in registry order.
    ///
    /// Built from the registry rather than spelled out, so registering a
    /// lint (a hazard lint, say) does not break a test about the shape.
    /// The level words are this test's own table, keyed on the group, so
    /// the mapping is still checked rather than recomputed.
    fn head() -> String {
        let rules: Vec<String> = LINTS.iter().map(|l| rule_row(*l)).collect();
        format!(
            concat!(
                r#"{{"$schema":"https://json.schemastore.org/"#,
                r#"sarif-2.1.0.json","version":"2.1.0","runs":[{{"tool":"#,
                r#"{{"driver":{{"name":"ctrm","version":"{}","#,
                r#""informationUri":"{}","rules":[{}]}}}},"#,
                r#""columnKind":"unicodeCodePoints","#,
            ),
            env!("CARGO_PKG_VERSION"),
            env!("CARGO_PKG_REPOSITORY"),
            rules.join(",")
        )
    }

    fn rule_row(lint: Lint) -> String {
        let (level, tag) = match lint.group {
            Group::Hazard => ("error", "hazard"),
            Group::Charset => ("error", "charset"),
            Group::Pedantic => ("none", "pedantic"),
        };
        format!(
            concat!(
                r#"{{"id":"{}","defaultConfiguration":{{"level":"{}"}},"#,
                r#""properties":{{"tags":["{}"]}}}}"#,
            ),
            lint.name, level, tag
        )
    }

    /// The one row a reader of this file can check by eye, pinned whole:
    /// the tool's ordinary violation. Pinned by CONTENT, not position: the
    /// registry lists the loudest group first, so the hazard lints precede
    /// it, and a test that assumed it led the list broke when they landed.
    #[test]
    fn the_ordinary_violation_is_a_rule() {
        let row = concat!(
            r#"{"id":"outside-set","#,
            r#""defaultConfiguration":{"level":"error"},"#,
            r#""properties":{"tags":["charset"]}}"#
        );
        assert!(head().contains(row), "{}", head());
        assert!(check(&[], &[]).starts_with(&head()));
    }

    #[test]
    fn the_log_is_the_contract() {
        let expected = concat!(
            r#""results":[{"ruleId":"outside-set","level":"error","#,
            r#""message":{"text":"U+2014 (set in force: ascii)"},"#,
            r#""locations":[{"physicalLocation":{"artifactLocation":{"#,
            r#""uri":"src/a.rs"},"region":{"startLine":2,"startColumn":5,"#,
            r#""endColumn":6}}}]}],"invocations":[{"#,
            r#""executionSuccessful":true,"#,
            r#""toolExecutionNotifications":[]}]}]}"#
        );
        let log = check(&[em_dash("src/a.rs", Level::Deny)], &[]);
        assert_eq!(log, format!("{}{expected}", head()));
    }

    /// Decoding stopped at byte 17, which is line 1, column 18.
    const NOT_UTF8: Unreadable = Unreadable::NotUtf8 {
        byte: 17,
        line: 1,
        column: 18,
    };

    #[test]
    fn unread_files_are_notifications_in_path_order() {
        let items =
            [skip("b.bin", Unreadable::Binary), skip("a.txt", NOT_UTF8)];
        let expected = concat!(
            r#""results":[],"invocations":[{"executionSuccessful":true,"#,
            r#""toolExecutionNotifications":[{"level":"warning","#,
            r#""message":{"text":"not read: invalid UTF-8 at 1:18"},"#,
            r#""locations":[{"physicalLocation":{"artifactLocation":{"#,
            r#""uri":"a.txt"},"region":{"startLine":1,"startColumn":18,"byteOffset":17}}}]},"#,
            r#"{"level":"note","message":{"text":"not read: binary"},"#,
            r#""locations":[{"physicalLocation":{"artifactLocation":{"#,
            r#""uri":"b.bin"}}}]}]}]}]}"#
        );
        assert_eq!(check(&[], &items), format!("{}{expected}", head()));
    }

    #[test]
    fn a_warned_finding_is_a_warning_and_results_keep_report_order() {
        let items =
            [em_dash("b.md", Level::Warn), em_dash("a.md", Level::Forbid)];
        let log = check(&items, &[]);
        let first = log.find(r#""uri":"a.md""#).unwrap_or(usize::MAX);
        let second = log.find(r#""uri":"b.md""#).unwrap_or(0);
        assert!(first < second, "{log}");
        assert!(log.contains(r#""level":"warning","message""#), "{log}");
    }

    #[test]
    fn levels_map_onto_the_sarif_words() {
        assert_eq!(level_word(Level::Forbid), "error");
        assert_eq!(level_word(Level::Deny), "error");
        assert_eq!(level_word(Level::Warn), "warning");
        assert_eq!(level_word(Level::Allow), "none");
    }

    #[test]
    fn a_path_is_percent_encoded_into_a_uri_reference() {
        assert_eq!(uri("src/a_b-c.d~e.rs"), "src/a_b-c.d~e.rs");
        assert_eq!(uri("my notes/100%.md"), "my%20notes/100%25.md");
        // A colon in the first segment would otherwise read as a scheme.
        assert_eq!(uri("c:d.md"), "c%3Ad.md");
        // U+00E9 LATIN SMALL LETTER E WITH ACUTE, as its two UTF-8 bytes.
        assert_eq!(uri("caf\u{e9}.md"), "caf%C3%A9.md");
    }

    #[test]
    fn an_absolute_path_becomes_a_file_uri() {
        assert_eq!(uri("/elsewhere/a.rs"), "file:///elsewhere/a.rs");
    }

    #[test]
    fn an_absent_repository_is_omitted_rather_than_empty() {
        assert_eq!(information_uri(""), None);
        assert_eq!(
            information_uri("https://x.test/r"),
            Some("https://x.test/r")
        );
    }
}