velociredactor 0.1.1

Redact secrets and PII from text and structured files, with stable numbered redactions
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143

//! Scanning comments, which are skipped unless asked for.

mod common;

use common::HIGH_ENTROPY_SECRET as S;
use velociredactor::{Allow, FormatHint, Redactor};

/// Redact `input` as `format`, with comment scanning on or off.
fn redact(format: &str, input: &str, comments: bool) -> String {
    let redactor = Redactor::builder().comments(comments).build();
    let redaction = redactor
        .redact(input.as_bytes(), FormatHint::Name(format))
        .unwrap_or_else(|e| panic!("redact as {format}: {e}"));
    String::from_utf8(redaction.render(&Allow::none()).unwrap()).unwrap()
}

/// Assert that the secret in a comment survives by default and is redacted
/// with comments on, and that redacting again changes nothing.
#[track_caller]
fn assert_comment_scanned(format: &str, input: &str, want: &str) {
    assert_eq!(redact(format, input, false), input, "{format}: default");
    let got = redact(format, input, true);
    assert_eq!(got, want, "{format}: with comments");
    assert_eq!(redact(format, &got, true), got, "{format}: not idempotent");
}

#[test]
fn json_line_and_block_comments() {
    assert_comment_scanned(
        "json",
        &format!("{{\n  // token {S}\n  \"a\": \"ok\" /* and {S} */\n}}"),
        "{\n  // token REDACTION-1\n  \"a\": \"ok\" /* and REDACTION-1 */\n}",
    );
}

#[test]
fn json_strings_that_look_like_comments_are_left_alone() {
    let input = format!(r#"{{"note": "// not a comment {S}"}}"#);
    // The string is a value, so it is redacted either way, and only once.
    let want = r#"{"note": "// not a comment REDACTION-1"}"#;
    assert_eq!(redact("json", &input, false), want);
    assert_eq!(redact("json", &input, true), want);
}

#[test]
fn yaml_comments() {
    assert_comment_scanned(
        "yaml",
        &format!("# lead {S}\na: ok # trail {S}\n"),
        "# lead REDACTION-1\na: ok # trail REDACTION-1\n",
    );
}

#[test]
fn yaml_hashes_inside_scalars_are_not_comments() {
    // The block scalar's body is a value: redacted once, and not spliced
    // twice by also being read as a comment.
    let input = format!("note: |\n  # inside {S}\nurl: \"http://x/#frag\"\n");
    let want = "note: |\n  # inside REDACTION-1\nurl: \"http://x/#frag\"\n";
    assert_eq!(redact("yaml", &input, true), want);
    assert_eq!(redact("yaml", &input, false), want);
}

#[test]
fn toml_comments() {
    assert_comment_scanned(
        "toml",
        &format!("# lead {S}\n[s]\na = \"ok\" # trail {S}\n"),
        "# lead REDACTION-1\n[s]\na = \"ok\" # trail REDACTION-1\n",
    );
}

#[test]
fn toml_hashes_inside_strings_are_not_comments() {
    let input = "a = \"x # y\"\nb = '''\n# z\n'''\n";
    assert_eq!(redact("toml", input, true), input);
}

#[test]
fn hcl_comments() {
    assert_comment_scanned(
        "hcl",
        &format!("# lead {S}\nvariable \"v\" {{\n  default = \"ok\" // trail {S}\n}}\n"),
        "# lead REDACTION-1\nvariable \"v\" {\n  default = \"ok\" // trail REDACTION-1\n}\n",
    );
}

#[test]
fn hcl_heredoc_bodies_are_values_not_comments() {
    let input = format!("a = <<EOT\n# inside {S}\nEOT\n");
    let want = "a = <<EOT\n# inside REDACTION-1\nEOT\n";
    assert_eq!(redact("hcl", &input, true), want);
    assert_eq!(redact("hcl", &input, false), want);
}

#[test]
fn ini_and_dotenv_comments() {
    assert_comment_scanned(
        "ini",
        &format!("; lead {S}\n[s]\nkey = ok\n# other {S}\n"),
        "; lead REDACTION-1\n[s]\nkey = ok\n# other REDACTION-1\n",
    );
    assert_comment_scanned(
        "dotenv",
        &format!("# lead {S}\nKEY=ok\n"),
        "# lead REDACTION-1\nKEY=ok\n",
    );
}

#[test]
fn xml_comments() {
    assert_comment_scanned(
        "xml",
        &format!("<r><!-- token {S} --><a>ok</a></r>"),
        "<r><!-- token REDACTION-1 --><a>ok</a></r>",
    );
}

#[test]
fn properties_comments() {
    assert_comment_scanned(
        "properties",
        &format!("# lead {S}\nkey=ok\n"),
        "# lead REDACTION-1\nkey=ok\n",
    );
}

#[test]
fn a_secret_in_a_comment_shares_the_token_with_the_same_value_elsewhere() {
    let input = format!("# see {S}\na = \"{S}\"\n");
    assert_eq!(
        redact("toml", &input, true),
        "# see REDACTION-1\na = \"REDACTION-1\"\n"
    );
}

#[test]
fn formats_without_comments_are_unaffected() {
    let input = format!("name,key\nx,{S}\n");
    let with = redact("csv", &input, true);
    assert_eq!(with, redact("csv", &input, false));
    assert_eq!(with, "name,key\nx,REDACTION-1\n");
}