speccoder_runtime/
secrets.rs

1//! Values copied into a worktree from the person's secret files (a `.env`
2//! that `[worktree] include` copies): masked in everything speccoder shows
3//! or hands on — command and check output, tool results, messages — as
4//! `[redacted:NAME]`, each in the forms it may be printed in (as is,
5//! base64, hex, URL-encoded).
6//!
7//! The values are held for the process, never written anywhere: masking
8//! more than one worktree's values everywhere is the safe direction.
9
10use std::sync::RwLock;
11
12use base64::Engine as _;
13
14/// The shortest value masked: shorter ones are flags and ports.
15pub const MIN_LEN: usize = 8;
16
17/// One value and the forms it is masked in.
18#[derive(Debug, Clone, PartialEq, Eq)]
19struct Masked {
20    /// The variable it is the value of.
21    name: String,
22    /// The value itself.
23    value: String,
24    /// The value and its encodings, longest first.
25    forms: Vec<String>,
26}
27
28static MASKED: RwLock<Vec<Masked>> = RwLock::new(Vec::new());
29
30/// Whether `value` is worth masking: at least [`MIN_LEN`] long and not a
31/// plain number.
32#[must_use]
33pub fn worth_masking(value: &str) -> bool {
34    value.len() >= MIN_LEN && !value.chars().all(|c| c.is_ascii_digit() || c == '.')
35}
36
37/// The forms `value` may be printed in.
38fn forms(value: &str) -> Vec<String> {
39    let b = value.as_bytes();
40    let mut out = vec![value.to_owned()];
41    let engines = [
42        base64::engine::general_purpose::STANDARD,
43        base64::engine::general_purpose::STANDARD_NO_PAD,
44        base64::engine::general_purpose::URL_SAFE,
45        base64::engine::general_purpose::URL_SAFE_NO_PAD,
46    ];
47    out.extend(engines.iter().map(|e| e.encode(b)));
48    let hex: String = b.iter().map(|x| format!("{x:02x}")).collect();
49    out.push(hex.to_uppercase());
50    out.push(hex);
51    out.push(url_encoded(value));
52    out.sort_by_key(|f| std::cmp::Reverse(f.len()));
53    out.dedup();
54    out
55}
56
57/// `value` percent-encoded (everything but RFC 3986's unreserved characters).
58fn url_encoded(value: &str) -> String {
59    value
60        .bytes()
61        .map(|c| {
62            if c.is_ascii_alphanumeric() || b"-._~".contains(&c) {
63                (c as char).to_string()
64            } else {
65                format!("%{c:02X}")
66            }
67        })
68        .collect()
69}
70
71/// Masks `value` (the variable `name`'s) from now on, in this process;
72/// once per name and value.
73pub fn register(name: &str, value: &str) {
74    if !worth_masking(value) {
75        return;
76    }
77    let Ok(mut m) = MASKED.write() else { return };
78    if m.iter().any(|x| x.name == name && x.value == value) {
79        return;
80    }
81    m.push(Masked {
82        name: name.to_owned(),
83        value: value.to_owned(),
84        forms: forms(value),
85    });
86    // Longer values first: one that contains another is masked whole.
87    m.sort_by_key(|x| std::cmp::Reverse(x.value.len()));
88}
89
90/// The longest form registered, in bytes: how far a value can straddle a
91/// cut.
92#[must_use]
93pub fn longest() -> usize {
94    MASKED
95        .read()
96        .ok()
97        .and_then(|m| m.iter().flat_map(|x| x.forms.iter().map(String::len)).max())
98        .unwrap_or(0)
99}
100
101/// Whether a variable's name says its value is secret: what the commands'
102/// environment drops ([`crate::verify::process::is_secret`]), and names
103/// with `KEY`, `PASS`, `PWD`, `CREDENTIAL`, `AUTH`, `PRIVATE`, `TOKEN` or
104/// `DSN`.
105#[must_use]
106pub fn secret_name(name: &str) -> bool {
107    let n = name.to_ascii_uppercase();
108    crate::verify::process::is_secret(&n)
109        || [
110            "KEY",
111            "PASS",
112            "PWD",
113            "CREDENTIAL",
114            "AUTH",
115            "PRIVATE",
116            "TOKEN",
117            "DSN",
118        ]
119        .iter()
120        .any(|w| n.contains(w))
121}
122
123/// The password of a URL with credentials (`scheme://user:pass@host`).
124fn url_password(value: &str) -> Option<&str> {
125    let (_, rest) = value.split_once("://")?;
126    let (userinfo, _) = rest.split_once('@')?;
127    let (_, pass) = userinfo.split_once(':')?;
128    (!pass.is_empty() && !pass.contains('/')).then_some(pass)
129}
130
131/// What a copied secret file's text gives to mask, as `(NAME, value)`. A
132/// dotenv file (named `.env*`): the variables whose name says secret
133/// ([`secret_name`]) — each line of a multi-line value too — and any URL
134/// with credentials, whole and its password. Any other file (a key, a
135/// `.netrc`): each line, without PEM's `-----BEGIN …-----` markers, under
136/// the file's name, and each line's last word.
137#[must_use]
138pub fn to_mask(file_name: &str, text: &str) -> Vec<(String, String)> {
139    let mut out = Vec::new();
140    let lines = |name: &str, text: &str, out: &mut Vec<(String, String)>| {
141        for l in text.lines().map(str::trim) {
142            if l.starts_with("-----") && l.ends_with("-----") {
143                continue;
144            }
145            out.push((name.to_owned(), l.to_owned()));
146            if let Some(last) = l.rsplit([' ', '=', '\t']).next() {
147                out.push((name.to_owned(), last.to_owned()));
148            }
149        }
150    };
151    if file_name.starts_with(".env") {
152        for (k, v) in dotenv(text) {
153            if let Some(pass) = url_password(&v) {
154                out.push((k.clone(), pass.to_owned()));
155                out.push((k.clone(), v.clone()));
156            }
157            if secret_name(&k) {
158                if v.contains('\n') {
159                    lines(&k, &v, &mut out);
160                }
161                out.push((k, v));
162            }
163        }
164    } else {
165        lines(file_name, text, &mut out);
166    }
167    out.retain(|(_, v)| worth_masking(v));
168    out.dedup();
169    out
170}
171/// `text` with every registered value, in each of its forms, replaced by
172/// `[redacted:NAME]`.
173#[must_use]
174pub fn mask(text: &str) -> String {
175    let Ok(m) = MASKED.read() else {
176        return text.to_owned();
177    };
178    let mut out = text.to_owned();
179    for x in m.iter() {
180        for f in &x.forms {
181            if out.contains(f.as_str()) {
182                out = out.replace(f.as_str(), &format!("[redacted:{}]", x.name));
183            }
184        }
185    }
186    out
187}
188
189/// [`mask`] over bytes that need not be UTF-8: each form replaced where it
190/// occurs, every other byte kept.
191#[must_use]
192pub fn mask_bytes(bytes: &[u8]) -> Vec<u8> {
193    let Ok(m) = MASKED.read() else {
194        return bytes.to_vec();
195    };
196    let mut out = bytes.to_vec();
197    for x in m.iter() {
198        let with = format!("[redacted:{}]", x.name);
199        for f in &x.forms {
200            if out.windows(f.len()).any(|w| w == f.as_bytes()) {
201                out = replace_bytes(&out, f.as_bytes(), with.as_bytes());
202            }
203        }
204    }
205    out
206}
207
208fn replace_bytes(hay: &[u8], needle: &[u8], with: &[u8]) -> Vec<u8> {
209    if needle.is_empty() || !hay.windows(needle.len()).any(|w| w == needle) {
210        return hay.to_vec();
211    }
212    let mut out = Vec::with_capacity(hay.len());
213    let mut i = 0;
214    while i < hay.len() {
215        if hay[i..].starts_with(needle) {
216            out.extend_from_slice(with);
217            i += needle.len();
218        } else {
219            out.push(hay[i]);
220            i += 1;
221        }
222    }
223    out
224}
225
226/// [`mask`] over every string in a JSON value.
227pub fn mask_json(v: &mut serde_json::Value) {
228    match v {
229        serde_json::Value::String(s) => {
230            let masked = mask(s);
231            if masked != *s {
232                *s = masked;
233            }
234        }
235        serde_json::Value::Array(a) => a.iter_mut().for_each(mask_json),
236        serde_json::Value::Object(o) => o.values_mut().for_each(mask_json),
237        _ => {}
238    }
239}
240
241/// The variable whose value (as is) `text` holds, if any.
242#[must_use]
243pub fn found_in(text: &str) -> Option<String> {
244    let m = MASKED.read().ok()?;
245    m.iter()
246        .find(|x| x.forms.iter().any(|f| text.contains(f.as_str())))
247        .map(|x| x.name.clone())
248}
249
250/// The `NAME=value` pairs of a dotenv file: `export` and quotes dropped
251/// (a quoted value may span lines), comments and blank lines skipped.
252#[must_use]
253pub fn dotenv(text: &str) -> Vec<(String, String)> {
254    let mut out = Vec::new();
255    let mut lines = text.lines();
256    while let Some(l) = lines.next() {
257        let l = l.trim();
258        if l.is_empty() || l.starts_with('#') {
259            continue;
260        }
261        let l = l.strip_prefix("export ").unwrap_or(l);
262        let Some((k, v)) = l.split_once('=') else {
263            continue;
264        };
265        let v = v.trim();
266        let v = match v.chars().next() {
267            Some(q @ ('"' | '\'')) => {
268                let mut value = v[1..].to_owned();
269                // Unclosed on its line: the value goes on to the closing quote.
270                while !value.contains(q) {
271                    let Some(next) = lines.next() else { break };
272                    value.push('\n');
273                    value.push_str(next);
274                }
275                value.split(q).next().unwrap_or_default().to_owned()
276            }
277            // An unquoted value ends at a comment.
278            _ => v.split(" #").next().unwrap_or_default().trim().to_owned(),
279        };
280        let k = k.trim();
281        if !k.is_empty() && !v.is_empty() {
282            out.push((k.to_owned(), v));
283        }
284    }
285    out
286}
287
288#[cfg(test)]
289mod tests {
290    use super::*;
291
292    #[test]
293    fn a_dotenv_file_gives_its_pairs() {
294        let pairs = dotenv(
295            "# db\nexport DB_PASSWORD=\"s3cr3t pass\"\nTOKEN='abc' # x\nPORT=5432 # local\n\nEMPTY=\nbad line\n",
296        );
297        assert_eq!(
298            pairs,
299            [
300                ("DB_PASSWORD".to_owned(), "s3cr3t pass".to_owned()),
301                ("TOKEN".to_owned(), "abc".to_owned()),
302                ("PORT".to_owned(), "5432".to_owned()),
303            ]
304        );
305        assert!(worth_masking("s3cr3t pass"));
306        assert!(!worth_masking("abc"));
307        assert!(!worth_masking("12345678"));
308    }
309
310    #[test]
311    fn only_secret_named_variables_and_credentials_in_urls_are_masked() {
312        let env = "NODE_ENV=development\nDB_HOST=localhost\nDB_USER=postgres\nDB_PASSWORD=hunter2hunter2\nSTRIPE_SECRET_KEY=sk_test_abcdef\nDATABASE_URL=postgres://u:pAss-w0rd-x@h/db\n";
313        let got: Vec<(String, String)> = to_mask(".env", env);
314        let values: Vec<&str> = got.iter().map(|(_, v)| v.as_str()).collect();
315        for kept in ["development", "localhost", "postgres"] {
316            assert!(!values.contains(&kept), "{kept} masked: {got:?}");
317        }
318        assert_eq!(
319            values,
320            [
321                "hunter2hunter2",
322                "sk_test_abcdef",
323                "pAss-w0rd-x",
324                "postgres://u:pAss-w0rd-x@h/db"
325            ]
326        );
327    }
328
329    #[test]
330    fn a_key_file_and_a_multi_line_value_are_masked_line_by_line() {
331        let pem = "-----BEGIN OPENSSH PRIVATE KEY-----\nb3BlbnNzaC1rZXktdjEAAAAA\nQyNTUxOQAAACBlYmxvY2s=\n-----END OPENSSH PRIVATE KEY-----\n";
332        let got = to_mask("id_ed25519", pem);
333        let values: Vec<&str> = got.iter().map(|(_, v)| v.as_str()).collect();
334        assert!(values.contains(&"b3BlbnNzaC1rZXktdjEAAAAA"), "{got:?}");
335        assert!(values.contains(&"QyNTUxOQAAACBlYmxvY2s="), "{got:?}");
336        assert!(!values.iter().any(|v| v.contains("BEGIN")), "{got:?}");
337        assert_eq!(got[0].0, "id_ed25519");
338        let env = format!("PRIVATE_KEY=\"{}\"\nOTHER=1\n", pem.trim_end());
339        let got = to_mask(".env", &env);
340        let values: Vec<&str> = got.iter().map(|(_, v)| v.as_str()).collect();
341        assert!(values.contains(&"b3BlbnNzaC1rZXktdjEAAAAA"), "{got:?}");
342        assert_eq!(dotenv(&env)[1], ("OTHER".to_owned(), "1".to_owned()));
343        let netrc = to_mask(".netrc", "machine h login me password n3trc-secret\n");
344        assert!(netrc.iter().any(|(_, v)| v == "n3trc-secret"), "{netrc:?}");
345    }
346
347    #[test]
348    fn a_value_registered_twice_is_kept_once() {
349        let count = || {
350            MASKED
351                .read()
352                .unwrap()
353                .iter()
354                .filter(|x| x.value == "twice-registered-v4l")
355                .count()
356        };
357        register("TWICE", "twice-registered-v4l");
358        register("TWICE", "twice-registered-v4l");
359        assert_eq!(count(), 1);
360    }
361
362    #[test]
363    fn a_value_is_masked_in_each_form_it_is_printed_in() {
364        let value = "masking-test-Zq9 value/+";
365        register("MASK_TEST", value);
366        let b64 = base64::engine::general_purpose::STANDARD.encode(value);
367        let hex: String = value.bytes().map(|x| format!("{x:02x}")).collect();
368        let text = format!("a {value} b {b64} c {hex} d {}", url_encoded(value));
369        assert_eq!(
370            mask(&text),
371            "a [redacted:MASK_TEST] b [redacted:MASK_TEST] c [redacted:MASK_TEST] d [redacted:MASK_TEST]"
372        );
373        assert_eq!(
374            mask_bytes(format!("\u{0}{value}\u{ff}").as_bytes()),
375            b"\0[redacted:MASK_TEST]\xc3\xbf".to_vec()
376        );
377        let mut v = serde_json::json!({"out": [format!("x{value}")]});
378        mask_json(&mut v);
379        assert_eq!(v["out"][0], "x[redacted:MASK_TEST]");
380        assert_eq!(found_in(&format!("+ {b64}")).as_deref(), Some("MASK_TEST"));
381        assert_eq!(found_in("nothing here"), None);
382    }
383}
speccoder_runtime/secrets.rs

speccoder_runtime/
secrets.rs
