dre_core/
secrets.rs

1//! Secret masking. The value of every environment variable named `DRE_SECRET_*`, and of every
2//! environment variable `profiles.yml` reads through `env_var()` into a secret field (one the
3//! plugin declares secret, or named like `password`, `token`, `api_key`), is replaced by
4//! `*****` in everything DRE writes for people to read: the console, `logs/dre.log`, JSON events,
5//! `run_results.json` and the compiled SQL in `target/compiled/`. The SQL actually sent to the
6//! database is unchanged. `mask_secrets: false` in `dre_project.yml` turns masking off.
7//!
8//! JSON is redacted structurally, before it is serialized: serializing escapes `"`, `\` and
9//! control characters, so masking the finished text would miss a secret that contains them. Use
10//! [`to_json_pretty`] / [`to_json_line`] for every JSON surface, never `mask` on JSON text.
11//!
12//! Every value is masked, however short. One shorter than 4 characters is masked where it stands
13//! alone (not inside a longer word or number), so a PIN of `12` masks `PIN 12` but leaves `2012`
14//! readable.
15
16use std::borrow::Cow;
17use std::sync::atomic::{AtomicBool, Ordering};
18use std::sync::{Arc, LazyLock, RwLock};
19
20pub const PREFIX: &str = "DRE_SECRET_";
21pub const MASK: &str = "*****";
22pub const MASKED_SOURCE_ERROR: &str =
23    "source error details omitted because the request contains a masked secret";
24const MIN_LEN: usize = 4;
25
26static ENABLED: AtomicBool = AtomicBool::new(true);
27
28fn eligible(value: &str) -> bool {
29    !value.is_empty()
30}
31
32/// Short values mask only where they stand alone.
33fn short(value: &str) -> bool {
34    value.chars().count() < MIN_LEN
35}
36
37/// `s` with each standalone `x` (not touching a letter or digit) replaced by the mask.
38fn replace_standalone(s: &str, x: &str) -> String {
39    let word = |c: Option<char>| c.is_some_and(char::is_alphanumeric);
40    let mut out = String::with_capacity(s.len());
41    let mut rest = s;
42    let mut before: Option<char> = None;
43    while let Some(i) = rest.find(x) {
44        let (head, tail) = rest.split_at(i);
45        out.push_str(head);
46        let prev = head.chars().next_back().or(before);
47        let next = tail[x.len()..].chars().next();
48        if word(prev) || word(next) {
49            out.push_str(x);
50        } else {
51            out.push_str(MASK);
52        }
53        before = x.chars().next_back();
54        rest = &tail[x.len()..];
55    }
56    out.push_str(rest);
57    out
58}
59
60/// Secret values, longest first so a secret containing another is masked whole. Starts with
61/// `DRE_SECRET_*`; [`register_secret_field`] adds the variables secret profile fields read.
62static SECRETS: LazyLock<RwLock<Arc<Secrets>>> = LazyLock::new(|| {
63    RwLock::new(Arc::new(Secrets::new(sorted(
64        std::env::vars()
65            .filter(|(k, _)| k.starts_with(PREFIX))
66            .map(|(_, v)| v),
67    ))))
68});
69
70/// The secret values, and the forms they take in Markdown and in collapsed titles.
71struct Secrets {
72    plain: Vec<String>,
73    markdown: Vec<String>,
74    title: Vec<String>,
75}
76
77impl Secrets {
78    fn new(plain: Vec<String>) -> Secrets {
79        let markdown = sorted(
80            plain
81                .iter()
82                .flat_map(|s| [s.clone(), dre_protocol::markdown::escape(s)]),
83        );
84        let title = sorted(
85            plain
86                .iter()
87                .flat_map(|s| [s.clone(), s.split_whitespace().collect::<Vec<_>>().join(" ")]),
88        );
89        Secrets {
90            plain,
91            markdown,
92            title,
93        }
94    }
95}
96
97fn sorted(values: impl IntoIterator<Item = String>) -> Vec<String> {
98    let mut v: Vec<String> = values.into_iter().filter(|v| eligible(v)).collect();
99    v.sort_by(|a, b| b.len().cmp(&a.len()).then_with(|| a.cmp(b)));
100    v.dedup();
101    v
102}
103
104fn secrets() -> Arc<Secrets> {
105    SECRETS.read().unwrap_or_else(|e| e.into_inner()).clone()
106}
107
108/// The environment variables a template reads through `env_var('NAME')` (either quote).
109pub fn env_references(text: &str) -> Vec<String> {
110    static CALL: LazyLock<regex::Regex> =
111        LazyLock::new(|| regex::Regex::new(r#"env_var\(\s*(?:'([^']+)'|"([^"]+)")"#).unwrap());
112    CALL.captures_iter(text)
113        .filter_map(|c| c.get(1).or_else(|| c.get(2)))
114        .map(|m| m.as_str().to_owned())
115        .collect()
116}
117
118/// Whether a profile field's name says it holds a secret. Paths, hosts and fingerprints don't
119/// count (`private_key_path`, `host_key_fingerprint`): masking them would hide what a message is
120/// about, and they aren't secrets.
121pub fn secret_field_name(name: &str) -> bool {
122    let name = name.to_ascii_lowercase();
123    ["password", "passphrase", "secret", "token", "credential"]
124        .iter()
125        .any(|w| name.contains(w))
126        || name == "key"
127        || name.ends_with("_key")
128        || name == "apikey"
129}
130
131/// Mask the values of the environment variables a secret profile field reads through
132/// `env_var()`, at any depth of `value`.
133pub fn register_secret_field(value: &serde_json::Value) {
134    let mut text = Vec::new();
135    collect_strings(value, &mut text);
136    register(text.iter().flat_map(|s| env_references(s)))
137}
138
139/// Mask the `env_var()` values that the secret fields of `fields` (a profile target) read,
140/// nested blocks (`ssh:`) included. `secret` adds names a plugin declares secret.
141pub fn register_secret_fields(fields: &serde_json::Map<String, serde_json::Value>, secret: &[String]) {
142    for (name, value) in fields {
143        if secret_field_name(name) || secret.contains(name) {
144            register_secret_field(value);
145        } else if let serde_json::Value::Object(nested) = value {
146            register_secret_fields(nested, &[]);
147        }
148    }
149}
150
151fn collect_strings<'a>(value: &'a serde_json::Value, out: &mut Vec<&'a str>) {
152    match value {
153        serde_json::Value::String(s) => out.push(s),
154        serde_json::Value::Array(a) => a.iter().for_each(|v| collect_strings(v, out)),
155        serde_json::Value::Object(o) => o.values().for_each(|v| collect_strings(v, out)),
156        _ => {}
157    }
158}
159
160fn register(names: impl Iterator<Item = String>) {
161    register_values(names.into_iter().filter_map(|name| std::env::var(name).ok()));
162}
163
164/// Mask these values from now on: a query's `secret_columns`.
165pub fn register_values(values: impl IntoIterator<Item = String>) {
166    let values: Vec<String> = values.into_iter().filter(|v| eligible(v)).collect();
167    if values.is_empty() {
168        return;
169    }
170    let mut guard = SECRETS.write().unwrap_or_else(|e| e.into_inner());
171    let merged = sorted(guard.plain.iter().cloned().chain(values));
172    *guard = Arc::new(Secrets::new(merged));
173}
174
175pub fn set_enabled(on: bool) {
176    ENABLED.store(on, Ordering::Relaxed);
177}
178
179pub(crate) fn enabled() -> bool {
180    ENABLED.load(Ordering::Relaxed)
181}
182
183fn mask_with<'a>(s: &'a str, secrets: &[String]) -> Cow<'a, str> {
184    if !secrets.iter().any(|x| s.contains(x.as_str())) {
185        return Cow::Borrowed(s);
186    }
187    let mut out = s.to_string();
188    for x in secrets {
189        out = if short(x) {
190            replace_standalone(&out, x)
191        } else {
192            out.replace(x.as_str(), MASK)
193        };
194    }
195    if out == s {
196        return Cow::Borrowed(s);
197    }
198    Cow::Owned(out)
199}
200
201/// `s` with every secret value replaced by `*****`.
202pub fn mask(s: &str) -> Cow<'_, str> {
203    if !ENABLED.load(Ordering::Relaxed) {
204        return Cow::Borrowed(s);
205    }
206    mask_with(s, &secrets().plain)
207}
208
209/// Mask a rendered message without changing the report payload. Markdown interpolation has
210/// already escaped punctuation, so both literal and escaped secret values must be matched.
211pub(crate) fn mask_markdown(s: &str) -> Cow<'_, str> {
212    if !ENABLED.load(Ordering::Relaxed) {
213        return Cow::Borrowed(s);
214    }
215    mask_with(s, &secrets().markdown)
216}
217
218/// Message titles collapse whitespace before delivery. Match that representation as well as
219/// the literal secret when protecting a title in logs and run metadata.
220pub(crate) fn mask_message_title(s: &str) -> Cow<'_, str> {
221    if !ENABLED.load(Ordering::Relaxed) {
222        return Cow::Borrowed(s);
223    }
224    mask_with(s, &secrets().title)
225}
226
227/// Whether masking is enabled and `s` contains a complete registered secret.
228pub fn contains_secret(s: &str) -> bool {
229    ENABLED.load(Ordering::Relaxed) && matches!(mask_with(s, &secrets().plain), Cow::Owned(_))
230}
231
232fn mask_source_error_with<'a>(error: &'a str, sensitive: bool, secrets: &[String]) -> Cow<'a, str> {
233    if sensitive {
234        Cow::Borrowed(MASKED_SOURCE_ERROR)
235    } else {
236        mask_with(error, secrets)
237    }
238}
239
240/// Mask a source error before it reaches logs or output. Source plugins can transform or truncate
241/// a request in their diagnostics, so exact replacement is not sufficient for a secret-bearing
242/// request. In that case the opaque detail is omitted instead.
243pub fn mask_source_error(error: &str, sensitive: bool) -> Cow<'_, str> {
244    if !ENABLED.load(Ordering::Relaxed) {
245        return Cow::Borrowed(error);
246    }
247    mask_source_error_with(error, sensitive, &secrets().plain)
248}
249
250fn unique_masked_names_with(names: &[String], secrets: &[String]) -> Vec<String> {
251    use std::collections::BTreeSet;
252
253    let masked: Vec<String> = names
254        .iter()
255        .map(|name| mask_with(name, secrets).into_owned())
256        .collect();
257    if names == masked {
258        return names.to_vec();
259    }
260    let mut used = BTreeSet::new();
261    let mut output = vec![String::new(); names.len()];
262    let mut pending = Vec::new();
263
264    // Literal names keep their public identity and reserve every suffix they already occupy.
265    for (index, (original, protected)) in names.iter().zip(&masked).enumerate() {
266        if original == protected {
267            used.insert(original.clone());
268            output[index].clone_from(original);
269        } else {
270            pending.push((protected.clone(), original.clone(), index));
271        }
272    }
273
274    // Original names make the assignment stable when source ordering changes.
275    pending.sort();
276    for (base, _, index) in pending {
277        let mut candidate = base.clone();
278        let mut suffix = 2;
279        while !used.insert(candidate.clone()) {
280            candidate = format!("{base}#{suffix}");
281            suffix += 1;
282        }
283        output[index] = candidate;
284    }
285    output
286}
287
288fn redact_json_with(v: &mut serde_json::Value, secrets: &[String]) {
289    use serde_json::Value;
290    match v {
291        Value::String(s) => {
292            if let Cow::Owned(m) = mask_with(s, secrets) {
293                *s = m;
294            }
295        }
296        Value::Array(a) => a.iter_mut().for_each(|v| redact_json_with(v, secrets)),
297        Value::Object(o) => {
298            let entries = std::mem::take(o);
299            let keys: Vec<String> = entries.keys().cloned().collect();
300            let protected_keys = unique_masked_names_with(&keys, secrets);
301            for ((_, mut val), protected_key) in entries.into_iter().zip(protected_keys) {
302                redact_json_with(&mut val, secrets);
303                o.insert(protected_key, val);
304            }
305        }
306        _ => {}
307    }
308}
309
310/// `v` with every secret value replaced by `*****` in all strings and object keys. Colliding
311/// masked keys receive `#2`, `#3`, and so on rather than silently replacing another entry.
312pub fn redact_json(v: &mut serde_json::Value) {
313    if ENABLED.load(Ordering::Relaxed) {
314        redact_json_with(v, &secrets().plain);
315    }
316}
317
318fn redacted<T: serde::Serialize>(v: &T) -> serde_json::Result<serde_json::Value> {
319    let mut v = serde_json::to_value(v)?;
320    redact_json(&mut v);
321    Ok(v)
322}
323
324/// Pretty JSON of `v` with secrets redacted first.
325pub fn to_json_pretty<T: serde::Serialize>(v: &T) -> serde_json::Result<String> {
326    serde_json::to_string_pretty(&redacted(v)?)
327}
328
329/// One-line JSON of `v` with secrets redacted first.
330pub fn to_json_line<T: serde::Serialize>(v: &T) -> serde_json::Result<String> {
331    serde_json::to_string(&redacted(v)?)
332}
333
334#[cfg(test)]
335mod tests {
336    use super::*;
337    use serde_json::json;
338
339    fn secrets(values: &[&str]) -> Vec<String> {
340        let mut values: Vec<String> = values.iter().map(|v| (*v).to_string()).collect();
341        values.sort_by_key(|s| std::cmp::Reverse(s.len()));
342        values
343    }
344
345    #[test]
346    fn redacts_nested_values_longest_first() {
347        let secrets = secrets(&["nested-secret", "nested-secret-and-more"]);
348        let mut value = json!({"items": ["nested-secret-and-more", {"value": "x nested-secret y"}]});
349        redact_json_with(&mut value, &secrets);
350        assert_eq!(value, json!({"items": ["*****", {"value": "x ***** y"}]}));
351    }
352
353    #[test]
354    fn preserves_entries_when_masked_keys_collide() {
355        let secrets = secrets(&["alpha-secret-key", "bravo-secret-key"]);
356        let mut value = json!({
357            "alpha-secret-key": 1,
358            "bravo-secret-key": 2,
359            "*****": 3,
360            "*****#2": 4,
361        });
362        redact_json_with(&mut value, &secrets);
363        assert_eq!(
364            value,
365            json!({
366                "*****": 3,
367                "*****#2": 4,
368                "*****#3": 1,
369                "*****#4": 2,
370            })
371        );
372    }
373
374    #[test]
375    fn masking_one_name_does_not_rename_duplicate_literal_names() {
376        let secrets = secrets(&["secret-column"]);
377        let names = ["a", "a", "secret-column"].map(str::to_string);
378        assert_eq!(unique_masked_names_with(&names, &secrets), ["a", "a", "*****"]);
379    }
380
381    #[test]
382    fn suppresses_transformed_source_errors_only_for_secret_requests() {
383        let secrets = secrets(&["long-secret-value-that-a-source-truncated"]);
384        let error = "LINE 1: select * from long-secret-value-that-a-source...";
385        assert_eq!(mask_source_error_with(error, true, &secrets), MASKED_SOURCE_ERROR);
386        assert_eq!(
387            mask_source_error_with("ordinary syntax error", false, &secrets),
388            "ordinary syntax error"
389        );
390    }
391
392    #[test]
393    fn finds_env_var_references_with_either_quote() {
394        assert_eq!(
395            env_references(
396                "password: \"{{ env_var('PG_PASSWORD') }}\"\ntoken: '{{ env_var(\"WAREHOUSE_TOKEN\", \"x\") }}'"
397            ),
398            ["PG_PASSWORD", "WAREHOUSE_TOKEN"]
399        );
400    }
401
402    #[test]
403    fn secret_field_names_leave_paths_and_fingerprints_alone() {
404        for name in [
405            "password",
406            "client_secret",
407            "token",
408            "api_key",
409            "private_key",
410            "key",
411            "passphrase",
412        ] {
413            assert!(secret_field_name(name), "{name}");
414        }
415        for name in [
416            "path",
417            "host",
418            "private_key_path",
419            "host_key_fingerprint",
420            "key_id",
421            "keyspace",
422            "schema",
423        ] {
424            assert!(!secret_field_name(name), "{name}");
425        }
426    }
427
428    #[test]
429    fn short_values_mask_where_they_stand_alone() {
430        assert!(short("ééé") && !short("éééé") && !eligible(""));
431        let secrets = sorted(["12".to_string(), "Ab9".to_string(), "s3cret".to_string()]);
432        assert_eq!(
433            mask_with("PIN 12, code Ab9; year 2012, x12y, Ab99, s3cretly", &secrets),
434            "PIN *****, code *****; year 2012, x12y, Ab99, *****ly"
435        );
436        assert_eq!(mask_with("12", &secrets), "*****");
437        assert_eq!(mask_with("12 12", &secrets), "***** *****");
438        assert!(matches!(mask_with("2012", &secrets), Cow::Borrowed(_)));
439    }
440}
dre_core/secrets.rs

dre_core/
secrets.rs
