dre_core/
secrets.rs

1//! Secret masking. The value of every environment variable named `DRE_SECRET_*`, and of every
2//! environment variable `profiles.yml` reads through `env_var()` into a secret field (one the
3//! plugin declares secret, or named like `password`, `token`, `api_key`), is replaced by
4//! `*****` in everything DRE writes for people to read: the console, `logs/dre.log`, JSON events,
5//! `run_results.json` and the compiled SQL in `target/compiled/`. The SQL actually sent to the
6//! database is unchanged. `mask_secrets: false` in `dre_project.yml` turns masking off.
7//!
8//! JSON is redacted structurally, before it is serialized: serializing escapes `"`, `\` and
9//! control characters, so masking the finished text would miss a secret that contains them. Use
10//! [`to_json_pretty`] / [`to_json_line`] for every JSON surface, never `mask` on JSON text.
11//!
12//! Values shorter than 4 characters aren't masked: replacing every `a` or `12` in a log would
13//! make it unreadable, and a value that short isn't a secret anyway.
14
15use std::borrow::Cow;
16use std::sync::atomic::{AtomicBool, Ordering};
17use std::sync::{Arc, LazyLock, RwLock};
18
19pub const PREFIX: &str = "DRE_SECRET_";
20pub const MASK: &str = "*****";
21pub const MASKED_SOURCE_ERROR: &str =
22    "source error details omitted because the request contains a masked secret";
23const MIN_LEN: usize = 4;
24
25static ENABLED: AtomicBool = AtomicBool::new(true);
26
27fn eligible(value: &str) -> bool {
28    value.chars().count() >= MIN_LEN
29}
30
31/// Secret values, longest first so a secret containing another is masked whole. Starts with
32/// `DRE_SECRET_*`; [`register_secret_field`] adds the variables secret profile fields read.
33static SECRETS: LazyLock<RwLock<Arc<Secrets>>> = LazyLock::new(|| {
34    RwLock::new(Arc::new(Secrets::new(sorted(
35        std::env::vars()
36            .filter(|(k, _)| k.starts_with(PREFIX))
37            .map(|(_, v)| v),
38    ))))
39});
40
41/// The secret values, and the forms they take in Markdown and in collapsed titles.
42struct Secrets {
43    plain: Vec<String>,
44    markdown: Vec<String>,
45    title: Vec<String>,
46}
47
48impl Secrets {
49    fn new(plain: Vec<String>) -> Secrets {
50        let markdown = sorted(
51            plain
52                .iter()
53                .flat_map(|s| [s.clone(), dre_protocol::markdown::escape(s)]),
54        );
55        let title = sorted(
56            plain
57                .iter()
58                .flat_map(|s| [s.clone(), s.split_whitespace().collect::<Vec<_>>().join(" ")]),
59        );
60        Secrets {
61            plain,
62            markdown,
63            title,
64        }
65    }
66}
67
68fn sorted(values: impl IntoIterator<Item = String>) -> Vec<String> {
69    let mut v: Vec<String> = values.into_iter().filter(|v| eligible(v)).collect();
70    v.sort_by(|a, b| b.len().cmp(&a.len()).then_with(|| a.cmp(b)));
71    v.dedup();
72    v
73}
74
75fn secrets() -> Arc<Secrets> {
76    SECRETS.read().unwrap_or_else(|e| e.into_inner()).clone()
77}
78
79/// The environment variables a template reads through `env_var('NAME')` (either quote).
80pub fn env_references(text: &str) -> Vec<String> {
81    static CALL: LazyLock<regex::Regex> =
82        LazyLock::new(|| regex::Regex::new(r#"env_var\(\s*(?:'([^']+)'|"([^"]+)")"#).unwrap());
83    CALL.captures_iter(text)
84        .filter_map(|c| c.get(1).or_else(|| c.get(2)))
85        .map(|m| m.as_str().to_owned())
86        .collect()
87}
88
89/// Whether a profile field's name says it holds a secret. Paths, hosts and fingerprints don't
90/// count (`private_key_path`, `host_key_fingerprint`): masking them would hide what a message is
91/// about, and they aren't secrets.
92pub fn secret_field_name(name: &str) -> bool {
93    let name = name.to_ascii_lowercase();
94    ["password", "passphrase", "secret", "token", "credential"]
95        .iter()
96        .any(|w| name.contains(w))
97        || name == "key"
98        || name.ends_with("_key")
99        || name == "apikey"
100}
101
102/// Mask the values of the environment variables a secret profile field reads through
103/// `env_var()`, at any depth of `value`.
104pub fn register_secret_field(value: &serde_json::Value) {
105    let mut text = Vec::new();
106    collect_strings(value, &mut text);
107    register(text.iter().flat_map(|s| env_references(s)))
108}
109
110/// Mask the `env_var()` values that the secret fields of `fields` (a profile target) read,
111/// nested blocks (`ssh:`) included. `secret` adds names a plugin declares secret.
112pub fn register_secret_fields(fields: &serde_json::Map<String, serde_json::Value>, secret: &[String]) {
113    for (name, value) in fields {
114        if secret_field_name(name) || secret.contains(name) {
115            register_secret_field(value);
116        } else if let serde_json::Value::Object(nested) = value {
117            register_secret_fields(nested, &[]);
118        }
119    }
120}
121
122fn collect_strings<'a>(value: &'a serde_json::Value, out: &mut Vec<&'a str>) {
123    match value {
124        serde_json::Value::String(s) => out.push(s),
125        serde_json::Value::Array(a) => a.iter().for_each(|v| collect_strings(v, out)),
126        serde_json::Value::Object(o) => o.values().for_each(|v| collect_strings(v, out)),
127        _ => {}
128    }
129}
130
131fn register(names: impl Iterator<Item = String>) {
132    register_values(names.into_iter().filter_map(|name| std::env::var(name).ok()));
133}
134
135/// Mask these values from now on: a query's `secret_columns`.
136pub fn register_values(values: impl IntoIterator<Item = String>) {
137    let values: Vec<String> = values.into_iter().filter(|v| eligible(v)).collect();
138    if values.is_empty() {
139        return;
140    }
141    let mut guard = SECRETS.write().unwrap_or_else(|e| e.into_inner());
142    let merged = sorted(guard.plain.iter().cloned().chain(values));
143    *guard = Arc::new(Secrets::new(merged));
144}
145
146pub fn set_enabled(on: bool) {
147    ENABLED.store(on, Ordering::Relaxed);
148}
149
150pub(crate) fn enabled() -> bool {
151    ENABLED.load(Ordering::Relaxed)
152}
153
154fn mask_with<'a>(s: &'a str, secrets: &[String]) -> Cow<'a, str> {
155    if !secrets.iter().any(|x| s.contains(x.as_str())) {
156        return Cow::Borrowed(s);
157    }
158    let mut out = s.to_string();
159    for x in secrets {
160        out = out.replace(x.as_str(), MASK);
161    }
162    Cow::Owned(out)
163}
164
165/// `s` with every secret value replaced by `*****`.
166pub fn mask(s: &str) -> Cow<'_, str> {
167    if !ENABLED.load(Ordering::Relaxed) {
168        return Cow::Borrowed(s);
169    }
170    mask_with(s, &secrets().plain)
171}
172
173/// Mask a rendered message without changing the report payload. Markdown interpolation has
174/// already escaped punctuation, so both literal and escaped secret values must be matched.
175pub(crate) fn mask_markdown(s: &str) -> Cow<'_, str> {
176    if !ENABLED.load(Ordering::Relaxed) {
177        return Cow::Borrowed(s);
178    }
179    mask_with(s, &secrets().markdown)
180}
181
182/// Message titles collapse whitespace before delivery. Match that representation as well as
183/// the literal secret when protecting a title in logs and run metadata.
184pub(crate) fn mask_message_title(s: &str) -> Cow<'_, str> {
185    if !ENABLED.load(Ordering::Relaxed) {
186        return Cow::Borrowed(s);
187    }
188    mask_with(s, &secrets().title)
189}
190
191/// Whether masking is enabled and `s` contains a complete registered secret.
192pub fn contains_secret(s: &str) -> bool {
193    ENABLED.load(Ordering::Relaxed) && secrets().plain.iter().any(|secret| s.contains(secret.as_str()))
194}
195
196fn mask_source_error_with<'a>(error: &'a str, sensitive: bool, secrets: &[String]) -> Cow<'a, str> {
197    if sensitive {
198        Cow::Borrowed(MASKED_SOURCE_ERROR)
199    } else {
200        mask_with(error, secrets)
201    }
202}
203
204/// Mask a source error before it reaches logs or output. Source plugins can transform or truncate
205/// a request in their diagnostics, so exact replacement is not sufficient for a secret-bearing
206/// request. In that case the opaque detail is omitted instead.
207pub fn mask_source_error(error: &str, sensitive: bool) -> Cow<'_, str> {
208    if !ENABLED.load(Ordering::Relaxed) {
209        return Cow::Borrowed(error);
210    }
211    mask_source_error_with(error, sensitive, &secrets().plain)
212}
213
214fn unique_masked_names_with(names: &[String], secrets: &[String]) -> Vec<String> {
215    use std::collections::BTreeSet;
216
217    let masked: Vec<String> = names
218        .iter()
219        .map(|name| mask_with(name, secrets).into_owned())
220        .collect();
221    if names == masked {
222        return names.to_vec();
223    }
224    let mut used = BTreeSet::new();
225    let mut output = vec![String::new(); names.len()];
226    let mut pending = Vec::new();
227
228    // Literal names keep their public identity and reserve every suffix they already occupy.
229    for (index, (original, protected)) in names.iter().zip(&masked).enumerate() {
230        if original == protected {
231            used.insert(original.clone());
232            output[index].clone_from(original);
233        } else {
234            pending.push((protected.clone(), original.clone(), index));
235        }
236    }
237
238    // Original names make the assignment stable when source ordering changes.
239    pending.sort();
240    for (base, _, index) in pending {
241        let mut candidate = base.clone();
242        let mut suffix = 2;
243        while !used.insert(candidate.clone()) {
244            candidate = format!("{base}#{suffix}");
245            suffix += 1;
246        }
247        output[index] = candidate;
248    }
249    output
250}
251
252fn redact_json_with(v: &mut serde_json::Value, secrets: &[String]) {
253    use serde_json::Value;
254    match v {
255        Value::String(s) => {
256            if let Cow::Owned(m) = mask_with(s, secrets) {
257                *s = m;
258            }
259        }
260        Value::Array(a) => a.iter_mut().for_each(|v| redact_json_with(v, secrets)),
261        Value::Object(o) => {
262            let entries = std::mem::take(o);
263            let keys: Vec<String> = entries.keys().cloned().collect();
264            let protected_keys = unique_masked_names_with(&keys, secrets);
265            for ((_, mut val), protected_key) in entries.into_iter().zip(protected_keys) {
266                redact_json_with(&mut val, secrets);
267                o.insert(protected_key, val);
268            }
269        }
270        _ => {}
271    }
272}
273
274/// `v` with every secret value replaced by `*****` in all strings and object keys. Colliding
275/// masked keys receive `#2`, `#3`, and so on rather than silently replacing another entry.
276pub fn redact_json(v: &mut serde_json::Value) {
277    if ENABLED.load(Ordering::Relaxed) {
278        redact_json_with(v, &secrets().plain);
279    }
280}
281
282fn redacted<T: serde::Serialize>(v: &T) -> serde_json::Result<serde_json::Value> {
283    let mut v = serde_json::to_value(v)?;
284    redact_json(&mut v);
285    Ok(v)
286}
287
288/// Pretty JSON of `v` with secrets redacted first.
289pub fn to_json_pretty<T: serde::Serialize>(v: &T) -> serde_json::Result<String> {
290    serde_json::to_string_pretty(&redacted(v)?)
291}
292
293/// One-line JSON of `v` with secrets redacted first.
294pub fn to_json_line<T: serde::Serialize>(v: &T) -> serde_json::Result<String> {
295    serde_json::to_string(&redacted(v)?)
296}
297
298#[cfg(test)]
299mod tests {
300    use super::*;
301    use serde_json::json;
302
303    fn secrets(values: &[&str]) -> Vec<String> {
304        let mut values: Vec<String> = values.iter().map(|v| (*v).to_string()).collect();
305        values.sort_by_key(|s| std::cmp::Reverse(s.len()));
306        values
307    }
308
309    #[test]
310    fn redacts_nested_values_longest_first() {
311        let secrets = secrets(&["nested-secret", "nested-secret-and-more"]);
312        let mut value = json!({"items": ["nested-secret-and-more", {"value": "x nested-secret y"}]});
313        redact_json_with(&mut value, &secrets);
314        assert_eq!(value, json!({"items": ["*****", {"value": "x ***** y"}]}));
315    }
316
317    #[test]
318    fn preserves_entries_when_masked_keys_collide() {
319        let secrets = secrets(&["alpha-secret-key", "bravo-secret-key"]);
320        let mut value = json!({
321            "alpha-secret-key": 1,
322            "bravo-secret-key": 2,
323            "*****": 3,
324            "*****#2": 4,
325        });
326        redact_json_with(&mut value, &secrets);
327        assert_eq!(
328            value,
329            json!({
330                "*****": 3,
331                "*****#2": 4,
332                "*****#3": 1,
333                "*****#4": 2,
334            })
335        );
336    }
337
338    #[test]
339    fn masking_one_name_does_not_rename_duplicate_literal_names() {
340        let secrets = secrets(&["secret-column"]);
341        let names = ["a", "a", "secret-column"].map(str::to_string);
342        assert_eq!(unique_masked_names_with(&names, &secrets), ["a", "a", "*****"]);
343    }
344
345    #[test]
346    fn suppresses_transformed_source_errors_only_for_secret_requests() {
347        let secrets = secrets(&["long-secret-value-that-a-source-truncated"]);
348        let error = "LINE 1: select * from long-secret-value-that-a-source...";
349        assert_eq!(mask_source_error_with(error, true, &secrets), MASKED_SOURCE_ERROR);
350        assert_eq!(
351            mask_source_error_with("ordinary syntax error", false, &secrets),
352            "ordinary syntax error"
353        );
354    }
355
356    #[test]
357    fn finds_env_var_references_with_either_quote() {
358        assert_eq!(
359            env_references(
360                "password: \"{{ env_var('PG_PASSWORD') }}\"\ntoken: '{{ env_var(\"WAREHOUSE_TOKEN\", \"x\") }}'"
361            ),
362            ["PG_PASSWORD", "WAREHOUSE_TOKEN"]
363        );
364    }
365
366    #[test]
367    fn secret_field_names_leave_paths_and_fingerprints_alone() {
368        for name in [
369            "password",
370            "client_secret",
371            "token",
372            "api_key",
373            "private_key",
374            "key",
375            "passphrase",
376        ] {
377            assert!(secret_field_name(name), "{name}");
378        }
379        for name in [
380            "path",
381            "host",
382            "private_key_path",
383            "host_key_fingerprint",
384            "key_id",
385            "keyspace",
386            "schema",
387        ] {
388            assert!(!secret_field_name(name), "{name}");
389        }
390    }
391
392    #[test]
393    fn minimum_length_counts_unicode_characters() {
394        assert!(!eligible("ééé"));
395        assert!(eligible("éééé"));
396    }
397}
dre_core/secrets.rs

dre_core/
secrets.rs
