dre_core/
secrets.rs

1//! Secret masking. The value of every environment variable named `DRE_SECRET_*`, and of every
2//! environment variable `profiles.yml` reads through `env_var()`, is replaced by
3//! `*****` in everything DRE writes for people to read: the console, `logs/dre.log`, JSON events,
4//! `run_results.json` and the compiled SQL in `target/compiled/`. The SQL actually sent to the
5//! database is unchanged. `mask_secrets: false` in `dre_project.yml` turns masking off.
6//!
7//! JSON is redacted structurally, before it is serialized: serializing escapes `"`, `\` and
8//! control characters, so masking the finished text would miss a secret that contains them. Use
9//! [`to_json_pretty`] / [`to_json_line`] for every JSON surface, never `mask` on JSON text.
10//!
11//! Values shorter than 4 characters aren't masked: replacing every `a` or `12` in a log would
12//! make it unreadable, and a value that short isn't a secret anyway.
13
14use std::borrow::Cow;
15use std::sync::atomic::{AtomicBool, Ordering};
16use std::sync::{Arc, LazyLock, RwLock};
17
18pub const PREFIX: &str = "DRE_SECRET_";
19pub const MASK: &str = "*****";
20pub const MASKED_SOURCE_ERROR: &str =
21    "source error details omitted because the request contains a masked secret";
22const MIN_LEN: usize = 4;
23
24static ENABLED: AtomicBool = AtomicBool::new(true);
25
26fn eligible(value: &str) -> bool {
27    value.chars().count() >= MIN_LEN
28}
29
30/// Secret values, longest first so a secret containing another is masked whole. Starts with
31/// `DRE_SECRET_*`; [`register_env_references`] adds the variables `profiles.yml` reads.
32static SECRETS: LazyLock<RwLock<Arc<Secrets>>> = LazyLock::new(|| {
33    RwLock::new(Arc::new(Secrets::new(sorted(
34        std::env::vars()
35            .filter(|(k, _)| k.starts_with(PREFIX))
36            .map(|(_, v)| v),
37    ))))
38});
39
40/// The secret values, and the forms they take in Markdown and in collapsed titles.
41struct Secrets {
42    plain: Vec<String>,
43    markdown: Vec<String>,
44    title: Vec<String>,
45}
46
47impl Secrets {
48    fn new(plain: Vec<String>) -> Secrets {
49        let markdown = sorted(
50            plain
51                .iter()
52                .flat_map(|s| [s.clone(), dre_protocol::markdown::escape(s)]),
53        );
54        let title = sorted(
55            plain
56                .iter()
57                .flat_map(|s| [s.clone(), s.split_whitespace().collect::<Vec<_>>().join(" ")]),
58        );
59        Secrets {
60            plain,
61            markdown,
62            title,
63        }
64    }
65}
66
67fn sorted(values: impl IntoIterator<Item = String>) -> Vec<String> {
68    let mut v: Vec<String> = values.into_iter().filter(|v| eligible(v)).collect();
69    v.sort_by_key(|s| std::cmp::Reverse(s.len()));
70    v.dedup();
71    v
72}
73
74fn secrets() -> Arc<Secrets> {
75    SECRETS.read().unwrap_or_else(|e| e.into_inner()).clone()
76}
77
78/// The environment variables a template reads through `env_var('NAME')` (either quote).
79pub fn env_references(text: &str) -> Vec<String> {
80    static CALL: LazyLock<regex::Regex> =
81        LazyLock::new(|| regex::Regex::new(r#"env_var\(\s*(?:'([^']+)'|"([^"]+)")"#).unwrap());
82    CALL.captures_iter(text)
83        .filter_map(|c| c.get(1).or_else(|| c.get(2)))
84        .map(|m| m.as_str().to_owned())
85        .collect()
86}
87
88/// Mask the values of the environment variables `text` (a `profiles.yml`) reads through
89/// `env_var()`: connection passwords and tokens are usually passed that way.
90pub fn register_env_references(text: &str) {
91    let values: Vec<String> = env_references(text)
92        .into_iter()
93        .filter_map(|name| std::env::var(name).ok())
94        .collect();
95    if values.is_empty() {
96        return;
97    }
98    let mut guard = SECRETS.write().unwrap_or_else(|e| e.into_inner());
99    let merged = sorted(guard.plain.iter().cloned().chain(values));
100    *guard = Arc::new(Secrets::new(merged));
101}
102
103pub fn set_enabled(on: bool) {
104    ENABLED.store(on, Ordering::Relaxed);
105}
106
107pub(crate) fn enabled() -> bool {
108    ENABLED.load(Ordering::Relaxed)
109}
110
111fn mask_with<'a>(s: &'a str, secrets: &[String]) -> Cow<'a, str> {
112    if !secrets.iter().any(|x| s.contains(x.as_str())) {
113        return Cow::Borrowed(s);
114    }
115    let mut out = s.to_string();
116    for x in secrets {
117        out = out.replace(x.as_str(), MASK);
118    }
119    Cow::Owned(out)
120}
121
122/// `s` with every secret value replaced by `*****`.
123pub fn mask(s: &str) -> Cow<'_, str> {
124    if !ENABLED.load(Ordering::Relaxed) {
125        return Cow::Borrowed(s);
126    }
127    mask_with(s, &secrets().plain)
128}
129
130/// Mask a rendered message without changing the report payload. Markdown interpolation has
131/// already escaped punctuation, so both literal and escaped secret values must be matched.
132pub(crate) fn mask_markdown(s: &str) -> Cow<'_, str> {
133    if !ENABLED.load(Ordering::Relaxed) {
134        return Cow::Borrowed(s);
135    }
136    mask_with(s, &secrets().markdown)
137}
138
139/// Message titles collapse whitespace before delivery. Match that representation as well as
140/// the literal secret when protecting a title in logs and run metadata.
141pub(crate) fn mask_message_title(s: &str) -> Cow<'_, str> {
142    if !ENABLED.load(Ordering::Relaxed) {
143        return Cow::Borrowed(s);
144    }
145    mask_with(s, &secrets().title)
146}
147
148/// Whether masking is enabled and `s` contains a complete registered secret.
149pub fn contains_secret(s: &str) -> bool {
150    ENABLED.load(Ordering::Relaxed) && secrets().plain.iter().any(|secret| s.contains(secret.as_str()))
151}
152
153fn mask_source_error_with<'a>(error: &'a str, sensitive: bool, secrets: &[String]) -> Cow<'a, str> {
154    if sensitive {
155        Cow::Borrowed(MASKED_SOURCE_ERROR)
156    } else {
157        mask_with(error, secrets)
158    }
159}
160
161/// Mask a source error before it reaches logs or output. Source plugins can transform or truncate
162/// a request in their diagnostics, so exact replacement is not sufficient for a secret-bearing
163/// request. In that case the opaque detail is omitted instead.
164pub fn mask_source_error(error: &str, sensitive: bool) -> Cow<'_, str> {
165    if !ENABLED.load(Ordering::Relaxed) {
166        return Cow::Borrowed(error);
167    }
168    mask_source_error_with(error, sensitive, &secrets().plain)
169}
170
171fn unique_masked_names_with(names: &[String], secrets: &[String]) -> Vec<String> {
172    use std::collections::BTreeSet;
173
174    let masked: Vec<String> = names
175        .iter()
176        .map(|name| mask_with(name, secrets).into_owned())
177        .collect();
178    if names == masked {
179        return names.to_vec();
180    }
181    let mut used = BTreeSet::new();
182    let mut output = vec![String::new(); names.len()];
183    let mut pending = Vec::new();
184
185    // Literal names keep their public identity and reserve every suffix they already occupy.
186    for (index, (original, protected)) in names.iter().zip(&masked).enumerate() {
187        if original == protected {
188            used.insert(original.clone());
189            output[index].clone_from(original);
190        } else {
191            pending.push((protected.clone(), original.clone(), index));
192        }
193    }
194
195    // Original names make the assignment stable when source ordering changes.
196    pending.sort();
197    for (base, _, index) in pending {
198        let mut candidate = base.clone();
199        let mut suffix = 2;
200        while !used.insert(candidate.clone()) {
201            candidate = format!("{base}#{suffix}");
202            suffix += 1;
203        }
204        output[index] = candidate;
205    }
206    output
207}
208
209fn redact_json_with(v: &mut serde_json::Value, secrets: &[String]) {
210    use serde_json::Value;
211    match v {
212        Value::String(s) => {
213            if let Cow::Owned(m) = mask_with(s, secrets) {
214                *s = m;
215            }
216        }
217        Value::Array(a) => a.iter_mut().for_each(|v| redact_json_with(v, secrets)),
218        Value::Object(o) => {
219            let entries = std::mem::take(o);
220            let keys: Vec<String> = entries.keys().cloned().collect();
221            let protected_keys = unique_masked_names_with(&keys, secrets);
222            for ((_, mut val), protected_key) in entries.into_iter().zip(protected_keys) {
223                redact_json_with(&mut val, secrets);
224                o.insert(protected_key, val);
225            }
226        }
227        _ => {}
228    }
229}
230
231/// `v` with every secret value replaced by `*****` in all strings and object keys. Colliding
232/// masked keys receive `#2`, `#3`, and so on rather than silently replacing another entry.
233pub fn redact_json(v: &mut serde_json::Value) {
234    if ENABLED.load(Ordering::Relaxed) {
235        redact_json_with(v, &secrets().plain);
236    }
237}
238
239fn redacted<T: serde::Serialize>(v: &T) -> serde_json::Result<serde_json::Value> {
240    let mut v = serde_json::to_value(v)?;
241    redact_json(&mut v);
242    Ok(v)
243}
244
245/// Pretty JSON of `v` with secrets redacted first.
246pub fn to_json_pretty<T: serde::Serialize>(v: &T) -> serde_json::Result<String> {
247    serde_json::to_string_pretty(&redacted(v)?)
248}
249
250/// One-line JSON of `v` with secrets redacted first.
251pub fn to_json_line<T: serde::Serialize>(v: &T) -> serde_json::Result<String> {
252    serde_json::to_string(&redacted(v)?)
253}
254
255#[cfg(test)]
256mod tests {
257    use super::*;
258    use serde_json::json;
259
260    fn secrets(values: &[&str]) -> Vec<String> {
261        let mut values: Vec<String> = values.iter().map(|v| (*v).to_string()).collect();
262        values.sort_by_key(|s| std::cmp::Reverse(s.len()));
263        values
264    }
265
266    #[test]
267    fn redacts_nested_values_longest_first() {
268        let secrets = secrets(&["nested-secret", "nested-secret-and-more"]);
269        let mut value = json!({"items": ["nested-secret-and-more", {"value": "x nested-secret y"}]});
270        redact_json_with(&mut value, &secrets);
271        assert_eq!(value, json!({"items": ["*****", {"value": "x ***** y"}]}));
272    }
273
274    #[test]
275    fn preserves_entries_when_masked_keys_collide() {
276        let secrets = secrets(&["alpha-secret-key", "bravo-secret-key"]);
277        let mut value = json!({
278            "alpha-secret-key": 1,
279            "bravo-secret-key": 2,
280            "*****": 3,
281            "*****#2": 4,
282        });
283        redact_json_with(&mut value, &secrets);
284        assert_eq!(
285            value,
286            json!({
287                "*****": 3,
288                "*****#2": 4,
289                "*****#3": 1,
290                "*****#4": 2,
291            })
292        );
293    }
294
295    #[test]
296    fn masking_one_name_does_not_rename_duplicate_literal_names() {
297        let secrets = secrets(&["secret-column"]);
298        let names = ["a", "a", "secret-column"].map(str::to_string);
299        assert_eq!(unique_masked_names_with(&names, &secrets), ["a", "a", "*****"]);
300    }
301
302    #[test]
303    fn suppresses_transformed_source_errors_only_for_secret_requests() {
304        let secrets = secrets(&["long-secret-value-that-a-source-truncated"]);
305        let error = "LINE 1: select * from long-secret-value-that-a-source...";
306        assert_eq!(mask_source_error_with(error, true, &secrets), MASKED_SOURCE_ERROR);
307        assert_eq!(
308            mask_source_error_with("ordinary syntax error", false, &secrets),
309            "ordinary syntax error"
310        );
311    }
312
313    #[test]
314    fn finds_env_var_references_with_either_quote() {
315        assert_eq!(
316            env_references(
317                "password: \"{{ env_var('PG_PASSWORD') }}\"\ntoken: '{{ env_var(\"WAREHOUSE_TOKEN\", \"x\") }}'"
318            ),
319            ["PG_PASSWORD", "WAREHOUSE_TOKEN"]
320        );
321    }
322
323    #[test]
324    fn minimum_length_counts_unicode_characters() {
325        assert!(!eligible("ééé"));
326        assert!(eligible("éééé"));
327    }
328}
dre_core/secrets.rs

dre_core/
secrets.rs
