memstead_base/ops/
redaction.rs

1//! Private-pattern redaction for portable authoring provenance.
2//!
3//! `memstead export --format mem` ships each entity's latest mutation
4//! rationale in the archive's `.memstead/provenance.json`. Those notes are
5//! written inside the private workspace and name what the public tree must
6//! never carry: internal `dev/` plan paths, the legacy domain, absolute
7//! user paths, and the rest of the classes `scripts/leak-scan.sh` refuses
8//! on the public repo. The export redacts every matched span to
9//! `[redacted:<class>]` and never strips the record (the decision on
10//! published anchors: a stripped note is indistinguishable from one never
11//! written, a sentinel keeps the rationale readable while naming nothing).
12//!
13//! One vocabulary. The classes below are the leak scan's `scan` lines,
14//! label and pattern verbatim; the test at the bottom reads the script and
15//! holds the two equal, so a class added to one without the other fails
16//! naming the class. Entity bodies are not touched here: the leak scan
17//! keeps guarding them, and an archive whose bodies carry a private string
18//! still refuses at the seal gate.
19
20use std::collections::BTreeMap;
21use std::sync::OnceLock;
22
23use regex::Regex;
24
25/// One redaction class: the leak scan's label, its extended-regex pattern
26/// verbatim, and whether the pattern's first capture group is a boundary
27/// prefix (a space, quote or line start) that must survive the redaction.
28#[derive(Debug, Clone, Copy)]
29pub struct RedactionClass {
30    pub name: &'static str,
31    pub pattern: &'static str,
32    /// The pattern opens with `(^|<boundary chars>)` so the match includes
33    /// one character that is not private; that group is kept.
34    pub keeps_leading_group: bool,
35}
36
37/// The redaction vocabulary, in the leak scan's order.
38pub const REDACTION_CLASSES: &[RedactionClass] = &[
39    RedactionClass {
40        name: "absolute-user-paths",
41        pattern: r"/Users/(dasboe|bjornbosenberg)",
42        keeps_leading_group: false,
43    },
44    RedactionClass {
45        name: "secrets",
46        pattern: r"(-----BEGIN [A-Z]+ PRIVATE KEY|ghp_[A-Za-z0-9]{30,}|github_pat_[A-Za-z0-9_]{30,}|gho_[A-Za-z0-9]{30,}|xox[baprs]-[A-Za-z0-9-]{10,}|AKIA[0-9A-Z]{16}|sk-ant-[A-Za-z0-9_-]{24,}|sk-[A-Za-z0-9]{24,})",
47        keeps_leading_group: false,
48    },
49    RedactionClass {
50        name: "private-infra",
51        pattern: r"(railway\.app|\.up\.railway\.app|railway\.json)",
52        keeps_leading_group: false,
53    },
54    RedactionClass {
55        name: "internal-refs",
56        pattern: r"(dev/plans|dev/strategy|dev/ci|LAUNCH\.md)",
57        keeps_leading_group: false,
58    },
59    RedactionClass {
60        name: "stale-product-name",
61        pattern: r"\b[Mm]emgno\b",
62        keeps_leading_group: false,
63    },
64    RedactionClass {
65        name: "excluded-private-dirs",
66        pattern: r#"(^|[[:space:]"'`(:,])(macos|websites|graph|inspector|local-ai)/"#,
67        keeps_leading_group: true,
68    },
69    RedactionClass {
70        name: "legacy-domain",
71        pattern: r"(mdgv\.io|dasboe/mdgv|dasboe\.github\.io)",
72        keeps_leading_group: false,
73    },
74];
75
76/// The sentinel a redacted span becomes.
77pub fn sentinel(class: &str) -> String {
78    format!("[redacted:{class}]")
79}
80
81fn compiled() -> &'static [(RedactionClass, Regex)] {
82    static COMPILED: OnceLock<Vec<(RedactionClass, Regex)>> = OnceLock::new();
83    COMPILED.get_or_init(|| {
84        REDACTION_CLASSES
85            .iter()
86            .map(|c| {
87                (
88                    *c,
89                    Regex::new(c.pattern).unwrap_or_else(|e| {
90                        panic!("redaction class {} has an invalid pattern: {e}", c.name)
91                    }),
92                )
93            })
94            .collect()
95    })
96}
97
98/// Redact every private span in `text`, class by class in vocabulary
99/// order; returns the redacted text and the per-class count of spans
100/// replaced (classes with no match are absent).
101pub fn redact(text: &str) -> (String, BTreeMap<&'static str, usize>) {
102    let mut out = text.to_string();
103    let mut counts: BTreeMap<&'static str, usize> = BTreeMap::new();
104    for (class, re) in compiled() {
105        let mut n = 0usize;
106        let replaced = re.replace_all(&out, |caps: &regex::Captures| {
107            n += 1;
108            if class.keeps_leading_group {
109                format!(
110                    "{}{}",
111                    caps.get(1).map(|m| m.as_str()).unwrap_or(""),
112                    sentinel(class.name)
113                )
114            } else {
115                sentinel(class.name)
116            }
117        });
118        if n > 0 {
119            out = replaced.into_owned();
120            counts.insert(class.name, n);
121        }
122    }
123    (out, counts)
124}
125
126/// Per-class redaction counts as the export result carries them.
127#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
128pub struct RedactionCount {
129    pub class: String,
130    pub count: usize,
131}
132
133/// Fold per-string counts into the export result's list, in vocabulary
134/// order.
135pub fn tally(into: &mut BTreeMap<&'static str, usize>, counts: BTreeMap<&'static str, usize>) {
136    for (k, v) in counts {
137        *into.entry(k).or_insert(0) += v;
138    }
139}
140
141pub fn counts_to_list(counts: &BTreeMap<&'static str, usize>) -> Vec<RedactionCount> {
142    REDACTION_CLASSES
143        .iter()
144        .filter_map(|c| {
145            counts.get(c.name).map(|n| RedactionCount {
146                class: c.name.to_string(),
147                count: *n,
148            })
149        })
150        .collect()
151}
152
153#[cfg(test)]
154mod tests {
155    use super::*;
156
157    #[test]
158    fn redacts_each_class_to_its_sentinel_and_keeps_the_rest() {
159        // Assembled at runtime: this file is exempt from the leak scan for
160        // its patterns, but the scan's file-type-scoped class still reads
161        // it, so no literal below may match a class.
162        let input = format!(
163            "see {}/x.md and {} from {}/w; reads a {}/ path prefix",
164            ["dev", "plans"].join("/"),
165            ["mdgv", "io"].join("."),
166            ["/Users", "bjornbosenberg"].join("/"),
167            "graph"
168        );
169        let (out, counts) = redact(&input);
170        assert_eq!(
171            out,
172            "see [redacted:internal-refs]/x.md and [redacted:legacy-domain] from [redacted:absolute-user-paths]/w; reads a [redacted:excluded-private-dirs] path prefix"
173        );
174        let list = counts_to_list(&counts);
175        assert_eq!(
176            list.iter()
177                .map(|c| (c.class.as_str(), c.count))
178                .collect::<Vec<_>>(),
179            vec![
180                ("absolute-user-paths", 1),
181                ("internal-refs", 1),
182                ("excluded-private-dirs", 1),
183                ("legacy-domain", 1)
184            ]
185        );
186        let (clean, none) = redact("an ordinary note about engine-graph/ and memstead.io");
187        assert_eq!(
188            clean,
189            "an ordinary note about engine-graph/ and memstead.io"
190        );
191        assert!(none.is_empty());
192    }
193
194    /// One vocabulary: the classes here equal the leak scan's `scan` lines,
195    /// label and pattern verbatim. A class added to either side alone
196    /// fails naming it.
197    #[test]
198    fn vocabulary_equals_the_leak_scan_classes() {
199        let script =
200            std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join("../../scripts/leak-scan.sh");
201        let text = std::fs::read_to_string(&script)
202            .unwrap_or_else(|e| panic!("read {}: {e}", script.display()));
203        // `scan "label" '<pattern>'` — the pattern is a single-quoted shell
204        // word; the one class whose pattern carries a literal quote spells
205        // it as '"'"' and is folded back here.
206        let re = Regex::new(r#"(?m)^scan\s+"([a-z-]+)"\s+'((?:[^']|'"'"')+)'"#).unwrap();
207        let scanned: Vec<(String, String)> = re
208            .captures_iter(&text)
209            .map(|c| (c[1].to_string(), c[2].replace("'\"'\"'", "'")))
210            .collect();
211        assert!(
212            !scanned.is_empty(),
213            "no scan lines parsed from {}",
214            script.display()
215        );
216        let ours: Vec<(String, String)> = REDACTION_CLASSES
217            .iter()
218            .map(|c| (c.name.to_string(), c.pattern.to_string()))
219            .collect();
220        for (name, pattern) in &scanned {
221            let mine = ours.iter().find(|(n, _)| n == name).unwrap_or_else(|| {
222                panic!(
223                    "leak-scan class `{name}` is not in the engine's redaction vocabulary (ops/redaction.rs)"
224                )
225            });
226            assert_eq!(
227                &mine.1, pattern,
228                "class `{name}`: the engine's pattern differs from the leak scan's"
229            );
230        }
231        for (name, _) in &ours {
232            assert!(
233                scanned.iter().any(|(n, _)| n == name),
234                "engine redaction class `{name}` has no leak-scan `scan` line"
235            );
236        }
237        assert_eq!(scanned.len(), ours.len());
238    }
239}
memstead_base/ops/redaction.rs

memstead_base/ops/
redaction.rs
