1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
//! Explicit, deterministic preservation of user-marked spans across every
//! compressor (read, shell, proxy, prose).
//!
//! Two complementary, fully deterministic mechanisms (#498 — pure functions of
//! their inputs, no model, no clock, no global state):
//!
//! 1. **Universal markers** `<lc_safe>…</lc_safe>`: any compressor wraps its
//! work in [`compress_preserving`]; the content between the markers passes
//! through verbatim and the markers themselves are stripped from the output.
//! 2. **`protect` token list** (`ctx_read` convenience): [`line_is_protected`]
//! lets the line-based lossy filters (entropy / information-bottleneck)
//! force-keep every line that contains one of the given tokens.
//!
//! Security: callers MUST run secret redaction *before* protect (`redact →
//! protect → compress`). Protect never re-introduces redacted secrets because
//! it only ever passes through bytes that already survived redaction.
/// Opening marker for a verbatim-preserved span.
pub const SAFE_OPEN: &str = "<lc_safe>";
/// Closing marker for a verbatim-preserved span.
pub const SAFE_CLOSE: &str = "</lc_safe>";
/// Cheap pre-check: does the input contain at least one protect marker? Lets
/// hot paths skip the span-splitting machinery entirely when nothing is marked.
#[must_use]
pub fn has_markers(s: &str) -> bool {
s.contains(SAFE_OPEN)
}
/// Compress only the *unprotected* regions of `input` with `f`; everything
/// between `<lc_safe>` and `</lc_safe>` passes through byte-for-byte and the
/// markers are stripped from the output.
///
/// Deterministic: pure function of `(input, f)`. An unterminated open marker
/// keeps the remainder of the input verbatim (fail-safe: never compress what the
/// user tried to protect). When there are no markers the input is handed to `f`
/// unchanged, so existing callers keep their exact byte output.
#[must_use]
pub fn compress_preserving<F: Fn(&str) -> String>(input: &str, f: F) -> String {
if !has_markers(input) {
return f(input);
}
let mut out = String::with_capacity(input.len());
let mut rest = input;
while let Some(start) = rest.find(SAFE_OPEN) {
out.push_str(&f(&rest[..start]));
let after = &rest[start + SAFE_OPEN.len()..];
let Some(end) = after.find(SAFE_CLOSE) else {
out.push_str(after); // unterminated → keep remainder verbatim
return out;
};
out.push_str(&after[..end]); // verbatim, markers dropped
rest = &after[end + SAFE_CLOSE.len()..];
}
out.push_str(&f(rest));
out
}
/// True if `line` must survive a lossy line filter because it contains one of
/// the explicit `protect` tokens. Empty tokens are ignored so an empty list (or
/// a list of empty strings) reproduces today's behaviour exactly.
#[must_use]
pub fn line_is_protected(line: &str, needles: &[String]) -> bool {
needles
.iter()
.any(|n| !n.is_empty() && line.contains(n.as_str()))
}
/// Stable cache-key fragment for a `protect` token list, or `""` when the list
/// is empty (so unprotected reads keep their current cache key).
///
/// The fragment is order- and duplicate-independent: force-keep matching is a
/// set operation, so `["a","b"]` and `["b","a","a"]` must map to the same key
/// (#498). Tokens are canonicalised (non-empty, sorted, deduped) before hashing.
#[must_use]
pub fn protect_fragment(needles: &[String]) -> String {
let mut canon: Vec<&str> = needles
.iter()
.map(String::as_str)
.filter(|s| !s.is_empty())
.collect();
if canon.is_empty() {
return String::new();
}
canon.sort_unstable();
canon.dedup();
// NUL separator: cannot occur inside a realistic source token, so distinct
// token sets cannot collide by joining (e.g. ["ab","c"] vs ["a","bc"]).
let joined = canon.join("\u{0}");
format!("p{}", &crate::core::hasher::hash_short(&joined)[..8])
}
#[cfg(test)]
mod tests {
use super::*;
/// A maximally aggressive test compressor: it deletes *everything*. If a
/// span survives this, it survives any real compressor.
fn drop_all(_: &str) -> String {
String::new()
}
#[test]
fn no_markers_passes_input_to_f() {
assert_eq!(
compress_preserving("plain text", str::to_uppercase),
"PLAIN TEXT"
);
assert!(!has_markers("plain text"));
}
#[test]
fn protect_spans_survive_compression() {
let input =
"noise before\n<lc_safe>CRITICAL = 42\nkeep me literally</lc_safe>\nnoise after";
let out = compress_preserving(input, drop_all);
// Protected content is byte-identical and the markers are gone.
assert_eq!(out, "CRITICAL = 42\nkeep me literally");
assert!(!out.contains(SAFE_OPEN));
assert!(!out.contains(SAFE_CLOSE));
}
#[test]
fn unprotected_regions_are_compressed_protected_are_not() {
let f = |s: &str| s.to_uppercase();
let out = compress_preserving("a<lc_safe>b</lc_safe>c", f);
assert_eq!(out, "AbC");
}
#[test]
fn multiple_spans_all_survive() {
let input = "x<lc_safe>one</lc_safe>y<lc_safe>two</lc_safe>z";
let out = compress_preserving(input, drop_all);
assert_eq!(out, "onetwo");
}
#[test]
fn unterminated_marker_keeps_remainder_verbatim() {
let input = "drop<lc_safe>tail without close\nstill kept";
let out = compress_preserving(input, drop_all);
assert_eq!(out, "tail without close\nstill kept");
}
#[test]
fn compress_preserving_is_deterministic() {
let input = "a<lc_safe>SAFE</lc_safe>b<lc_safe>X</lc_safe>c";
let a = compress_preserving(input, str::to_uppercase);
let b = compress_preserving(input, str::to_uppercase);
assert_eq!(a, b);
}
#[test]
fn line_is_protected_matches_token_and_ignores_empty() {
let needles = vec!["TODO".to_string(), String::new()];
assert!(line_is_protected(" // TODO: fix", &needles));
assert!(!line_is_protected("nothing here", &needles));
// An empty needle must never match every line.
assert!(!line_is_protected("anything", &[String::new()]));
assert!(!line_is_protected("anything", &[]));
}
#[test]
fn protect_fragment_empty_is_blank() {
assert_eq!(protect_fragment(&[]), "");
assert_eq!(protect_fragment(&[String::new()]), "");
}
#[test]
fn protect_fragment_is_order_and_dup_independent() {
let a = protect_fragment(&["alpha".to_string(), "beta".to_string()]);
let b = protect_fragment(&["beta".to_string(), "alpha".to_string(), "alpha".to_string()]);
assert_eq!(a, b);
assert!(a.starts_with('p'));
assert_eq!(a.len(), 9); // 'p' + 8 hex chars
}
#[test]
fn protect_fragment_distinguishes_sets() {
let a = protect_fragment(&["alpha".to_string()]);
let b = protect_fragment(&["beta".to_string()]);
assert_ne!(a, b);
}
}