lean_ctx/core/context_quality/
retention.rs

1// SPDX-License-Identifier: Apache-2.0
2//! Retention probes: did the facts a reader needs survive a transformation?
3//!
4//! A probe is one atomic, deterministically extracted fact of the original text (an
5//! error code, a failing-test count, a `file:line` location, a path, a hash, a URL).
6//! Each probe resolves to [`Retention::Retained`] (present in what the model receives),
7//! [`Retention::Recoverable`] (absent, but an exact verified recovery path to the
8//! original exists), or [`Retention::Lost`]. A lost *critical* probe fails the check.
9//!
10//! Matching is by exact token, not substring: the delivered text is scanned with the
11//! same extractors and a probe is retained only if the same fact appears there as a
12//! whole token (`src/lib.rs:42` is not retained by `archive/src/lib.rs:42`). Atomic
13//! tokens are used instead of whole lines because compressors legitimately rewrite
14//! prose; the known reversible rewrites of terse compression (the auto-dictionary
15//! legend, `test result: FAILED` → `FAIL`) are resolved before matching.
16//!
17//! Lines that carry a detected secret are not probed at all: removing a secret is a
18//! security action, not context loss, and must never be reported as recoverable.
19
20use std::borrow::Cow;
21use std::collections::{BTreeSet, HashSet};
22use std::sync::OnceLock;
23
24use regex::Regex;
25use serde::{Deserialize, Serialize};
26
27/// Cap on non-critical probes per text (they only feed report detail).
28pub const MAX_PROBES: usize = 256;
29/// Cap on critical probes per text. Every critical fact up to this bound is checked;
30/// beyond it the check fails closed instead of sampling.
31pub const MAX_CRITICAL_PROBES: usize = 4096;
32
33/// What kind of fact a probe captures.
34#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)]
35#[serde(rename_all = "snake_case")]
36pub enum ProbeKind {
37    /// Compiler/runtime error code (`E0308`, `TS2345`, `ERROR-503`).
38    ErrorCode,
39    /// Test-runner outcome (`2 failed`, `test result: FAILED`).
40    TestResult,
41    /// Severity/status keyword on its line (`FAILED`, `panicked`, `OOMKilled`).
42    Status,
43    /// Source location with line number (`src/main.rs:42:5`).
44    Location,
45    /// File path.
46    Path,
47    /// Commit/content hash.
48    Hash,
49    Url,
50}
51
52#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)]
53#[serde(rename_all = "snake_case")]
54pub enum Criticality {
55    Informational,
56    Important,
57    Critical,
58}
59
60#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)]
61#[serde(rename_all = "snake_case")]
62pub enum Retention {
63    Retained,
64    Recoverable,
65    Lost,
66    NotApplicable,
67}
68
69#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord)]
70pub struct Probe {
71    pub kind: ProbeKind,
72    pub criticality: Criticality,
73    pub value: String,
74}
75
76/// How the full original can be obtained again if the delivered text dropped something.
77#[derive(Debug, Clone, Copy, PartialEq, Eq)]
78pub enum RecoveryPath {
79    /// No exact recovery: whatever is missing from the delivered text is lost.
80    None,
81    /// The original bytes are reachable through a recovery handle or source re-read
82    /// that was verified (resolves, digest matches) for this delivery.
83    Verified,
84}
85
86/// Per-criticality counts of probe outcomes.
87#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize)]
88pub struct RetentionCounts {
89    pub retained: usize,
90    pub recoverable: usize,
91    pub lost: usize,
92}
93
94impl RetentionCounts {
95    fn add(&mut self, r: Retention) {
96        match r {
97            Retention::Retained => self.retained += 1,
98            Retention::Recoverable => self.recoverable += 1,
99            Retention::Lost => self.lost += 1,
100            Retention::NotApplicable => {}
101        }
102    }
103
104    pub fn total(&self) -> usize {
105        self.retained + self.recoverable + self.lost
106    }
107}
108
109/// Outcome of probing one transformation.
110#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
111pub struct RetentionReport {
112    pub critical: RetentionCounts,
113    pub important: RetentionCounts,
114    pub informational: RetentionCounts,
115    /// Kinds of the critical probes that were lost (sorted, deduplicated). Values are
116    /// deliberately not recorded: receipts must not copy content.
117    pub lost_critical_kinds: Vec<ProbeKind>,
118    /// Probe-bearing lines skipped because they carry a detected secret (security, not
119    /// retention).
120    pub secret_lines_withheld: usize,
121    /// More than [`MAX_PROBES`] non-critical probes existed; the rest was not checked.
122    pub truncated: bool,
123    /// More than [`MAX_CRITICAL_PROBES`] critical facts existed. They could not all be
124    /// checked, so the report fails closed.
125    #[serde(default)]
126    pub critical_unchecked: bool,
127}
128
129impl RetentionReport {
130    /// The hard invariant: no critical fact may be lost without a recovery path, and
131    /// critical facts that were not checked count as a failure, never as a pass.
132    pub fn passes(&self) -> bool {
133        self.critical.lost == 0 && !self.critical_unchecked
134    }
135}
136
137/// Probes of one text plus the extraction bookkeeping.
138#[derive(Debug, Clone, Default, PartialEq, Eq)]
139pub struct Extraction {
140    pub probes: Vec<Probe>,
141    pub truncated: bool,
142    pub critical_unchecked: bool,
143    pub secret_lines: usize,
144}
145
146fn re(cell: &'static OnceLock<Regex>, pattern: &str) -> &'static Regex {
147    cell.get_or_init(|| Regex::new(pattern).expect("static retention probe regex"))
148}
149
150fn error_code_re() -> &'static Regex {
151    static R: OnceLock<Regex> = OnceLock::new();
152    re(
153        &R,
154        r"(?-u:\b)(?:E\d{4}|TS\d{4}|CS\d{4}|[A-Z]{2,12}[-_]\d{2,6}|[A-Z]{3,}\d{3,5})(?-u:\b)",
155    )
156}
157
158fn test_result_re() -> &'static Regex {
159    static R: OnceLock<Regex> = OnceLock::new();
160    re(
161        &R,
162        r"(?i)(?-u:\b)test result: (?:ok|failed)(?-u:\b)|(?-u:\b)\d+ (?:passed|failed|failing|errors?|skipped)(?-u:\b)",
163    )
164}
165
166fn status_re() -> &'static Regex {
167    static R: OnceLock<Regex> = OnceLock::new();
168    re(
169        &R,
170        r"(?-u:\b)(?:FAILED|FAILURE|FATAL|CRITICAL|PANIC|panicked|OOMKilled|SIGSEGV|SIGKILL|Segmentation fault|CrashLoopBackOff|CVE-\d{4}-\d{4,})(?-u:\b)",
171    )
172}
173
174fn location_re() -> &'static Regex {
175    static R: OnceLock<Regex> = OnceLock::new();
176    re(&R, r"[\w./\\-]+\.[A-Za-z]{1,6}:\d+(?::\d+)?")
177}
178
179fn path_re() -> &'static Regex {
180    static R: OnceLock<Regex> = OnceLock::new();
181    re(&R, r"(?:[\w.-]+/)+[\w.-]+\.[A-Za-z0-9]{1,6}(?-u:\b)")
182}
183
184fn hash_re() -> &'static Regex {
185    static R: OnceLock<Regex> = OnceLock::new();
186    re(&R, r"(?-u:\b)[0-9a-f]{12,64}(?-u:\b)")
187}
188
189fn url_re() -> &'static Regex {
190    static R: OnceLock<Regex> = OnceLock::new();
191    re(&R, r#"https?://[^\s"'<>)\]]+"#)
192}
193
194/// Abbreviations terse's Cargo dictionary applies to runner summaries.
195fn dictionary_outcome_re() -> &'static Regex {
196    static R: OnceLock<Regex> = OnceLock::new();
197    re(&R, r"(?-u:\b)(?:FAIL|PASS)(?-u:\b)")
198}
199
200/// Whether a line reports a problem, which upgrades codes/locations on it to critical.
201fn is_problem_line(line: &str) -> bool {
202    status_re().is_match(line) || {
203        let lower = line.to_ascii_lowercase();
204        lower.contains("error") || lower.contains("fail") || lower.contains("panic")
205    }
206}
207
208/// Upper bound on hits per extractor and line, so one pathological line cannot
209/// allocate without bound. Overflow is reported, never silently dropped.
210const MAX_HITS_PER_LINE: usize = 64;
211
212/// Every fact token on one line, with its kind (criticality is decided by the
213/// caller), plus whether any extractor hit [`MAX_HITS_PER_LINE`].
214fn scan_line(line: &str) -> (Vec<(ProbeKind, &str)>, bool) {
215    let extractors: [(ProbeKind, &Regex); 7] = [
216        (ProbeKind::TestResult, test_result_re()),
217        (ProbeKind::Status, status_re()),
218        (ProbeKind::ErrorCode, error_code_re()),
219        (ProbeKind::Location, location_re()),
220        (ProbeKind::Path, path_re()),
221        (ProbeKind::Hash, hash_re()),
222        (ProbeKind::Url, url_re()),
223    ];
224    let mut out = Vec::new();
225    let mut overflow = false;
226    for (kind, re) in extractors {
227        let mut hits = 0usize;
228        for m in re.find_iter(line) {
229            let v = m.as_str();
230            if kind == ProbeKind::Hash
231                && !(v.bytes().any(|b| b.is_ascii_digit())
232                    && v.bytes().any(|b| b.is_ascii_alphabetic()))
233            {
234                continue;
235            }
236            if hits == MAX_HITS_PER_LINE {
237                overflow = true;
238                break;
239            }
240            hits += 1;
241            out.push((kind, v));
242        }
243    }
244    (out, overflow)
245}
246
247/// Canonical form used for matching: runner summaries and statuses may change case
248/// (`FAILED` / `failed`), every other fact must match exactly.
249fn canonical(kind: ProbeKind, value: &str) -> String {
250    match kind {
251        ProbeKind::TestResult | ProbeKind::Status => value.to_ascii_lowercase(),
252        _ => value.to_string(),
253    }
254}
255
256fn criticality_of(kind: ProbeKind, problem_line: bool) -> Criticality {
257    match kind {
258        ProbeKind::TestResult | ProbeKind::Status => Criticality::Critical,
259        ProbeKind::ErrorCode | ProbeKind::Location if problem_line => Criticality::Critical,
260        _ => Criticality::Important,
261    }
262}
263
264/// Extracts the probes of `text`, sorted and deduplicated.
265///
266/// The whole text is scanned: the facts that matter most (a test summary, the final
267/// error) are often on the last lines. All critical probes are kept up to
268/// [`MAX_CRITICAL_PROBES`]; past that bound extraction reports `critical_unchecked`.
269pub fn extract_probes(text: &str) -> Extraction {
270    let mut ex = Extraction::default();
271    let mut critical: BTreeSet<Probe> = BTreeSet::new();
272    let mut other: BTreeSet<Probe> = BTreeSet::new();
273
274    for line in text.lines() {
275        let (hits, overflow) = scan_line(line);
276        if overflow {
277            // Facts on this line went unchecked: fail closed and stop scanning.
278            ex.critical_unchecked = true;
279            break;
280        }
281        if hits.is_empty() {
282            continue;
283        }
284        // Secret detection only runs on lines that would be probed, keeping the
285        // check cheap on large outputs.
286        if !crate::core::secret_detection::detect_secrets(line).is_empty() {
287            ex.secret_lines += 1;
288            continue;
289        }
290        let problem = is_problem_line(line);
291        for (kind, value) in hits {
292            let probe = Probe {
293                kind,
294                criticality: criticality_of(kind, problem),
295                value: value.to_string(),
296            };
297            if probe.criticality == Criticality::Critical {
298                if critical.len() < MAX_CRITICAL_PROBES {
299                    critical.insert(probe);
300                } else if !critical.contains(&probe) {
301                    ex.critical_unchecked = true;
302                    break;
303                }
304            } else if other.len() < MAX_PROBES {
305                other.insert(probe);
306            } else if !other.contains(&probe) {
307                ex.truncated = true;
308            }
309        }
310        if ex.critical_unchecked {
311            // The result already fails closed; scanning on is wasted work.
312            break;
313        }
314    }
315
316    // A value captured under several kinds/criticalities keeps only its strongest entry.
317    let mut ordered: Vec<Probe> = critical.into_iter().chain(other).collect();
318    ordered.sort_by(|a, b| b.criticality.cmp(&a.criticality).then(a.cmp(b)));
319    let mut seen: HashSet<String> = HashSet::new();
320    ordered.retain(|p| seen.insert(p.value.clone()));
321    ordered.sort();
322    ex.probes = ordered;
323    ex
324}
325
326/// The delivered text with terse's reversible auto-dictionary legend resolved.
327fn resolve_reversible_rewrites(delivered: &str) -> Cow<'_, str> {
328    match crate::core::terse::auto_dict::expand(delivered) {
329        Some(expanded) => Cow::Owned(expanded),
330        None => Cow::Borrowed(delivered),
331    }
332}
333
334/// Canonical fact tokens present in what the model receives.
335fn delivered_facts(delivered: &str) -> HashSet<(ProbeKind, String)> {
336    let text = resolve_reversible_rewrites(delivered);
337    let mut facts = HashSet::new();
338    for line in text.lines() {
339        for (kind, value) in scan_line(line).0 {
340            facts.insert((kind, canonical(kind, value)));
341        }
342        for m in dictionary_outcome_re().find_iter(line) {
343            if m.as_str() == "FAIL" {
344                facts.insert((ProbeKind::TestResult, "test result: failed".into()));
345                facts.insert((ProbeKind::Status, "failed".into()));
346            } else {
347                facts.insert((ProbeKind::TestResult, "test result: ok".into()));
348            }
349        }
350    }
351    facts
352}
353
354/// Probes `original` and checks each fact against what the model receives.
355pub fn assess(original: &str, delivered: &str, recovery: RecoveryPath) -> RetentionReport {
356    let ex = extract_probes(original);
357    let facts = delivered_facts(delivered);
358    let mut report = RetentionReport {
359        critical: RetentionCounts::default(),
360        important: RetentionCounts::default(),
361        informational: RetentionCounts::default(),
362        lost_critical_kinds: Vec::new(),
363        secret_lines_withheld: ex.secret_lines,
364        truncated: ex.truncated,
365        critical_unchecked: ex.critical_unchecked,
366    };
367    let mut lost_kinds: BTreeSet<ProbeKind> = BTreeSet::new();
368    for probe in &ex.probes {
369        let r = if facts.contains(&(probe.kind, canonical(probe.kind, &probe.value))) {
370            Retention::Retained
371        } else if recovery == RecoveryPath::Verified {
372            Retention::Recoverable
373        } else {
374            Retention::Lost
375        };
376        match probe.criticality {
377            Criticality::Critical => {
378                report.critical.add(r);
379                if r == Retention::Lost {
380                    lost_kinds.insert(probe.kind);
381                }
382            }
383            Criticality::Important => report.important.add(r),
384            Criticality::Informational => report.informational.add(r),
385        }
386    }
387    report.lost_critical_kinds = lost_kinds.into_iter().collect();
388    report
389}
390
391#[cfg(test)]
392mod tests {
393    use super::*;
394
395    const CARGO_FAILURE: &str = "\
396   Compiling app v0.1.0
397error[E0308]: mismatched types
398  --> src/handlers/order.rs:42:17
399running 12 tests
400test orders::totals ... FAILED
401thread 'orders::totals' panicked at src/handlers/order.rs:88:9
402test result: FAILED. 11 passed; 1 failed; 0 ignored
403see https://doc.rust-lang.org/error_codes/E0308.html
404commit 3f9c2a7b81d4e6f0
405";
406
407    fn kinds_of(text: &str, kind: ProbeKind) -> Vec<String> {
408        extract_probes(text)
409            .probes
410            .into_iter()
411            .filter(|p| p.kind == kind)
412            .map(|p| p.value)
413            .collect()
414    }
415
416    #[test]
417    fn extracts_the_facts_a_reader_of_a_failed_build_needs() {
418        assert_eq!(kinds_of(CARGO_FAILURE, ProbeKind::ErrorCode), vec!["E0308"]);
419        assert!(kinds_of(CARGO_FAILURE, ProbeKind::TestResult).contains(&"1 failed".into()));
420        assert!(
421            kinds_of(CARGO_FAILURE, ProbeKind::Location)
422                .contains(&"src/handlers/order.rs:88:9".into())
423        );
424        assert_eq!(
425            kinds_of(CARGO_FAILURE, ProbeKind::Hash),
426            vec!["3f9c2a7b81d4e6f0"]
427        );
428        let probes = extract_probes(CARGO_FAILURE).probes;
429        let e0308 = probes.iter().find(|p| p.value == "E0308").unwrap();
430        assert_eq!(
431            e0308.criticality,
432            Criticality::Critical,
433            "code on an error line"
434        );
435    }
436
437    #[test]
438    fn identity_retains_everything_and_passes() {
439        let r = assess(CARGO_FAILURE, CARGO_FAILURE, RecoveryPath::None);
440        assert!(r.passes());
441        assert_eq!(r.critical.lost + r.important.lost, 0);
442        assert!(r.critical.retained >= 5, "{r:?}");
443    }
444
445    #[test]
446    fn dropping_the_failure_lines_is_a_critical_loss() {
447        let summarized = "Compiling app v0.1.0\nrunning 12 tests\n11 passed\n";
448        let r = assess(CARGO_FAILURE, summarized, RecoveryPath::None);
449        assert!(!r.passes());
450        assert!(r.lost_critical_kinds.contains(&ProbeKind::ErrorCode));
451        assert!(r.lost_critical_kinds.contains(&ProbeKind::TestResult));
452    }
453
454    #[test]
455    fn a_location_inside_a_different_path_is_not_retained() {
456        let original = "error[E0308]: mismatched types at src/lib.rs:42\n";
457        let moved = "error[E0308]: mismatched types at archive/src/lib.rs:42.old\n";
458        let r = assess(original, moved, RecoveryPath::None);
459        assert!(!r.passes(), "pointing at another file is a loss: {r:?}");
460        assert_eq!(r.lost_critical_kinds, vec![ProbeKind::Location]);
461    }
462
463    #[test]
464    fn missing_facts_with_a_verified_recovery_path_are_recoverable_not_lost() {
465        let r = assess(CARGO_FAILURE, "build output elided", RecoveryPath::Verified);
466        assert!(r.passes());
467        assert_eq!(r.critical.lost, 0);
468        assert!(r.critical.recoverable > 0);
469    }
470
471    #[test]
472    fn terse_reversible_rewrites_do_not_count_as_loss() {
473        let original = "warning: unused variable in src/config/env.rs:42\n\
474                        error[E0425]: not found in deploy 3f9c2a7b81d4e6f0\n\
475                        retry 3f9c2a7b81d4e6f0 failed\n\
476                        rollback 3f9c2a7b81d4e6f0\n\
477                        test result: FAILED. 3 passed; 2 failed\n";
478        let filtered = original.replace("warning", "W").replace("variable", "var");
479        // Run the real terse rewrites: Cargo dictionary (FAIL) and auto-dictionary
480        // (the repeated hash becomes @D0 plus a legend line).
481        let dict = crate::core::terse::dictionaries::apply_dictionaries(
482            &filtered,
483            crate::core::terse::dictionaries::DictLevel::Full,
484        );
485        let delivered = crate::core::terse::auto_dict::apply(&dict).expect("hash repeats");
486        assert!(
487            delivered.starts_with("[dict: @D0=3f9c2a7b81d4e6f0]"),
488            "{delivered}"
489        );
490        assert!(delivered.contains("FAIL."), "{delivered}");
491        let r = assess(original, &delivered, RecoveryPath::None);
492        assert!(r.passes(), "{r:?}\n{delivered}");
493    }
494
495    #[test]
496    fn secret_lines_are_withheld_not_probed() {
497        let key = format!("{}{}", concat!("AK", "IA"), "Q".repeat(16));
498        let original = format!("export AWS_ACCESS_KEY_ID={key} # see src/env/aws.rs:3\nok\n");
499        let r = assess(
500            &original,
501            "export AWS_ACCESS_KEY_ID=[REDACTED]\nok\n",
502            RecoveryPath::None,
503        );
504        assert_eq!(r.secret_lines_withheld, 1);
505        assert!(r.passes(), "a redacted secret is not context loss: {r:?}");
506        assert_eq!(r.important.lost, 0, "nothing on the secret line is probed");
507    }
508
509    #[test]
510    fn every_critical_fact_of_a_long_log_is_checked() {
511        let mut log = String::from("error[E0433]: failed to resolve: use of undeclared crate\n");
512        for i in 0..3000 {
513            log.push_str(&format!("error: unused import at src/gen/m{i}.rs:{i}:1\n"));
514        }
515        log.push_str("test result: FAILED. 12 passed; 3 failed\n");
516        let ex = extract_probes(&log);
517        assert!(!ex.critical_unchecked);
518        // Dropping one line from the middle is caught, not sampled away.
519        let without_middle = log.replace("src/gen/m1500.rs:1500:1", "");
520        assert!(!assess(&log, &without_middle, RecoveryPath::None).passes());
521        let without_summary = log.replace("test result: FAILED. 12 passed; 3 failed\n", "");
522        assert!(!assess(&log, &without_summary, RecoveryPath::None).passes());
523    }
524
525    #[test]
526    fn critical_overflow_fails_closed() {
527        let log = (0..=MAX_CRITICAL_PROBES)
528            .map(|i| format!("error at src/m{i}.rs:{i}"))
529            .collect::<Vec<_>>()
530            .join("\n");
531        let r = assess(&log, &log, RecoveryPath::None);
532        assert!(r.critical_unchecked);
533        assert_eq!(r.critical.lost, 0);
534        assert!(!r.passes(), "unchecked critical facts never pass");
535
536        // One pathological line: per-line work is bounded and the overflow fails
537        // closed instead of dropping facts silently.
538        let line = (0..=MAX_HITS_PER_LINE)
539            .map(|i| format!("E{:04}", 1000 + i))
540            .collect::<Vec<_>>()
541            .join(" error ");
542        let r = assess(&line, &line, RecoveryPath::None);
543        assert!(r.critical_unchecked);
544        assert!(!r.passes());
545    }
546}
lean_ctx/core/context_quality/retention.rs

lean_ctx/core/context_quality/
retention.rs
