vyre_libs/scan/
post_process.rs

1//! Match post-processing: dedup, entropy, and confidence in one pass.
2//!
3//! The module is the canonical host reference for matcher output shaping.
4//! Consumers that need device-resident post-processing use the same field
5//! contract: sorted non-overlapping `(pattern_id, start, end)` spans plus
6//! deterministic entropy and confidence signals.
7
8use vyre_foundation::match_result::Match;
9use vyre_primitives::matching::region::{dedup_regions_inplace, RegionTriple};
10
11/// Post-processing contract violation.
12#[derive(Debug, Clone, Copy, PartialEq, Eq)]
13pub enum PostProcessError {
14    /// A match range does not fit inside the haystack that was scanned.
15    InvalidRange {
16        /// Pattern id attached to the invalid match.
17        pattern_id: u32,
18        /// Inclusive start byte offset.
19        start: u32,
20        /// Exclusive end byte offset.
21        end: u32,
22        /// Haystack length in bytes.
23        haystack_len: usize,
24    },
25}
26
27impl std::fmt::Display for PostProcessError {
28    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
29        match *self {
30            Self::InvalidRange {
31                pattern_id,
32                start,
33                end,
34                haystack_len,
35            } => write!(
36                f,
37                "match range is outside the scanned haystack: pattern_id={pattern_id}, start={start}, end={end}, haystack_len={haystack_len}. Fix: preserve matcher readback bounds and reject corrupt hit triples before scoring."
38            ),
39        }
40    }
41}
42
43impl std::error::Error for PostProcessError {}
44
45/// Output of [`try_reference_post_process`]. Carries the deduped match and the
46/// two derived signals every downstream consumer reads.
47#[derive(Debug, Clone, Copy, PartialEq)]
48pub struct PostProcessedMatch {
49    /// Pattern id from the original `Match`.
50    pub pattern_id: u32,
51    /// Inclusive start byte offset.
52    pub start: u32,
53    /// Exclusive end byte offset.
54    pub end: u32,
55    /// Shannon entropy in bits/byte over `haystack[start..end]`. `0.0`
56    /// for zero-width matches.
57    pub entropy_bits_per_byte: f32,
58    /// `[0.0, 1.0]` confidence score combining length + entropy.
59    /// Specifically `min(1, len/16) * (entropy / 8)`  -  the same
60    /// heuristic a scan consumer's per-match scorer applies. The factor of 16
61    /// matches the typical AKIA / ghp_ token width; entropy is
62    /// normalised against the 8 bits/byte ceiling for binary-uniform
63    /// data.
64    pub confidence: f32,
65}
66
67/// Fuse `dedup_regions_inplace`, entropy-per-span, and confidence into one
68/// Reference oracle pass over the input.
69///
70/// Returned vector is sorted by `(pid, start, end)` (the dedup
71/// post-condition). `haystack` is the same byte buffer the matcher scanned.
72///
73/// # Errors
74///
75/// Returns [`PostProcessError::InvalidRange`] if any deduped match points
76/// outside `haystack`.
77#[cfg(any(test, feature = "cpu-parity"))]
78pub fn try_reference_post_process(
79    matches: &[Match],
80    haystack: &[u8],
81) -> Result<Vec<PostProcessedMatch>, PostProcessError> {
82    let mut triples = Vec::new();
83    let mut out = Vec::new();
84    try_reference_post_process_into(matches, haystack, &mut triples, &mut out)?;
85    Ok(out)
86}
87
88/// Caller-owned variant of [`try_reference_post_process`].
89///
90/// Reuses `triples` and `out` across scans. This is the hot-path API for
91/// daemons and benchmark loops that post-process thousands of small readbacks.
92///
93/// # Errors
94///
95/// Returns [`PostProcessError::InvalidRange`] if any deduped match points
96/// outside `haystack`.
97#[cfg(any(test, feature = "cpu-parity"))]
98pub fn try_reference_post_process_into(
99    matches: &[Match],
100    haystack: &[u8],
101    triples: &mut Vec<RegionTriple>,
102    out: &mut Vec<PostProcessedMatch>,
103) -> Result<(), PostProcessError> {
104    triples.clear();
105    out.clear();
106    if matches.is_empty() {
107        return Ok(());
108    }
109
110    triples.reserve(matches.len());
111    triples.extend(
112        matches
113            .iter()
114            .map(|m| RegionTriple::new(m.pattern_id, m.start, m.end)),
115    );
116    dedup_regions_inplace(triples);
117
118    out.reserve(triples.len());
119    for &t in triples.iter() {
120        let s = t.start as usize;
121        let e = t.end as usize;
122        if e > haystack.len() || s > e {
123            out.clear();
124            return Err(PostProcessError::InvalidRange {
125                pattern_id: t.pid,
126                start: t.start,
127                end: t.end,
128                haystack_len: haystack.len(),
129            });
130        }
131        let bytes = &haystack[s..e];
132        let entropy = shannon_entropy_bits_per_byte(bytes);
133        let len_score = (bytes.len() as f32 / 16.0).min(1.0);
134        let entropy_score = entropy / 8.0;
135        let confidence = (len_score * entropy_score).clamp(0.0, 1.0);
136        out.push(PostProcessedMatch {
137            pattern_id: t.pid,
138            start: t.start,
139            end: t.end,
140            entropy_bits_per_byte: entropy,
141            confidence,
142        });
143    }
144    Ok(())
145}
146
147/// Infallible reference wrapper for callers whose matcher contract has
148/// already proved all ranges are within `haystack`.
149///
150/// Panics on corrupt match triples. Callers that need recoverable diagnostics
151/// use [`try_reference_post_process`].
152#[must_use]
153#[cfg(any(test, feature = "cpu-parity"))]
154pub fn reference_post_process(matches: &[Match], haystack: &[u8]) -> Vec<PostProcessedMatch> {
155    try_reference_post_process(matches, haystack).unwrap_or_else(|error| {
156        panic!("vyre-libs scan Reference oracle post-process contract failed: {error}")
157    })
158}
159
160/// Shannon entropy in bits/byte. Returns `0.0` on an empty slice. The
161/// implementation is straight `-sum(p_i log2 p_i)` over a 256-bucket
162/// histogram  -  match cost is dominated by the haystack scan, so a
163/// fixed stack histogram here is amortised on every realistic input.
164#[must_use]
165#[cfg(any(test, feature = "cpu-parity"))]
166pub fn shannon_entropy_bits_per_byte(bytes: &[u8]) -> f32 {
167    if bytes.is_empty() {
168        return 0.0;
169    }
170    let counts = vyre_primitives::text::byte_histogram::reference_byte_histogram(bytes);
171    let n = bytes.len() as f32;
172    let mut h = 0.0_f32;
173    for c in counts {
174        if c == 0 {
175            continue;
176        }
177        let p = c as f32 / n;
178        h -= p * p.log2();
179    }
180    h
181}
182
183#[cfg(test)]
184mod tests {
185    use super::*;
186
187    #[test]
188    fn into_reuses_scratch_and_matches_allocating_api() {
189        let haystack = b"AKIA1234567890ZZ";
190        let matches = [
191            Match::new(7, 0, 8),
192            Match::new(7, 0, 8),
193            Match::new(8, 4, 12),
194        ];
195
196        let expected = try_reference_post_process(&matches, haystack).unwrap();
197        let mut triples = Vec::with_capacity(16);
198        let triples_ptr = triples.as_ptr();
199        let mut out = Vec::with_capacity(16);
200        let out_ptr = out.as_ptr();
201
202        try_reference_post_process_into(&matches, haystack, &mut triples, &mut out).unwrap();
203
204        assert_eq!(out, expected);
205        assert_eq!(triples.as_ptr(), triples_ptr);
206        assert_eq!(out.as_ptr(), out_ptr);
207    }
208
209    #[test]
210    fn into_clears_outputs_on_empty_input() {
211        let mut triples = vec![RegionTriple::new(1, 0, 1)];
212        let mut out = vec![PostProcessedMatch {
213            pattern_id: 1,
214            start: 0,
215            end: 1,
216            entropy_bits_per_byte: 0.0,
217            confidence: 0.0,
218        }];
219
220        try_reference_post_process_into(&[], b"", &mut triples, &mut out).unwrap();
221
222        assert!(triples.is_empty());
223        assert!(out.is_empty());
224    }
225
226    #[test]
227    fn into_reports_invalid_ranges_without_partial_output() {
228        let mut triples = Vec::new();
229        let mut out = Vec::new();
230        let err = try_reference_post_process_into(
231            &[Match::new(1, 10, 12)],
232            b"short",
233            &mut triples,
234            &mut out,
235        )
236        .unwrap_err();
237
238        assert_eq!(
239            err,
240            PostProcessError::InvalidRange {
241                pattern_id: 1,
242                start: 10,
243                end: 12,
244                haystack_len: 5,
245            }
246        );
247        assert!(out.is_empty());
248    }
249
250    #[test]
251    #[should_panic(expected = "post-process contract failed")]
252    fn infallible_wrapper_panics_on_corrupt_ranges() {
253        let _ = reference_post_process(&[Match::new(1, 10, 12)], b"short");
254    }
255}
vyre_libs/scan/post_process.rs

vyre_libs/scan/
post_process.rs
