Skip to main content

tokencompact_core/
session.rs

1use std::collections::{BTreeMap, BTreeSet};
2use std::error::Error;
3use std::fmt::{Display, Formatter};
4
5use crate::{
6    Diagnostic, EndReason, LogLine, OutputPolicy, Provenance, Reduction, ReductionStats, Scope,
7    SessionOptions, Stream, TestFailure, finalize_findings, normalize_terminal_text,
8};
9
10/// Parser lifecycle notification for a caller-owned scope.
11#[derive(Clone, Debug)]
12pub enum ScopeBoundary<'a> {
13    Begin(&'a Scope),
14    End {
15        scope: &'a Scope,
16        reason: EndReason,
17        input_truncated: bool,
18    },
19}
20
21/// Deterministic, synchronous parser contract.
22pub trait Parser: Send {
23    fn id(&self) -> &'static str;
24
25    fn observe(&mut self, line: &LogLine<'_>, emitter: &mut Emitter<'_>);
26
27    fn boundary(&mut self, _boundary: ScopeBoundary<'_>, _emitter: &mut Emitter<'_>) {}
28
29    fn finish(&mut self, _emitter: &mut Emitter<'_>) {}
30}
31
32/// Immutable ordered set of trusted native parsers.
33#[derive(Default)]
34pub struct ParserPlan {
35    parsers: Vec<Box<dyn Parser>>,
36    ids: BTreeSet<&'static str>,
37}
38
39impl ParserPlan {
40    #[must_use]
41    pub fn new() -> Self {
42        Self::default()
43    }
44
45    pub fn push<P: Parser + 'static>(&mut self, parser: P) -> Result<(), ParserPlanError> {
46        let id = parser.id();
47        if id.is_empty() {
48            return Err(ParserPlanError::EmptyId);
49        }
50        if !self.ids.insert(id) {
51            return Err(ParserPlanError::DuplicateId(id));
52        }
53        self.parsers.push(Box::new(parser));
54        Ok(())
55    }
56
57    #[must_use]
58    pub fn len(&self) -> usize {
59        self.parsers.len()
60    }
61
62    #[must_use]
63    pub fn is_empty(&self) -> bool {
64        self.parsers.is_empty()
65    }
66}
67
68/// Invalid immutable parser-plan construction.
69#[derive(Clone, Copy, Debug, Eq, PartialEq)]
70pub enum ParserPlanError {
71    EmptyId,
72    DuplicateId(&'static str),
73}
74
75impl Display for ParserPlanError {
76    fn fmt(&self, formatter: &mut Formatter<'_>) -> std::fmt::Result {
77        match self {
78            Self::EmptyId => formatter.write_str("parser identifiers must not be empty"),
79            Self::DuplicateId(id) => write!(formatter, "duplicate parser identifier {id:?}"),
80        }
81    }
82}
83
84impl Error for ParserPlanError {}
85
86#[derive(Default)]
87struct CandidateStore {
88    diagnostics: Vec<Diagnostic>,
89    test_failures: Vec<TestFailure>,
90    dropped: u64,
91}
92
93/// Bounded parser output sink. Contextual provenance is supplied by the engine.
94pub struct Emitter<'a> {
95    store: &'a mut CandidateStore,
96    maximum: usize,
97    provenance: Option<Provenance>,
98}
99
100impl Emitter<'_> {
101    pub fn diagnostic(&mut self, mut diagnostic: Diagnostic) {
102        if self
103            .store
104            .diagnostics
105            .len()
106            .saturating_add(self.store.test_failures.len())
107            >= self.maximum
108        {
109            self.store.dropped = self.store.dropped.saturating_add(1);
110            return;
111        }
112        if diagnostic.provenance.is_none() {
113            diagnostic.provenance = self.provenance.clone();
114        }
115        self.store.diagnostics.push(diagnostic);
116    }
117
118    pub fn test_failure(&mut self, mut failure: TestFailure) {
119        if self
120            .store
121            .diagnostics
122            .len()
123            .saturating_add(self.store.test_failures.len())
124            >= self.maximum
125        {
126            self.store.dropped = self.store.dropped.saturating_add(1);
127            return;
128        }
129        if failure.provenance.is_none() {
130            failure.provenance = self.provenance.clone();
131        }
132        self.store.test_failures.push(failure);
133    }
134}
135
136#[derive(Clone, Debug)]
137struct ScopeState {
138    scope: Scope,
139    retained_bytes: usize,
140    truncated: bool,
141    stream_lines: BTreeMap<Stream, u64>,
142}
143
144#[derive(Clone, Debug, Default)]
145struct LineBuffer {
146    bytes: Vec<u8>,
147    truncated: bool,
148    previous_was_cr: bool,
149}
150
151/// Incremental, chunk-invariant reduction session.
152pub struct ReductionSession<'a> {
153    plan: ParserPlan,
154    options: SessionOptions,
155    policy: OutputPolicy<'a>,
156    scopes: BTreeMap<String, ScopeState>,
157    buffers: BTreeMap<(String, Stream), LineBuffer>,
158    candidates: CandidateStore,
159    stats: ReductionStats,
160    next_ordinal: u64,
161}
162
163impl<'a> ReductionSession<'a> {
164    #[must_use]
165    pub fn new(plan: ParserPlan, options: SessionOptions, policy: OutputPolicy<'a>) -> Self {
166        Self {
167            plan,
168            options,
169            policy,
170            scopes: BTreeMap::new(),
171            buffers: BTreeMap::new(),
172            candidates: CandidateStore::default(),
173            stats: ReductionStats::default(),
174            next_ordinal: 0,
175        }
176    }
177
178    pub fn begin_scope(&mut self, scope: Scope) -> bool {
179        if self.scopes.contains_key(&scope.id)
180            || self.scopes.len() >= self.options.limits.max_scopes
181        {
182            self.stats.candidates_dropped = self.stats.candidates_dropped.saturating_add(1);
183            return false;
184        }
185        let id = scope.id.clone();
186        self.scopes.insert(
187            id,
188            ScopeState {
189                scope: scope.clone(),
190                retained_bytes: 0,
191                truncated: false,
192                stream_lines: BTreeMap::new(),
193            },
194        );
195        self.stats.scopes_started = self.stats.scopes_started.saturating_add(1);
196        self.dispatch_boundary(ScopeBoundary::Begin(&scope));
197        true
198    }
199
200    pub fn push_chunk(&mut self, scope_id: &str, stream: Stream, chunk: &[u8]) -> bool {
201        let Some(state) = self.scopes.get_mut(scope_id) else {
202            return false;
203        };
204        self.stats.bytes_seen = self
205            .stats
206            .bytes_seen
207            .saturating_add(u64::try_from(chunk.len()).unwrap_or(u64::MAX));
208        let available = self
209            .options
210            .limits
211            .max_scope_bytes
212            .saturating_sub(state.retained_bytes);
213        let retained = chunk.len().min(available);
214        state.retained_bytes = state.retained_bytes.saturating_add(retained);
215        self.stats.bytes_retained = self
216            .stats
217            .bytes_retained
218            .saturating_add(u64::try_from(retained).unwrap_or(u64::MAX));
219        if retained < chunk.len() {
220            state.truncated = true;
221        }
222
223        let key = (scope_id.to_owned(), stream);
224        let buffer = self.buffers.entry(key).or_default();
225        let mut completed = Vec::<(Vec<u8>, bool)>::new();
226        for byte in &chunk[..retained] {
227            let boundary = *byte == b'\n' || *byte == b'\r';
228            if boundary {
229                if *byte == b'\n' && buffer.previous_was_cr {
230                    buffer.previous_was_cr = false;
231                    continue;
232                }
233                completed.push((std::mem::take(&mut buffer.bytes), buffer.truncated));
234                buffer.truncated = false;
235                buffer.previous_was_cr = *byte == b'\r';
236            } else {
237                buffer.previous_was_cr = false;
238                if buffer.bytes.len() < self.options.limits.max_line_bytes {
239                    buffer.bytes.push(*byte);
240                } else {
241                    buffer.truncated = true;
242                }
243            }
244        }
245        for (bytes, truncated) in completed {
246            self.dispatch_bytes(scope_id, stream, bytes, truncated);
247        }
248        true
249    }
250
251    /// Adds a trusted structured finding to the same redaction and budget path.
252    pub fn emit_structured(&mut self, scope_id: &str, mut diagnostic: Diagnostic) -> bool {
253        let Some(state) = self.scopes.get(scope_id) else {
254            return false;
255        };
256        if diagnostic.provenance.is_none() {
257            diagnostic.provenance = Some(
258                Provenance::new("structured")
259                    .with_scope(&state.scope)
260                    .with_parser("adapter"),
261            );
262        }
263        let mut emitter = Emitter {
264            store: &mut self.candidates,
265            maximum: self.options.limits.max_candidates,
266            provenance: None,
267        };
268        emitter.diagnostic(diagnostic);
269        true
270    }
271
272    pub fn end_scope(&mut self, scope_id: &str, reason: EndReason) -> bool {
273        let Some(mut state) = self.scopes.remove(scope_id) else {
274            return false;
275        };
276        let keys = self
277            .buffers
278            .keys()
279            .filter(|(id, _)| id == scope_id)
280            .cloned()
281            .collect::<Vec<_>>();
282        for key in keys {
283            if let Some(buffer) = self.buffers.remove(&key)
284                && (!buffer.bytes.is_empty() || buffer.truncated)
285            {
286                self.scopes.insert(scope_id.to_owned(), state.clone());
287                self.dispatch_bytes(scope_id, key.1, buffer.bytes, buffer.truncated);
288                state = self
289                    .scopes
290                    .remove(scope_id)
291                    .expect("temporary scope state must remain present");
292            }
293        }
294        let effective_reason = if state.truncated && reason == EndReason::Complete {
295            EndReason::Truncated
296        } else {
297            reason
298        };
299        self.dispatch_boundary(ScopeBoundary::End {
300            scope: &state.scope,
301            reason: effective_reason,
302            input_truncated: state.truncated,
303        });
304        self.stats.scopes_completed = self.stats.scopes_completed.saturating_add(1);
305        true
306    }
307
308    #[must_use]
309    pub fn finish(mut self) -> Reduction {
310        let scope_ids = self.scopes.keys().cloned().collect::<Vec<_>>();
311        for scope_id in scope_ids {
312            self.end_scope(&scope_id, EndReason::Interrupted);
313        }
314        for parser in &mut self.plan.parsers {
315            let provenance = Some(Provenance::new("parser").with_parser(parser.id()));
316            let mut emitter = Emitter {
317                store: &mut self.candidates,
318                maximum: self.options.limits.max_candidates,
319                provenance,
320            };
321            parser.finish(&mut emitter);
322        }
323        self.stats.candidates_emitted = u64::try_from(
324            self.candidates
325                .diagnostics
326                .len()
327                .saturating_add(self.candidates.test_failures.len()),
328        )
329        .unwrap_or(u64::MAX);
330        self.stats.candidates_dropped = self
331            .stats
332            .candidates_dropped
333            .saturating_add(self.candidates.dropped);
334        finalize_findings(
335            self.candidates.diagnostics,
336            self.candidates.test_failures,
337            self.options.budget,
338            self.policy,
339            self.stats,
340        )
341    }
342
343    fn dispatch_bytes(&mut self, scope_id: &str, stream: Stream, bytes: Vec<u8>, truncated: bool) {
344        let normalized = normalize_terminal_text(&bytes);
345        if normalized.is_empty() {
346            return;
347        }
348        for text in normalized.lines() {
349            let Some(state) = self.scopes.get_mut(scope_id) else {
350                return;
351            };
352            let stream_line = state.stream_lines.entry(stream).or_default();
353            *stream_line = stream_line.saturating_add(1);
354            let stream_line = *stream_line;
355            let scope = state.scope.clone();
356            let ordinal = self.next_ordinal;
357            self.next_ordinal = self.next_ordinal.saturating_add(1);
358            self.stats.lines_seen = self.stats.lines_seen.saturating_add(1);
359            if truncated {
360                self.stats.truncated_lines = self.stats.truncated_lines.saturating_add(1);
361                state.truncated = true;
362            }
363            let line = LogLine {
364                scope: &scope,
365                stream,
366                ordinal,
367                stream_line,
368                text,
369                truncated,
370            };
371            for parser in &mut self.plan.parsers {
372                let provenance = Some(
373                    Provenance::new(stream_name(stream))
374                        .with_scope(&scope)
375                        .with_span(stream, stream_line, stream_line)
376                        .with_parser(parser.id()),
377                );
378                let mut emitter = Emitter {
379                    store: &mut self.candidates,
380                    maximum: self.options.limits.max_candidates,
381                    provenance,
382                };
383                parser.observe(&line, &mut emitter);
384            }
385        }
386    }
387
388    fn dispatch_boundary(&mut self, boundary: ScopeBoundary<'_>) {
389        for parser in &mut self.plan.parsers {
390            let scope = match &boundary {
391                ScopeBoundary::Begin(scope) | ScopeBoundary::End { scope, .. } => scope,
392            };
393            let provenance = Some(
394                Provenance::new("boundary")
395                    .with_scope(scope)
396                    .with_parser(parser.id()),
397            );
398            let mut emitter = Emitter {
399                store: &mut self.candidates,
400                maximum: self.options.limits.max_candidates,
401                provenance,
402            };
403            parser.boundary(boundary.clone(), &mut emitter);
404        }
405    }
406}
407
408fn stream_name(stream: Stream) -> &'static str {
409    match stream {
410        Stream::Stdout => "stdout",
411        Stream::Stderr => "stderr",
412        Stream::Combined => "combined",
413        Stream::Annotation => "annotation",
414        Stream::Structured => "structured",
415    }
416}
417
418#[cfg(test)]
419mod tests {
420    use super::*;
421    use crate::{
422        Budget, DiagnosticClass, EvidenceQuality, GenericRanker, NoPathMapping, NoRedaction,
423        OutputPolicy, Severity,
424    };
425
426    #[derive(Default)]
427    struct ErrorParser;
428
429    impl Parser for ErrorParser {
430        fn id(&self) -> &'static str {
431            "test.error"
432        }
433
434        fn observe(&mut self, line: &LogLine<'_>, emitter: &mut Emitter<'_>) {
435            if line.text.contains("error:") {
436                emitter.diagnostic(Diagnostic {
437                    severity: Severity::Error,
438                    class: DiagnosticClass::Tool,
439                    code: Some("test.error".to_owned()),
440                    message: line.text.to_owned(),
441                    location: None,
442                    provenance: None,
443                    quality: EvidenceQuality::Structured,
444                    repetition_count: 1,
445                });
446            }
447        }
448    }
449
450    fn run(chunks: &[&[u8]]) -> Reduction {
451        let mut plan = ParserPlan::new();
452        plan.push(ErrorParser).unwrap();
453        let mut session = ReductionSession::new(
454            plan,
455            SessionOptions {
456                budget: Budget::unbounded(),
457                ..SessionOptions::default()
458            },
459            OutputPolicy::new(&NoRedaction, &NoPathMapping, &GenericRanker),
460        );
461        assert!(session.begin_scope(Scope::step("compile")));
462        for chunk in chunks {
463            assert!(session.push_chunk("compile", Stream::Stderr, chunk));
464        }
465        assert!(session.end_scope("compile", EndReason::Complete));
466        session.finish()
467    }
468
469    #[test]
470    fn arbitrary_chunking_produces_the_same_findings() {
471        let whole = run(&[b"note\nerror: broken\n"]);
472        let split = run(&[b"no", b"te\ner", b"ror: bro", b"ken\n"]);
473        assert_eq!(whole.diagnostics, split.diagnostics);
474        assert_eq!(whole.diagnostics.len(), 1);
475    }
476
477    #[test]
478    fn enforces_scope_and_line_bounds() {
479        let mut plan = ParserPlan::new();
480        plan.push(ErrorParser).unwrap();
481        let mut session = ReductionSession::new(
482            plan,
483            SessionOptions {
484                budget: Budget::unbounded(),
485                limits: crate::Limits {
486                    max_scope_bytes: 12,
487                    max_line_bytes: 8,
488                    ..crate::Limits::default()
489                },
490            },
491            OutputPolicy::new(&NoRedaction, &NoPathMapping, &GenericRanker),
492        );
493        session.begin_scope(Scope::step("compile"));
494        session.push_chunk("compile", Stream::Stderr, b"error: this line is too long\n");
495        session.end_scope("compile", EndReason::Complete);
496        let reduction = session.finish();
497        assert!(reduction.truncated);
498        assert!(reduction.stats.bytes_seen > reduction.stats.bytes_retained);
499    }
500
501    #[test]
502    fn rejects_duplicate_parser_ids() {
503        let mut plan = ParserPlan::new();
504        plan.push(ErrorParser).unwrap();
505        assert_eq!(
506            plan.push(ErrorParser).unwrap_err(),
507            ParserPlanError::DuplicateId("test.error")
508        );
509    }
510}