Skip to main content

tokencompact_builtins/
streaming.rs

1use std::collections::BTreeMap;
2
3use tokencompact_core::{
4    Emitter, EndReason, EvidenceQuality, FallbackPolicy, LogLine, Parser, ParserPlan, Provenance,
5    Scope, ScopeBoundary, Stream,
6};
7
8use crate::diagnostics;
9use crate::test_log::TestLogReducer;
10
11/// Fixed options for the built-in parser pack in a streaming session.
12#[derive(Clone, Copy, Debug, Eq, PartialEq)]
13pub struct BuiltinParserOptions {
14    pub fallback: FallbackPolicy,
15    pub max_buffered_scope_bytes: usize,
16}
17
18impl Default for BuiltinParserOptions {
19    fn default() -> Self {
20        Self {
21            fallback: FallbackPolicy::Generic,
22            max_buffered_scope_bytes: 1024 * 1024,
23        }
24    }
25}
26
27/// Creates the immutable, versioned built-in parser order.
28#[must_use]
29pub fn builtin_parser_plan(options: BuiltinParserOptions) -> ParserPlan {
30    let mut plan = ParserPlan::new();
31    plan.push(BuiltinDiagnosticParser::new(options))
32        .expect("the built-in diagnostic parser identifier is unique");
33    plan.push(TestFailureParser::default())
34        .expect("the built-in test parser identifier is unique");
35    plan
36}
37
38#[derive(Clone, Debug)]
39struct BufferedStream {
40    text: String,
41    first_line: u64,
42    last_line: u64,
43    truncated: bool,
44}
45
46/// Bounded bridge from incremental normalized lines to the built-in parser
47/// registry. Individual language parsers retain their deterministic prepass and
48/// line-arbitration order while callers gain chunk-invariant session framing.
49pub struct BuiltinDiagnosticParser {
50    options: BuiltinParserOptions,
51    scopes: BTreeMap<(String, Stream), BufferedStream>,
52}
53
54impl BuiltinDiagnosticParser {
55    #[must_use]
56    pub fn new(options: BuiltinParserOptions) -> Self {
57        Self {
58            options,
59            scopes: BTreeMap::new(),
60        }
61    }
62
63    fn flush_scope(&mut self, scope: &Scope, emitter: &mut Emitter<'_>) {
64        let keys = self
65            .scopes
66            .keys()
67            .filter(|(scope_id, _)| scope_id == &scope.id)
68            .cloned()
69            .collect::<Vec<_>>();
70        for key in keys {
71            let Some(buffer) = self.scopes.remove(&key) else {
72                continue;
73            };
74            let mut parsed = Vec::new();
75            diagnostics::add_text_diagnostics(
76                buffer.text.as_bytes(),
77                &mut parsed,
78                self.options.fallback,
79            );
80            for mut diagnostic in parsed {
81                if diagnostic.location.is_some() {
82                    diagnostic.quality = EvidenceQuality::Located;
83                }
84                diagnostic.provenance = Some(
85                    Provenance::new(stream_name(key.1))
86                        .with_scope(scope)
87                        .with_span(key.1, buffer.first_line, buffer.last_line)
88                        .with_parser(self.id()),
89                );
90                emitter.diagnostic(diagnostic);
91            }
92        }
93    }
94}
95
96impl Parser for BuiltinDiagnosticParser {
97    fn id(&self) -> &'static str {
98        "builtin.diagnostics.v1"
99    }
100
101    fn observe(&mut self, line: &LogLine<'_>, _emitter: &mut Emitter<'_>) {
102        let key = (line.scope.id.clone(), line.stream);
103        let buffer = self.scopes.entry(key).or_insert_with(|| BufferedStream {
104            text: String::new(),
105            first_line: line.stream_line,
106            last_line: line.stream_line,
107            truncated: false,
108        });
109        buffer.last_line = line.stream_line;
110        let required = line.text.len().saturating_add(1);
111        let available = self
112            .options
113            .max_buffered_scope_bytes
114            .saturating_sub(buffer.text.len());
115        if required <= available {
116            if !buffer.text.is_empty() {
117                buffer.text.push('\n');
118            }
119            buffer.text.push_str(line.text);
120        } else {
121            buffer.truncated = true;
122            let retained = available.min(line.text.len());
123            let mut boundary = retained;
124            while boundary > 0 && !line.text.is_char_boundary(boundary) {
125                boundary -= 1;
126            }
127            if boundary > 0 {
128                if !buffer.text.is_empty() {
129                    buffer.text.push('\n');
130                }
131                buffer.text.push_str(&line.text[..boundary]);
132            }
133        }
134        buffer.truncated |= line.truncated;
135    }
136
137    fn boundary(&mut self, boundary: ScopeBoundary<'_>, emitter: &mut Emitter<'_>) {
138        if let ScopeBoundary::End { scope, .. } = boundary {
139            self.flush_scope(scope, emitter);
140        }
141    }
142}
143
144#[derive(Default)]
145struct TestFailureParser {
146    scopes: BTreeMap<String, TestLogReducer>,
147}
148
149impl Parser for TestFailureParser {
150    fn id(&self) -> &'static str {
151        "builtin.test-log.v1"
152    }
153
154    fn observe(&mut self, line: &LogLine<'_>, _emitter: &mut Emitter<'_>) {
155        let provenance = Provenance::new(stream_name(line.stream))
156            .with_scope(line.scope)
157            .with_span(line.stream, line.stream_line, line.stream_line)
158            .with_parser(self.id());
159        self.scopes
160            .entry(line.scope.id.clone())
161            .or_default()
162            .observe_line(line.text, &provenance);
163    }
164
165    fn boundary(&mut self, boundary: ScopeBoundary<'_>, emitter: &mut Emitter<'_>) {
166        let ScopeBoundary::End { scope, reason, .. } = boundary else {
167            return;
168        };
169        let Some(mut reducer) = self.scopes.remove(&scope.id) else {
170            return;
171        };
172        reducer.finish_log(reason == EndReason::Complete);
173        let result = reducer.finish();
174        for failure in result.failures {
175            emitter.test_failure(failure);
176        }
177    }
178}
179
180fn stream_name(stream: Stream) -> &'static str {
181    match stream {
182        Stream::Stdout => "stdout",
183        Stream::Stderr => "stderr",
184        Stream::Combined => "combined",
185        Stream::Annotation => "annotation",
186        Stream::Structured => "structured",
187    }
188}
189
190#[cfg(test)]
191mod tests {
192    use tokencompact_core::{
193        Budget, EndReason, GenericRanker, NoPathMapping, NoRedaction, OutputPolicy,
194        ReductionSession, Scope, SessionOptions,
195    };
196
197    use super::*;
198
199    fn reduce_chunks(chunks: &[&[u8]]) -> tokencompact_core::Reduction {
200        let mut session = ReductionSession::new(
201            builtin_parser_plan(BuiltinParserOptions::default()),
202            SessionOptions {
203                budget: Budget::unbounded(),
204                ..SessionOptions::default()
205            },
206            OutputPolicy::new(&NoRedaction, &NoPathMapping, &GenericRanker),
207        );
208        session.begin_scope(Scope::step("tests"));
209        for chunk in chunks {
210            session.push_chunk("tests", Stream::Stderr, chunk);
211        }
212        session.end_scope("tests", EndReason::Complete);
213        session.finish()
214    }
215
216    #[test]
217    fn batch_and_arbitrary_stream_chunks_are_equivalent() {
218        let text = b"error[E0308]: mismatched types\n --> src/lib.rs:7:5\n  |\n7 | value\n  | ^ expected u32, found &str\n";
219        let whole = reduce_chunks(&[text]);
220        let split = reduce_chunks(&[&text[..9], &text[9..31], &text[31..]]);
221        assert_eq!(whole.diagnostics, split.diagnostics);
222        assert_eq!(
223            whole.diagnostics[0].location.as_ref().unwrap().path,
224            "src/lib.rs"
225        );
226    }
227
228    #[test]
229    fn emits_structured_test_failures() {
230        let reduction = reduce_chunks(&[
231            b"test invoice::fails ... FAILED\n---- invoice::fails stdout ----\nthread 'invoice::fails' panicked at src/lib.rs:7:3:\nassertion `left == right` failed\n",
232        ]);
233        assert_eq!(reduction.test_failures.len(), 1);
234        assert_eq!(reduction.test_failures[0].name, "invoice::fails");
235    }
236}