Skip to main content

typesayer_parser/
parser.rs

1// Copyright 2026 Thomas Santerre and Moderately AI Inc.
2//
3// SPDX-License-Identifier: MIT OR Apache-2.0
4
5//! Top-level streaming parser for chat-adapter responses.
6//!
7//! Composes `MarkerDetector` (recognises
8//! `[[ ## name ## ]]` boundaries) with per-field
9//! `FieldParser` sub-parsers (handles the
10//! per-FieldType content semantics). Emits a typed
11//! [`super::event::ParseEvent`] stream that mirrors the schema tree
12//! as values arrive.
13//!
14//! Mirrors the buffered `ChatAdapter::parse` (in `typesayer`)
15//! contract:
16//!
17//! - Field order is what the LM emits, not declared order.
18//! - First occurrence wins if a field marker repeats.
19//! - `[[ ## completed ## ]]` is the stream-end sentinel; events after it are silently consumed.
20//! - Single-string-output signature + no markers ever → the entire completion is the value (the
21//!   buffered parser's `single output + no markers` fallback).
22
23use std::collections::{BTreeMap, HashSet};
24
25use typesayer_types::{
26    field::{FieldType, FieldValue},
27    signature::Signature,
28};
29
30use super::{
31    event::ParseEvent,
32    field_parsers::{FieldParser, parser_for},
33    marker::{MarkerDetector, MarkerScan},
34};
35
36/// Push-driven streaming parser for chat-adapter completions.
37///
38/// Hold one per LM call; push provider `StreamDelta.content` chunks
39/// via [`Self::push`]; call [`Self::finish`] when the provider
40/// stream ends. Both return [`Vec<ParseEvent>`] for the caller (the
41/// streaming consumer) to forward over SSE.
42pub struct ChatStreamParser<'a> {
43    signature: &'a Signature,
44    output_names: HashSet<&'a str>,
45    marker_detector: MarkerDetector,
46    state: ParserState,
47    completed_fields: BTreeMap<String, FieldValue>,
48    /// Accumulator for the no-marker fallback case — content seen
49    /// before any marker arrived, in case finish() needs to wrap it
50    /// as a single string value.
51    pre_marker_buffer: String,
52    /// Sticky flag: set once we've seen any marker. Disables the
53    /// no-marker fallback on finish().
54    saw_any_marker: bool,
55    /// Sticky flag: set once we've emitted `stream_start`. Mirrors
56    /// the buffered parser's contract that the first event is always
57    /// StreamStart.
58    sent_stream_start: bool,
59    /// Set once `[[ ## completed ## ]]` arrives so subsequent push()
60    /// calls silently consume trailing tokens.
61    seen_completed: bool,
62}
63
64enum ParserState {
65    BeforeFirstField,
66    InField {
67        name: String,
68        path: String,
69        parser: Box<dyn FieldParser>,
70    },
71    AfterCompleted,
72}
73
74impl<'a> ChatStreamParser<'a> {
75    /// Build a parser for `signature`. Field types are read at marker
76    /// time via the signature's `output_fields()` lookup.
77    #[must_use]
78    pub fn new(signature: &'a Signature) -> Self {
79        let output_names: HashSet<&'a str> =
80            signature.output_fields().map(|f| f.name.as_str()).collect();
81        Self {
82            signature,
83            output_names,
84            marker_detector: MarkerDetector::new(),
85            state: ParserState::BeforeFirstField,
86            completed_fields: BTreeMap::new(),
87            pre_marker_buffer: String::new(),
88            saw_any_marker: false,
89            sent_stream_start: false,
90            seen_completed: false,
91        }
92    }
93
94    /// Feed one chunk of provider text. Returns the events produced
95    /// by this chunk in emission order.
96    pub fn push(&mut self, chunk: &str) -> Vec<ParseEvent> {
97        let mut events = Vec::new();
98        if !self.sent_stream_start {
99            events.push(ParseEvent::StreamStart {
100                output_schema_hash: output_schema_hash(self.signature),
101            });
102            self.sent_stream_start = true;
103        }
104        if self.seen_completed || matches!(self.state, ParserState::AfterCompleted) {
105            return events;
106        }
107        self.process(chunk, &mut events);
108        events
109    }
110
111    /// Finalize at end of provider stream. Returns any closing
112    /// events (FieldComplete for an unclosed final field, the
113    /// no-marker fallback if applicable, StreamComplete).
114    #[must_use]
115    pub fn finish(mut self) -> Vec<ParseEvent> {
116        let mut events = Vec::new();
117        if !self.sent_stream_start {
118            events.push(ParseEvent::StreamStart {
119                output_schema_hash: output_schema_hash(self.signature),
120            });
121            self.sent_stream_start = true;
122        }
123        // Drain any trailing marker-tail content into the active path
124        // before finalising — handles the case where the LM ended with
125        // bytes the detector was holding against a possible marker.
126        let trailing = self.marker_detector.drain_tail();
127        if !trailing.is_empty() {
128            self.consume_content(&trailing, &mut events);
129        }
130
131        // Take state out so &mut self method calls remain valid in
132        // arm bodies; AfterCompleted is the natural terminal placeholder.
133        let state = std::mem::replace(&mut self.state, ParserState::AfterCompleted);
134        match state {
135            ParserState::AfterCompleted => {
136                // StreamComplete already emitted; nothing more.
137            }
138            ParserState::InField { name, path, parser } => {
139                let (parser_events, value) = parser.finish();
140                events.extend(parser_events);
141                events.push(ParseEvent::FieldComplete {
142                    path,
143                    value: value.clone(),
144                });
145                self.completed_fields.insert(name, value);
146                events.push(ParseEvent::StreamComplete {
147                    output: FieldValue::Object(self.completed_fields),
148                });
149            }
150            ParserState::BeforeFirstField => {
151                // No marker ever appeared. If the signature has exactly
152                // one String output field, fall back to "the whole
153                // completion is that field's value" — mirrors the
154                // buffered parser's single-output-no-marker case.
155                if !self.saw_any_marker && self.try_synthesize_single_string_output(&mut events) {
156                    let output = FieldValue::Object(std::mem::take(&mut self.completed_fields));
157                    events.push(ParseEvent::StreamComplete { output });
158                } else {
159                    // No fallback applies — surface as an error.
160                    events.push(ParseEvent::StreamError {
161                        path: None,
162                        message: "stream ended without any field markers".to_owned(),
163                    });
164                }
165            }
166        }
167        events
168    }
169
170    fn process(&mut self, chunk: &str, events: &mut Vec<ParseEvent>) {
171        // Fast path: most streaming pushes carry no marker. The first
172        // detector call accepts the borrowed `chunk` directly, so the
173        // hot no-marker case doesn't allocate a working copy. Only
174        // when a marker is found AND its remainder is non-empty do we
175        // need to loop, holding the remainder as an owned String.
176        let mut remaining: String = match self.marker_detector.push(chunk) {
177            MarkerScan::NoMarker { content_before } => {
178                if !content_before.is_empty() {
179                    self.consume_content(&content_before, events);
180                }
181                return;
182            }
183            MarkerScan::MarkerFound {
184                content_before,
185                name,
186                remainder,
187            } => {
188                self.saw_any_marker = true;
189                if !content_before.is_empty() {
190                    self.consume_content(&content_before, events);
191                }
192                self.handle_marker(&name, events);
193                if matches!(self.state, ParserState::AfterCompleted) || self.seen_completed {
194                    return;
195                }
196                remainder
197            }
198        };
199
200        // Slow path: the chunk contained one marker AND trailing
201        // content (and possibly further markers). Drain it.
202        loop {
203            if remaining.is_empty() {
204                return;
205            }
206            let scan = self.marker_detector.push(&remaining);
207            match scan {
208                MarkerScan::NoMarker { content_before } => {
209                    if !content_before.is_empty() {
210                        self.consume_content(&content_before, events);
211                    }
212                    return;
213                }
214                MarkerScan::MarkerFound {
215                    content_before,
216                    name,
217                    remainder: next,
218                } => {
219                    self.saw_any_marker = true;
220                    if !content_before.is_empty() {
221                        self.consume_content(&content_before, events);
222                    }
223                    self.handle_marker(&name, events);
224                    if matches!(self.state, ParserState::AfterCompleted) || self.seen_completed {
225                        return;
226                    }
227                    remaining = next;
228                }
229            }
230        }
231    }
232
233    /// Route a slice of plain content (no markers) to wherever it
234    /// belongs — the active field's parser, or the pre-marker
235    /// buffer if we haven't seen the first marker yet.
236    fn consume_content(&mut self, content: &str, events: &mut Vec<ParseEvent>) {
237        match &mut self.state {
238            ParserState::InField { parser, .. } => {
239                let sub_events = parser.push(content);
240                events.extend(sub_events);
241            }
242            ParserState::BeforeFirstField => {
243                self.pre_marker_buffer.push_str(content);
244            }
245            ParserState::AfterCompleted => {
246                // Silently ignore trailing content after the sentinel.
247            }
248        }
249    }
250
251    /// Handle a marker arrival: close the current field (if any),
252    /// then either transition to a new field or to AfterCompleted.
253    fn handle_marker(&mut self, name: &str, events: &mut Vec<ParseEvent>) {
254        // First, close any current field.
255        let current_state = std::mem::replace(&mut self.state, ParserState::BeforeFirstField);
256        let was_in_field = matches!(current_state, ParserState::InField { .. });
257        if let ParserState::InField {
258            name: prev_name,
259            path: prev_path,
260            parser,
261        } = current_state
262        {
263            let (parser_events, value) = parser.finish();
264            events.extend(parser_events);
265            events.push(ParseEvent::FieldComplete {
266                path: prev_path,
267                value: value.clone(),
268            });
269            self.completed_fields.entry(prev_name).or_insert(value);
270        }
271
272        // Now dispatch on the new marker name.
273        if name == "completed" {
274            // No-real-marker case (sentinel arrived without any prior
275            // field marker AND nothing previously completed). Two
276            // sub-cases mirror the buffered parser's behaviour:
277            //
278            // 1. Single-String-output signature → rescue: promote `pre_marker_buffer` to that
279            //    field's value (commit 1fb557e8 in chat.rs). Instruction-tuned models routinely
280            //    emit the visible "I'm done" sentinel while dropping the opening field marker on
281            //    single-output signatures as boilerplate.
282            //
283            // 2. Multi-output (or non-String single-output) signature → emit StreamError. The
284            //    buffered `ChatAdapter::parse` returns `PredictError::NoFieldMarkers` for the same
285            //    shape (chat.rs::parse_no_markers_multiple_outputs_fails); without this branch the
286            //    streaming parser would silently emit `StreamComplete` with an empty `Object` and
287            //    the consumer would return success-with-empty- output, masking what the buffered
288            //    path treats as an error.
289            if !was_in_field && self.completed_fields.is_empty() {
290                let rescued = self.try_synthesize_single_string_output(events);
291                if !rescued {
292                    events.push(ParseEvent::StreamError {
293                        path: None,
294                        message: "stream ended without any field markers".to_owned(),
295                    });
296                }
297            }
298            let output = FieldValue::Object(std::mem::take(&mut self.completed_fields));
299            events.push(ParseEvent::StreamComplete { output });
300            self.state = ParserState::AfterCompleted;
301            self.seen_completed = true;
302            return;
303        }
304
305        // Is this a declared output field?
306        if !self.output_names.contains(name) {
307            // Unknown marker name — could be an input echo or a
308            // hallucinated field name. The buffered parser silently
309            // skips these (`if !output_names.contains(field_name) ||
310            // field_name == "completed" { continue; }`). Mirror that.
311            // We don't transition; we stay in BeforeFirstField.
312            return;
313        }
314
315        // First occurrence wins — if we've already completed this
316        // field, ignore subsequent markers for it (mirrors the
317        // buffered parser's `if fields.contains_key(field_name) {
318        // continue; }`).
319        if self.completed_fields.contains_key(name) {
320            return;
321        }
322
323        // Look up the declared FieldType for this name. The
324        // contains check above guarantees the find will succeed.
325        let field_type = self
326            .signature
327            .output_fields()
328            .find(|f| f.name == name)
329            .map(|f| f.field_type.clone());
330        let Some(field_type) = field_type else {
331            return; // unreachable per the contains check, defensively skip
332        };
333
334        let path = format!("/{name}");
335        let parser = parser_for(&field_type, path.clone());
336        events.push(ParseEvent::FieldStart {
337            path: path.clone(),
338            field_type,
339        });
340        self.state = ParserState::InField {
341            name: name.to_owned(),
342            path,
343            parser,
344        };
345    }
346
347    /// Emit a synthetic `FieldStart` + `ValueAppend` + `FieldComplete`
348    /// triple for a single-String-output signature, sourcing the value
349    /// from `pre_marker_buffer` and inserting it into
350    /// `completed_fields`. Used by two rescue paths that mirror the
351    /// buffered parser's single-output-no-real-marker fallback:
352    /// `finish()` (no marker ever arrived) and `handle_marker()` (only
353    /// the `completed` sentinel arrived). Returns true if the rescue
354    /// applied; false if the signature isn't shaped as
355    /// single-String-output.
356    fn try_synthesize_single_string_output(&mut self, events: &mut Vec<ParseEvent>) -> bool {
357        let single_string_field_name = {
358            let mut iter = self.signature.output_fields();
359            let first = iter.next();
360            let rest_exists = iter.next().is_some();
361            match (first, rest_exists) {
362                (Some(f), false) if matches!(f.field_type, FieldType::String) => {
363                    Some(f.name.clone())
364                }
365                _ => None,
366            }
367        };
368        let Some(field_name) = single_string_field_name else {
369            return false;
370        };
371
372        let path = format!("/{field_name}");
373        let trimmed_value = FieldValue::Str(self.pre_marker_buffer.trim().to_owned());
374        events.push(ParseEvent::FieldStart {
375            path: path.clone(),
376            field_type: FieldType::String,
377        });
378        if !self.pre_marker_buffer.is_empty() {
379            events.push(ParseEvent::ValueAppend {
380                path: path.clone(),
381                delta: std::mem::take(&mut self.pre_marker_buffer),
382            });
383        }
384        events.push(ParseEvent::FieldComplete {
385            path,
386            value: trimmed_value.clone(),
387        });
388        self.completed_fields.insert(field_name, trimmed_value);
389        true
390    }
391}
392
393/// Compute a stable identity hash of the signature's output spec.
394/// Not a cryptographic hash — clients use this to verify they're
395/// rendering against the schema they expected, not for any security
396/// property. SipHash via `DefaultHasher` is fine for this purpose.
397fn output_schema_hash(signature: &Signature) -> String {
398    use std::{
399        collections::hash_map::DefaultHasher,
400        hash::{Hash, Hasher},
401    };
402    let mut h = DefaultHasher::new();
403    for f in signature.output_fields() {
404        f.name.hash(&mut h);
405        f.field_type.type_label().hash(&mut h);
406    }
407    format!("{:016x}", h.finish())
408}
409
410#[cfg(test)]
411mod tests {
412    use typesayer_types::field::FieldDef;
413
414    use super::*;
415
416    fn simple_string_signature() -> Signature {
417        Signature::builder("test")
418            .input(FieldDef::input("q", FieldType::String, "question"))
419            .output(FieldDef::output("answer", FieldType::String, "the answer"))
420            .build()
421            .unwrap()
422    }
423
424    fn two_string_signature() -> Signature {
425        // Two String outputs — drives the multi-output sentinel-only
426        // rescue path that must fall back to StreamError parity with
427        // the buffered parser's `NoFieldMarkers` behaviour.
428        Signature::builder("test")
429            .input(FieldDef::input("q", FieldType::String, "question"))
430            .output(FieldDef::output("a", FieldType::String, "first"))
431            .output(FieldDef::output("b", FieldType::String, "second"))
432            .build()
433            .unwrap()
434    }
435
436    fn list_of_strings_signature() -> Signature {
437        Signature::builder("test")
438            .input(FieldDef::input("q", FieldType::String, "question"))
439            .output(FieldDef::output(
440                "items",
441                FieldType::List(Box::new(FieldType::String)),
442                "items",
443            ))
444            .build()
445            .unwrap()
446    }
447
448    fn collect_events(events: Vec<ParseEvent>) -> Vec<String> {
449        events.iter().map(|e| e.event_type().to_owned()).collect()
450    }
451
452    #[test]
453    fn simple_string_schema_emits_expected_sequence() {
454        let sig = simple_string_signature();
455        let mut parser = ChatStreamParser::new(&sig);
456        let mut events = Vec::new();
457        events.extend(parser.push("[[ ## answer ## ]]\nhello"));
458        events.extend(parser.push(" world\n[[ ## completed ## ]]"));
459        events.extend(parser.finish());
460
461        let types = collect_events(events.clone());
462        // Should be: stream_start, field_start, value_append*, field_complete, stream_complete
463        assert_eq!(types[0], "stream_start");
464        assert_eq!(types[1], "field_start");
465        let value_append_count = types.iter().filter(|t| *t == "value_append").count();
466        assert!(value_append_count >= 1, "expected at least 1 value_append");
467        assert_eq!(types[types.len() - 2], "field_complete");
468        assert_eq!(types[types.len() - 1], "stream_complete");
469
470        // The final StreamComplete should carry the correct output.
471        match events.last().unwrap() {
472            ParseEvent::StreamComplete { output } => {
473                let FieldValue::Object(map) = output else {
474                    panic!("expected Object")
475                };
476                let answer = map.get("answer").unwrap();
477                assert_eq!(answer, &FieldValue::Str("hello world".into()));
478            }
479            other => panic!("expected StreamComplete, got {other:?}"),
480        }
481    }
482
483    #[test]
484    fn no_marker_fallback_for_single_string_output() {
485        let sig = simple_string_signature();
486        let mut parser = ChatStreamParser::new(&sig);
487        let mut events = Vec::new();
488        events.extend(parser.push("Paris"));
489        events.extend(parser.finish());
490
491        let types = collect_events(events.clone());
492        // Should be: stream_start, field_start, value_append, field_complete, stream_complete
493        assert_eq!(
494            types,
495            vec![
496                "stream_start",
497                "field_start",
498                "value_append",
499                "field_complete",
500                "stream_complete"
501            ]
502        );
503
504        // Final value should be "Paris"
505        match events.last().unwrap() {
506            ParseEvent::StreamComplete {
507                output: FieldValue::Object(map),
508            } => {
509                assert_eq!(map.get("answer"), Some(&FieldValue::Str("Paris".into())));
510            }
511            other => panic!("got {other:?}"),
512        }
513    }
514
515    #[test]
516    fn list_of_strings_schema_emits_open_added_close() {
517        let sig = list_of_strings_signature();
518        let mut parser = ChatStreamParser::new(&sig);
519        let mut events = Vec::new();
520        events.extend(parser.push(
521            r#"[[ ## items ## ]]
522["a", "b", "c"]
523[[ ## completed ## ]]"#,
524        ));
525        events.extend(parser.finish());
526
527        let types = collect_events(events.clone());
528        // Expect: stream_start, field_start, list_open, element_added*3, list_close,
529        // field_complete, stream_complete
530        let element_added_count = types.iter().filter(|t| *t == "element_added").count();
531        assert_eq!(element_added_count, 3);
532        assert!(types.contains(&"list_open".to_owned()));
533        assert!(types.contains(&"list_close".to_owned()));
534        assert!(types.contains(&"stream_complete".to_owned()));
535
536        match events.last().unwrap() {
537            ParseEvent::StreamComplete {
538                output: FieldValue::Object(map),
539            } => {
540                let items = map.get("items").unwrap();
541                let FieldValue::List(v) = items else {
542                    panic!("expected List")
543                };
544                let strs: Vec<&str> = v
545                    .iter()
546                    .map(|x| match x {
547                        FieldValue::Str(s) => s.as_str(),
548                        _ => panic!(),
549                    })
550                    .collect();
551                assert_eq!(strs, vec!["a", "b", "c"]);
552            }
553            other => panic!("got {other:?}"),
554        }
555    }
556
557    fn string_and_list_signature() -> Signature {
558        // Two-output signature: one String + one List<String>. Used
559        // by the multi-field event-ordering test to verify the
560        // top-level parser correctly composes the two FieldParsers
561        // across a single LM response.
562        Signature::builder("test")
563            .input(FieldDef::input("q", FieldType::String, "question"))
564            .output(FieldDef::output("a", FieldType::String, "first text"))
565            .output(FieldDef::output(
566                "b",
567                FieldType::List(Box::new(FieldType::String)),
568                "second list",
569            ))
570            .build()
571            .unwrap()
572    }
573
574    #[test]
575    fn two_field_schema_full_event_sequence_ordering() {
576        // Schema {a: String, b: List<String>}. Verify the canonical
577        // event sequence: stream_start, field_start(a), value_append(a),
578        // field_complete(a), field_start(b), list_open(b),
579        // element_added(b, 0), element_added(b, 1), list_close(b),
580        // field_complete(b), stream_complete. This pins the multi-
581        // field composition contract from the original plan.
582        let sig = string_and_list_signature();
583        let mut parser = ChatStreamParser::new(&sig);
584        let mut events = Vec::new();
585        events.extend(parser.push(
586            r#"[[ ## a ## ]]
587first
588[[ ## b ## ]]
589["x", "y"]
590[[ ## completed ## ]]"#,
591        ));
592        events.extend(parser.finish());
593
594        let types = collect_events(events.clone());
595        assert_eq!(
596            types,
597            vec![
598                "stream_start",
599                "field_start",
600                "value_append",
601                "field_complete",
602                "field_start",
603                "list_open",
604                "element_added",
605                "element_added",
606                "list_close",
607                "field_complete",
608                "stream_complete",
609            ]
610        );
611
612        // Verify the final StreamComplete carries both fields
613        // correctly mapped.
614        match events.last().unwrap() {
615            ParseEvent::StreamComplete {
616                output: FieldValue::Object(map),
617            } => {
618                assert_eq!(map.get("a"), Some(&FieldValue::Str("first".into())));
619                assert_eq!(
620                    map.get("b"),
621                    Some(&FieldValue::List(vec![
622                        FieldValue::Str("x".into()),
623                        FieldValue::Str("y".into()),
624                    ]))
625                );
626            }
627            other => panic!("expected StreamComplete, got {other:?}"),
628        }
629    }
630
631    #[test]
632    fn out_of_declared_order_field_emission() {
633        // Schema declares a-then-b, but the LM emits b before a.
634        // The streaming parser must follow LM emission order on the
635        // wire (FieldStart(/b) before FieldStart(/a)) — clients
636        // render in arrival order, not schema order. The final
637        // StreamComplete output is a BTreeMap which orders by key
638        // (sorts a, b alphabetically), but the EVENT STREAM order
639        // is what consumers see live.
640        let sig = Signature::builder("test")
641            .input(FieldDef::input("q", FieldType::String, "question"))
642            .output(FieldDef::output("a", FieldType::String, "first"))
643            .output(FieldDef::output("b", FieldType::String, "second"))
644            .build()
645            .unwrap();
646        let mut parser = ChatStreamParser::new(&sig);
647        let mut events = Vec::new();
648        events.extend(
649            parser.push("[[ ## b ## ]]\nbcontent\n[[ ## a ## ]]\nacontent\n[[ ## completed ## ]]"),
650        );
651        events.extend(parser.finish());
652
653        // Extract field_start events in order. The /b field must
654        // appear FIRST on the wire even though the signature
655        // declared a-then-b.
656        let field_start_paths: Vec<String> = events
657            .iter()
658            .filter_map(|e| match e {
659                ParseEvent::FieldStart { path, .. } => Some(path.clone()),
660                _ => None,
661            })
662            .collect();
663        assert_eq!(field_start_paths, vec!["/b", "/a"]);
664
665        // StreamComplete output is by BTreeMap key order (a, b).
666        match events.last().unwrap() {
667            ParseEvent::StreamComplete {
668                output: FieldValue::Object(map),
669            } => {
670                assert_eq!(map.get("a"), Some(&FieldValue::Str("acontent".into())));
671                assert_eq!(map.get("b"), Some(&FieldValue::Str("bcontent".into())));
672            }
673            other => panic!("expected StreamComplete, got {other:?}"),
674        }
675    }
676
677    #[test]
678    fn stream_error_mid_stream_halts_field_emission() {
679        // A malformed JSON value inside a List<String> field should
680        // emit StreamError and stop emitting per-element events for
681        // that field — the JsonArrayParser transitions to the
682        // errored state and silently consumes further input. The
683        // wrapping ChatStreamParser must propagate this without
684        // crashing, and the eventual StreamComplete still fires
685        // (the sentinel is unconditional once seen).
686        let sig = list_of_strings_signature();
687        let mut parser = ChatStreamParser::new(&sig);
688        let mut events = Vec::new();
689        events.extend(parser.push(
690            r#"[[ ## items ## ]]
691["a", bad_token, "c"]
692[[ ## completed ## ]]"#,
693        ));
694        events.extend(parser.finish());
695
696        // Must have at least one StreamError.
697        assert!(
698            events
699                .iter()
700                .any(|e| matches!(e, ParseEvent::StreamError { .. })),
701            "malformed JSON in list must emit at least one StreamError"
702        );
703
704        // Must NOT have emitted element_added events for `c` (the
705        // value after the error position) — the JsonArrayParser
706        // entered errored state after `bad_token` and won't emit
707        // further per-element events.
708        let c_was_added = events.iter().any(
709            |e| matches!(e, ParseEvent::ElementAdded { value: FieldValue::Str(s), .. } if s == "c"),
710        );
711        assert!(!c_was_added, "no element_added for `c` after StreamError");
712
713        // Parser doesn't crash; StreamComplete still arrives.
714        assert!(
715            events
716                .iter()
717                .any(|e| matches!(e, ParseEvent::StreamComplete { .. })),
718            "StreamComplete must still fire after mid-stream StreamError"
719        );
720    }
721
722    #[test]
723    fn unknown_field_marker_silently_skipped() {
724        // The LM hallucinates a field name not in the signature — the
725        // buffered parser ignores it; we mirror that.
726        let sig = simple_string_signature();
727        let mut parser = ChatStreamParser::new(&sig);
728        let mut events = Vec::new();
729        events.extend(parser.push(
730            "[[ ## hallucinated ## ]]\nignored\n[[ ## answer ## ]]\nactual\n[[ ## completed ## ]]",
731        ));
732        events.extend(parser.finish());
733
734        let field_start_paths: Vec<String> = events
735            .iter()
736            .filter_map(|e| match e {
737                ParseEvent::FieldStart { path, .. } => Some(path.clone()),
738                _ => None,
739            })
740            .collect();
741        assert_eq!(field_start_paths, vec!["/answer"]);
742
743        match events.last().unwrap() {
744            ParseEvent::StreamComplete {
745                output: FieldValue::Object(map),
746            } => {
747                assert_eq!(map.get("answer"), Some(&FieldValue::Str("actual".into())));
748                assert!(!map.contains_key("hallucinated"));
749            }
750            other => panic!("got {other:?}"),
751        }
752    }
753
754    #[test]
755    fn completed_sentinel_only_rescues_single_string_output() {
756        // Mirrors chat.rs::parse_single_output_only_completed_marker_fallback
757        // (commit 1fb557e8): instruction-tuned models routinely emit the
758        // closing sentinel while dropping the opening field marker on
759        // single-output signatures. Both parsers must produce the same
760        // final value rather than silently dropping the buffered content.
761        let sig = simple_string_signature();
762        let mut parser = ChatStreamParser::new(&sig);
763        let mut events = Vec::new();
764        events.extend(parser.push("Paris is the capital of France.\n[[ ## completed ## ]]"));
765        events.extend(parser.finish());
766
767        // Expect the rescue path to synthesise field events before the
768        // stream-end sentinel: stream_start, field_start, value_append,
769        // field_complete, stream_complete.
770        let types = collect_events(events.clone());
771        assert_eq!(
772            types,
773            vec![
774                "stream_start",
775                "field_start",
776                "value_append",
777                "field_complete",
778                "stream_complete",
779            ]
780        );
781
782        match events.last().unwrap() {
783            ParseEvent::StreamComplete {
784                output: FieldValue::Object(map),
785            } => {
786                assert_eq!(
787                    map.get("answer"),
788                    Some(&FieldValue::Str("Paris is the capital of France.".into()))
789                );
790            }
791            other => panic!("expected StreamComplete with rescued value, got {other:?}"),
792        }
793    }
794
795    #[test]
796    fn completed_sentinel_only_multi_output_emits_stream_error() {
797        // Mirrors chat.rs::parse_no_markers_multiple_outputs_fails:
798        // the buffered parser returns `PredictError::NoFieldMarkers`
799        // when the LM emits content + only the closing sentinel
800        // (no real field markers) on a signature with multiple
801        // output fields — there's no rescue heuristic for multi-
802        // output, and silently emitting an empty `StreamComplete`
803        // would mask what the buffered path treats as a parse error.
804        // The streaming parser must emit a StreamError event before
805        // the StreamComplete so the consumer returns
806        // structured parse error instead of success-with-empty-output.
807        let sig = two_string_signature();
808        let mut parser = ChatStreamParser::new(&sig);
809        let mut events = Vec::new();
810        events.extend(parser.push("Some content\n[[ ## completed ## ]]"));
811        events.extend(parser.finish());
812
813        // The exact sequence: stream_start, stream_error, stream_complete.
814        let types = collect_events(events.clone());
815        assert_eq!(
816            types,
817            vec!["stream_start", "stream_error", "stream_complete"]
818        );
819
820        // StreamError carries a message referencing the missing markers.
821        let error_msg = events
822            .iter()
823            .find_map(|e| match e {
824                ParseEvent::StreamError { message, path } if path.is_none() => Some(message),
825                _ => None,
826            })
827            .expect("multi-output sentinel-only must emit a path-less StreamError");
828        assert!(
829            error_msg.contains("without any field markers"),
830            "StreamError message should describe the missing-marker condition; got {error_msg:?}"
831        );
832
833        // StreamComplete still fires (terminal sentinel acknowledged),
834        // but the output is the empty object — no fields could be
835        // synthesised. The accompanying StreamError is the signal
836        // that the consumer should surface as structured parse error.
837        match events.last().unwrap() {
838            ParseEvent::StreamComplete {
839                output: FieldValue::Object(map),
840            } => {
841                assert!(
842                    map.is_empty(),
843                    "multi-output sentinel-only yields empty output map"
844                );
845            }
846            other => panic!("expected StreamComplete, got {other:?}"),
847        }
848    }
849
850    #[test]
851    fn schema_hash_stable_for_same_signature() {
852        let sig1 = simple_string_signature();
853        let sig2 = simple_string_signature();
854        assert_eq!(output_schema_hash(&sig1), output_schema_hash(&sig2));
855    }
856
857    #[test]
858    fn schema_hash_differs_for_different_signatures() {
859        let s1 = simple_string_signature();
860        let s2 = list_of_strings_signature();
861        assert_ne!(output_schema_hash(&s1), output_schema_hash(&s2));
862    }
863
864    use proptest::prelude::*;
865
866    use crate::proptest_strategies::{
867        DriftPerturbation, arb_chunking_positions, arb_supported_type_and_value,
868        arb_unsupported_field_type, chunk_at_positions, serialize_completion,
869        serialize_completion_with_drift, single_output_signature,
870    };
871
872    #[test]
873    fn parity_fence_json_chunked_at_various_points() {
874        // Same shape as the proptest failure: List<List<Object<f1:
875        // String>>> + FenceJson + various chunk boundaries.
876        use typesayer_types::ObjectField;
877        let field_type = FieldType::List(Box::new(FieldType::List(Box::new(FieldType::Object(
878            vec![ObjectField {
879                name: "f1".into(),
880                description: String::new(),
881                field_type: FieldType::String,
882            }],
883        )))));
884        // Just two inner lists with two objects each.
885        let value = FieldValue::List(vec![
886            FieldValue::List(vec![
887                FieldValue::Object(
888                    std::iter::once(("f1".to_string(), FieldValue::Str("abc".to_string())))
889                        .collect(),
890                ),
891                FieldValue::Object(
892                    std::iter::once(("f1".to_string(), FieldValue::Str("def".to_string())))
893                        .collect(),
894                ),
895            ]),
896            FieldValue::List(vec![FieldValue::Object(
897                std::iter::once(("f1".to_string(), FieldValue::Str("ghi".to_string()))).collect(),
898            )]),
899        ]);
900        let signature = single_output_signature("answer", field_type.clone());
901        let completion = serialize_completion_with_drift(
902            "answer",
903            &field_type,
904            &value,
905            DriftPerturbation::FenceJson,
906        );
907        // Try every split point.
908        for split in 1..completion.len() {
909            if !completion.is_char_boundary(split) {
910                continue;
911            }
912            let mut parser = ChatStreamParser::new(&signature);
913            let mut events = Vec::new();
914            events.extend(parser.push(&completion[..split]));
915            events.extend(parser.push(&completion[split..]));
916            events.extend(parser.finish());
917            let streamed_value = match events.last() {
918                Some(ParseEvent::StreamComplete {
919                    output: FieldValue::Object(map),
920                }) => map.get("answer").cloned(),
921                _ => None,
922            };
923            assert_eq!(
924                streamed_value.as_ref(),
925                Some(&value),
926                "split={split} input={:?}",
927                &completion[..split]
928            );
929        }
930    }
931
932    #[test]
933    fn parity_fence_json_with_empty_string_first() {
934        use typesayer_types::ObjectField;
935        // The counterexample's first inner-list element had an empty
936        // f1 — i.e. `[[{"f1": ""}, ...]]`. Test if empty strings inside
937        // nested JSON throw off the parser.
938        let field_type = FieldType::List(Box::new(FieldType::List(Box::new(FieldType::Object(
939            vec![ObjectField {
940                name: "f1".into(),
941                description: String::new(),
942                field_type: FieldType::String,
943            }],
944        )))));
945        let value = FieldValue::List(vec![FieldValue::List(vec![
946            FieldValue::Object(
947                std::iter::once(("f1".to_string(), FieldValue::Str(String::new()))).collect(),
948            ),
949            FieldValue::Object(
950                std::iter::once((
951                    "f1".to_string(),
952                    FieldValue::Str(
953                        "CPcb.7tmqOtkv4WyIny0D9vfIcR!LpVu-aY6fmGOl HKNNVtt,S".to_string(),
954                    ),
955                ))
956                .collect(),
957            ),
958        ])]);
959        let signature = single_output_signature("answer", field_type.clone());
960        let completion = serialize_completion_with_drift(
961            "answer",
962            &field_type,
963            &value,
964            DriftPerturbation::FenceJson,
965        );
966        let mut parser = ChatStreamParser::new(&signature);
967        let mut events = Vec::new();
968        events.extend(parser.push(&completion));
969        events.extend(parser.finish());
970        let streamed_value = match events.last() {
971            Some(ParseEvent::StreamComplete {
972                output: FieldValue::Object(map),
973            }) => map.get("answer").cloned(),
974            _ => None,
975        };
976        assert_eq!(streamed_value.as_ref(), Some(&value));
977    }
978
979    #[test]
980    fn parity_fence_json_nested_list_of_list_of_object() {
981        use typesayer_types::ObjectField;
982        // Minimal reproducer for the proptest FenceJson failure mode.
983        // List<List<Object<f1: String>>> with FenceJson perturbation.
984        let field_type = FieldType::List(Box::new(FieldType::List(Box::new(FieldType::Object(
985            vec![ObjectField {
986                name: "f1".into(),
987                description: String::new(),
988                field_type: FieldType::String,
989            }],
990        )))));
991        let value = FieldValue::List(vec![FieldValue::List(vec![FieldValue::Object(
992            std::iter::once(("f1".to_string(), FieldValue::Str("hi".to_string()))).collect(),
993        )])]);
994        let signature = single_output_signature("answer", field_type.clone());
995        let completion = serialize_completion_with_drift(
996            "answer",
997            &field_type,
998            &value,
999            DriftPerturbation::FenceJson,
1000        );
1001        let mut parser = ChatStreamParser::new(&signature);
1002        let mut events = Vec::new();
1003        events.extend(parser.push(&completion));
1004        events.extend(parser.finish());
1005        let streamed_value = match events.last() {
1006            Some(ParseEvent::StreamComplete {
1007                output: FieldValue::Object(map),
1008            }) => map.get("answer").cloned(),
1009            _ => None,
1010        };
1011        assert_eq!(streamed_value.as_ref(), Some(&value));
1012    }
1013
1014    /// Run the top-level parser against `(field_type, value)` serialised
1015    /// to the LM wire format, split into chunks, then re-assemble. The
1016    /// final `StreamComplete` output must carry the input value back —
1017    /// modulo the well-known top-level-`String` trim (`StringParser`
1018    /// trims whitespace at finish; mirrored by `value.trim()` here).
1019    fn parse_value_through_chunks(
1020        field_type: &FieldType,
1021        value: &FieldValue,
1022        positions: &[u8],
1023    ) -> Vec<ParseEvent> {
1024        let signature = single_output_signature("answer", field_type.clone());
1025        let completion = serialize_completion("answer", field_type, value);
1026        let chunks = chunk_at_positions(&completion, positions);
1027
1028        let mut parser = ChatStreamParser::new(&signature);
1029        let mut events = Vec::new();
1030        for chunk in chunks {
1031            events.extend(parser.push(chunk));
1032        }
1033        events.extend(parser.finish());
1034        events
1035    }
1036
1037    proptest! {
1038        /// Round-trip: any supported `(FieldType, FieldValue)` serialised
1039        /// to the LM wire format and arbitrarily chunked, parses back
1040        /// to the original value via `ChatStreamParser`.
1041        ///
1042        /// Exercises the full marker layer + per-FieldType dispatch +
1043        /// chunk-boundary handling end-to-end. The per-parser
1044        /// proptests (string, array) cover the lower layers; this one
1045        /// verifies that the top-level orchestration composes them
1046        /// correctly across the marker boundary.
1047        #[test]
1048        fn chat_stream_parser_roundtrips_any_supported_value(
1049            (field_type, value) in arb_supported_type_and_value(),
1050            positions in arb_chunking_positions(),
1051        ) {
1052            let events = parse_value_through_chunks(&field_type, &value, &positions);
1053
1054            // Top-level String trims at finish (see
1055            // `surrounding_whitespace_trimmed_in_final_value`); match
1056            // that for the expected comparison.
1057            let expected = match (&field_type, &value) {
1058                (FieldType::String, FieldValue::Str(s)) => FieldValue::Str(s.trim().to_owned()),
1059                _ => value.clone(),
1060            };
1061
1062            match events.last() {
1063                Some(ParseEvent::StreamComplete { output: FieldValue::Object(map) }) => {
1064                    prop_assert_eq!(map.get("answer"), Some(&expected));
1065                }
1066                Some(other) => {
1067                    prop_assert!(false, "expected StreamComplete, got {other:?}");
1068                }
1069                None => {
1070                    prop_assert!(false, "no events emitted");
1071                }
1072            }
1073        }
1074
1075        /// Degrade-gracefully: any FieldType the streaming codec does
1076        /// NOT support, fed through the same wire format with a stub
1077        /// value, must emit a `StreamError` without panic and without
1078        /// hanging. The synthesised content is a single space (valid
1079        /// for any wire shape but won't parse as a structured value).
1080        #[test]
1081        fn unsupported_field_type_emits_stream_error_without_panic(
1082            field_type in arb_unsupported_field_type(),
1083            positions in arb_chunking_positions(),
1084        ) {
1085            let signature = single_output_signature("answer", field_type.clone());
1086            // For unsupported types we can't construct a matching
1087            // FieldValue cleanly, so synthesise the completion
1088            // directly with a stub content body. The space mid-marker
1089            // is intentional — it ensures the StringParser sees at
1090            // least one byte to push to its sub-parser.
1091            let completion = "[[ ## answer ## ]]\n \n[[ ## completed ## ]]";
1092            let chunks = chunk_at_positions(completion, &positions);
1093
1094            let mut parser = ChatStreamParser::new(&signature);
1095            let mut events = Vec::new();
1096            for chunk in chunks {
1097                events.extend(parser.push(chunk));
1098            }
1099            events.extend(parser.finish());
1100
1101            // The unsupported sub-parser is required to emit at least
1102            // one StreamError. Either the parser itself emits it (for
1103            // top-level FieldTypes routed to UnsupportedParser like
1104            // Media), or the JSON layer emits it (for FieldTypes
1105            // routed through JsonFieldParser → JsonUnsupportedParser
1106            // like Int/Map/Nullable). Both are acceptable; the
1107            // property is "never panics, always reports".
1108            let has_stream_error = events
1109                .iter()
1110                .any(|e| matches!(e, ParseEvent::StreamError { .. }));
1111            prop_assert!(
1112                has_stream_error,
1113                "expected StreamError for unsupported type {field_type:?}, events: {events:?}"
1114            );
1115        }
1116
1117    }
1118}