1use std::collections::{BTreeMap, HashSet};
24
25use typesayer_types::{
26 field::{FieldType, FieldValue},
27 signature::Signature,
28};
29
30use super::{
31 event::ParseEvent,
32 field_parsers::{FieldParser, parser_for},
33 marker::{MarkerDetector, MarkerScan},
34};
35
36pub struct ChatStreamParser<'a> {
43 signature: &'a Signature,
44 output_names: HashSet<&'a str>,
45 marker_detector: MarkerDetector,
46 state: ParserState,
47 completed_fields: BTreeMap<String, FieldValue>,
48 pre_marker_buffer: String,
52 saw_any_marker: bool,
55 sent_stream_start: bool,
59 seen_completed: bool,
62}
63
64enum ParserState {
65 BeforeFirstField,
66 InField {
67 name: String,
68 path: String,
69 parser: Box<dyn FieldParser>,
70 },
71 AfterCompleted,
72}
73
74impl<'a> ChatStreamParser<'a> {
75 #[must_use]
78 pub fn new(signature: &'a Signature) -> Self {
79 let output_names: HashSet<&'a str> =
80 signature.output_fields().map(|f| f.name.as_str()).collect();
81 Self {
82 signature,
83 output_names,
84 marker_detector: MarkerDetector::new(),
85 state: ParserState::BeforeFirstField,
86 completed_fields: BTreeMap::new(),
87 pre_marker_buffer: String::new(),
88 saw_any_marker: false,
89 sent_stream_start: false,
90 seen_completed: false,
91 }
92 }
93
94 pub fn push(&mut self, chunk: &str) -> Vec<ParseEvent> {
97 let mut events = Vec::new();
98 if !self.sent_stream_start {
99 events.push(ParseEvent::StreamStart {
100 output_schema_hash: output_schema_hash(self.signature),
101 });
102 self.sent_stream_start = true;
103 }
104 if self.seen_completed || matches!(self.state, ParserState::AfterCompleted) {
105 return events;
106 }
107 self.process(chunk, &mut events);
108 events
109 }
110
111 #[must_use]
115 pub fn finish(mut self) -> Vec<ParseEvent> {
116 let mut events = Vec::new();
117 if !self.sent_stream_start {
118 events.push(ParseEvent::StreamStart {
119 output_schema_hash: output_schema_hash(self.signature),
120 });
121 self.sent_stream_start = true;
122 }
123 let trailing = self.marker_detector.drain_tail();
127 if !trailing.is_empty() {
128 self.consume_content(&trailing, &mut events);
129 }
130
131 let state = std::mem::replace(&mut self.state, ParserState::AfterCompleted);
134 match state {
135 ParserState::AfterCompleted => {
136 }
138 ParserState::InField { name, path, parser } => {
139 let (parser_events, value) = parser.finish();
140 events.extend(parser_events);
141 events.push(ParseEvent::FieldComplete {
142 path,
143 value: value.clone(),
144 });
145 self.completed_fields.insert(name, value);
146 events.push(ParseEvent::StreamComplete {
147 output: FieldValue::Object(self.completed_fields),
148 });
149 }
150 ParserState::BeforeFirstField => {
151 if !self.saw_any_marker && self.try_synthesize_single_string_output(&mut events) {
156 let output = FieldValue::Object(std::mem::take(&mut self.completed_fields));
157 events.push(ParseEvent::StreamComplete { output });
158 } else {
159 events.push(ParseEvent::StreamError {
161 path: None,
162 message: "stream ended without any field markers".to_owned(),
163 });
164 }
165 }
166 }
167 events
168 }
169
170 fn process(&mut self, chunk: &str, events: &mut Vec<ParseEvent>) {
171 let mut remaining: String = match self.marker_detector.push(chunk) {
177 MarkerScan::NoMarker { content_before } => {
178 if !content_before.is_empty() {
179 self.consume_content(&content_before, events);
180 }
181 return;
182 }
183 MarkerScan::MarkerFound {
184 content_before,
185 name,
186 remainder,
187 } => {
188 self.saw_any_marker = true;
189 if !content_before.is_empty() {
190 self.consume_content(&content_before, events);
191 }
192 self.handle_marker(&name, events);
193 if matches!(self.state, ParserState::AfterCompleted) || self.seen_completed {
194 return;
195 }
196 remainder
197 }
198 };
199
200 loop {
203 if remaining.is_empty() {
204 return;
205 }
206 let scan = self.marker_detector.push(&remaining);
207 match scan {
208 MarkerScan::NoMarker { content_before } => {
209 if !content_before.is_empty() {
210 self.consume_content(&content_before, events);
211 }
212 return;
213 }
214 MarkerScan::MarkerFound {
215 content_before,
216 name,
217 remainder: next,
218 } => {
219 self.saw_any_marker = true;
220 if !content_before.is_empty() {
221 self.consume_content(&content_before, events);
222 }
223 self.handle_marker(&name, events);
224 if matches!(self.state, ParserState::AfterCompleted) || self.seen_completed {
225 return;
226 }
227 remaining = next;
228 }
229 }
230 }
231 }
232
233 fn consume_content(&mut self, content: &str, events: &mut Vec<ParseEvent>) {
237 match &mut self.state {
238 ParserState::InField { parser, .. } => {
239 let sub_events = parser.push(content);
240 events.extend(sub_events);
241 }
242 ParserState::BeforeFirstField => {
243 self.pre_marker_buffer.push_str(content);
244 }
245 ParserState::AfterCompleted => {
246 }
248 }
249 }
250
251 fn handle_marker(&mut self, name: &str, events: &mut Vec<ParseEvent>) {
254 let current_state = std::mem::replace(&mut self.state, ParserState::BeforeFirstField);
256 let was_in_field = matches!(current_state, ParserState::InField { .. });
257 if let ParserState::InField {
258 name: prev_name,
259 path: prev_path,
260 parser,
261 } = current_state
262 {
263 let (parser_events, value) = parser.finish();
264 events.extend(parser_events);
265 events.push(ParseEvent::FieldComplete {
266 path: prev_path,
267 value: value.clone(),
268 });
269 self.completed_fields.entry(prev_name).or_insert(value);
270 }
271
272 if name == "completed" {
274 if !was_in_field && self.completed_fields.is_empty() {
290 let rescued = self.try_synthesize_single_string_output(events);
291 if !rescued {
292 events.push(ParseEvent::StreamError {
293 path: None,
294 message: "stream ended without any field markers".to_owned(),
295 });
296 }
297 }
298 let output = FieldValue::Object(std::mem::take(&mut self.completed_fields));
299 events.push(ParseEvent::StreamComplete { output });
300 self.state = ParserState::AfterCompleted;
301 self.seen_completed = true;
302 return;
303 }
304
305 if !self.output_names.contains(name) {
307 return;
313 }
314
315 if self.completed_fields.contains_key(name) {
320 return;
321 }
322
323 let field_type = self
326 .signature
327 .output_fields()
328 .find(|f| f.name == name)
329 .map(|f| f.field_type.clone());
330 let Some(field_type) = field_type else {
331 return; };
333
334 let path = format!("/{name}");
335 let parser = parser_for(&field_type, path.clone());
336 events.push(ParseEvent::FieldStart {
337 path: path.clone(),
338 field_type,
339 });
340 self.state = ParserState::InField {
341 name: name.to_owned(),
342 path,
343 parser,
344 };
345 }
346
347 fn try_synthesize_single_string_output(&mut self, events: &mut Vec<ParseEvent>) -> bool {
357 let single_string_field_name = {
358 let mut iter = self.signature.output_fields();
359 let first = iter.next();
360 let rest_exists = iter.next().is_some();
361 match (first, rest_exists) {
362 (Some(f), false) if matches!(f.field_type, FieldType::String) => {
363 Some(f.name.clone())
364 }
365 _ => None,
366 }
367 };
368 let Some(field_name) = single_string_field_name else {
369 return false;
370 };
371
372 let path = format!("/{field_name}");
373 let trimmed_value = FieldValue::Str(self.pre_marker_buffer.trim().to_owned());
374 events.push(ParseEvent::FieldStart {
375 path: path.clone(),
376 field_type: FieldType::String,
377 });
378 if !self.pre_marker_buffer.is_empty() {
379 events.push(ParseEvent::ValueAppend {
380 path: path.clone(),
381 delta: std::mem::take(&mut self.pre_marker_buffer),
382 });
383 }
384 events.push(ParseEvent::FieldComplete {
385 path,
386 value: trimmed_value.clone(),
387 });
388 self.completed_fields.insert(field_name, trimmed_value);
389 true
390 }
391}
392
393fn output_schema_hash(signature: &Signature) -> String {
398 use std::{
399 collections::hash_map::DefaultHasher,
400 hash::{Hash, Hasher},
401 };
402 let mut h = DefaultHasher::new();
403 for f in signature.output_fields() {
404 f.name.hash(&mut h);
405 f.field_type.type_label().hash(&mut h);
406 }
407 format!("{:016x}", h.finish())
408}
409
410#[cfg(test)]
411mod tests {
412 use typesayer_types::field::FieldDef;
413
414 use super::*;
415
416 fn simple_string_signature() -> Signature {
417 Signature::builder("test")
418 .input(FieldDef::input("q", FieldType::String, "question"))
419 .output(FieldDef::output("answer", FieldType::String, "the answer"))
420 .build()
421 .unwrap()
422 }
423
424 fn two_string_signature() -> Signature {
425 Signature::builder("test")
429 .input(FieldDef::input("q", FieldType::String, "question"))
430 .output(FieldDef::output("a", FieldType::String, "first"))
431 .output(FieldDef::output("b", FieldType::String, "second"))
432 .build()
433 .unwrap()
434 }
435
436 fn list_of_strings_signature() -> Signature {
437 Signature::builder("test")
438 .input(FieldDef::input("q", FieldType::String, "question"))
439 .output(FieldDef::output(
440 "items",
441 FieldType::List(Box::new(FieldType::String)),
442 "items",
443 ))
444 .build()
445 .unwrap()
446 }
447
448 fn collect_events(events: Vec<ParseEvent>) -> Vec<String> {
449 events.iter().map(|e| e.event_type().to_owned()).collect()
450 }
451
452 #[test]
453 fn simple_string_schema_emits_expected_sequence() {
454 let sig = simple_string_signature();
455 let mut parser = ChatStreamParser::new(&sig);
456 let mut events = Vec::new();
457 events.extend(parser.push("[[ ## answer ## ]]\nhello"));
458 events.extend(parser.push(" world\n[[ ## completed ## ]]"));
459 events.extend(parser.finish());
460
461 let types = collect_events(events.clone());
462 assert_eq!(types[0], "stream_start");
464 assert_eq!(types[1], "field_start");
465 let value_append_count = types.iter().filter(|t| *t == "value_append").count();
466 assert!(value_append_count >= 1, "expected at least 1 value_append");
467 assert_eq!(types[types.len() - 2], "field_complete");
468 assert_eq!(types[types.len() - 1], "stream_complete");
469
470 match events.last().unwrap() {
472 ParseEvent::StreamComplete { output } => {
473 let FieldValue::Object(map) = output else {
474 panic!("expected Object")
475 };
476 let answer = map.get("answer").unwrap();
477 assert_eq!(answer, &FieldValue::Str("hello world".into()));
478 }
479 other => panic!("expected StreamComplete, got {other:?}"),
480 }
481 }
482
483 #[test]
484 fn no_marker_fallback_for_single_string_output() {
485 let sig = simple_string_signature();
486 let mut parser = ChatStreamParser::new(&sig);
487 let mut events = Vec::new();
488 events.extend(parser.push("Paris"));
489 events.extend(parser.finish());
490
491 let types = collect_events(events.clone());
492 assert_eq!(
494 types,
495 vec![
496 "stream_start",
497 "field_start",
498 "value_append",
499 "field_complete",
500 "stream_complete"
501 ]
502 );
503
504 match events.last().unwrap() {
506 ParseEvent::StreamComplete {
507 output: FieldValue::Object(map),
508 } => {
509 assert_eq!(map.get("answer"), Some(&FieldValue::Str("Paris".into())));
510 }
511 other => panic!("got {other:?}"),
512 }
513 }
514
515 #[test]
516 fn list_of_strings_schema_emits_open_added_close() {
517 let sig = list_of_strings_signature();
518 let mut parser = ChatStreamParser::new(&sig);
519 let mut events = Vec::new();
520 events.extend(parser.push(
521 r#"[[ ## items ## ]]
522["a", "b", "c"]
523[[ ## completed ## ]]"#,
524 ));
525 events.extend(parser.finish());
526
527 let types = collect_events(events.clone());
528 let element_added_count = types.iter().filter(|t| *t == "element_added").count();
531 assert_eq!(element_added_count, 3);
532 assert!(types.contains(&"list_open".to_owned()));
533 assert!(types.contains(&"list_close".to_owned()));
534 assert!(types.contains(&"stream_complete".to_owned()));
535
536 match events.last().unwrap() {
537 ParseEvent::StreamComplete {
538 output: FieldValue::Object(map),
539 } => {
540 let items = map.get("items").unwrap();
541 let FieldValue::List(v) = items else {
542 panic!("expected List")
543 };
544 let strs: Vec<&str> = v
545 .iter()
546 .map(|x| match x {
547 FieldValue::Str(s) => s.as_str(),
548 _ => panic!(),
549 })
550 .collect();
551 assert_eq!(strs, vec!["a", "b", "c"]);
552 }
553 other => panic!("got {other:?}"),
554 }
555 }
556
557 fn string_and_list_signature() -> Signature {
558 Signature::builder("test")
563 .input(FieldDef::input("q", FieldType::String, "question"))
564 .output(FieldDef::output("a", FieldType::String, "first text"))
565 .output(FieldDef::output(
566 "b",
567 FieldType::List(Box::new(FieldType::String)),
568 "second list",
569 ))
570 .build()
571 .unwrap()
572 }
573
574 #[test]
575 fn two_field_schema_full_event_sequence_ordering() {
576 let sig = string_and_list_signature();
583 let mut parser = ChatStreamParser::new(&sig);
584 let mut events = Vec::new();
585 events.extend(parser.push(
586 r#"[[ ## a ## ]]
587first
588[[ ## b ## ]]
589["x", "y"]
590[[ ## completed ## ]]"#,
591 ));
592 events.extend(parser.finish());
593
594 let types = collect_events(events.clone());
595 assert_eq!(
596 types,
597 vec![
598 "stream_start",
599 "field_start",
600 "value_append",
601 "field_complete",
602 "field_start",
603 "list_open",
604 "element_added",
605 "element_added",
606 "list_close",
607 "field_complete",
608 "stream_complete",
609 ]
610 );
611
612 match events.last().unwrap() {
615 ParseEvent::StreamComplete {
616 output: FieldValue::Object(map),
617 } => {
618 assert_eq!(map.get("a"), Some(&FieldValue::Str("first".into())));
619 assert_eq!(
620 map.get("b"),
621 Some(&FieldValue::List(vec![
622 FieldValue::Str("x".into()),
623 FieldValue::Str("y".into()),
624 ]))
625 );
626 }
627 other => panic!("expected StreamComplete, got {other:?}"),
628 }
629 }
630
631 #[test]
632 fn out_of_declared_order_field_emission() {
633 let sig = Signature::builder("test")
641 .input(FieldDef::input("q", FieldType::String, "question"))
642 .output(FieldDef::output("a", FieldType::String, "first"))
643 .output(FieldDef::output("b", FieldType::String, "second"))
644 .build()
645 .unwrap();
646 let mut parser = ChatStreamParser::new(&sig);
647 let mut events = Vec::new();
648 events.extend(
649 parser.push("[[ ## b ## ]]\nbcontent\n[[ ## a ## ]]\nacontent\n[[ ## completed ## ]]"),
650 );
651 events.extend(parser.finish());
652
653 let field_start_paths: Vec<String> = events
657 .iter()
658 .filter_map(|e| match e {
659 ParseEvent::FieldStart { path, .. } => Some(path.clone()),
660 _ => None,
661 })
662 .collect();
663 assert_eq!(field_start_paths, vec!["/b", "/a"]);
664
665 match events.last().unwrap() {
667 ParseEvent::StreamComplete {
668 output: FieldValue::Object(map),
669 } => {
670 assert_eq!(map.get("a"), Some(&FieldValue::Str("acontent".into())));
671 assert_eq!(map.get("b"), Some(&FieldValue::Str("bcontent".into())));
672 }
673 other => panic!("expected StreamComplete, got {other:?}"),
674 }
675 }
676
677 #[test]
678 fn stream_error_mid_stream_halts_field_emission() {
679 let sig = list_of_strings_signature();
687 let mut parser = ChatStreamParser::new(&sig);
688 let mut events = Vec::new();
689 events.extend(parser.push(
690 r#"[[ ## items ## ]]
691["a", bad_token, "c"]
692[[ ## completed ## ]]"#,
693 ));
694 events.extend(parser.finish());
695
696 assert!(
698 events
699 .iter()
700 .any(|e| matches!(e, ParseEvent::StreamError { .. })),
701 "malformed JSON in list must emit at least one StreamError"
702 );
703
704 let c_was_added = events.iter().any(
709 |e| matches!(e, ParseEvent::ElementAdded { value: FieldValue::Str(s), .. } if s == "c"),
710 );
711 assert!(!c_was_added, "no element_added for `c` after StreamError");
712
713 assert!(
715 events
716 .iter()
717 .any(|e| matches!(e, ParseEvent::StreamComplete { .. })),
718 "StreamComplete must still fire after mid-stream StreamError"
719 );
720 }
721
722 #[test]
723 fn unknown_field_marker_silently_skipped() {
724 let sig = simple_string_signature();
727 let mut parser = ChatStreamParser::new(&sig);
728 let mut events = Vec::new();
729 events.extend(parser.push(
730 "[[ ## hallucinated ## ]]\nignored\n[[ ## answer ## ]]\nactual\n[[ ## completed ## ]]",
731 ));
732 events.extend(parser.finish());
733
734 let field_start_paths: Vec<String> = events
735 .iter()
736 .filter_map(|e| match e {
737 ParseEvent::FieldStart { path, .. } => Some(path.clone()),
738 _ => None,
739 })
740 .collect();
741 assert_eq!(field_start_paths, vec!["/answer"]);
742
743 match events.last().unwrap() {
744 ParseEvent::StreamComplete {
745 output: FieldValue::Object(map),
746 } => {
747 assert_eq!(map.get("answer"), Some(&FieldValue::Str("actual".into())));
748 assert!(!map.contains_key("hallucinated"));
749 }
750 other => panic!("got {other:?}"),
751 }
752 }
753
754 #[test]
755 fn completed_sentinel_only_rescues_single_string_output() {
756 let sig = simple_string_signature();
762 let mut parser = ChatStreamParser::new(&sig);
763 let mut events = Vec::new();
764 events.extend(parser.push("Paris is the capital of France.\n[[ ## completed ## ]]"));
765 events.extend(parser.finish());
766
767 let types = collect_events(events.clone());
771 assert_eq!(
772 types,
773 vec![
774 "stream_start",
775 "field_start",
776 "value_append",
777 "field_complete",
778 "stream_complete",
779 ]
780 );
781
782 match events.last().unwrap() {
783 ParseEvent::StreamComplete {
784 output: FieldValue::Object(map),
785 } => {
786 assert_eq!(
787 map.get("answer"),
788 Some(&FieldValue::Str("Paris is the capital of France.".into()))
789 );
790 }
791 other => panic!("expected StreamComplete with rescued value, got {other:?}"),
792 }
793 }
794
795 #[test]
796 fn completed_sentinel_only_multi_output_emits_stream_error() {
797 let sig = two_string_signature();
808 let mut parser = ChatStreamParser::new(&sig);
809 let mut events = Vec::new();
810 events.extend(parser.push("Some content\n[[ ## completed ## ]]"));
811 events.extend(parser.finish());
812
813 let types = collect_events(events.clone());
815 assert_eq!(
816 types,
817 vec!["stream_start", "stream_error", "stream_complete"]
818 );
819
820 let error_msg = events
822 .iter()
823 .find_map(|e| match e {
824 ParseEvent::StreamError { message, path } if path.is_none() => Some(message),
825 _ => None,
826 })
827 .expect("multi-output sentinel-only must emit a path-less StreamError");
828 assert!(
829 error_msg.contains("without any field markers"),
830 "StreamError message should describe the missing-marker condition; got {error_msg:?}"
831 );
832
833 match events.last().unwrap() {
838 ParseEvent::StreamComplete {
839 output: FieldValue::Object(map),
840 } => {
841 assert!(
842 map.is_empty(),
843 "multi-output sentinel-only yields empty output map"
844 );
845 }
846 other => panic!("expected StreamComplete, got {other:?}"),
847 }
848 }
849
850 #[test]
851 fn schema_hash_stable_for_same_signature() {
852 let sig1 = simple_string_signature();
853 let sig2 = simple_string_signature();
854 assert_eq!(output_schema_hash(&sig1), output_schema_hash(&sig2));
855 }
856
857 #[test]
858 fn schema_hash_differs_for_different_signatures() {
859 let s1 = simple_string_signature();
860 let s2 = list_of_strings_signature();
861 assert_ne!(output_schema_hash(&s1), output_schema_hash(&s2));
862 }
863
864 use proptest::prelude::*;
865
866 use crate::proptest_strategies::{
867 DriftPerturbation, arb_chunking_positions, arb_supported_type_and_value,
868 arb_unsupported_field_type, chunk_at_positions, serialize_completion,
869 serialize_completion_with_drift, single_output_signature,
870 };
871
872 #[test]
873 fn parity_fence_json_chunked_at_various_points() {
874 use typesayer_types::ObjectField;
877 let field_type = FieldType::List(Box::new(FieldType::List(Box::new(FieldType::Object(
878 vec![ObjectField {
879 name: "f1".into(),
880 description: String::new(),
881 field_type: FieldType::String,
882 }],
883 )))));
884 let value = FieldValue::List(vec![
886 FieldValue::List(vec![
887 FieldValue::Object(
888 std::iter::once(("f1".to_string(), FieldValue::Str("abc".to_string())))
889 .collect(),
890 ),
891 FieldValue::Object(
892 std::iter::once(("f1".to_string(), FieldValue::Str("def".to_string())))
893 .collect(),
894 ),
895 ]),
896 FieldValue::List(vec![FieldValue::Object(
897 std::iter::once(("f1".to_string(), FieldValue::Str("ghi".to_string()))).collect(),
898 )]),
899 ]);
900 let signature = single_output_signature("answer", field_type.clone());
901 let completion = serialize_completion_with_drift(
902 "answer",
903 &field_type,
904 &value,
905 DriftPerturbation::FenceJson,
906 );
907 for split in 1..completion.len() {
909 if !completion.is_char_boundary(split) {
910 continue;
911 }
912 let mut parser = ChatStreamParser::new(&signature);
913 let mut events = Vec::new();
914 events.extend(parser.push(&completion[..split]));
915 events.extend(parser.push(&completion[split..]));
916 events.extend(parser.finish());
917 let streamed_value = match events.last() {
918 Some(ParseEvent::StreamComplete {
919 output: FieldValue::Object(map),
920 }) => map.get("answer").cloned(),
921 _ => None,
922 };
923 assert_eq!(
924 streamed_value.as_ref(),
925 Some(&value),
926 "split={split} input={:?}",
927 &completion[..split]
928 );
929 }
930 }
931
932 #[test]
933 fn parity_fence_json_with_empty_string_first() {
934 use typesayer_types::ObjectField;
935 let field_type = FieldType::List(Box::new(FieldType::List(Box::new(FieldType::Object(
939 vec![ObjectField {
940 name: "f1".into(),
941 description: String::new(),
942 field_type: FieldType::String,
943 }],
944 )))));
945 let value = FieldValue::List(vec![FieldValue::List(vec![
946 FieldValue::Object(
947 std::iter::once(("f1".to_string(), FieldValue::Str(String::new()))).collect(),
948 ),
949 FieldValue::Object(
950 std::iter::once((
951 "f1".to_string(),
952 FieldValue::Str(
953 "CPcb.7tmqOtkv4WyIny0D9vfIcR!LpVu-aY6fmGOl HKNNVtt,S".to_string(),
954 ),
955 ))
956 .collect(),
957 ),
958 ])]);
959 let signature = single_output_signature("answer", field_type.clone());
960 let completion = serialize_completion_with_drift(
961 "answer",
962 &field_type,
963 &value,
964 DriftPerturbation::FenceJson,
965 );
966 let mut parser = ChatStreamParser::new(&signature);
967 let mut events = Vec::new();
968 events.extend(parser.push(&completion));
969 events.extend(parser.finish());
970 let streamed_value = match events.last() {
971 Some(ParseEvent::StreamComplete {
972 output: FieldValue::Object(map),
973 }) => map.get("answer").cloned(),
974 _ => None,
975 };
976 assert_eq!(streamed_value.as_ref(), Some(&value));
977 }
978
979 #[test]
980 fn parity_fence_json_nested_list_of_list_of_object() {
981 use typesayer_types::ObjectField;
982 let field_type = FieldType::List(Box::new(FieldType::List(Box::new(FieldType::Object(
985 vec![ObjectField {
986 name: "f1".into(),
987 description: String::new(),
988 field_type: FieldType::String,
989 }],
990 )))));
991 let value = FieldValue::List(vec![FieldValue::List(vec![FieldValue::Object(
992 std::iter::once(("f1".to_string(), FieldValue::Str("hi".to_string()))).collect(),
993 )])]);
994 let signature = single_output_signature("answer", field_type.clone());
995 let completion = serialize_completion_with_drift(
996 "answer",
997 &field_type,
998 &value,
999 DriftPerturbation::FenceJson,
1000 );
1001 let mut parser = ChatStreamParser::new(&signature);
1002 let mut events = Vec::new();
1003 events.extend(parser.push(&completion));
1004 events.extend(parser.finish());
1005 let streamed_value = match events.last() {
1006 Some(ParseEvent::StreamComplete {
1007 output: FieldValue::Object(map),
1008 }) => map.get("answer").cloned(),
1009 _ => None,
1010 };
1011 assert_eq!(streamed_value.as_ref(), Some(&value));
1012 }
1013
1014 fn parse_value_through_chunks(
1020 field_type: &FieldType,
1021 value: &FieldValue,
1022 positions: &[u8],
1023 ) -> Vec<ParseEvent> {
1024 let signature = single_output_signature("answer", field_type.clone());
1025 let completion = serialize_completion("answer", field_type, value);
1026 let chunks = chunk_at_positions(&completion, positions);
1027
1028 let mut parser = ChatStreamParser::new(&signature);
1029 let mut events = Vec::new();
1030 for chunk in chunks {
1031 events.extend(parser.push(chunk));
1032 }
1033 events.extend(parser.finish());
1034 events
1035 }
1036
1037 proptest! {
1038 #[test]
1048 fn chat_stream_parser_roundtrips_any_supported_value(
1049 (field_type, value) in arb_supported_type_and_value(),
1050 positions in arb_chunking_positions(),
1051 ) {
1052 let events = parse_value_through_chunks(&field_type, &value, &positions);
1053
1054 let expected = match (&field_type, &value) {
1058 (FieldType::String, FieldValue::Str(s)) => FieldValue::Str(s.trim().to_owned()),
1059 _ => value.clone(),
1060 };
1061
1062 match events.last() {
1063 Some(ParseEvent::StreamComplete { output: FieldValue::Object(map) }) => {
1064 prop_assert_eq!(map.get("answer"), Some(&expected));
1065 }
1066 Some(other) => {
1067 prop_assert!(false, "expected StreamComplete, got {other:?}");
1068 }
1069 None => {
1070 prop_assert!(false, "no events emitted");
1071 }
1072 }
1073 }
1074
1075 #[test]
1081 fn unsupported_field_type_emits_stream_error_without_panic(
1082 field_type in arb_unsupported_field_type(),
1083 positions in arb_chunking_positions(),
1084 ) {
1085 let signature = single_output_signature("answer", field_type.clone());
1086 let completion = "[[ ## answer ## ]]\n \n[[ ## completed ## ]]";
1092 let chunks = chunk_at_positions(completion, &positions);
1093
1094 let mut parser = ChatStreamParser::new(&signature);
1095 let mut events = Vec::new();
1096 for chunk in chunks {
1097 events.extend(parser.push(chunk));
1098 }
1099 events.extend(parser.finish());
1100
1101 let has_stream_error = events
1109 .iter()
1110 .any(|e| matches!(e, ParseEvent::StreamError { .. }));
1111 prop_assert!(
1112 has_stream_error,
1113 "expected StreamError for unsupported type {field_type:?}, events: {events:?}"
1114 );
1115 }
1116
1117 }
1118}