use super::*;
fn scan(stream: &str, max: usize) -> Vec<String> {
let mut lines: Vec<&[u8]> = stream.as_bytes().split(|b| *b == b'\n').collect();
if stream.ends_with('\n') {
lines.pop();
}
let mut scanner = DataLines::new(max);
let mut out = Vec::new();
for line in lines {
scanner.push(line.strip_suffix(b"\r").unwrap_or(line), &mut out);
}
scanner.finish(&mut out);
out.iter()
.map(|l| String::from_utf8(l.line.to_vec()).unwrap())
.collect()
}
#[test]
fn assembles_event_data() {
let marker = "body-bytes-must-not-be-logged";
let cases: Vec<(&str, String, Vec<String>)> = vec![
(
"one-line events pass through as read",
"event: response.created\ndata: {\"type\":\"response.created\"}\n\n: keep-alive\n\ndata: {\"type\":\"x\",\"v\":\"a b\"}\n\n".into(),
vec!["event: response.created".into(), r#"data: {"type":"response.created"}"#.into(), "".into(), ": keep-alive".into(), "".into(), r#"data: {"type":"x","v":"a b"}"#.into(), "".into()],
),
(
"CRLF line endings",
"event: a\r\ndata: {\"type\":\"a\"}\r\n\r\n".into(),
vec!["event: a".into(), r#"data: {"type":"a"}"#.into(), "".into()],
),
(
"a value after a Unicode space starts an event",
"data:\u{a0}{\"type\":\"response.completed\",\ndata: \"response\":{\"output\":[]}}\n\n".into(),
vec!["data:\u{a0}{\"type\":\"response.completed\",\ndata: \"response\":{\"output\":[]}}".into(), "".into()],
),
(
"data lines of one event are joined and compacted",
"event: response.completed\ndata: {\"type\":\"response.completed\",\ndata: \"response\":{\"id\":\"r\",\"output\":[]}}\n\n".into(),
vec!["event: response.completed".into(), r#"data: {"type":"response.completed","response":{"id":"r","output":[]}}"#.into(), "".into()],
),
(
"joined CRLF event",
"data: {\"a\":\r\ndata: [1,\r\ndata: 2]}\r\n\r\n".into(),
vec![r#"data: {"a":[1,2]}"#.into(), "".into()],
),
(
"comment and field lines inside an event come before its data",
"event: e\ndata: {\"a\":\n: keep-alive\nid: 7\nretry: 3000\ndata: 1}\n\n".into(),
vec!["event: e".into(), ": keep-alive".into(), "id: 7".into(), "retry: 3000".into(), r#"data: {"a":1}"#.into(), "".into()],
),
(
"unterminated one-line event at end of stream",
"data: {\"type\":\"response.completed\"}".into(),
vec![r#"data: {"type":"response.completed"}"#.into()],
),
(
"unterminated multi-line event at end of stream",
"data: {\"type\":\ndata: \"response.completed\"}\n".into(),
vec![r#"data: {"type":"response.completed"}"#.into()],
),
(
"complete events without blank separators stay separate",
"data: {\"a\":1}\ndata: {\"b\":2}\n".into(),
vec![r#"data: {"a":1}"#.into(), r#"data: {"b":2}"#.into()],
),
(
"split event followed by an event without a blank separator",
"data: {\"a\":\ndata: 1}\ndata: {\"b\":2}\n".into(),
vec![r#"data: {"a":1}"#.into(), r#"data: {"b":2}"#.into()],
),
(
"an invalid fragment absorbs the following data lines up to the blank line",
format!("data: {{\"a\":\"{marker}\ndata: {{\"b\":2}}\n\n"),
vec![format!("data: {{\"a\":\"{marker}\ndata: {{\"b\":2}}"), "".into()],
),
(
"only the first invalid event in a stream is described",
"data: {\"a\":\n\ndata: {\"b\":\n\ndata: {\"c\":3}\n\n".into(),
vec!["data: {\"a\":".into(), "".into(), "data: {\"b\":".into(), "".into(), r#"data: {"c":3}"#.into(), "".into()],
),
(
"a complete object on its own line inside an open array stays in the event",
"data: {\"type\":\"response.completed\",\"response\":{\"output\":[\ndata: {\"id\":\"m\",\"type\":\"message\"}\ndata: ]}}\n\n".into(),
vec![r#"data: {"type":"response.completed","response":{"output":[{"id":"m","type":"message"}]}}"#.into(), "".into()],
),
(
"unterminated incomplete event at end of stream is still delivered",
format!("data: {{\"type\":\"response.completed\",\"t\":\"{marker}\","),
vec![format!("data: {{\"type\":\"response.completed\",\"t\":\"{marker}\",")],
),
(
"empty and non-JSON data lines take the stock path",
"data:\n\ndata: \n\ndata: ping\n\n".into(),
vec!["data:".into(), "".into(), "data: ".into(), "".into(), "data: ping".into(), "".into()],
),
("done sentinel", "data: [DONE]\n\n".into(), vec!["data: [DONE]".into(), "".into()]),
(
"invalid one-line event passes through with one warning",
format!("data: {{\"type\":\"response.completed\",\"t\":\"{marker}\",\n\n"),
vec![format!("data: {{\"type\":\"response.completed\",\"t\":\"{marker}\","), "".into()],
),
(
"event split inside a string stays invalid and keeps its data lines",
format!("data: {{\"t\":\"{marker}\ndata: {marker}\"}}\n\n"),
vec![format!("data: {{\"t\":\"{marker}\ndata: {marker}\"}}"), "".into()],
),
];
for (name, stream, want) in cases {
assert_eq!(scan(&stream, 1 << 20), want, "{name}");
}
}
#[test]
fn bounds_an_event_to_the_line_limit() {
let (x, y, z) = ("x".repeat(40), "y".repeat(40), "z".repeat(40));
let stream = format!("data: {{\"a\":\"{x}\ndata: {y}\ndata: {z}\"}}\n\ndata: {{\"b\":2}}\n\n");
let want = vec![
format!("data: {{\"a\":\"{x}"),
format!("data: {y}"),
format!("data: {z}\"}}"),
String::new(),
r#"data: {"b":2}"#.to_owned(),
String::new(),
];
assert_eq!(scan(&stream, 64), want);
}
#[test]
fn assembles_an_event_of_thousands_of_lines() {
let item = |i: usize| {
let letter = (b'a' + (i % 26) as u8) as char;
format!(
r#"{{"type":"message","content":[{{"type":"output_text","text":"{}"}}]}}"#,
letter.to_string().repeat(960)
)
};
let head = r#"{"type":"response.completed","response":{"id":"resp_big","output":["#;
let (mut payload, mut stream) = (format!("{head}{}", item(0)), format!("data: {head}{}\n", item(0)));
for i in 1..2000 {
payload.push_str(&format!(",{}", item(i)));
stream.push_str(&format!("data: ,{}\n", item(i)));
}
payload.push_str("]}}");
stream.push_str("data: ]}}\n\n");
let started = std::time::Instant::now();
let got = scan(&stream, 52_428_800);
assert_eq!(got, vec![format!("data: {payload}"), String::new()]);
assert!(
started.elapsed() < std::time::Duration::from_secs(2),
"{:?}",
started.elapsed()
);
}
#[test]
fn matches_line_splitting_on_well_formed_streams() {
let mut stream =
"event: response.created\ndata: {\"type\":\"response.created\",\"response\":{\"id\":\"resp_1\",\"model\":\"gpt-5.4\"}}\n\n"
.to_owned();
for i in 0..6 {
stream.push_str(&format!(
": comment-{i:04}\nevent: response.output_text.delta\ndata: {{\"type\":\"response.output_text.delta\",\"output_index\":0,\"content_index\":0,\"delta\":\"{}\"}}\n\n",
"x".repeat(64)
));
}
stream.push_str("data: {\"type\":\"response.completed\",\"response\":{\"id\":\"resp_1\",\"status\":\"completed\",\"output\":[]}}\n\n");
stream.push_str(
"data: {\"type\":\"x\"}\ndata: {\"type\":\"y\"}\n: trailing comment\n\nevent: done\ndata: [DONE]\n\n",
);
for newline in ["\n", "\r\n"] {
let input = stream.replace('\n', newline);
let want: Vec<String> = input.lines().map(str::to_owned).collect();
assert_eq!(scan(&input, 1 << 20), want, "{newline:?}");
}
}
#[test]
fn an_invalid_split_event_stays_invalid() {
let mut processor = crate::codex_response::Processor::new(false, false);
let event = b"event: response.completed\ndata: {\"type\":\"response.completed\",\"response\":{\"id\":\"resp_bad\",\"status\":\"comp\ndata: leted\",\"output\":[]}}\n\n";
let crate::codex_response::Step::Terminal(out) = processor.event(event) else {
panic!("a terminal event");
};
assert_eq!(&out[..], &event[..]);
}