use std::time::Duration;
use bytes::BytesMut;
use crate::{Error, Event, Result};
const EVENT_LIMIT: usize = 32 * 1024 * 1024;
#[derive(Debug)]
pub(super) struct Decoder {
framing: Framing,
}
#[derive(Debug)]
enum Framing {
Json(JsonDecoder),
Ndjson(NdjsonDecoder),
Sse(SseDecoder),
}
impl Decoder {
pub(super) fn for_content_type(content_type: Option<&str>) -> Result<Self> {
let content_type = content_type.ok_or_else(|| {
Error::decode("response Content-Type is required to infer event framing")
})?;
let content_type = content_type
.split(';')
.next()
.map(str::trim)
.unwrap_or_default()
.to_ascii_lowercase();
let framing = match content_type.as_str() {
"text/event-stream" => Framing::Sse(SseDecoder::new(EVENT_LIMIT)),
"application/x-ndjson"
| "application/ndjson"
| "application/x-jsonl"
| "application/jsonl" => Framing::Ndjson(NdjsonDecoder::new(EVENT_LIMIT)),
"application/json" => Framing::Json(JsonDecoder::new(EVENT_LIMIT)),
_ if content_type
.split_once('/')
.is_some_and(|(_, subtype)| subtype.ends_with("+json")) =>
{
Framing::Json(JsonDecoder::new(EVENT_LIMIT))
}
_ => {
return Err(Error::decode(
"response Content-Type does not describe JSON, NDJSON, JSON Lines, or SSE events",
));
}
};
Ok(Self { framing })
}
pub(super) fn push_one(&mut self, bytes: &[u8], offset: &mut usize) -> Result<Option<Event>> {
match &mut self.framing {
Framing::Json(decoder) => decoder.push_one(bytes, offset),
Framing::Ndjson(decoder) => decoder.push_one(bytes, offset),
Framing::Sse(decoder) => decoder.push_one(bytes, offset),
}
}
pub(super) fn finish(&mut self) -> Result<Option<Event>> {
match &mut self.framing {
Framing::Json(decoder) => decoder.finish(),
Framing::Ndjson(decoder) => decoder.finish(),
Framing::Sse(decoder) => decoder.finish(),
}
}
}
#[derive(Debug)]
struct JsonDecoder {
data: BytesMut,
limit: usize,
}
impl JsonDecoder {
fn new(limit: usize) -> Self {
Self {
data: BytesMut::new(),
limit,
}
}
fn push_one(&mut self, bytes: &[u8], offset: &mut usize) -> Result<Option<Event>> {
let data = &bytes[*offset..];
let new_len = self
.data
.len()
.checked_add(data.len())
.filter(|length| *length <= self.limit)
.ok_or_else(|| Error::decode("JSON event exceeds the 32 MiB limit"))?;
self.data.reserve(new_len.saturating_sub(self.data.len()));
self.data.extend_from_slice(data);
*offset = bytes.len();
Ok(None)
}
fn finish(&mut self) -> Result<Option<Event>> {
let data = self.data.split().freeze();
let data = String::from_utf8(data.to_vec())
.map_err(|_| Error::decode("JSON event is not valid UTF-8"))?;
Ok(Some(Event::new(data)))
}
}
#[derive(Debug)]
struct NdjsonDecoder {
record: BytesMut,
limit: usize,
}
impl NdjsonDecoder {
fn new(limit: usize) -> Self {
Self {
record: BytesMut::new(),
limit,
}
}
fn push_one(&mut self, bytes: &[u8], offset: &mut usize) -> Result<Option<Event>> {
let remaining = &bytes[*offset..];
if let Some(newline) = remaining.iter().position(|byte| *byte == b'\n') {
self.record.extend_from_slice(&remaining[..newline]);
*offset += newline + 1;
self.ensure_record_fits()?;
return self.finish_record().map(Some);
}
self.record.extend_from_slice(remaining);
*offset = bytes.len();
self.ensure_record_fits()?;
Ok(None)
}
fn finish(&mut self) -> Result<Option<Event>> {
if self.record.is_empty() {
return Ok(None);
}
self.finish_record().map(Some)
}
fn finish_record(&mut self) -> Result<Event> {
if self.record.last() == Some(&b'\r') {
self.record.truncate(self.record.len() - 1);
}
if self.record.is_empty() {
return Err(Error::decode("NDJSON event is empty"));
}
if self.record.len() > self.limit {
return Err(Error::decode("NDJSON event exceeds the 32 MiB limit"));
}
let record = self.record.split().freeze();
let data = String::from_utf8(record.to_vec())
.map_err(|_| Error::decode("NDJSON event is not valid UTF-8"))?;
Ok(Event::new(data))
}
fn ensure_record_fits(&self) -> Result<()> {
let may_end_with_cr =
self.record.len() == self.limit + 1 && self.record.last() == Some(&b'\r');
if self.record.len() > self.limit && !may_end_with_cr {
return Err(Error::decode("NDJSON event exceeds the 32 MiB limit"));
}
Ok(())
}
}
#[derive(Debug, Default)]
enum BomState {
#[default]
Start,
Ef,
EfBb,
Done,
}
#[derive(Debug)]
struct SseDecoder {
line: Vec<u8>,
data: String,
event: Option<String>,
last_event_id: Option<String>,
retry: Option<Duration>,
event_bytes: usize,
event_limit: usize,
pending_cr: bool,
bom: BomState,
}
impl SseDecoder {
fn new(event_limit: usize) -> Self {
Self {
line: Vec::new(),
data: String::new(),
event: None,
last_event_id: None,
retry: None,
event_bytes: 0,
event_limit,
pending_cr: false,
bom: BomState::default(),
}
}
fn push_one(&mut self, bytes: &[u8], offset: &mut usize) -> Result<Option<Event>> {
while let Some(&byte) = bytes.get(*offset) {
*offset += 1;
if let Some(event) = self.byte(byte)? {
return Ok(Some(event));
}
}
Ok(None)
}
fn finish(&mut self) -> Result<Option<Event>> {
match std::mem::replace(&mut self.bom, BomState::Done) {
BomState::Ef => {
self.event_byte(0xef)?;
}
BomState::EfBb => {
self.event_byte(0xef)?;
self.event_byte(0xbb)?;
}
BomState::Start | BomState::Done => {}
}
let event = if self.pending_cr {
self.pending_cr = false;
self.finish_line()?
} else {
if !self.line.is_empty() {
std::str::from_utf8(&self.line)
.map_err(|_| Error::decode("server-sent event field is not valid UTF-8"))?;
}
None
};
self.reset();
Ok(event)
}
fn byte(&mut self, byte: u8) -> Result<Option<Event>> {
match std::mem::replace(&mut self.bom, BomState::Done) {
BomState::Start if byte == 0xef => {
self.bom = BomState::Ef;
return Ok(None);
}
BomState::Ef if byte == 0xbb => {
self.bom = BomState::EfBb;
return Ok(None);
}
BomState::EfBb if byte == 0xbf => return Ok(None),
BomState::Ef => {
self.event_byte(0xef)?;
}
BomState::EfBb => {
self.event_byte(0xef)?;
self.event_byte(0xbb)?;
}
BomState::Start | BomState::Done => {}
}
self.event_byte(byte)
}
fn event_byte(&mut self, byte: u8) -> Result<Option<Event>> {
let mut event = None;
if self.pending_cr {
self.pending_cr = false;
if byte == b'\n' {
self.count_byte()?;
return self.finish_line();
}
event = self.finish_line()?;
}
self.count_byte()?;
match byte {
b'\r' => self.pending_cr = true,
b'\n' => return self.finish_line(),
_ => self.line.push(byte),
}
Ok(event)
}
fn finish_line(&mut self) -> Result<Option<Event>> {
let line = std::mem::take(&mut self.line);
if line.is_empty() {
let event = self.dispatch();
self.event_bytes = 0;
return Ok(event);
}
let line = std::str::from_utf8(&line)
.map_err(|_| Error::decode("server-sent event field is not valid UTF-8"))?;
if line.starts_with(':') {
return Ok(None);
}
let (field, mut value) = line.split_once(':').unwrap_or((line, ""));
if let Some(stripped) = value.strip_prefix(' ') {
value = stripped;
}
match field {
"data" => {
self.data.push_str(value);
self.data.push('\n');
}
"event" => self.event = (!value.is_empty()).then(|| value.to_owned()),
"id" if !value.contains('\0') => self.last_event_id = Some(value.to_owned()),
"retry" if !value.is_empty() && value.bytes().all(|byte| byte.is_ascii_digit()) => {
if let Ok(retry) = value.parse::<u64>() {
self.retry = Some(Duration::from_millis(retry));
}
}
_ => {}
}
Ok(None)
}
fn dispatch(&mut self) -> Option<Event> {
if self.data.is_empty() {
self.event = None;
return None;
}
self.data.pop();
Some(Event::with_metadata(
std::mem::take(&mut self.data),
self.event.take(),
self.last_event_id.clone(),
self.retry,
))
}
fn count_byte(&mut self) -> Result<()> {
self.event_bytes = self
.event_bytes
.checked_add(1)
.filter(|bytes| *bytes <= self.event_limit)
.ok_or_else(|| Error::decode("server-sent event exceeds the 32 MiB limit"))?;
Ok(())
}
fn reset(&mut self) {
self.line.clear();
self.data.clear();
self.event = None;
self.event_bytes = 0;
self.pending_cr = false;
}
}
#[cfg(test)]
mod tests {
use super::*;
fn decode_json_document(decoder: &mut JsonDecoder, chunks: &[&[u8]]) -> Result<Event> {
for chunk in chunks {
let mut offset = 0;
assert!(decoder.push_one(chunk, &mut offset)?.is_none());
assert_eq!(offset, chunk.len());
}
decoder
.finish()?
.ok_or_else(|| Error::decode("JSON decoder did not emit its document"))
}
fn decode_ndjson(decoder: &mut NdjsonDecoder, chunks: &[&[u8]]) -> Result<Vec<Event>> {
let mut events = Vec::new();
for chunk in chunks {
let mut offset = 0;
while offset < chunk.len() {
if let Some(event) = decoder.push_one(chunk, &mut offset)? {
events.push(event);
}
}
}
if let Some(event) = decoder.finish()? {
events.push(event);
}
Ok(events)
}
fn decode(decoder: &mut SseDecoder, chunks: &[&[u8]]) -> Result<Vec<Event>> {
let mut events = Vec::new();
for chunk in chunks {
let mut offset = 0;
while offset < chunk.len() {
if let Some(event) = decoder.push_one(chunk, &mut offset)? {
events.push(event);
}
}
}
if let Some(event) = decoder.finish()? {
events.push(event);
}
Ok(events)
}
#[test]
fn json_decoder_preserves_a_document_split_across_body_frames() {
let event = decode_json_document(
&mut JsonDecoder::new(1024),
&[br#"{"text":"fir"#, br#"st"}"#],
)
.unwrap();
assert_eq!(event.data(), r#"{"text":"first"}"#);
}
#[test]
fn ndjson_decoder_preserves_records_split_across_body_frames() {
let events = decode_ndjson(
&mut NdjsonDecoder::new(1024),
&[
br#"{"text":"fir"#,
b"st\"}\n{\"text\":\"second\"}\r",
b"\n{\"text\":\"third\"}",
],
)
.unwrap();
let data = events.iter().map(Event::data).collect::<Vec<_>>();
assert_eq!(
data,
[
r#"{"text":"first"}"#,
r#"{"text":"second"}"#,
r#"{"text":"third"}"#
]
);
}
#[test]
fn json_and_ndjson_decoders_enforce_the_event_limit() {
let exact = decode_json_document(&mut JsonDecoder::new(3), &[b"123"]).unwrap();
assert_eq!(exact.data(), "123");
assert!(
decode_json_document(&mut JsonDecoder::new(2), &[b"123"])
.unwrap_err()
.is_decode()
);
let exact = decode_ndjson(&mut NdjsonDecoder::new(3), &[b"123\r\n"]).unwrap();
assert_eq!(exact[0].data(), "123");
assert!(
decode_ndjson(&mut NdjsonDecoder::new(2), &[b"123\n"])
.unwrap_err()
.is_decode()
);
}
#[test]
fn sse_decoder_handles_chunked_bom_line_endings_and_persistent_fields() {
let mut decoder = SseDecoder::new(1024);
let events = decode(
&mut decoder,
&[
b"\xef",
b"\xbb",
b"\xbf: ready\r",
b"\nid: event-1\revent: update\rretry: 1500\rdata: first\rdata:",
b" second\r\rdata: third\n\n",
],
)
.unwrap();
assert_eq!(events.len(), 2);
assert_eq!(events[0].event(), Some("update"));
assert_eq!(events[0].data(), "first\nsecond");
assert_eq!(events[0].id(), Some("event-1"));
assert_eq!(events[0].retry(), Some(Duration::from_millis(1500)));
assert_eq!(events[1].event(), None);
assert_eq!(events[1].data(), "third");
assert_eq!(events[1].id(), Some("event-1"));
assert_eq!(events[1].retry(), Some(Duration::from_millis(1500)));
}
#[test]
fn sse_decoder_classifies_invalid_utf8_and_oversized_events_as_decode_errors() {
let mut invalid_utf8 = SseDecoder::new(1024);
let error = decode(&mut invalid_utf8, &[b"data: \xff\n\n"]).unwrap_err();
assert!(error.is_decode());
let mut incomplete_utf8 = SseDecoder::new(1024);
let error = decode(&mut incomplete_utf8, &[b"\xff"]).unwrap_err();
assert!(error.is_decode());
let mut oversized = SseDecoder::new(8);
let error = decode(&mut oversized, &[b"data: abc\n\n"]).unwrap_err();
assert!(error.is_decode());
}
#[test]
fn sse_decoder_follows_event_stream_conformance_cases() {
type Expected<'a> = (&'a str, Option<&'a str>);
type Case<'a> = (&'a str, &'a [&'a [u8]], &'a [Expected<'a>]);
let cases: &[Case<'_>] = &[
(
"empty data fields",
&[b"data\n\ndata\ndata:\n\n"],
&[("", None), ("\n", None)],
),
(
"comments and unknown fields",
&[b": heartbeat\nignored: value\ndata: kept\n\n"],
&[("kept", None)],
),
(
"empty and NUL identifiers",
&[
b"id: before\ndata: first\n\nid\ndata: second\n\nid: ignored\0value\ndata: third\n\n",
],
&[
("first", Some("before")),
("second", Some("")),
("third", Some("")),
],
),
("unterminated event at EOF", &[b"data: discarded"], &[]),
(
"UTF-8 split across chunks",
&[b"data: \xe2", b"\x82", b"\xac\n\n"],
&[("\u{20ac}", None)],
),
];
for (name, chunks, expected) in cases {
let events = decode(&mut SseDecoder::new(1024), chunks)
.unwrap_or_else(|error| panic!("{name}: {error}"));
let actual = events
.iter()
.map(|event| (event.data(), event.id()))
.collect::<Vec<_>>();
assert_eq!(actual, *expected, "{name}");
}
let exact = b"data:\n\n";
assert_eq!(
decode(&mut SseDecoder::new(exact.len()), &[exact])
.unwrap()
.len(),
1
);
let error = decode(&mut SseDecoder::new(exact.len() - 1), &[exact]).unwrap_err();
assert!(error.is_decode());
}
fn decode_in_random_chunks(
decoder: &mut SseDecoder,
input: &[u8],
rng: &mut fastrand::Rng,
) -> Result<()> {
let mut start = 0;
while start < input.len() {
let end = (start + rng.usize(1..128)).min(input.len());
let chunk = &input[start..end];
let mut offset = 0;
while offset < chunk.len() {
let _ = decoder.push_one(chunk, &mut offset)?;
}
start = end;
}
let _ = decoder.finish()?;
Ok(())
}
#[test]
fn sse_decoder_handles_seeded_mutations_across_chunk_boundaries() {
const SEEDS: &[&[u8]] = &[
b"data: message\n\n",
b": heartbeat\r\nid: event-1\r\nevent: update\r\nretry: 1500\r\ndata: first\r\ndata: second\r\n\r\n",
b"\xef\xbb\xbfdata: \xe2\x82\xac\n\n",
b"id\nretry: 0\ndata:\n\n",
];
let mut rng = fastrand::Rng::with_seed(0);
for (index, seed) in SEEDS.iter().enumerate() {
for _ in 0..32 {
decode_in_random_chunks(&mut SseDecoder::new(EVENT_LIMIT), seed, &mut rng)
.unwrap_or_else(|error| panic!("valid SSE seed {index} failed: {error}"));
}
}
for _ in 0..1024 {
let mut input = Vec::new();
for _ in 0..rng.usize(1..8) {
input.extend_from_slice(SEEDS[rng.usize(0..SEEDS.len())]);
}
for _ in 0..rng.usize(0..16) {
match rng.usize(0..3) {
0 if !input.is_empty() => {
let index = rng.usize(0..input.len());
input[index] = rng.usize(0x20..0x7f) as u8;
}
1 => {
let index = rng.usize(0..input.len() + 1);
input.insert(index, rng.usize(0x20..0x7f) as u8);
}
_ if !input.is_empty() => {
input.remove(rng.usize(0..input.len()));
}
_ => {}
}
}
let _ = decode_in_random_chunks(&mut SseDecoder::new(EVENT_LIMIT), &input, &mut rng);
}
}
}