use serde::{Deserialize, Serialize};
use std::borrow::Cow;
#[derive(Debug, Clone, PartialEq, Eq, Hash)]
pub enum StreamFormat {
TextEventStream,
NdJson,
JsonLines,
Custom(String),
}
impl StreamFormat {
pub fn mime_type(&self) -> &str {
match self {
StreamFormat::TextEventStream => "text/event-stream",
StreamFormat::NdJson => "application/x-ndjson",
StreamFormat::JsonLines => "application/jsonl",
StreamFormat::Custom(mime) => mime,
}
}
pub fn is_sse(&self) -> bool {
matches!(self, StreamFormat::TextEventStream)
}
pub fn is_json_lines(&self) -> bool {
matches!(self, StreamFormat::NdJson | StreamFormat::JsonLines)
}
pub fn default_parser_config(&self) -> LineParserConfig {
match self {
StreamFormat::TextEventStream => LineParserConfig {
line_prefix: Some("data: ".to_string()),
event_prefix: Some("event: ".to_string()),
empty_line_behavior: EmptyLineBehavior::EventDelimiter,
comment_prefix: Some(": ".to_string()),
},
StreamFormat::NdJson | StreamFormat::JsonLines => LineParserConfig {
line_prefix: None,
event_prefix: None,
empty_line_behavior: EmptyLineBehavior::Ignore,
comment_prefix: None,
},
StreamFormat::Custom(_) => LineParserConfig::default(),
}
}
}
impl Default for StreamFormat {
fn default() -> Self {
StreamFormat::TextEventStream
}
}
impl std::fmt::Display for StreamFormat {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
write!(f, "{}", self.mime_type())
}
}
impl Serialize for StreamFormat {
fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
where
S: serde::Serializer,
{
serializer.serialize_str(self.mime_type())
}
}
impl<'de> Deserialize<'de> for StreamFormat {
fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
where
D: serde::Deserializer<'de>,
{
let s = String::deserialize(deserializer)?;
Ok(match s.as_str() {
"text/event-stream" => StreamFormat::TextEventStream,
"application/x-ndjson" => StreamFormat::NdJson,
"application/jsonl" => StreamFormat::JsonLines,
other => StreamFormat::Custom(other.to_string()),
})
}
}
#[derive(Debug, Clone, PartialEq, Eq, Hash, Default, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum EmptyLineBehavior {
#[default]
Ignore,
EventDelimiter,
Include,
}
impl EmptyLineBehavior {
pub fn is_event_delimiter(&self) -> bool {
matches!(self, EmptyLineBehavior::EventDelimiter)
}
pub fn is_ignored(&self) -> bool {
matches!(self, EmptyLineBehavior::Ignore)
}
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct LineParserConfig {
#[serde(default, skip_serializing_if = "Option::is_none")]
pub line_prefix: Option<String>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub event_prefix: Option<String>,
#[serde(default)]
pub empty_line_behavior: EmptyLineBehavior,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub comment_prefix: Option<String>,
}
impl Default for LineParserConfig {
fn default() -> Self {
Self {
line_prefix: None,
event_prefix: None,
empty_line_behavior: EmptyLineBehavior::default(),
comment_prefix: None,
}
}
}
impl LineParserConfig {
pub fn new() -> Self {
Self::default()
}
pub fn sse() -> Self {
Self {
line_prefix: Some("data: ".to_string()),
event_prefix: Some("event: ".to_string()),
empty_line_behavior: EmptyLineBehavior::EventDelimiter,
comment_prefix: Some(": ".to_string()),
}
}
pub fn ndjson() -> Self {
Self {
line_prefix: None,
event_prefix: None,
empty_line_behavior: EmptyLineBehavior::Ignore,
comment_prefix: None,
}
}
pub fn is_comment(&self, line: &str) -> bool {
self.comment_prefix
.as_ref()
.is_some_and(|prefix| line.starts_with(prefix))
}
pub fn is_event_line(&self, line: &str) -> bool {
self.event_prefix
.as_ref()
.is_some_and(|prefix| line.starts_with(prefix))
}
pub fn is_data_line(&self, line: &str) -> bool {
match &self.line_prefix {
Some(prefix) => line.starts_with(prefix),
None => !line.is_empty() && !self.is_comment(line) && !self.is_event_line(line),
}
}
pub fn extract_event_type<'a>(&self, line: &'a str) -> Option<&'a str> {
self.event_prefix
.as_ref()
.and_then(|prefix| line.strip_prefix(prefix))
}
pub fn strip_prefix<'a>(&self, line: &'a str) -> Cow<'a, str> {
match &self.line_prefix {
Some(prefix) => {
if let Some(stripped) = line.strip_prefix(prefix) {
Cow::Borrowed(stripped)
} else {
Cow::Borrowed(line)
}
}
None => Cow::Borrowed(line),
}
}
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum ParsedLine<'a> {
Data(Cow<'a, str>),
EventType(&'a str),
Comment,
Empty,
Unknown(&'a str),
}
impl<'a> ParsedLine<'a> {
pub fn as_data(&self) -> Option<&str> {
match self {
ParsedLine::Data(data) => Some(data),
_ => None,
}
}
pub fn as_event_type(&self) -> Option<&str> {
match self {
ParsedLine::EventType(t) => Some(t),
_ => None,
}
}
pub fn is_empty(&self) -> bool {
matches!(self, ParsedLine::Empty)
}
pub fn should_skip(&self, config: &LineParserConfig) -> bool {
match self {
ParsedLine::Comment => true,
ParsedLine::Empty => config.empty_line_behavior.is_ignored(),
_ => false,
}
}
}
#[derive(Debug, Clone)]
pub struct LineParser {
config: LineParserConfig,
}
impl LineParser {
pub fn new(config: LineParserConfig) -> Self {
Self { config }
}
pub fn sse() -> Self {
Self::new(LineParserConfig::sse())
}
pub fn ndjson() -> Self {
Self::new(LineParserConfig::ndjson())
}
pub fn config(&self) -> &LineParserConfig {
&self.config
}
pub fn parse_line<'a>(&self, line: &'a str) -> ParsedLine<'a> {
let trimmed = line.trim_end_matches(['\r', '\n']);
if trimmed.is_empty() {
return ParsedLine::Empty;
}
if self.config.is_comment(trimmed) {
return ParsedLine::Comment;
}
if let Some(event_type) = self.config.extract_event_type(trimmed) {
return ParsedLine::EventType(event_type);
}
if self.config.is_data_line(trimmed) {
return ParsedLine::Data(self.config.strip_prefix(trimmed));
}
ParsedLine::Unknown(trimmed)
}
pub fn extract_data<'a>(&self, line: &'a str) -> Option<Cow<'a, str>> {
match self.parse_line(line) {
ParsedLine::Data(data) => Some(data),
_ => None,
}
}
pub fn is_end_marker(&self, line: &str, end_markers: &[&str]) -> bool {
if let ParsedLine::Data(data) = self.parse_line(line) {
end_markers.iter().any(|marker| data.trim() == *marker)
} else {
false
}
}
}
impl Default for LineParser {
fn default() -> Self {
Self::new(LineParserConfig::default())
}
}
impl From<LineParserConfig> for LineParser {
fn from(config: LineParserConfig) -> Self {
Self::new(config)
}
}
impl From<StreamFormat> for LineParser {
fn from(format: StreamFormat) -> Self {
Self::new(format.default_parser_config())
}
}
#[derive(Debug, Clone, Default)]
pub struct LineParserBuilder {
line_prefix: Option<String>,
event_prefix: Option<String>,
empty_line_behavior: EmptyLineBehavior,
comment_prefix: Option<String>,
}
impl LineParserBuilder {
pub fn new() -> Self {
Self::default()
}
pub fn from_sse() -> Self {
Self {
line_prefix: Some("data: ".to_string()),
event_prefix: Some("event: ".to_string()),
empty_line_behavior: EmptyLineBehavior::EventDelimiter,
comment_prefix: Some(": ".to_string()),
}
}
pub fn from_ndjson() -> Self {
Self {
line_prefix: None,
event_prefix: None,
empty_line_behavior: EmptyLineBehavior::Ignore,
comment_prefix: None,
}
}
pub fn line_prefix(mut self, prefix: impl Into<String>) -> Self {
self.line_prefix = Some(prefix.into());
self
}
pub fn no_line_prefix(mut self) -> Self {
self.line_prefix = None;
self
}
pub fn event_prefix(mut self, prefix: impl Into<String>) -> Self {
self.event_prefix = Some(prefix.into());
self
}
pub fn no_event_prefix(mut self) -> Self {
self.event_prefix = None;
self
}
pub fn empty_line_behavior(mut self, behavior: EmptyLineBehavior) -> Self {
self.empty_line_behavior = behavior;
self
}
pub fn ignore_empty_lines(mut self) -> Self {
self.empty_line_behavior = EmptyLineBehavior::Ignore;
self
}
pub fn empty_lines_delimit_events(mut self) -> Self {
self.empty_line_behavior = EmptyLineBehavior::EventDelimiter;
self
}
pub fn include_empty_lines(mut self) -> Self {
self.empty_line_behavior = EmptyLineBehavior::Include;
self
}
pub fn comment_prefix(mut self, prefix: impl Into<String>) -> Self {
self.comment_prefix = Some(prefix.into());
self
}
pub fn no_comment_prefix(mut self) -> Self {
self.comment_prefix = None;
self
}
pub fn build_config(self) -> LineParserConfig {
LineParserConfig {
line_prefix: self.line_prefix,
event_prefix: self.event_prefix,
empty_line_behavior: self.empty_line_behavior,
comment_prefix: self.comment_prefix,
}
}
pub fn build(self) -> LineParser {
LineParser::new(self.build_config())
}
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct StreamFormatConfig {
pub format: StreamFormat,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub parser: Option<LineParserConfig>,
#[serde(default, skip_serializing_if = "Vec::is_empty")]
pub end_markers: Vec<String>,
}
impl StreamFormatConfig {
pub fn new(format: StreamFormat) -> Self {
Self {
format,
parser: None,
end_markers: Vec::new(),
}
}
pub fn sse() -> Self {
Self {
format: StreamFormat::TextEventStream,
parser: None,
end_markers: vec!["[DONE]".to_string()],
}
}
pub fn ndjson() -> Self {
Self {
format: StreamFormat::NdJson,
parser: None,
end_markers: Vec::new(),
}
}
pub fn with_parser(mut self, config: LineParserConfig) -> Self {
self.parser = Some(config);
self
}
pub fn with_end_marker(mut self, marker: impl Into<String>) -> Self {
self.end_markers.push(marker.into());
self
}
pub fn parser_config(&self) -> LineParserConfig {
self.parser
.clone()
.unwrap_or_else(|| self.format.default_parser_config())
}
pub fn create_parser(&self) -> LineParser {
LineParser::new(self.parser_config())
}
pub fn is_end_marker(&self, data: &str) -> bool {
let trimmed = data.trim();
self.end_markers.iter().any(|m| m == trimmed)
}
}
impl Default for StreamFormatConfig {
fn default() -> Self {
Self::sse()
}
}
impl From<StreamFormat> for StreamFormatConfig {
fn from(format: StreamFormat) -> Self {
Self::new(format)
}
}
#[cfg(test)]
mod tests {
use super::*;
mod stream_format {
use super::*;
#[test]
fn test_mime_types() {
assert_eq!(StreamFormat::TextEventStream.mime_type(), "text/event-stream");
assert_eq!(StreamFormat::NdJson.mime_type(), "application/x-ndjson");
assert_eq!(StreamFormat::JsonLines.mime_type(), "application/jsonl");
assert_eq!(
StreamFormat::Custom("text/plain".to_string()).mime_type(),
"text/plain"
);
}
#[test]
fn test_is_sse() {
assert!(StreamFormat::TextEventStream.is_sse());
assert!(!StreamFormat::NdJson.is_sse());
assert!(!StreamFormat::JsonLines.is_sse());
assert!(!StreamFormat::Custom("text/plain".to_string()).is_sse());
}
#[test]
fn test_is_json_lines() {
assert!(!StreamFormat::TextEventStream.is_json_lines());
assert!(StreamFormat::NdJson.is_json_lines());
assert!(StreamFormat::JsonLines.is_json_lines());
assert!(!StreamFormat::Custom("text/plain".to_string()).is_json_lines());
}
#[test]
fn test_display() {
assert_eq!(format!("{}", StreamFormat::TextEventStream), "text/event-stream");
assert_eq!(format!("{}", StreamFormat::NdJson), "application/x-ndjson");
}
#[test]
fn test_serialize_deserialize() {
let format = StreamFormat::TextEventStream;
let json = serde_json::to_string(&format).unwrap();
assert_eq!(json, "\"text/event-stream\"");
let parsed: StreamFormat = serde_json::from_str(&json).unwrap();
assert_eq!(parsed, StreamFormat::TextEventStream);
}
#[test]
fn test_deserialize_custom() {
let json = "\"text/plain\"";
let parsed: StreamFormat = serde_json::from_str(json).unwrap();
assert_eq!(parsed, StreamFormat::Custom("text/plain".to_string()));
}
#[test]
fn test_default_parser_config_sse() {
let config = StreamFormat::TextEventStream.default_parser_config();
assert_eq!(config.line_prefix, Some("data: ".to_string()));
assert_eq!(config.event_prefix, Some("event: ".to_string()));
assert_eq!(config.empty_line_behavior, EmptyLineBehavior::EventDelimiter);
assert_eq!(config.comment_prefix, Some(": ".to_string()));
}
#[test]
fn test_default_parser_config_ndjson() {
let config = StreamFormat::NdJson.default_parser_config();
assert_eq!(config.line_prefix, None);
assert_eq!(config.event_prefix, None);
assert_eq!(config.empty_line_behavior, EmptyLineBehavior::Ignore);
assert_eq!(config.comment_prefix, None);
}
}
mod empty_line_behavior {
use super::*;
#[test]
fn test_default() {
assert_eq!(EmptyLineBehavior::default(), EmptyLineBehavior::Ignore);
}
#[test]
fn test_is_event_delimiter() {
assert!(!EmptyLineBehavior::Ignore.is_event_delimiter());
assert!(EmptyLineBehavior::EventDelimiter.is_event_delimiter());
assert!(!EmptyLineBehavior::Include.is_event_delimiter());
}
#[test]
fn test_is_ignored() {
assert!(EmptyLineBehavior::Ignore.is_ignored());
assert!(!EmptyLineBehavior::EventDelimiter.is_ignored());
assert!(!EmptyLineBehavior::Include.is_ignored());
}
#[test]
fn test_serialize_deserialize() {
let behavior = EmptyLineBehavior::EventDelimiter;
let json = serde_json::to_string(&behavior).unwrap();
assert_eq!(json, "\"event_delimiter\"");
let parsed: EmptyLineBehavior = serde_json::from_str(&json).unwrap();
assert_eq!(parsed, EmptyLineBehavior::EventDelimiter);
}
}
mod line_parser_config {
use super::*;
#[test]
fn test_sse_config() {
let config = LineParserConfig::sse();
assert_eq!(config.line_prefix, Some("data: ".to_string()));
assert_eq!(config.event_prefix, Some("event: ".to_string()));
assert_eq!(config.empty_line_behavior, EmptyLineBehavior::EventDelimiter);
assert_eq!(config.comment_prefix, Some(": ".to_string()));
}
#[test]
fn test_ndjson_config() {
let config = LineParserConfig::ndjson();
assert_eq!(config.line_prefix, None);
assert_eq!(config.event_prefix, None);
assert_eq!(config.empty_line_behavior, EmptyLineBehavior::Ignore);
assert_eq!(config.comment_prefix, None);
}
#[test]
fn test_is_comment() {
let config = LineParserConfig::sse();
assert!(config.is_comment(": this is a comment"));
assert!(config.is_comment(": keepalive"));
assert!(!config.is_comment("data: some data"));
assert!(!config.is_comment("event: message"));
}
#[test]
fn test_is_event_line() {
let config = LineParserConfig::sse();
assert!(config.is_event_line("event: message"));
assert!(config.is_event_line("event: content_block_delta"));
assert!(!config.is_event_line("data: some data"));
assert!(!config.is_event_line(": comment"));
}
#[test]
fn test_is_data_line() {
let config = LineParserConfig::sse();
assert!(config.is_data_line("data: {\"content\": \"hello\"}"));
assert!(config.is_data_line("data: [DONE]"));
assert!(!config.is_data_line("event: message"));
assert!(!config.is_data_line(": comment"));
assert!(!config.is_data_line(""));
}
#[test]
fn test_extract_event_type() {
let config = LineParserConfig::sse();
assert_eq!(config.extract_event_type("event: message"), Some("message"));
assert_eq!(
config.extract_event_type("event: content_block_delta"),
Some("content_block_delta")
);
assert_eq!(config.extract_event_type("data: something"), None);
}
#[test]
fn test_strip_prefix() {
let config = LineParserConfig::sse();
assert_eq!(
config.strip_prefix("data: {\"content\": \"hello\"}"),
Cow::Borrowed("{\"content\": \"hello\"}")
);
assert_eq!(
config.strip_prefix("no prefix here"),
Cow::Borrowed("no prefix here")
);
}
#[test]
fn test_ndjson_is_data_line() {
let config = LineParserConfig::ndjson();
assert!(config.is_data_line("{\"content\": \"hello\"}"));
assert!(!config.is_data_line(""));
}
}
mod parsed_line {
use super::*;
#[test]
fn test_as_data() {
let line = ParsedLine::Data(Cow::Borrowed("test data"));
assert_eq!(line.as_data(), Some("test data"));
let line = ParsedLine::EventType("message");
assert_eq!(line.as_data(), None);
}
#[test]
fn test_as_event_type() {
let line = ParsedLine::EventType("message");
assert_eq!(line.as_event_type(), Some("message"));
let line = ParsedLine::Data(Cow::Borrowed("data"));
assert_eq!(line.as_event_type(), None);
}
#[test]
fn test_is_empty() {
assert!(ParsedLine::<'_>::Empty.is_empty());
assert!(!ParsedLine::Data(Cow::Borrowed("")).is_empty());
assert!(!ParsedLine::Comment.is_empty());
}
#[test]
fn test_should_skip() {
let config = LineParserConfig::sse();
assert!(ParsedLine::<'_>::Comment.should_skip(&config));
assert!(!ParsedLine::<'_>::Empty.should_skip(&config)); assert!(!ParsedLine::Data(Cow::Borrowed("data")).should_skip(&config));
let ndjson_config = LineParserConfig::ndjson();
assert!(ParsedLine::<'_>::Empty.should_skip(&ndjson_config)); }
}
mod line_parser {
use super::*;
#[test]
fn test_parse_sse_data_line() {
let parser = LineParser::sse();
match parser.parse_line("data: {\"content\": \"hello\"}") {
ParsedLine::Data(data) => assert_eq!(data, "{\"content\": \"hello\"}"),
other => panic!("Expected Data, got {:?}", other),
}
}
#[test]
fn test_parse_sse_event_line() {
let parser = LineParser::sse();
match parser.parse_line("event: message") {
ParsedLine::EventType(t) => assert_eq!(t, "message"),
other => panic!("Expected EventType, got {:?}", other),
}
}
#[test]
fn test_parse_sse_comment() {
let parser = LineParser::sse();
match parser.parse_line(": keepalive") {
ParsedLine::Comment => {}
other => panic!("Expected Comment, got {:?}", other),
}
}
#[test]
fn test_parse_sse_empty() {
let parser = LineParser::sse();
match parser.parse_line("") {
ParsedLine::Empty => {}
other => panic!("Expected Empty, got {:?}", other),
}
match parser.parse_line("\n") {
ParsedLine::Empty => {}
other => panic!("Expected Empty, got {:?}", other),
}
}
#[test]
fn test_parse_ndjson_line() {
let parser = LineParser::ndjson();
match parser.parse_line("{\"content\": \"hello\"}") {
ParsedLine::Data(data) => assert_eq!(data, "{\"content\": \"hello\"}"),
other => panic!("Expected Data, got {:?}", other),
}
}
#[test]
fn test_extract_data() {
let parser = LineParser::sse();
assert_eq!(
parser.extract_data("data: hello"),
Some(Cow::Borrowed("hello"))
);
assert_eq!(parser.extract_data("event: message"), None);
assert_eq!(parser.extract_data(": comment"), None);
assert_eq!(parser.extract_data(""), None);
}
#[test]
fn test_is_end_marker() {
let parser = LineParser::sse();
assert!(parser.is_end_marker("data: [DONE]", &["[DONE]"]));
assert!(parser.is_end_marker("data: [DONE] ", &["[DONE]"])); assert!(!parser.is_end_marker("data: {\"content\": \"hello\"}", &["[DONE]"]));
assert!(!parser.is_end_marker("event: message", &["[DONE]"]));
}
#[test]
fn test_from_stream_format() {
let parser: LineParser = StreamFormat::TextEventStream.into();
assert_eq!(parser.config().line_prefix, Some("data: ".to_string()));
let parser: LineParser = StreamFormat::NdJson.into();
assert_eq!(parser.config().line_prefix, None);
}
}
mod line_parser_builder {
use super::*;
#[test]
fn test_build_empty() {
let parser = LineParserBuilder::new().build();
assert_eq!(parser.config().line_prefix, None);
assert_eq!(parser.config().event_prefix, None);
assert_eq!(parser.config().empty_line_behavior, EmptyLineBehavior::Ignore);
assert_eq!(parser.config().comment_prefix, None);
}
#[test]
fn test_build_custom() {
let parser = LineParserBuilder::new()
.line_prefix("msg: ")
.event_prefix("type: ")
.comment_prefix("# ")
.empty_lines_delimit_events()
.build();
assert_eq!(parser.config().line_prefix, Some("msg: ".to_string()));
assert_eq!(parser.config().event_prefix, Some("type: ".to_string()));
assert_eq!(parser.config().comment_prefix, Some("# ".to_string()));
assert_eq!(
parser.config().empty_line_behavior,
EmptyLineBehavior::EventDelimiter
);
}
#[test]
fn test_from_sse() {
let parser = LineParserBuilder::from_sse().build();
assert_eq!(parser.config().line_prefix, Some("data: ".to_string()));
assert_eq!(parser.config().event_prefix, Some("event: ".to_string()));
}
#[test]
fn test_from_ndjson() {
let parser = LineParserBuilder::from_ndjson().build();
assert_eq!(parser.config().line_prefix, None);
assert_eq!(parser.config().event_prefix, None);
}
#[test]
fn test_chain_methods() {
let parser = LineParserBuilder::from_sse()
.no_event_prefix() .ignore_empty_lines() .build();
assert_eq!(parser.config().line_prefix, Some("data: ".to_string())); assert_eq!(parser.config().event_prefix, None); assert_eq!(parser.config().empty_line_behavior, EmptyLineBehavior::Ignore);
}
}
mod stream_format_config {
use super::*;
#[test]
fn test_sse_config() {
let config = StreamFormatConfig::sse();
assert_eq!(config.format, StreamFormat::TextEventStream);
assert_eq!(config.end_markers, vec!["[DONE]".to_string()]);
}
#[test]
fn test_ndjson_config() {
let config = StreamFormatConfig::ndjson();
assert_eq!(config.format, StreamFormat::NdJson);
assert!(config.end_markers.is_empty());
}
#[test]
fn test_with_parser() {
let custom_parser = LineParserConfig {
line_prefix: Some("custom: ".to_string()),
..Default::default()
};
let config = StreamFormatConfig::sse().with_parser(custom_parser);
assert_eq!(
config.parser_config().line_prefix,
Some("custom: ".to_string())
);
}
#[test]
fn test_with_end_marker() {
let config = StreamFormatConfig::ndjson()
.with_end_marker("[END]")
.with_end_marker("[STOP]");
assert_eq!(config.end_markers, vec!["[END]", "[STOP]"]);
}
#[test]
fn test_is_end_marker() {
let config = StreamFormatConfig::sse();
assert!(config.is_end_marker("[DONE]"));
assert!(config.is_end_marker(" [DONE] ")); assert!(!config.is_end_marker("{\"content\": \"hello\"}"));
}
#[test]
fn test_create_parser() {
let config = StreamFormatConfig::sse();
let parser = config.create_parser();
assert_eq!(parser.config().line_prefix, Some("data: ".to_string()));
}
#[test]
fn test_serialize_deserialize() {
let config = StreamFormatConfig::sse();
let json = serde_json::to_string(&config).unwrap();
let parsed: StreamFormatConfig = serde_json::from_str(&json).unwrap();
assert_eq!(parsed.format, StreamFormat::TextEventStream);
assert_eq!(parsed.end_markers, vec!["[DONE]".to_string()]);
}
}
mod integration {
use super::*;
#[test]
fn test_parse_openai_sse_stream() {
let parser = LineParser::sse();
let lines = vec![
"data: {\"choices\":[{\"delta\":{\"role\":\"assistant\"}}]}",
"",
"data: {\"choices\":[{\"delta\":{\"content\":\"Hello\"}}]}",
"",
"data: {\"choices\":[{\"delta\":{\"content\":\" world\"}}]}",
"",
"data: [DONE]",
];
let mut data_lines = Vec::new();
let mut empty_count = 0;
for line in lines {
match parser.parse_line(line) {
ParsedLine::Data(data) => data_lines.push(data.to_string()),
ParsedLine::Empty => empty_count += 1,
_ => {}
}
}
assert_eq!(data_lines.len(), 4);
assert_eq!(empty_count, 3);
assert!(data_lines[0].contains("role"));
assert!(data_lines[1].contains("Hello"));
assert!(data_lines[2].contains("world"));
assert_eq!(data_lines[3], "[DONE]");
}
#[test]
fn test_parse_anthropic_sse_stream() {
let parser = LineParser::sse();
let lines = vec![
"event: message_start",
"data: {\"type\":\"message_start\",\"message\":{}}",
"",
"event: content_block_delta",
"data: {\"type\":\"content_block_delta\",\"delta\":{\"text\":\"Hello\"}}",
"",
"event: message_stop",
"data: {\"type\":\"message_stop\"}",
];
let mut events = Vec::new();
let mut data_lines = Vec::new();
for line in lines {
match parser.parse_line(line) {
ParsedLine::EventType(t) => events.push(t.to_string()),
ParsedLine::Data(d) => data_lines.push(d.to_string()),
_ => {}
}
}
assert_eq!(events, vec!["message_start", "content_block_delta", "message_stop"]);
assert_eq!(data_lines.len(), 3);
}
#[test]
fn test_parse_ndjson_stream() {
let parser = LineParser::ndjson();
let lines = vec![
"{\"content\":\"Hello\"}",
"{\"content\":\" world\"}",
"{\"done\":true}",
];
let data_lines: Vec<_> = lines
.iter()
.filter_map(|line| parser.extract_data(line))
.map(|d| d.to_string())
.collect();
assert_eq!(data_lines.len(), 3);
assert!(data_lines[0].contains("Hello"));
assert!(data_lines[1].contains("world"));
assert!(data_lines[2].contains("done"));
}
#[test]
fn test_end_to_end_with_format_config() {
let config = StreamFormatConfig::sse();
let parser = config.create_parser();
let line = "data: {\"choices\":[{\"delta\":{\"content\":\"Hello\"}}]}";
let data = parser.extract_data(line).unwrap();
assert!(data.contains("Hello"));
let end_line = "data: [DONE]";
let end_data = parser.extract_data(end_line).unwrap();
assert!(config.is_end_marker(&end_data));
}
}
}