use crate::api::ApiClient;
use crate::cancel::CancelSignal;
use crate::message::Message;
use crate::stream::{StreamAccumulator, StreamEvent, StreamStopReason, Usage};
use crate::tool::ToolSchema;
use futures::StreamExt;
use std::fmt;
use std::sync::Arc;
use std::time::{Duration, Instant};
#[derive(Debug, Clone)]
pub struct StreamTimeoutConfig {
pub initial_event_timeout: Duration,
pub per_event_timeout: Duration,
pub total_stream_timeout: Duration,
pub max_consecutive_timeouts: u32,
pub progress_interval: Duration,
pub fallback_to_non_streaming: bool,
}
impl Default for StreamTimeoutConfig {
fn default() -> Self {
Self {
initial_event_timeout: Duration::from_secs(120),
per_event_timeout: Duration::from_secs(300),
total_stream_timeout: Duration::from_secs(900),
max_consecutive_timeouts: 10,
progress_interval: Duration::from_secs(30),
fallback_to_non_streaming: true,
}
}
}
impl StreamTimeoutConfig {
pub fn validate(&self) -> Result<(), String> {
if self.initial_event_timeout.is_zero() {
return Err("initial_event_timeout must be non-zero".to_string());
}
if self.per_event_timeout.is_zero() {
return Err("per_event_timeout must be non-zero".to_string());
}
if self.total_stream_timeout.is_zero() {
return Err("total_stream_timeout must be non-zero".to_string());
}
if self.total_stream_timeout < self.initial_event_timeout {
return Err(format!(
"total_stream_timeout ({:?}) must be >= initial_event_timeout ({:?})",
self.total_stream_timeout, self.initial_event_timeout
));
}
if self.progress_interval.is_zero() {
return Err("progress_interval must be non-zero".to_string());
}
if self.max_consecutive_timeouts == 0 {
return Err("max_consecutive_timeouts must be >= 1".to_string());
}
Ok(())
}
}
#[derive(Debug, Clone)]
pub struct StreamRetryConfig {
pub max_retries: u32,
pub base_delay_ms: u64,
pub max_delay_ms: u64,
pub jitter_factor: f64,
}
impl Default for StreamRetryConfig {
fn default() -> Self {
Self {
max_retries: 3,
base_delay_ms: 100,
max_delay_ms: 10_000,
jitter_factor: 0.1,
}
}
}
impl StreamRetryConfig {
#[must_use]
pub fn base_delay(&self, attempt: u32) -> Duration {
let delay_ms = self
.base_delay_ms
.saturating_mul(1u64.checked_shl(attempt).unwrap_or(u64::MAX));
Duration::from_millis(delay_ms.min(self.max_delay_ms))
}
pub fn validate(&self) -> Result<(), String> {
if self.base_delay_ms == 0 {
return Err("base_delay_ms must be non-zero".to_string());
}
if self.max_delay_ms == 0 {
return Err("max_delay_ms must be non-zero".to_string());
}
if self.max_delay_ms < self.base_delay_ms {
return Err(format!(
"max_delay_ms ({}) must be >= base_delay_ms ({})",
self.max_delay_ms, self.base_delay_ms
));
}
if !self.jitter_factor.is_finite() {
return Err(format!(
"jitter_factor must be finite, got {}",
self.jitter_factor
));
}
if !(0.0..=1.0).contains(&self.jitter_factor) {
return Err(format!(
"jitter_factor must be in 0.0..=1.0, got {}",
self.jitter_factor
));
}
Ok(())
}
}
#[derive(Debug, Clone)]
#[non_exhaustive]
pub enum StreamOutcome {
Completed {
events_processed: u64,
duration: Duration,
},
TotalTimeout {
has_partial_data: bool,
events_processed: u64,
duration: Duration,
},
EventTimeout {
has_partial_data: bool,
consecutive_timeouts: u32,
},
InitFailed {
last_error: String,
attempts: u32,
},
FallbackToNonStreaming,
Cancelled,
}
impl fmt::Display for StreamOutcome {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
match self {
Self::Completed {
events_processed,
duration,
} => {
write!(
f,
"stream completed ({events_processed} events in {:.1}s)",
duration.as_secs_f64()
)
}
Self::TotalTimeout {
has_partial_data,
events_processed,
duration,
} => {
let partial = if *has_partial_data {
" (partial data)"
} else {
""
};
write!(
f,
"total timeout after {:.1}s, {events_processed} events{partial}",
duration.as_secs_f64()
)
}
Self::EventTimeout {
has_partial_data,
consecutive_timeouts,
} => {
let partial = if *has_partial_data {
" (partial data)"
} else {
""
};
write!(
f,
"event timeout after {consecutive_timeouts} consecutive timeouts{partial}"
)
}
Self::InitFailed {
last_error,
attempts,
} => {
write!(f, "init failed after {attempts} attempts: {last_error}")
}
Self::FallbackToNonStreaming => {
write!(f, "fell back to non-streaming request")
}
Self::Cancelled => write!(f, "cancelled"),
}
}
}
#[derive(Debug)]
#[non_exhaustive]
pub enum StreamHandlerError {
InitFailed(StreamOutcome),
StreamFailed(StreamOutcome),
FallbackFailed {
stream_outcome: StreamOutcome,
fallback_error: String,
},
Cancelled,
}
impl fmt::Display for StreamHandlerError {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
match self {
Self::InitFailed(outcome) => write!(f, "stream init failed: {outcome}"),
Self::StreamFailed(outcome) => write!(f, "stream failed: {outcome}"),
Self::FallbackFailed {
stream_outcome,
fallback_error,
} => {
write!(
f,
"stream failed ({stream_outcome}) and fallback also failed: {fallback_error}"
)
}
Self::Cancelled => write!(f, "cancelled"),
}
}
}
impl std::error::Error for StreamHandlerError {}
#[derive(Debug, Clone)]
pub struct StreamProgress {
pub elapsed: Duration,
pub events_processed: u64,
}
pub struct StreamHandler {
timeout_config: StreamTimeoutConfig,
retry_config: StreamRetryConfig,
}
impl fmt::Debug for StreamHandler {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
f.debug_struct("StreamHandler")
.field("timeout_config", &self.timeout_config)
.field("retry_config", &self.retry_config)
.finish()
}
}
impl Default for StreamHandler {
fn default() -> Self {
Self::new()
}
}
impl StreamHandler {
#[must_use]
pub fn new() -> Self {
Self {
timeout_config: StreamTimeoutConfig::default(),
retry_config: StreamRetryConfig::default(),
}
}
#[must_use]
pub fn with_config(mut self, timeout: StreamTimeoutConfig, retry: StreamRetryConfig) -> Self {
self.timeout_config = timeout;
self.retry_config = retry;
self
}
#[must_use]
pub fn timeout_config(&self) -> &StreamTimeoutConfig {
&self.timeout_config
}
#[must_use]
pub fn retry_config(&self) -> &StreamRetryConfig {
&self.retry_config
}
pub async fn stream_turn<C: ApiClient>(
&self,
client: &C,
conversation: Vec<Message>,
system: Option<String>,
tool_schemas: Option<Vec<ToolSchema>>,
cancel: &Arc<CancelSignal>,
) -> Result<StreamTurnResult, StreamHandlerError> {
let total_deadline = Some(
Instant::now()
.checked_add(self.timeout_config.total_stream_timeout)
.unwrap_or(Instant::now()),
);
let max_attempts = self.retry_config.max_retries.saturating_add(1);
let mut last_stream_outcome: Option<StreamOutcome> = None;
for attempt in 0..max_attempts {
if cancel.is_cancelled() {
return Err(StreamHandlerError::Cancelled);
}
if let Some(deadline) = total_deadline {
if Instant::now() >= deadline {
return Err(StreamHandlerError::InitFailed(
StreamOutcome::TotalTimeout {
has_partial_data: false,
events_processed: 0,
duration: self.timeout_config.total_stream_timeout,
},
));
}
}
match self
.try_stream_once(
client,
conversation.clone(),
system.clone(),
tool_schemas.clone(),
cancel,
total_deadline,
)
.await
{
Ok(result) => return Ok(result),
Err(StreamHandlerError::Cancelled) => {
return Err(StreamHandlerError::Cancelled);
}
Err(e) => {
let outcome = match &e {
StreamHandlerError::InitFailed(o) | StreamHandlerError::StreamFailed(o) => {
Some(o.clone())
}
_ => None,
};
last_stream_outcome = outcome;
if attempt >= max_attempts.saturating_sub(1) {
if self.timeout_config.fallback_to_non_streaming {
return self
.fallback_non_streaming(
client,
conversation,
system,
tool_schemas,
cancel,
last_stream_outcome,
)
.await;
}
return Err(e);
}
let delay = self.retry_config.base_delay(attempt);
tokio::time::sleep(delay).await;
}
}
}
Err(StreamHandlerError::InitFailed(
last_stream_outcome.unwrap_or(StreamOutcome::InitFailed {
attempts: self.retry_config.max_retries,
last_error: "all attempts failed".to_string(),
}),
))
}
async fn try_stream_once<C: ApiClient>(
&self,
client: &C,
conversation: Vec<Message>,
system: Option<String>,
tool_schemas: Option<Vec<ToolSchema>>,
cancel: &Arc<CancelSignal>,
total_deadline: Option<Instant>,
) -> Result<StreamTurnResult, StreamHandlerError> {
let stream = client.stream_messages(conversation, system, tool_schemas);
self.process_events(stream, cancel, total_deadline).await
}
async fn process_events<S>(
&self,
mut stream: S,
cancel: &Arc<CancelSignal>,
total_deadline: Option<Instant>,
) -> Result<StreamTurnResult, StreamHandlerError>
where
S: futures::Stream<Item = Result<crate::stream::StreamEvent, crate::api::error::ApiError>>
+ Unpin,
{
let mut accumulator = StreamAccumulator::new();
let mut stop_reason = StreamStopReason::EndTurn;
let mut consecutive_timeouts: usize = 0;
let mut events_processed: u64 = 0;
let stream_start = Instant::now();
loop {
if let Some(deadline) = total_deadline {
if Instant::now() >= deadline {
let has_partial_data = !accumulator.peek_parts().is_empty();
return Err(StreamHandlerError::StreamFailed(
StreamOutcome::TotalTimeout {
has_partial_data,
events_processed,
duration: stream_start.elapsed(),
},
));
}
}
let per_event_timeout = self.timeout_config.per_event_timeout;
let event_deadline = Instant::now()
.checked_add(per_event_timeout)
.unwrap_or(Instant::now());
let event_result = tokio::select! {
event = stream.next() => event,
() = cancel.notified() => {
return Err(StreamHandlerError::Cancelled);
}
() = tokio::time::sleep_until(event_deadline.into()) => {
consecutive_timeouts = consecutive_timeouts.saturating_add(1);
let max_consecutive = self.timeout_config.max_consecutive_timeouts as usize;
if consecutive_timeouts >= max_consecutive {
let has_partial_data = !accumulator.peek_parts().is_empty();
return Err(StreamHandlerError::StreamFailed(
StreamOutcome::EventTimeout {
has_partial_data,
consecutive_timeouts: u32::try_from(consecutive_timeouts).unwrap_or(u32::MAX),
},
));
}
continue;
}
};
match event_result {
Some(Ok(event)) => {
consecutive_timeouts = 0;
events_processed = events_processed.saturating_add(1);
if let StreamEvent::MessageDelta(delta) = &event {
if let Some(ref reason_str) = delta.delta.stop_reason {
stop_reason =
StreamStopReason::from_api_str(reason_str).unwrap_or(stop_reason);
}
}
if let Err(e) = accumulator.process(&event) {
return Err(StreamHandlerError::StreamFailed(
StreamOutcome::InitFailed {
attempts: 1,
last_error: e.to_string(),
},
));
}
}
Some(Err(api_error)) => {
return Err(StreamHandlerError::StreamFailed(
StreamOutcome::InitFailed {
attempts: 1,
last_error: api_error.to_string(),
},
));
}
None => break,
}
}
let usage = accumulator.usage().copied();
let message = accumulator.build();
let elapsed = stream_start.elapsed();
Ok(StreamTurnResult {
message,
usage,
stop_reason,
from_fallback: false,
elapsed,
})
}
async fn fallback_non_streaming<C: ApiClient>(
&self,
client: &C,
conversation: Vec<Message>,
system: Option<String>,
tool_schemas: Option<Vec<ToolSchema>>,
cancel: &Arc<CancelSignal>,
stream_outcome: Option<StreamOutcome>,
) -> Result<StreamTurnResult, StreamHandlerError> {
if cancel.is_cancelled() {
return Err(StreamHandlerError::Cancelled);
}
let start = Instant::now();
let result = tokio::select! {
res = client.create_message(conversation, system, tool_schemas) => res,
() = cancel.notified() => {
return Err(StreamHandlerError::Cancelled);
}
};
match result {
Ok(value) => {
let text = value
.get("content")
.and_then(|c| c.as_array())
.and_then(|parts| {
parts
.iter()
.find_map(|p| p.get("text").and_then(|t| t.as_str()).map(String::from))
})
.unwrap_or_default();
let stop_reason = value
.get("stop_reason")
.and_then(|r| r.as_str())
.and_then(StreamStopReason::from_api_str)
.unwrap_or(StreamStopReason::EndTurn);
Ok(StreamTurnResult {
message: Message::assistant(&text),
usage: None,
stop_reason,
from_fallback: true,
elapsed: start.elapsed(),
})
}
Err(e) => Err(StreamHandlerError::FallbackFailed {
stream_outcome: stream_outcome.unwrap_or(StreamOutcome::InitFailed {
attempts: 0,
last_error: "unknown".to_string(),
}),
fallback_error: e.to_string(),
}),
}
}
}
#[derive(Debug, Clone)]
#[non_exhaustive]
pub struct StreamTurnResult {
pub message: Message,
pub usage: Option<Usage>,
pub stop_reason: StreamStopReason,
pub from_fallback: bool,
pub elapsed: Duration,
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn timeout_config_default_values() {
let config = StreamTimeoutConfig::default();
assert_eq!(config.initial_event_timeout, Duration::from_secs(120));
assert_eq!(config.per_event_timeout, Duration::from_secs(300));
assert_eq!(config.total_stream_timeout, Duration::from_secs(900));
assert_eq!(config.max_consecutive_timeouts, 10);
assert_eq!(config.progress_interval, Duration::from_secs(30));
assert!(config.fallback_to_non_streaming);
}
#[test]
fn timeout_config_custom_values() {
let config = StreamTimeoutConfig {
initial_event_timeout: Duration::from_secs(30),
per_event_timeout: Duration::from_secs(60),
total_stream_timeout: Duration::from_secs(300),
max_consecutive_timeouts: 5,
progress_interval: Duration::from_secs(10),
fallback_to_non_streaming: false,
};
assert_eq!(config.initial_event_timeout, Duration::from_secs(30));
assert!(!config.fallback_to_non_streaming);
}
#[test]
fn retry_config_default_values() {
let config = StreamRetryConfig::default();
assert_eq!(config.max_retries, 3);
assert_eq!(config.base_delay_ms, 100);
assert_eq!(config.max_delay_ms, 10_000);
assert!((config.jitter_factor - 0.1).abs() < f64::EPSILON);
}
#[test]
fn retry_config_base_delay_exponential() {
let config = StreamRetryConfig::default();
assert_eq!(config.base_delay(0), Duration::from_millis(100));
assert_eq!(config.base_delay(1), Duration::from_millis(200));
assert_eq!(config.base_delay(2), Duration::from_millis(400));
assert_eq!(config.base_delay(3), Duration::from_millis(800));
}
#[test]
fn retry_config_base_delay_capped_at_max() {
let config = StreamRetryConfig {
base_delay_ms: 1000,
max_delay_ms: 5000,
..Default::default()
};
assert_eq!(config.base_delay(3), Duration::from_secs(5));
}
#[test]
fn outcome_completed_display() {
let outcome = StreamOutcome::Completed {
events_processed: 42,
duration: Duration::from_secs(5),
};
let s = outcome.to_string();
assert!(s.contains("42 events"));
assert!(s.contains("5.0s"));
}
#[test]
fn outcome_total_timeout_display() {
let outcome = StreamOutcome::TotalTimeout {
has_partial_data: true,
events_processed: 10,
duration: Duration::from_secs(900),
};
let s = outcome.to_string();
assert!(s.contains("partial data"));
assert!(s.contains("900.0s"));
}
#[test]
fn outcome_event_timeout_display() {
let outcome = StreamOutcome::EventTimeout {
has_partial_data: false,
consecutive_timeouts: 10,
};
let s = outcome.to_string();
assert!(s.contains("10 consecutive"));
assert!(!s.contains("partial data"));
}
#[test]
fn outcome_init_failed_display() {
let outcome = StreamOutcome::InitFailed {
last_error: "connection refused".to_string(),
attempts: 3,
};
let s = outcome.to_string();
assert!(s.contains("3 attempts"));
assert!(s.contains("connection refused"));
}
#[test]
fn outcome_fallback_display() {
let outcome = StreamOutcome::FallbackToNonStreaming;
let s = outcome.to_string();
assert!(s.contains("non-streaming"));
}
#[test]
fn outcome_cancelled_display() {
let outcome = StreamOutcome::Cancelled;
assert_eq!(outcome.to_string(), "cancelled");
}
#[test]
fn error_init_failed_display() {
let outcome = StreamOutcome::InitFailed {
last_error: "timeout".to_string(),
attempts: 3,
};
let err = StreamHandlerError::InitFailed(outcome);
let s = err.to_string();
assert!(s.contains("init failed"));
}
#[test]
fn error_stream_failed_display() {
let outcome = StreamOutcome::EventTimeout {
has_partial_data: true,
consecutive_timeouts: 5,
};
let err = StreamHandlerError::StreamFailed(outcome);
let s = err.to_string();
assert!(s.contains("stream failed"));
}
#[test]
fn error_fallback_failed_display() {
let stream_outcome = StreamOutcome::TotalTimeout {
has_partial_data: false,
events_processed: 0,
duration: Duration::from_secs(900),
};
let err = StreamHandlerError::FallbackFailed {
stream_outcome,
fallback_error: "api error 429".to_string(),
};
let s = err.to_string();
assert!(s.contains("fallback also failed"));
assert!(s.contains("429"));
}
#[test]
fn error_cancelled_display() {
let err = StreamHandlerError::Cancelled;
assert_eq!(err.to_string(), "cancelled");
}
#[test]
fn progress_fields() {
let progress = StreamProgress {
elapsed: Duration::from_secs(45),
events_processed: 127,
};
assert_eq!(progress.elapsed, Duration::from_secs(45));
assert_eq!(progress.events_processed, 127);
}
#[test]
fn handler_new_defaults() {
let handler = StreamHandler::new();
assert_eq!(
handler.timeout_config().initial_event_timeout,
Duration::from_secs(120),
);
assert_eq!(handler.retry_config().max_retries, 3);
}
#[test]
fn handler_with_config() {
let handler = StreamHandler::new().with_config(
StreamTimeoutConfig {
initial_event_timeout: Duration::from_secs(60),
..Default::default()
},
StreamRetryConfig {
max_retries: 5,
..Default::default()
},
);
assert_eq!(
handler.timeout_config().initial_event_timeout,
Duration::from_secs(60),
);
assert_eq!(handler.retry_config().max_retries, 5);
}
#[test]
fn handler_default_trait() {
let handler = StreamHandler::default();
assert_eq!(
handler.timeout_config().initial_event_timeout,
Duration::from_secs(120),
);
}
#[test]
fn handler_debug_format() {
let handler = StreamHandler::new();
let debug = format!("{handler:?}");
assert!(debug.contains("StreamHandler"));
assert!(debug.contains("timeout_config"));
}
#[test]
fn timeout_config_validate_default_ok() {
assert!(StreamTimeoutConfig::default().validate().is_ok());
}
#[test]
fn timeout_config_validate_zero_initial() {
let config = StreamTimeoutConfig {
initial_event_timeout: Duration::ZERO,
..Default::default()
};
let err = config.validate().unwrap_err();
assert!(err.contains("initial_event_timeout"));
}
#[test]
fn timeout_config_validate_zero_per_event() {
let config = StreamTimeoutConfig {
per_event_timeout: Duration::ZERO,
..Default::default()
};
let err = config.validate().unwrap_err();
assert!(err.contains("per_event_timeout"));
}
#[test]
fn timeout_config_validate_zero_total() {
let config = StreamTimeoutConfig {
total_stream_timeout: Duration::ZERO,
..Default::default()
};
let err = config.validate().unwrap_err();
assert!(err.contains("total_stream_timeout"));
}
#[test]
fn timeout_config_validate_total_less_than_initial() {
let config = StreamTimeoutConfig {
initial_event_timeout: Duration::from_secs(120),
total_stream_timeout: Duration::from_secs(60),
..Default::default()
};
let err = config.validate().unwrap_err();
assert!(err.contains("total_stream_timeout"));
assert!(err.contains("initial_event_timeout"));
}
#[test]
fn timeout_config_validate_zero_progress() {
let config = StreamTimeoutConfig {
progress_interval: Duration::ZERO,
..Default::default()
};
let err = config.validate().unwrap_err();
assert!(err.contains("progress_interval"));
}
#[test]
fn retry_config_validate_default_ok() {
assert!(StreamRetryConfig::default().validate().is_ok());
}
#[test]
fn retry_config_validate_zero_base_delay() {
let config = StreamRetryConfig {
base_delay_ms: 0,
..Default::default()
};
let err = config.validate().unwrap_err();
assert!(err.contains("base_delay_ms"));
}
#[test]
fn retry_config_validate_zero_max_delay() {
let config = StreamRetryConfig {
max_delay_ms: 0,
..Default::default()
};
let err = config.validate().unwrap_err();
assert!(err.contains("max_delay_ms"));
}
#[test]
fn retry_config_validate_max_less_than_base() {
let config = StreamRetryConfig {
base_delay_ms: 1000,
max_delay_ms: 500,
..Default::default()
};
let err = config.validate().unwrap_err();
assert!(err.contains("max_delay_ms"));
assert!(err.contains("base_delay_ms"));
}
#[test]
fn retry_config_validate_jitter_nan() {
let config = StreamRetryConfig {
jitter_factor: f64::NAN,
..Default::default()
};
let err = config.validate().unwrap_err();
assert!(err.contains("finite"));
}
#[test]
fn retry_config_validate_jitter_infinity() {
let config = StreamRetryConfig {
jitter_factor: f64::INFINITY,
..Default::default()
};
let err = config.validate().unwrap_err();
assert!(err.contains("finite"));
}
#[test]
fn retry_config_validate_jitter_above_one() {
let config = StreamRetryConfig {
jitter_factor: 1.5,
..Default::default()
};
let err = config.validate().unwrap_err();
assert!(err.contains("0.0..=1.0"));
}
#[test]
fn retry_config_validate_jitter_negative() {
let config = StreamRetryConfig {
jitter_factor: -0.1,
..Default::default()
};
let err = config.validate().unwrap_err();
assert!(err.contains("0.0..=1.0"));
}
#[test]
fn retry_config_validate_jitter_boundaries() {
let config = StreamRetryConfig {
jitter_factor: 0.0,
..Default::default()
};
assert!(config.validate().is_ok());
let config = StreamRetryConfig {
jitter_factor: 1.0,
..Default::default()
};
assert!(config.validate().is_ok());
}
#[test]
fn stream_turn_result_fields() {
let result = StreamTurnResult {
message: Message::assistant("hello"),
usage: Some(Usage::new(10, 5)),
stop_reason: StreamStopReason::EndTurn,
from_fallback: false,
elapsed: Duration::from_millis(100),
};
assert!(!result.from_fallback);
assert_eq!(result.stop_reason, StreamStopReason::EndTurn);
assert!(result.usage.is_some());
}
#[test]
fn stream_turn_result_fallback_flag() {
let result = StreamTurnResult {
message: Message::assistant("fallback text"),
usage: None,
stop_reason: StreamStopReason::EndTurn,
from_fallback: true,
elapsed: Duration::from_millis(200),
};
assert!(result.from_fallback);
assert!(result.usage.is_none());
}
use crate::api::error::ApiError;
use crate::stream::{
DeltaPart, IndexedDelta, MessageDelta, MessageDeltaPayload, MessageMetadata, MessageStart,
PartStart, StreamEvent,
};
fn happy_stream_events() -> Vec<Result<StreamEvent, ApiError>> {
vec![
Ok(StreamEvent::MessageStart(MessageStart {
message: MessageMetadata {
id: "msg_test".to_string(),
role: "assistant".to_string(),
model: "test-model".to_string(),
},
})),
Ok(StreamEvent::PartStart(PartStart {
index: 0,
part: Some(crate::stream::MessagePart::text("")),
})),
Ok(StreamEvent::IndexedDelta(IndexedDelta {
index: 0,
delta: DeltaPart::Text {
text: "hi".to_string(),
},
})),
Ok(StreamEvent::PartStop),
Ok(StreamEvent::MessageDelta(MessageDelta {
delta: MessageDeltaPayload {
stop_reason: Some("end_turn".to_string()),
},
usage: None,
})),
Ok(StreamEvent::MessageStop),
]
}
fn event_stream(
events: Vec<Result<StreamEvent, ApiError>>,
) -> std::pin::Pin<
Box<dyn futures::Stream<Item = Result<StreamEvent, ApiError>> + Send + 'static>,
> {
Box::pin(futures::stream::iter(events))
}
#[tokio::test]
async fn process_events_happy_path() {
let handler = StreamHandler::new();
let cancel = Arc::new(CancelSignal::new());
let stream = event_stream(happy_stream_events());
let result = handler
.process_events(stream, &cancel, None)
.await
.expect("should succeed");
assert!(!result.from_fallback);
assert_eq!(result.stop_reason, StreamStopReason::EndTurn);
assert!(result.elapsed > Duration::ZERO);
}
#[tokio::test]
async fn process_events_api_error_mid_stream() {
let handler = StreamHandler::new();
let cancel = Arc::new(CancelSignal::new());
let events = vec![
Ok(StreamEvent::MessageStart(MessageStart {
message: MessageMetadata {
id: "msg_test".to_string(),
role: "assistant".to_string(),
model: "test-model".to_string(),
},
})),
Err(ApiError::api("connection lost")),
];
let stream = event_stream(events);
let err = handler
.process_events(stream, &cancel, None)
.await
.expect_err("should fail on API error");
match err {
StreamHandlerError::StreamFailed(outcome) => {
let s = outcome.to_string();
assert!(s.contains("connection lost"), "unexpected: {s}");
}
other => panic!("expected StreamFailed, got: {other}"),
}
}
#[tokio::test]
async fn process_events_total_timeout() {
let handler = StreamHandler::new().with_config(
StreamTimeoutConfig {
total_stream_timeout: Duration::from_millis(1),
per_event_timeout: Duration::from_secs(300),
initial_event_timeout: Duration::from_secs(120),
max_consecutive_timeouts: 10,
progress_interval: Duration::from_secs(30),
fallback_to_non_streaming: false,
},
StreamRetryConfig::default(),
);
let cancel = Arc::new(CancelSignal::new());
let deadline = Some(
Instant::now()
.checked_sub(Duration::from_secs(1))
.unwrap_or(Instant::now()),
);
let pending_stream: std::pin::Pin<
Box<dyn futures::Stream<Item = Result<StreamEvent, ApiError>> + Send + 'static>,
> = Box::pin(futures::stream::pending());
let err = handler
.process_events(pending_stream, &cancel, deadline)
.await
.expect_err("should fail on total timeout");
match err {
StreamHandlerError::StreamFailed(StreamOutcome::TotalTimeout { .. }) => {}
other => panic!("expected StreamFailed(TotalTimeout), got: {other}"),
}
}
#[tokio::test]
async fn process_events_cancelled() {
let handler = StreamHandler::new().with_config(
StreamTimeoutConfig {
per_event_timeout: Duration::from_secs(300),
..Default::default()
},
StreamRetryConfig::default(),
);
let cancel = Arc::new(CancelSignal::new());
cancel.cancel();
let pending_stream: std::pin::Pin<
Box<dyn futures::Stream<Item = Result<StreamEvent, ApiError>> + Send + 'static>,
> = Box::pin(futures::stream::pending());
let err = handler
.process_events(pending_stream, &cancel, None)
.await
.expect_err("should fail on cancellation");
assert!(
matches!(err, StreamHandlerError::Cancelled),
"expected Cancelled, got: {err}"
);
}
#[tokio::test]
async fn process_events_empty_stream() {
let handler = StreamHandler::new();
let cancel = Arc::new(CancelSignal::new());
let stream = event_stream(vec![]);
let result = handler
.process_events(stream, &cancel, None)
.await
.expect("empty stream should succeed");
assert!(!result.from_fallback);
}
struct HandlerMock {
create_error: Option<String>,
create_response: Option<serde_json::Value>,
}
impl HandlerMock {
fn new() -> Self {
Self {
create_error: None,
create_response: None,
}
}
fn with_text_response(mut self, text: &str) -> Self {
self.create_response = Some(serde_json::json!({
"content": [{"type": "text", "text": text}],
"stop_reason": "end_turn"
}));
self
}
fn with_create_error(mut self, msg: &str) -> Self {
self.create_error = Some(msg.to_string());
self
}
}
impl ApiClient for HandlerMock {
fn model(&self) -> String {
"test-model".to_string()
}
fn stream_messages(
&self,
_messages: Vec<Message>,
_system: Option<String>,
_tools: Option<Vec<ToolSchema>>,
) -> std::pin::Pin<
Box<dyn futures::Stream<Item = Result<StreamEvent, ApiError>> + Send + 'static>,
> {
Box::pin(futures::stream::iter(happy_stream_events()))
}
fn create_message(
&self,
_messages: Vec<Message>,
_system: Option<String>,
_tools: Option<Vec<ToolSchema>>,
) -> std::pin::Pin<
Box<dyn std::future::Future<Output = Result<serde_json::Value, ApiError>> + Send + '_>,
> {
if let Some(ref err) = self.create_error {
let err = err.clone();
return Box::pin(async move { Err(ApiError::api(&err)) });
}
let val = self.create_response.clone().unwrap_or(serde_json::json!({
"content": [{"type": "text", "text": "default"}],
"stop_reason": "end_turn"
}));
Box::pin(async move { Ok(val) })
}
}
#[tokio::test]
async fn fallback_non_streaming_success() {
let handler = StreamHandler::new().with_config(
StreamTimeoutConfig {
fallback_to_non_streaming: true,
..Default::default()
},
StreamRetryConfig::default(),
);
let client = HandlerMock::new().with_text_response("fallback works");
let cancel = Arc::new(CancelSignal::new());
let result = handler
.fallback_non_streaming(
&client,
vec![],
None,
None,
&cancel,
Some(StreamOutcome::InitFailed {
last_error: "stream failed".to_string(),
attempts: 3,
}),
)
.await
.expect("fallback should succeed");
assert!(result.from_fallback);
assert_eq!(result.stop_reason, StreamStopReason::EndTurn);
}
#[tokio::test]
async fn fallback_non_streaming_cancelled_before_start() {
let handler = StreamHandler::new().with_config(
StreamTimeoutConfig {
fallback_to_non_streaming: true,
..Default::default()
},
StreamRetryConfig::default(),
);
let client = HandlerMock::new().with_text_response("fallback works");
let cancel = Arc::new(CancelSignal::new());
cancel.cancel();
let err = handler
.fallback_non_streaming(&client, vec![], None, None, &cancel, None)
.await
.expect_err("should fail on cancellation");
assert!(
matches!(err, StreamHandlerError::Cancelled),
"expected Cancelled, got: {err}"
);
}
#[tokio::test]
async fn fallback_non_streaming_error() {
let handler = StreamHandler::new().with_config(
StreamTimeoutConfig {
fallback_to_non_streaming: true,
..Default::default()
},
StreamRetryConfig::default(),
);
let client = HandlerMock::new().with_create_error("service unavailable");
let cancel = Arc::new(CancelSignal::new());
let err = handler
.fallback_non_streaming(
&client,
vec![],
None,
None,
&cancel,
Some(StreamOutcome::InitFailed {
last_error: "stream timeout".to_string(),
attempts: 2,
}),
)
.await
.expect_err("should fail when fallback also errors");
match err {
StreamHandlerError::FallbackFailed {
stream_outcome,
fallback_error,
} => {
let stream_s = stream_outcome.to_string();
assert!(
stream_s.contains("stream timeout"),
"unexpected: {stream_s}"
);
assert!(
fallback_error.contains("service unavailable"),
"unexpected: {fallback_error}"
);
}
other => panic!("expected FallbackFailed, got: {other}"),
}
}
#[tokio::test]
async fn stream_turn_happy_path() {
let handler = StreamHandler::new();
let client = HandlerMock::new().with_text_response("hello world");
let cancel = Arc::new(CancelSignal::new());
let result = handler
.stream_turn(&client, vec![], None, None, &cancel)
.await
.expect("stream_turn should succeed");
assert!(!result.from_fallback);
assert_eq!(result.stop_reason, StreamStopReason::EndTurn);
}
#[tokio::test]
async fn stream_turn_cancelled_at_start() {
let handler = StreamHandler::new();
let client = HandlerMock::new().with_text_response("hello");
let cancel = Arc::new(CancelSignal::new());
cancel.cancel();
let err = handler
.stream_turn(&client, vec![], None, None, &cancel)
.await
.expect_err("should fail on cancellation");
assert!(
matches!(err, StreamHandlerError::Cancelled),
"expected Cancelled, got: {err}"
);
}
#[tokio::test]
async fn stream_turn_fallback_after_stream_error() {
struct ErrorMock;
impl ApiClient for ErrorMock {
fn model(&self) -> String {
"test-model".to_string()
}
fn stream_messages(
&self,
_messages: Vec<Message>,
_system: Option<String>,
_tools: Option<Vec<ToolSchema>>,
) -> std::pin::Pin<
Box<dyn futures::Stream<Item = Result<StreamEvent, ApiError>> + Send + 'static>,
> {
Box::pin(futures::stream::once(async {
Err(ApiError::api("API down"))
}))
}
fn create_message(
&self,
_messages: Vec<Message>,
_system: Option<String>,
_tools: Option<Vec<ToolSchema>>,
) -> std::pin::Pin<
Box<
dyn std::future::Future<Output = Result<serde_json::Value, ApiError>>
+ Send
+ '_,
>,
> {
Box::pin(async { Err(ApiError::api("unreachable")) })
}
}
let handler = StreamHandler::new().with_config(
StreamTimeoutConfig {
fallback_to_non_streaming: false,
..Default::default()
},
StreamRetryConfig {
max_retries: 0,
..Default::default()
},
);
let client = ErrorMock;
let cancel = Arc::new(CancelSignal::new());
let err = handler
.stream_turn(&client, vec![], None, None, &cancel)
.await
.expect_err("should fail when streaming errors and fallback is disabled");
match err {
StreamHandlerError::StreamFailed(outcome) => {
let s = outcome.to_string();
assert!(s.contains("API down"), "unexpected: {s}");
}
other => panic!("expected StreamFailed, got: {other}"),
}
}
}