oxify_connect_llm/
streaming.rs

1//! Streaming LLM support with Server-Sent Events (SSE)
2
3use async_trait::async_trait;
4use futures::stream::Stream;
5use serde::{Deserialize, Serialize};
6use std::pin::Pin;
7
8use crate::{LlmRequest, Result};
9
10/// A chunk of streamed LLM response
11#[derive(Debug, Clone, Serialize, Deserialize)]
12pub struct LlmChunk {
13    /// The text content of this chunk
14    pub content: String,
15
16    /// Whether this is the final chunk
17    pub done: bool,
18
19    /// Model identifier (only in final chunk)
20    pub model: Option<String>,
21
22    /// Usage statistics (only in final chunk)
23    pub usage: Option<StreamUsage>,
24}
25
26#[derive(Debug, Clone, Serialize, Deserialize)]
27pub struct StreamUsage {
28    pub prompt_tokens: Option<u32>,
29    pub completion_tokens: Option<u32>,
30    pub total_tokens: Option<u32>,
31}
32
33/// Type alias for streaming response
34pub type LlmStream = Pin<Box<dyn Stream<Item = Result<LlmChunk>> + Send>>;
35
36/// Trait for LLM providers that support streaming
37#[async_trait]
38pub trait StreamingLlmProvider: Send + Sync {
39    /// Stream completion responses token-by-token
40    async fn complete_stream(&self, request: LlmRequest) -> Result<LlmStream>;
41}
oxify_connect_llm/streaming.rs

oxify_connect_llm/
streaming.rs
