Expand description
MidStream: Real-Time Large Language Model Streaming Platform
This library provides functionality for real-time LLM response streaming, inflight data analysis, and integration with external tools.
§Example
use midstream::{Midstream, HyprSettings, HyprServiceImpl, StreamProcessor, LLMClient};
use bytes::Bytes;
use futures::stream::BoxStream;
use futures::stream::iter;
use std::time::Duration;
// Example LLM client implementation
struct ExampleLLMClient;
impl LLMClient for ExampleLLMClient {
fn stream(&self) -> BoxStream<'static, Bytes> {
Box::pin(iter(vec![
Bytes::from_static(b"Processing"),
Bytes::from_static(b"the"),
Bytes::from_static(b"stream"),
]))
}
}
#[tokio::main]
async fn main() -> Result<(), Box<dyn std::error::Error>> {
// Initialize settings
let settings = HyprSettings::new()?;
// Create hyprstream service
let hypr_service = HyprServiceImpl::new(&settings).await?;
// Create LLM client
let llm_client = ExampleLLMClient;
// Initialize Midstream
let midstream = Midstream::new(
Box::new(llm_client),
Box::new(hypr_service),
);
// Process stream
let messages = midstream.process_stream().await?;
println!("Processed messages: {:?}", messages);
// Get metrics
let metrics = midstream.get_metrics().await;
println!("Collected metrics: {:?}", metrics);
// Get average sentiment for last 5 minutes
let avg = midstream.get_average_sentiment(Duration::from_secs(300)).await?;
println!("Average sentiment: {}", avg);
Ok(())
}Re-exports§
pub use config::HyprSettings;pub use midstream::Midstream;pub use midstream::StreamProcessor;pub use midstream::LLMMessage;pub use midstream::LLMClient;pub use midstream::HyprService;pub use midstream::ToolIntegration;pub use midstream::Intent;pub use midstream::MetricRecord;pub use midstream::TimeWindow;pub use midstream::AggregateFunction;pub use hypr_service::HyprServiceImpl;