allama 0.2.0

Low-level, provider-agnostic LLM abstractions for Rust
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26

//! Consume a stream event by event and collect the final response.

use allama::mock::MockStreamingProvider;
use allama::prelude::*;

#[tokio::main]
async fn main() -> anyhow::Result<()> {
    let provider =
        MockStreamingProvider::build(|b| b.text("Deterministic streamed hello from allama."));
    let mut stream = provider
        .chat_stream(&ChatRequest::new("demo-model").user("Say hello"))
        .await?
        .with_collector();

    while let Some(item) = stream.next_event().await {
        let item = item?;
        if let StreamEvent::TextDelta { text, .. } = &item.event {
            print!("{text}");
        }
    }
    println!();

    let response = stream.finish()?;
    println!("final text: {}", response.text_or_empty());
    Ok(())
}