anyllm 0.1.0

Low-level, generic LLM provider abstraction for Rust
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30

use anyllm::prelude::*;

fn build_provider() -> MockStreamingProvider {
    MockStreamingProvider::build(|builder| {
        builder.text("Deterministic streamed hello from anyllm.")
    })
}

fn build_request() -> ChatRequest {
    ChatRequest::new("demo-model").user("Say hello")
}

#[tokio::main]
async fn main() -> anyllm::Result<()> {
    let provider = build_provider();
    let request = build_request();
    let mut stream = provider.chat_stream(&request).await?;
    let mut collector = StreamCollector::new();

    while let Some(event) = stream.next().await {
        let event = event?;
        println!("stream event: {event:?}");
        collector.push(event)?;
    }

    let response = collector.finish()?;
    println!("final text: {}", response.text_or_empty());
    println!("stream calls recorded: {}", provider.call_count());
    Ok(())
}