allama 0.2.0

Low-level, provider-agnostic LLM abstractions for Rust
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54

//! Log every chat call as a JSON line, streams included.
//!
//! `RecordingProvider` hands a `ChatExchange` (request, response or error,
//! and the stream transcript) to a sink when each call's outcome is known.
//! `TimeoutProvider` bounds the calls; a stalled stream is recorded with its
//! partial output and the timeout.

use std::time::Duration;

use allama::mock::{MockStreamEvent, MockStreamingProvider};
use allama::prelude::*;
use allama::wrappers::{RecordingProvider, TimeoutPolicy, TimeoutProvider};

#[tokio::main]
async fn main() -> anyhow::Result<()> {
    let mut stalled = MockStreamEvent::text_response("Hello");
    // Pause before `ResponseStop` for longer than the idle bound.
    stalled.insert(
        stalled.len() - 1,
        MockStreamEvent::delayed(Duration::from_secs(5)),
    );
    let provider =
        MockStreamingProvider::new([MockStreamEvent::text_response("Hi there"), stalled]);

    let provider = RecordingProvider::new(
        TimeoutProvider::new(
            provider,
            TimeoutPolicy::new().stream_idle(Duration::from_millis(100)),
        ),
        |exchange| {
            println!(
                "{}",
                serde_json::to_string(&exchange).expect("exchanges serialize")
            );
        },
    );

    let request = ChatRequest::new("demo-model").user("Say hello");
    let response = provider
        .chat_stream(&request)
        .await?
        .collect_response()
        .await?;
    eprintln!("first: {}", response.text_or_empty());

    let error = provider
        .chat_stream(&request)
        .await?
        .collect_response()
        .await
        .unwrap_err();
    eprintln!("second: {error}");
    Ok(())
}