anyllm 0.1.0

Low-level, generic LLM provider abstraction for Rust
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50

use anyllm::prelude::*;
use tracing_subscriber::FmtSubscriber;

fn init_tracing() {
    let subscriber = FmtSubscriber::builder()
        .with_target(false)
        .without_time()
        .finish();

    let _ = tracing::subscriber::set_global_default(subscriber);
}

fn build_provider() -> MockStreamingProvider {
    MockStreamingProvider::build(|builder| {
        builder
            .text("Traced hello from anyllm.")
            .text("Traced hello from anyllm.")
    })
}

fn build_request() -> ChatRequest {
    ChatRequest::new("demo-model")
        .system("You are concise.")
        .user("Say hello")
}

#[tokio::main]
async fn main() -> anyllm::Result<()> {
    init_tracing();

    let provider =
        TracingChatProvider::new(build_provider()).with_content_capture(TracingContentConfig {
            capture_input_messages: true,
            capture_output_messages: true,
            max_payload_chars: 128,
        });

    let request = build_request();
    let response = provider.chat(&request).await?;
    println!("traced chat text: {}", response.text_or_empty());

    let streamed = provider
        .chat_stream(&request)
        .await?
        .collect_response()
        .await?;
    println!("traced stream text: {}", streamed.text_or_empty());

    Ok(())
}