llm_stream 0.3.1

A very simple Rust library to simplify streaming api interaction with LLMs, free from complex async operations and redundant dependencies.
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33

use anyhow::Result;
use futures::stream::TryStreamExt;
use llm_stream::google::{Auth, Client, Content, MessageBody, Part, Role};
use std::io::Write;

#[tokio::main]
async fn main() -> Result<()> {
    env_logger::init();

    let key = std::env::var("GOOGLE_API_KEY")?;

    let auth = Auth::new(key);
    let client = Client::new(auth, "https://generativelanguage.googleapis.com/v1beta");

    let messages = vec![Content {
        parts: vec![Part {
            text: "What is the capital of the United States?".to_string(),
        }],
        role: Role::User,
    }];

    let body = MessageBody::new("gemini-1.5-flash", messages);

    // let mut stream = client.message_stream(&body)?;
    let mut stream = client.delta(&body)?;

    while let Ok(Some(text)) = stream.try_next().await {
        print!("{text}");
        std::io::stdout().flush()?;
    }

    Ok(())
}