modelbridge 0.5.2

One provider-neutral Rust client for OpenAI, Anthropic, Gemini, Groq, Ollama and friends: streaming, tool calling, an agent loop, usage accounting, and subscription sign-in
Documentation
//! Streaming onto a worker thread, which is how a TUI or GUI uses this crate.
//!
//! `cargo run --example streaming -- anthropic`
//!
//! The model call blocks, so it runs on its own thread and pushes `Chunk`s
//! down a channel. The main thread drains the channel — here in a loop, in a
//! real application once per frame — and stays responsive the whole time. The
//! cancel flag is the same one an escape key would raise.

use std::sync::atomic::Ordering;
use std::sync::mpsc::channel;
use std::thread;
use std::time::Instant;

use modelbridge::{Cancel, Chunk, Client, Provider, Request};

/// Stop after this much text, standing in for a user hitting escape.
const PATIENCE: usize = 280;

fn main() -> Result<(), String> {
    let provider = std::env::args().nth(1).unwrap_or_else(|| "openai".into());
    let client = Client::default_for(&provider)?;

    let cancel = Cancel::default();
    let request = Request::new("Answer in a few plain paragraphs.")
        .user("Why did the Portuguese empire decline?")
        .max_output_tokens(600)
        .cancel(cancel.clone());

    let (tx, rx) = channel::<Chunk>();
    let started = Instant::now();

    // The worker owns the request and the sender; dropping the sender when it
    // returns is what ends the loop below.
    let worker = thread::spawn(move || {
        let result = client.stream(&request, &tx);
        drop(tx);
        (result, client.usage())
    });

    let mut answer = String::new();
    let mut first_token = None;
    for chunk in rx {
        match chunk {
            Chunk::Text(text) => {
                if first_token.is_none() {
                    first_token = Some(started.elapsed());
                }
                print!("{text}");
                answer.push_str(&text);

                // Anything the interface would do between frames goes here.
                if answer.len() > PATIENCE && !cancel.load(Ordering::Relaxed) {
                    cancel.store(true, Ordering::Relaxed);
                    eprintln!("\n\n[that's enough — cancelling]");
                }
            }
            Chunk::Tool(call) => eprintln!("\n[tool: {}]", call.name),
            Chunk::Done {
                input_tokens,
                output_tokens,
            } => eprintln!("\n\n[done: {input_tokens} in, {output_tokens} out]"),
            // Cancelling ends the stream this way, which is not a failure.
            Chunk::Failed(modelbridge::Error::Cancelled) => {
                eprintln!("[stopped early, keeping {} chars]", answer.len())
            }
            Chunk::Failed(error) => eprintln!("\n[failed: {error}]"),
        }
    }

    let (result, usage) = worker.join().map_err(|_| "the worker thread panicked")?;
    match result {
        // A cancelled stream is the caller getting what it asked for.
        Err(modelbridge::Error::Cancelled) | Ok(()) => {}
        Err(error) => return Err(error.to_string()),
    }

    if let Some(latency) = first_token {
        eprintln!("first token after {} ms", latency.as_millis());
    }
    eprintln!("{} tokens, about ${:.4}", usage.total(), usage.dollars());
    Ok(())
}