modelbridge 0.5.2

One provider-neutral Rust client for OpenAI, Anthropic, Gemini, Groq, Ollama and friends: streaming, tool calling, an agent loop, usage accounting, and subscription sign-in
Documentation
//! One small Rust API for OpenAI, Anthropic, OpenRouter, Groq, DeepSeek,
//! Gemini, xAI, Mistral, Ollama, and any OpenAI-compatible endpoint.
//!
//! ```no_run
//! use modelbridge::{Client, Request};
//!
//! let client = Client::default_for("anthropic")?;
//! let answer = client.complete(&Request::new("Be brief").user("Why is the sky blue?"))?;
//! println!("{} ({} tokens)", answer.text, answer.usage.total());
//! # Ok::<(), modelbridge::Error>(())
//! ```
//!
//! Everything is synchronous and runtime-free. [`Provider::stream`] blocks and
//! pushes [`Chunk`]s down an `mpsc` channel, which is what a TUI or GUI wants:
//! run it on a worker thread and drain the channel each frame.
//!
//! - [`PROVIDERS`] and [`MODELS`] know the endpoint, credential variable, wire
//!   format, and price of the providers people actually use, without ever
//!   restricting the model id you pass.
//! - [`agent`] runs the call-tools-until-answered loop, feeding results back in
//!   each provider's native format.
//! - `oauth` is the whole browser sign-in flow for the ChatGPT and Claude
//!   subscription routes, including refresh and storage. On by default; turn it
//!   off with `default-features = false` and three dependencies go with it.

mod catalog;
mod client;
mod encode;
mod error;
mod message;
mod meter;
mod request;
mod stream;

pub mod agent;
#[cfg(feature = "oauth")]
pub mod oauth;

pub use catalog::{
    model, models_for, provider, ApiStyle, Auth, ModelSpec, ProviderSpec, MODELS, PROVIDERS,
};
pub use client::{Client, ClientBuilder, Retry};
pub use error::{Error, Result};
pub use message::{Message, Role, ToolCall, ToolResult};
pub use meter::Usage;
pub use request::{Cancel, Request, ToolSchema};

use std::collections::HashMap;
use std::sync::mpsc::{channel, Sender};

use serde_json::Value;

/// Supplies OAuth access tokens while the host application owns login,
/// persistence, expiry, scopes, and refresh-token security.
///
/// `access_token` is called before each request. If a server answers 401 or
/// 403, the client calls `refresh_access_token` and retries exactly once.
///
/// The `oauth` feature implements this for the ChatGPT and Claude subscription
/// routes; implement it yourself for an enterprise gateway or a provider whose
/// identity flow this crate does not know.
pub trait OAuthTokenProvider: Send + Sync {
    fn access_token(&self) -> std::result::Result<String, String>;
    fn refresh_access_token(&self) -> std::result::Result<String, String>;
}

/// One piece of a reply, as it arrives.
#[derive(Clone, Debug, PartialEq)]
pub enum Chunk {
    Text(String),
    Tool(ToolCall),
    /// Always the last chunk of a successful stream.
    Done {
        input_tokens: u32,
        output_tokens: u32,
    },
    /// Always the last chunk of a failed one.
    Failed(Error),
}

/// A finished reply.
#[derive(Clone, Debug, Default, PartialEq)]
pub struct Response {
    pub text: String,
    pub tool_calls: Vec<ToolCall>,
    pub usage: Usage,
}

impl Response {
    /// The assistant turn to append to a conversation, tool calls included, so
    /// the next request carries what this one asked for.
    pub fn as_message(&self) -> Message {
        Message::tool_calls(self.text.clone(), self.tool_calls.clone())
    }
}

/// Anything that can answer a [`Request`].
///
/// Implement it for a fake in tests, a recorded transcript, or a protocol this
/// crate has never heard of; nothing above the trait needs to change.
pub trait Provider: Send + Sync {
    fn stream(&self, request: &Request, sender: &Sender<Chunk>) -> Result<()>;

    /// How to name this route to a user, conventionally `provider/model`.
    fn label(&self) -> String;

    fn usage(&self) -> Usage {
        Usage::default()
    }

    /// Usage split by `provider/model`. One total hides the fact that a single
    /// background model is quietly responsible for most of a bill.
    fn usage_by_model(&self) -> HashMap<String, Usage> {
        HashMap::new()
    }
}

/// Consume a streaming provider into one [`Response`].
///
/// A stream that produced text or tool calls before failing returns what it
/// managed; a stream that produced nothing returns the error.
pub fn complete(provider: &dyn Provider, request: &Request) -> Result<Response> {
    let (tx, rx) = channel();
    provider.stream(request, &tx)?;
    drop(tx);

    let mut response = Response::default();
    let mut failure = None;
    for chunk in rx {
        match chunk {
            Chunk::Text(text) => response.text.push_str(&text),
            Chunk::Tool(call) => response.tool_calls.push(call),
            Chunk::Done {
                input_tokens,
                output_tokens,
            } => {
                response.usage.input = input_tokens as u64;
                response.usage.output = output_tokens as u64;
                response.usage.calls = 1;
            }
            Chunk::Failed(error) => failure = Some(error),
        }
    }
    match failure {
        Some(error) if response.text.trim().is_empty() && response.tool_calls.is_empty() => {
            Err(error)
        }
        _ => Ok(response),
    }
}

/// [`complete`], keeping only the text.
pub fn collect_text(provider: &dyn Provider, request: &Request) -> Result<String> {
    complete(provider, request).map(|response| response.text)
}

/// The first well-formed JSON object in a string, tolerating code fences and
/// whatever prose a model wrapped around it.
pub fn extract_json(text: &str) -> Option<Value> {
    if let Ok(value) = serde_json::from_str::<Value>(text.trim()) {
        return Some(value);
    }
    let chars: Vec<char> = text.chars().collect();
    let start = chars.iter().position(|&ch| ch == '{')?;
    let (mut depth, mut in_string, mut escaped) = (0_i32, false, false);
    for index in start..chars.len() {
        let ch = chars[index];
        if in_string {
            if escaped {
                escaped = false;
            } else if ch == '\\' {
                escaped = true;
            } else if ch == '"' {
                in_string = false;
            }
            continue;
        }
        match ch {
            '"' => in_string = true,
            '{' => depth += 1,
            '}' => {
                depth -= 1;
                if depth == 0 {
                    let candidate: String = chars[start..=index].iter().collect();
                    return serde_json::from_str(&candidate).ok();
                }
            }
            _ => {}
        }
    }
    None
}

#[cfg(test)]
mod tests {
    use super::*;
    use serde_json::json;

    struct ScriptedProvider;

    impl Provider for ScriptedProvider {
        fn stream(&self, _: &Request, sender: &Sender<Chunk>) -> Result<()> {
            sender.send(Chunk::Text("bon".into())).unwrap();
            sender.send(Chunk::Text("jour".into())).unwrap();
            sender
                .send(Chunk::Tool(ToolCall::new(
                    "define",
                    json!({"word": "bonjour"}),
                )))
                .unwrap();
            sender
                .send(Chunk::Done {
                    input_tokens: 8,
                    output_tokens: 4,
                })
                .unwrap();
            Ok(())
        }

        fn label(&self) -> String {
            "scripted".into()
        }
    }

    struct FailingProvider(&'static str);

    impl Provider for FailingProvider {
        fn stream(&self, _: &Request, sender: &Sender<Chunk>) -> Result<()> {
            if !self.0.is_empty() {
                sender.send(Chunk::Text(self.0.into())).unwrap();
            }
            sender
                .send(Chunk::Failed(Error::Stream("cut short".into())))
                .unwrap();
            Ok(())
        }

        fn label(&self) -> String {
            "failing".into()
        }
    }

    #[test]
    fn extracts_fenced_json() {
        assert_eq!(
            extract_json("result:\n```json\n{\"ok\":true}\n```").unwrap()["ok"],
            true
        );
    }

    #[test]
    fn extracts_json_with_braces_inside_strings() {
        let value =
            extract_json("prefix {\"example\":\"use {braces}\",\"ok\":true} suffix").unwrap();
        assert_eq!(value["example"], "use {braces}");
        assert_eq!(value["ok"], true);
    }

    #[test]
    fn completion_collects_text_tools_and_usage() {
        let response = complete(&ScriptedProvider, &Request::default()).unwrap();
        assert_eq!(response.text, "bonjour");
        assert_eq!(response.tool_calls[0].name, "define");
        assert_eq!(response.tool_calls[0].args["word"], "bonjour");
        assert_eq!(response.usage.total(), 12);
        assert_eq!(response.usage.calls, 1);
    }

    #[test]
    fn a_response_replays_as_the_assistant_turn_that_produced_it() {
        let message = complete(&ScriptedProvider, &Request::default())
            .unwrap()
            .as_message();
        assert_eq!(message.role, Role::Assistant);
        assert_eq!(message.text, "bonjour");
        assert!(!message.calls[0].id.is_empty());
    }

    #[test]
    fn partial_output_survives_a_broken_stream_but_nothing_does_not() {
        assert_eq!(
            complete(&FailingProvider("half an ans"), &Request::default())
                .unwrap()
                .text,
            "half an ans"
        );
        assert!(matches!(
            complete(&FailingProvider(""), &Request::default()),
            Err(Error::Stream(_))
        ));
    }
}