openheim 0.2.0

A fast, multi-provider LLM agent runtime written in Rust
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56

mod anthropic;
mod gemini;
mod openai;
mod openai_compatible;
mod retry;

use async_trait::async_trait;
use tokio::sync::mpsc;

use crate::core::models::{Choice, Message, Tool};
use crate::error::Result;

/// A single streaming chunk produced during an LLM call.
#[derive(Debug)]
pub enum LlmChunk {
    /// A token or partial text from the model's response.
    Text(String),
    /// A chunk from the model's extended thinking (reasoning).
    Thinking(String),
}

/// Abstraction over a chat-completion API.
///
/// Implement this trait to add a custom provider. The built-in implementations
/// are [`AnthropicClient`], [`GeminiClient`], [`OpenAiClient`], and
/// [`OpenAiCompatibleClient`] (for any OpenAI-compatible endpoint).
#[async_trait]
pub trait LlmClient: Send + Sync {
    /// Send a chat request and return the first choice from the provider.
    async fn send(&self, messages: &[Message], tools: &[Tool]) -> Result<Choice>;

    /// Streaming variant: sends [`LlmChunk`]s to `chunk_tx` as they arrive,
    /// then returns the complete [`Choice`] once the response is finished.
    ///
    /// The default implementation calls [`LlmClient::send`] and emits the full
    /// response content as a single [`LlmChunk::Text`]. Override this to enable
    /// real token-by-token streaming.
    async fn send_streaming(
        &self,
        messages: &[Message],
        tools: &[Tool],
        chunk_tx: mpsc::UnboundedSender<LlmChunk>,
    ) -> Result<Choice> {
        let choice = self.send(messages, tools).await?;
        if let Some(ref content) = choice.message.content {
            let _ = chunk_tx.send(LlmChunk::Text(content.clone()));
        }
        Ok(choice)
    }
}

pub use anthropic::AnthropicClient;
pub use gemini::GeminiClient;
pub use openai::OpenAiClient;
pub use openai_compatible::OpenAiCompatibleClient;
pub use retry::RetryClient;