llm-relay 0.2.0

Shared types, format conversion, and HTTP client for Anthropic and OpenAI LLM APIs
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93

use std::time::Duration;

use crate::types::common::Provider;

pub mod chat;
#[cfg(feature = "embeddings")]
pub mod embeddings;
pub mod error;

pub use chat::ChatOptions;
#[cfg(feature = "embeddings")]
pub use embeddings::{EmbeddingsClient, EmbeddingsConfig};
pub use error::LlmError;

/// Configuration for the LLM client.
#[derive(Debug, Clone)]
pub struct ClientConfig {
    pub provider: Provider,
    pub base_url: String,
    pub api_key: String,
    pub timeout: Duration,
    pub model: String,
    pub max_tokens: u32,
}

impl ClientConfig {
    /// Create config for the Anthropic API.
    pub fn anthropic(api_key: impl Into<String>, model: impl Into<String>) -> Self {
        Self {
            provider: Provider::Anthropic,
            base_url: "https://api.anthropic.com".to_string(),
            api_key: api_key.into(),
            timeout: Duration::from_secs(180),
            model: model.into(),
            max_tokens: 16384,
        }
    }

    /// Create config for an OpenAI-compatible API (OpenRouter, OpenAI, Ollama, etc.).
    pub fn openai_compatible(
        base_url: impl Into<String>,
        api_key: impl Into<String>,
        model: impl Into<String>,
    ) -> Self {
        Self {
            provider: Provider::OpenAiCompatible,
            base_url: base_url.into(),
            api_key: api_key.into(),
            timeout: Duration::from_secs(60),
            model: model.into(),
            max_tokens: 16384,
        }
    }

    #[must_use]
    pub fn timeout(mut self, timeout: Duration) -> Self {
        self.timeout = timeout;
        self
    }

    #[must_use]
    pub fn max_tokens(mut self, max_tokens: u32) -> Self {
        self.max_tokens = max_tokens;
        self
    }

    #[must_use]
    pub fn base_url(mut self, base_url: impl Into<String>) -> Self {
        self.base_url = base_url.into();
        self
    }
}

/// The main LLM client.
pub struct LlmClient {
    pub(crate) http: reqwest::Client,
    pub(crate) config: ClientConfig,
}

impl LlmClient {
    pub fn new(config: ClientConfig) -> Result<Self, LlmError> {
        let http = reqwest::Client::builder()
            .timeout(config.timeout)
            .build()
            .map_err(|e| LlmError::Client(e.to_string()))?;
        Ok(Self { http, config })
    }

    /// Get a reference to the client config.
    pub fn config(&self) -> &ClientConfig {
        &self.config
    }
}