flyllm 0.3.1

A Rust library for unifying LLM backends as an abstraction layer with load balancing.
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86

use serde::{Serialize, Deserialize};

/// Enum representing the different LLM providers supported
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, Hash)]
pub enum ProviderType {
    Anthropic,
    OpenAI,
    Mistral,
    Google,
    Ollama
}

/// Unified request structure used across all providers
#[derive(Debug, Serialize, Deserialize, Clone)]
pub struct LlmRequest {
    pub messages: Vec<Message>,
    pub model: Option<String>,
    pub max_tokens: Option<u32>,
    pub temperature: Option<f32>,
}

/// Standard message format used across providers
#[derive(Debug, Serialize, Deserialize, Clone)]
pub struct Message {
    pub role: String,
    pub content: String,
}

/// Unified response structure returned by all providers
#[derive(Debug, Serialize, Deserialize, Clone)]
pub struct LlmResponse {
    pub content: String,
    pub model: String,
    pub usage: Option<TokenUsage>,
}

/// Token usage information returned by providers
#[derive(Debug, Serialize, Deserialize, Clone)]
pub struct TokenUsage {
    pub prompt_tokens: u32,
    pub completion_tokens: u32,
    pub total_tokens: u32,
}

impl Default for TokenUsage {
    fn default() -> Self {
        Self {
            prompt_tokens: 0,
            completion_tokens: 0,
            total_tokens: 0
        }
    }
}

/// Information about an LLM model
#[derive(Debug, Serialize, Deserialize, Clone)]
pub struct ModelInfo {
    pub name: String,
    pub provider: ProviderType,
}

/// Display implementation for ProviderType
impl std::fmt::Display for ProviderType {
    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
        match self {
            ProviderType::Anthropic => write!(f, "Anthropic"),
            ProviderType::OpenAI => write!(f, "OpenAI"),
            ProviderType::Mistral => write!(f, "Mistral"),
            ProviderType::Google => write!(f, "Google"),
            ProviderType::Ollama => write!(f, "Ollama"),
        }
    }
}

impl From<&str> for ProviderType {
    fn from(value: &str) -> Self {
        match value {
            "Anthropic" => ProviderType::Anthropic,
            "OpenAI" => ProviderType::OpenAI,
            "Mistral" => ProviderType::Mistral,
            "Google" => ProviderType::Google,
            "Ollama" => ProviderType::Ollama,
            _ => panic!("Unknown provider: {}", value),
        }
    }
}