flyllm 0.3.1

A Rust library for unifying LLM backends as an abstraction layer with load balancing.
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71

//! FlyLLM is a Rust library that provides a load-balanced, multi-provider client for Large Language Models.
//! 
//! It enables developers to seamlessly work with multiple LLM providers (OpenAI, Anthropic, Google, Mistral...)
//! through a unified API with request routing, load balancing, and failure handling.
//!
//! # Features
//!
//! - **Multi-provider support**: Integrate with OpenAI, Anthropic, Google, and Mistral
//! - **Load balancing**: Distribute requests across multiple providers
//! - **Automatic retries**: Handle provider failures with configurable retry policies
//! - **Task routing**: Route specific tasks to the most suitable providers
//! - **Metrics tracking**: Monitor response times, error rates, and token usage
//!
//! # Example
//!
//! ```
//! use flyllm::{LlmManager, ProviderType, GenerationRequest, TaskDefinition};
//! 
//! async fn example() {
//!     // Create a manager
//!     let mut manager = LlmManager::new();
//!     
//!     // Add providers
//!     manager.add_provider(
//!         ProviderType::OpenAI,
//!         "api-key".to_string(),
//!         "gpt-4-turbo".to_string(),
//!         vec![],
//!         true
//!     );
//!     
//!     // Generate a response
//!     let request = GenerationRequest {
//!         prompt: "Explain Rust in one paragraph".to_string(),
//!         task: None,
//!         params: None,
//!     };
//!     
//!     let responses = manager.generate_sequentially(vec![request]).await;
//!     println!("{}", responses[0].content);
//! }
//! ```

pub mod providers;
pub mod errors;
pub mod constants;
pub mod load_balancer;

pub use providers::{
    ProviderType, 
    LlmRequest, 
    LlmResponse,
    LlmInstance,
    create_instance,
    AnthropicInstance,
    OpenAIInstance,
    ModelInfo,
    ModelDiscovery
};

pub use errors::{LlmError, LlmResult};

pub use load_balancer::{LlmManager, GenerationRequest, LlmManagerResponse, TaskDefinition};

/// Initialize the logging system
///
/// This should be called at the start of your application in case
/// you want to activate the library's debug and info logging.
pub fn use_logging() {
    env_logger::init();
}