flyllm 0.4.0

A Rust library for unifying LLM backends as an abstraction layer with load balancing.
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72

//! FlyLLM is a Rust library that provides a load-balanced, multi-provider client for Large Language Models.
//!
//! It enables developers to seamlessly work with multiple LLM providers through a unified API
//! with request routing, load balancing, and failure handling.
//!
//! # Features
//!
//! - **Multi-provider support**: Integrate with OpenAI, Anthropic, Google, Mistral, Ollama,
//!   LM Studio, Groq, Cohere, Together AI, and Perplexity
//! - **Load balancing**: Distribute requests across multiple providers
//! - **Automatic retries**: Handle provider failures with configurable retry policies
//! - **Task routing**: Route specific tasks to the most suitable providers
//! - **Metrics tracking**: Monitor response times, error rates, and token usage
//!
//! # Example
//!
//! ```no_run
//! use flyllm::{LlmManager, ProviderType, GenerationRequest, TaskDefinition};
//!
//! async fn example() {
//!     // Create a manager using the builder pattern
//!     let manager = LlmManager::builder()
//!         .define_task(TaskDefinition::new("chat"))
//!         .add_instance(ProviderType::OpenAI, "gpt-4-turbo", "api-key")
//!         .supports("chat")
//!         .build()
//!         .await
//!         .expect("Failed to build manager");
//!
//!     // Generate a response
//!     let request = GenerationRequest {
//!         prompt: "Explain Rust in one paragraph".to_string(),
//!         task: None,
//!         params: None,
//!     };
//!
//!     let responses = manager.generate_sequentially(vec![request]).await;
//!     println!("{}", responses[0].content);
//! }
//! ```

pub mod providers;
pub mod errors;
pub mod constants;
pub mod load_balancer;
pub mod config;

pub use providers::{
    ProviderType,
    LlmRequest,
    LlmResponse,
    LlmInstance,
    create_instance,
    AnthropicInstance,
    OpenAIInstance,
    ModelInfo,
    ModelDiscovery,
    StreamChunk,
    LlmStream
};

pub use errors::{LlmError, LlmResult};

pub use load_balancer::{LlmManager, GenerationRequest, LlmManagerResponse, TaskDefinition};

/// Initialize the logging system
///
/// This should be called at the start of your application in case
/// you want to activate the library's debug and info logging.
pub fn use_logging() {
    env_logger::init();
}