liter-llm 1.2.2

Universal LLM API client — 142+ providers, streaming, tool calling. Rust-powered, type-safe, compiled.
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77

//! Tower middleware integration for [`crate::client::LlmClient`].
//!
//! This module is only compiled when the `tower` feature is enabled.  It
//! provides:
//!
//! - [`types::LlmRequest`] / [`types::LlmResponse`] — the request/response
//!   enums that cross the tower `Service` boundary.
//! - [`service::LlmService`] — a thin `tower::Service` wrapper around any
//!   [`crate::client::LlmClient`].
//! - [`tracing::TracingLayer`] / [`tracing::TracingService`] — OTEL-compatible
//!   tracing middleware.
//! - [`fallback::FallbackLayer`] / [`fallback::FallbackService`] — route to a
//!   backup service on transient errors.
//! - [`cost::CostTrackingLayer`] / [`cost::CostTrackingService`] — emit
//!   `gen_ai.usage.cost` tracing span attribute from embedded pricing data.
//! - [`rate_limit::ModelRateLimitLayer`] / [`rate_limit::ModelRateLimitService`]
//!   — per-model RPM / TPM rate limiting.
//! - [`cache::CacheLayer`] / [`cache::CacheService`] — in-memory response
//!   caching for non-streaming requests.
//! - [`cooldown::CooldownLayer`] / [`cooldown::CooldownService`] — deployment
//!   cooldowns after transient errors.
//! - [`health::HealthCheckLayer`] / [`health::HealthCheckService`] — periodic
//!   health probes with automatic request rejection on failure.
//! - [`budget::BudgetLayer`] / [`budget::BudgetService`] — global and per-model
//!   spending budget enforcement (hard reject or soft warn).
//! - [`hooks::HooksLayer`] / [`hooks::HooksService`] — user-defined pre/post
//!   request hooks for guardrails, logging, and auditing.
//!
//! # Example
//!
//! ```rust,ignore
//! use liter_llm::tower::{CostTrackingLayer, LlmService, TracingLayer};
//! use tower::ServiceBuilder;
//!
//! let client = liter_llm::DefaultClient::new(config, None)?;
//! let service = ServiceBuilder::new()
//!     .layer(TracingLayer)
//!     .layer(CostTrackingLayer)
//!     .service(LlmService::new(client));
//! ```

pub mod budget;
pub mod cache;
#[cfg(feature = "opendal-cache")]
pub mod cache_opendal;
pub mod cooldown;
pub mod cost;
pub mod fallback;
pub mod health;
pub mod hooks;
pub mod rate_limit;
pub mod router;
pub mod service;
#[cfg(test)]
mod tests;
#[cfg(test)]
pub(crate) mod tests_common;
pub mod tracing;
pub mod types;

// Re-export tower core types for convenient access
pub use tower::ServiceExt;

pub use budget::{BudgetConfig, BudgetLayer, BudgetService, BudgetState, Enforcement};
pub use cache::{CacheBackend, CacheConfig, CacheLayer, CacheService, CacheStore, CachedResponse, InMemoryStore};
#[cfg(feature = "opendal-cache")]
pub use cache_opendal::OpenDalCacheStore;
pub use cooldown::{CooldownLayer, CooldownService};
pub use cost::{CostTrackingLayer, CostTrackingService};
pub use fallback::{FallbackLayer, FallbackService};
pub use health::{HealthCheckLayer, HealthCheckService};
pub use hooks::{HooksLayer, HooksService, LlmHook};
pub use rate_limit::{ModelRateLimitLayer, ModelRateLimitService, RateLimitConfig};
pub use router::{Router, RoutingStrategy};
pub use service::LlmService;
pub use tracing::{TracingLayer, TracingService};
pub use types::{LlmRequest, LlmResponse};