edgequake-llm 0.10.2

Multi-provider LLM abstraction library with caching, rate limiting, and cost tracking
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176

//! EdgeQuake LLM - LLM and Embedding Provider Abstraction
//!
//! # Implements
//!
//! - **FEAT0017**: Multi-Provider LLM Support
//! - **FEAT0018**: Embedding Provider Abstraction
//! - **FEAT0019**: LLM Response Caching
//! - **FEAT0020**: API Rate Limiting
//! - **FEAT0005**: Embedding Generation (via providers)
//!
//! # Enforces
//!
//! - **BR0301**: LLM API rate limits (configurable per provider)
//! - **BR0302**: Document size limits (context window awareness)
//! - **BR0303**: Cost tracking per request
//! - **BR0010**: Embedding dimension validated (1536 default)
//!
//! This crate provides traits and implementations for:
//! - Text completion (LLM providers)
//! - Text embedding (embedding providers)
//! - Token counting and management
//! - Rate limiting for API calls
//! - Response caching for cost reduction
//!
//! # Providers
//!
//! | Provider | FEAT0017 | Chat | Embeddings | Notes |
//! |----------|----------|------|------------|-------|
//! | OpenAI | ✓ | ✓ | ✓ | Primary production provider |
//! | Azure OpenAI | ✓ | ✓ | ✓ | Enterprise deployments |
//! | Ollama | ✓ | ✓ | ✓ | Local/on-prem models |
//! | LM Studio | ✓ | ✓ | ✓ | Local OpenAI-compatible API |
//! | Gemini | ✓ | ✓ | ✓ | Google AI |
//! | Mock | ✓ | ✓ | ✓ | Testing (no API calls) |
//!
//! # Architecture
//!
//! The crate uses trait-based abstraction to support multiple LLM backends:
//! - OpenAI (GPT-4, GPT-3.5)
//! - OpenAI-compatible APIs (Ollama, LM Studio, etc.)
//! - Anthropic (Claude 3.5, Claude 3)
//! - Future: Mistral, local models
//!
//! # Example
//!
//! ```ignore
//! use edgequake_llm::{LLMProvider, OpenAIProvider};
//!
//! let provider = OpenAIProvider::new("your-api-key");
//! let response = provider.complete("Hello, world!").await?;
//! ```
//!
//! # See Also
//!
//! - [`crate::traits`] for provider trait definitions
//! - [`crate::providers`] for concrete implementations
//! - [`crate::cache`] for response caching

pub mod application_context;
pub mod cache;
pub mod cache_prompt;
pub mod cost_tracker; // OODA-21: Session-level cost tracking
pub mod discovery;
pub mod error;
pub mod factory;
pub mod http;
pub mod imagegen;
pub mod inference_metrics; // OODA-33: Unified streaming metrics
pub mod middleware;
pub mod model_config;
pub mod provider_catalog;
pub mod providers;
pub mod rate_limiter;
pub mod registry;
pub mod reranker;
pub mod retry;
pub mod stream_tool_calls;
pub mod stream_tools;
pub mod tokenizer;
pub mod trace_context;
pub mod traits;

pub use application_context::{
    ApplicationContext, ApplicationContextBuilder, AttributionPolicy, AttributionProviderKind,
    AttributionWarning,
};
pub use cache::{CacheConfig, CacheStats, CachedProvider, LLMCache};
pub use cache_prompt::{
    apply_cache_control, parse_cache_stats, CachePromptConfig, CacheStats as PromptCacheStats,
};
pub use cost_tracker::{
    format_cost, format_tokens, CostEntry, CostSummary, ModelPricing, SessionCostTracker,
};
pub use discovery::{
    find_static_models, search_models, search_static_models, static_lookup_by_name,
    CapabilityFilter, DiscoveredModel, DiscoveryError, DiscoverySource, DiscoveryStrategy,
    ModelCapability, ModelDiscoveryProvider, ModelDiscoveryService, ModelMatchKind,
    ModelSearchMatch, ModelSearchQuery,
};
pub use error::{LlmError, Result, RetryStrategy};
pub use factory::{ProviderFactory, ProviderType};
#[cfg(feature = "bedrock")]
pub use imagegen::BedrockStabilityImageGen;
pub use imagegen::{
    AspectRatio, AzureImageGen, FalImageGen, GeminiImageGenProvider, GeneratedImage, ImageFormat,
    ImageGenData, ImageGenError, ImageGenFactory, ImageGenOptions, ImageGenProvider,
    ImageGenRequest, ImageGenResponse, ImageResolution, MockImageGenProvider, NvidiaImageGen,
    OpenAIImageGen, SafetyLevel, ThinkingLevel, VertexAIImageGen, XAIImageGen,
};
pub use inference_metrics::InferenceMetrics; // OODA-33
pub use middleware::{
    LLMMiddleware, LLMMiddlewareStack, LLMRequest, LogLevel, LoggingLLMMiddleware,
    MetricsLLMMiddleware, MetricsSummary,
};
pub use model_config::{
    DefaultsConfig, ModelCapabilities, ModelCard, ModelConfigError, ModelCost, ModelType,
    ModelsConfig, ProviderConfig, ProviderType as ConfigProviderType,
};
pub use provider_catalog::{
    AttributionSupport, ProviderCatalog, ProviderDescriptor, ProviderFeatures,
};
pub use providers::azure_openai::{AzureCredential, AzureOpenAIProvider};
pub use providers::gemini::GeminiProvider;
pub use providers::jina::JinaProvider;
pub use providers::lmstudio::{LMStudioProvider, LmStudioModelMetadata};
pub use providers::mock::MockProvider;
pub use providers::ollama::{
    OllamaModelDetails, OllamaModelInfo, OllamaModelsResponse, OllamaProvider,
};
pub use providers::openai::OpenAIProvider;
// FEAT-007: Mistral AI provider
pub use providers::mistral::MistralProvider;
// FEAT-020: AWS Bedrock provider (feature-gated)
#[cfg(feature = "bedrock")]
pub use providers::bedrock::BedrockProvider;
// OODA-01: Anthropic (Claude) provider
pub use providers::anthropic::AnthropicProvider;
// OODA-02: OpenRouter provider (200+ models)
// OODA-72: Dynamic model discovery with caching
pub use providers::openrouter::{
    ModelArchitecture as OpenRouterModelArchitecture, ModelInfo as OpenRouterModelInfo,
    ModelPricing as OpenRouterModelPricing, ModelsResponse as OpenRouterModelsResponse,
    OpenRouterProvider,
};
// OODA-200: Configurable OpenAI-compatible provider
pub use providers::openai_compatible::OpenAICompatibleProvider;
// OODA-71: xAI Grok provider (api.x.ai)
pub use providers::vscode::{
    Model as CopilotModel, ModelsResponse as CopilotModelsResponse, VsCodeCopilotProvider,
};
pub use providers::xai::XAIProvider;
pub use providers::{trace_llm_arc, TracingProvider};
pub use stream_tool_calls::StreamToolCallAccumulator;
pub use stream_tools::{
    finalize_streamed_tool_calls_with_repair, FinalizeStreamToolCallsOptions, PartialStreamToolCall,
};
pub use trace_context::with_trace_context;
// FEAT-030: NVIDIA NIM provider (integrate.api.nvidia.com)
pub use providers::nvidia::{NvidiaModelInfo, NvidiaModelsResponse, NvidiaProvider};
// FEAT-040: Cohere provider (Command A/R series, Embed v4, Rerank)
pub use providers::cohere::CohereProvider;
pub use rate_limiter::{RateLimitedProvider, RateLimiter, RateLimiterConfig};
pub use registry::ProviderRegistry;
pub use reranker::{
    create_bm25_reranker, create_cross_encoder_reranker, create_production_reranker,
    try_http_cross_encoder_reranker, BM25Reranker, BiEncoderReranker, HttpReranker, HybridReranker,
    MockReranker, RRFReranker, RerankConfig, RerankResult, Reranker, ScoreAggregation,
    TermOverlapReranker,
};
pub use retry::RetryExecutor;
pub use tokenizer::Tokenizer;
pub use traits::{
    CacheControl, ChatMessage, ChatRole, CompletionOptions, EmbeddingProvider, FunctionCall,
    FunctionDefinition, ImageData, LLMProvider, LLMResponse, ToolCall, ToolChoice, ToolDefinition,
    ToolResult,
};