zentinel-proxy 0.6.11

A security-first reverse proxy built on Pingora with sleepable ops at the edge
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72

//! Inference routing module for LLM/AI traffic patterns
//!
//! This module provides:
//! - Token-based rate limiting (tokens/minute instead of requests/second)
//! - Token budget tracking (cumulative usage per period)
//! - Cost attribution (per-model pricing)
//! - Multi-provider token counting (OpenAI, Anthropic, generic)
//! - Model-aware load balancing (LeastTokensQueued strategy)
//!
//! # Example Usage
//!
//! ```kdl
//! route "/v1/chat/completions" {
//!     inference {
//!         provider "openai"
//!         rate-limit {
//!             tokens-per-minute 100000
//!             burst-tokens 10000
//!         }
//!         budget {
//!             period "daily"
//!             limit 1000000
//!             enforce true
//!         }
//!         cost-attribution {
//!             pricing {
//!                 model "gpt-4*" {
//!                     input-cost-per-million 30.0
//!                     output-cost-per-million 60.0
//!                 }
//!             }
//!         }
//!         routing {
//!             strategy "least-tokens-queued"
//!         }
//!     }
//!     upstream "llm-pool" { ... }
//! }
//! ```

mod budget;
mod cost;
mod guardrails;
mod manager;
mod metrics;
mod providers;
mod rate_limit;
mod streaming;
mod tiktoken;
mod tokens;

pub use budget::TokenBudgetTracker;
pub use cost::CostCalculator;
pub use guardrails::{
    extract_inference_content, GuardrailProcessor, PiiCheckResult, PromptInjectionResult,
};
pub use manager::{InferenceCheckResult, InferenceRateLimitManager, InferenceRouteStats};
pub use metrics::InferenceMetrics;
pub use providers::{create_provider, InferenceProviderAdapter};
pub use rate_limit::{TokenRateLimitResult, TokenRateLimiter};
pub use streaming::{
    is_sse_response, StreamingTokenCounter, StreamingTokenResult, TokenCountSource,
};
pub use tiktoken::{tiktoken_manager, TiktokenEncoding, TiktokenManager};
pub use tokens::{TokenCounter, TokenEstimate, TokenSource};

use zentinel_config::{InferenceConfig, InferenceProvider};

/// Create a provider adapter based on the configured provider type
pub fn create_inference_provider(config: &InferenceConfig) -> Box<dyn InferenceProviderAdapter> {
    create_provider(&config.provider)
}