ferrum-server 0.12.2

OpenAI-compatible HTTP API server for Ferrum inference
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53

//! # Ferrum Server
//!
//! HTTP API server abstractions for LLM inference services.
//!
//! ## Overview
//!
//! This module defines the core traits for implementing HTTP API servers
//! that can serve LLM inference requests with OpenAI API compatibility.
//!
//! ## Design Principles
//!
//! - **Framework Agnostic**: Abstract interfaces that work with any HTTP framework
//! - **OpenAI Compatible**: Support for OpenAI Chat Completions API
//! - **Middleware Support**: Pluggable middleware for auth, logging, rate limiting
//! - **Streaming Support**: Server-Sent Events for streaming responses
//! - **Monitoring**: Built-in metrics and health checks

pub mod axum_server;
pub mod chat_template;
pub mod middleware;
pub mod model_registry;
pub mod openai;
pub mod traits;
pub mod types;

// Re-exports
pub use traits::{
    AuthProvider, HttpServer, MiddlewareStack, RateLimiter as ServerRateLimiter, RequestHandler,
    ResponseBuilder, StreamingHandler,
};

pub use types::{
    ApiVersion, Headers, HealthStatus, HttpMethod, HttpRequest, HttpResponse, RequestContext,
    ServerConfig, ServerMetrics, StatusCode,
};

pub use openai::{
    ChatCompletionsRequest, ChatCompletionsResponse, ChatFunction, ChatFunctionCall, ChatMessage,
    ChatTool, ChatToolCall, CompletionPrompt, CompletionsRequest, CompletionsResponse,
    FunctionCallChoice, ModelListResponse, OpenAiError, OpenAiErrorType, OpenAiResponseFormat,
    StreamOptions, ToolChoice,
};

pub use middleware::{
    AuthConfig, CompressionConfig, CorsConfig, LoggingConfig, MiddlewareConfig,
    RateLimitConfig as MiddlewareRateLimitConfig,
};

pub use axum_server::{default_chat_sampling_params, init_prometheus_recorder, AxumServer};
pub use model_registry::{
    LoraAdapterModel, ServedModelEntry, ServedModelKind, ServedModelName, ServedModelRegistry,
    ServedModelRegistryError,
};