Skip to main content

vera2_serve/
lib.rs

1//! Vera HTTP API server.
2//!
3//! Exposes OpenAI-compatible inference endpoints for standard vera clients:
4//!
5//! ```text
6//! POST /v1/embeddings   OpenAI format  (EMBEDDING_MODEL_BASE_URL)
7//! POST /v1/rerank       Cohere/Jina format  (RERANKER_MODEL_BASE_URL)
8//! GET  /v1/health       liveness + model info
9//! ```
10//!
11//! A regular vera client configured with `vera setup --api` pointing at
12//! `http://host:port/v1` will work without any modifications.
13
14mod handlers;
15mod provider_cache;
16pub mod types;
17
18use std::sync::Arc;
19use std::time::Duration;
20
21use anyhow::Result;
22use axum::{
23    Json, Router,
24    http::StatusCode,
25    routing::{get, post},
26};
27use vera_core::config::{InferenceBackend, VeraConfig};
28use vera_core::embedding::DynamicProvider;
29use vera_core::retrieval::DynamicReranker;
30
31pub use provider_cache::CacheMode;
32use provider_cache::ModelSlot;
33use types::ApiError;
34
35pub(crate) type AcquireError = (StatusCode, Json<ApiError>);
36
37/// Shared state injected into every handler.
38pub struct AppState {
39    pub api_key: Option<String>,
40    /// Config used to create providers on-demand.
41    pub config: VeraConfig,
42    pub backend: InferenceBackend,
43    /// Human-readable model name reported in /v1/health and embeddings responses.
44    pub model_name: String,
45    /// Whether a reranker is available (probed at startup).
46    pub reranker_available: bool,
47    /// The embedding model, cached independently of the reranker.
48    pub(crate) embedding: Arc<ModelSlot<DynamicProvider, AcquireError>>,
49    /// The reranker, cached independently of the embedding model.
50    pub(crate) reranker: Arc<ModelSlot<DynamicReranker, AcquireError>>,
51}
52
53/// Start the Vera HTTP server.
54///
55/// Probes the embedding model and reranker at startup to validate the config,
56/// then listens for connections on `host:port`. The embedding probe is kept and
57/// becomes the first resident model; the reranker is loaded on its first request,
58/// since holding it costs memory a server that only embeds never uses. Each model
59/// is then held as `cache_mode` dictates.
60///
61/// - `config`     — vera retrieval/embedding config
62/// - `backend`    — compute backend (API, CPU, GPU)
63/// - `api_key`    — optional bearer token; `None` disables auth
64/// - `host`       — bind address (e.g. `"127.0.0.1"` or `"0.0.0.0"`)
65/// - `port`       — TCP port to listen on
66/// - `cache_mode` — how long a loaded model stays resident
67pub async fn run_server(
68    config: VeraConfig,
69    backend: InferenceBackend,
70    api_key: Option<String>,
71    host: &str,
72    port: u16,
73    cache_mode: CacheMode,
74) -> Result<()> {
75    eprintln!(
76        "vera serve: initializing {} backend…",
77        backend_label(backend)
78    );
79
80    // Probe-load to validate the config and obtain the model name.
81    let (probe, model_name) = vera_core::embedding::create_dynamic_provider(&config, backend)
82        .await
83        .map_err(|e| anyhow::anyhow!("failed to load embedding model: {e}"))?;
84
85    eprintln!("vera serve: embedding model ready ({})", model_name);
86
87    let embedding: Arc<ModelSlot<DynamicProvider, AcquireError>> =
88        Arc::new(ModelSlot::new(cache_mode));
89    let reranker: Arc<ModelSlot<DynamicReranker, AcquireError>> =
90        Arc::new(ModelSlot::new(cache_mode));
91
92    // The probe already paid for a full load, so hand it to the slot rather than
93    // dropping it and making the first request load the same model again. Seeded
94    // here, before the reranker probe, so `PerRequest` — where `seed` is a no-op —
95    // still releases it before anything else is loaded.
96    embedding.seed(Arc::new(probe)).await;
97
98    // The reranker probe is deliberately not seeded: it is built here, read for
99    // `reranker_available`, and dropped, so the first /v1/rerank loads it again.
100    // Keeping it would cost ~670 MB resident on a server that only answers
101    // /v1/embeddings and never reranks at all.
102    let reranker_available = vera_core::retrieval::create_dynamic_reranker(&config, backend)
103        .await
104        .unwrap_or_else(|e| {
105            eprintln!("vera serve: reranker unavailable ({e}), reranking disabled");
106            None
107        })
108        .is_some();
109
110    if reranker_available {
111        eprintln!("vera serve: reranker ready");
112    }
113
114    let api_key = api_key.filter(|k| !k.is_empty());
115
116    if api_key.is_some() {
117        eprintln!("vera serve: API key authentication enabled");
118    } else {
119        eprintln!("vera serve: no API key set — unauthenticated access allowed");
120    }
121
122    match cache_mode {
123        CacheMode::PerRequest => eprintln!(
124            "vera serve: model cache disabled (--idle-timeout 0); every request rebuilds the model"
125        ),
126        CacheMode::Forever => eprintln!("vera serve: models stay loaded (no idle timeout)"),
127        CacheMode::Idle(d) => eprintln!(
128            "vera serve: models unload after {}s of inactivity",
129            d.as_secs()
130        ),
131    }
132
133    // Background task: unload each model after `cache_mode`'s idle timeout.
134    if let CacheMode::Idle(timeout) = cache_mode {
135        let embedding = Arc::clone(&embedding);
136        let reranker = Arc::clone(&reranker);
137        tokio::spawn(async move {
138            let check_interval = (timeout / 4).max(Duration::from_secs(1));
139            loop {
140                tokio::time::sleep(check_interval).await;
141                if embedding.evict_if_idle().await {
142                    eprintln!("vera serve: embedding model unloaded (idle timeout reached)");
143                }
144                if reranker.evict_if_idle().await {
145                    eprintln!("vera serve: reranker unloaded (idle timeout reached)");
146                }
147            }
148        });
149    }
150
151    let state = Arc::new(AppState {
152        api_key,
153        config,
154        backend,
155        model_name,
156        reranker_available,
157        embedding,
158        reranker,
159    });
160
161    let app = Router::new()
162        .route("/v1/embeddings", post(handlers::embeddings))
163        .route("/v1/rerank", post(handlers::rerank))
164        .route("/v1/health", get(handlers::health))
165        .with_state(state);
166
167    let addr = format!("{host}:{port}");
168    let listener = tokio::net::TcpListener::bind(&addr).await?;
169    eprintln!("vera serve: listening on http://{addr}");
170    eprintln!();
171    eprintln!("  Client setup:");
172    eprintln!("    vera setup --api  (then set EMBEDDING_MODEL_BASE_URL=http://{addr}/v1)");
173    axum::serve(listener, app).await?;
174    Ok(())
175}
176
177fn backend_label(backend: InferenceBackend) -> &'static str {
178    use vera_core::config::OnnxExecutionProvider;
179    match backend {
180        InferenceBackend::Api => "api",
181        InferenceBackend::PotionCode => "potion-code (CPU)",
182        InferenceBackend::OnnxJina(OnnxExecutionProvider::Cpu) => "cpu",
183        InferenceBackend::OnnxJina(OnnxExecutionProvider::Cuda) => "cuda (GPU)",
184        InferenceBackend::OnnxJina(OnnxExecutionProvider::Rocm) => "rocm (AMD GPU)",
185        InferenceBackend::OnnxJina(OnnxExecutionProvider::DirectMl) => "directml (GPU)",
186        InferenceBackend::OnnxJina(OnnxExecutionProvider::CoreMl) => "coreml (Apple GPU)",
187        InferenceBackend::OnnxJina(OnnxExecutionProvider::OpenVino) => "openvino (Intel GPU)",
188    }
189}