1mod handlers;
15mod provider_cache;
16pub mod types;
17
18use std::sync::Arc;
19use std::time::Duration;
20
21use anyhow::Result;
22use axum::{
23 Json, Router,
24 http::StatusCode,
25 routing::{get, post},
26};
27use vera_core::config::{InferenceBackend, VeraConfig};
28use vera_core::embedding::DynamicProvider;
29use vera_core::retrieval::DynamicReranker;
30
31pub use provider_cache::CacheMode;
32use provider_cache::ModelSlot;
33use types::ApiError;
34
35pub(crate) type AcquireError = (StatusCode, Json<ApiError>);
36
37pub struct AppState {
39 pub api_key: Option<String>,
40 pub config: VeraConfig,
42 pub backend: InferenceBackend,
43 pub model_name: String,
45 pub reranker_available: bool,
47 pub(crate) embedding: Arc<ModelSlot<DynamicProvider, AcquireError>>,
49 pub(crate) reranker: Arc<ModelSlot<DynamicReranker, AcquireError>>,
51}
52
53pub async fn run_server(
68 config: VeraConfig,
69 backend: InferenceBackend,
70 api_key: Option<String>,
71 host: &str,
72 port: u16,
73 cache_mode: CacheMode,
74) -> Result<()> {
75 eprintln!(
76 "vera serve: initializing {} backend…",
77 backend_label(backend)
78 );
79
80 let (probe, model_name) = vera_core::embedding::create_dynamic_provider(&config, backend)
82 .await
83 .map_err(|e| anyhow::anyhow!("failed to load embedding model: {e}"))?;
84
85 eprintln!("vera serve: embedding model ready ({})", model_name);
86
87 let embedding: Arc<ModelSlot<DynamicProvider, AcquireError>> =
88 Arc::new(ModelSlot::new(cache_mode));
89 let reranker: Arc<ModelSlot<DynamicReranker, AcquireError>> =
90 Arc::new(ModelSlot::new(cache_mode));
91
92 embedding.seed(Arc::new(probe)).await;
97
98 let reranker_available = vera_core::retrieval::create_dynamic_reranker(&config, backend)
103 .await
104 .unwrap_or_else(|e| {
105 eprintln!("vera serve: reranker unavailable ({e}), reranking disabled");
106 None
107 })
108 .is_some();
109
110 if reranker_available {
111 eprintln!("vera serve: reranker ready");
112 }
113
114 let api_key = api_key.filter(|k| !k.is_empty());
115
116 if api_key.is_some() {
117 eprintln!("vera serve: API key authentication enabled");
118 } else {
119 eprintln!("vera serve: no API key set — unauthenticated access allowed");
120 }
121
122 match cache_mode {
123 CacheMode::PerRequest => eprintln!(
124 "vera serve: model cache disabled (--idle-timeout 0); every request rebuilds the model"
125 ),
126 CacheMode::Forever => eprintln!("vera serve: models stay loaded (no idle timeout)"),
127 CacheMode::Idle(d) => eprintln!(
128 "vera serve: models unload after {}s of inactivity",
129 d.as_secs()
130 ),
131 }
132
133 if let CacheMode::Idle(timeout) = cache_mode {
135 let embedding = Arc::clone(&embedding);
136 let reranker = Arc::clone(&reranker);
137 tokio::spawn(async move {
138 let check_interval = (timeout / 4).max(Duration::from_secs(1));
139 loop {
140 tokio::time::sleep(check_interval).await;
141 if embedding.evict_if_idle().await {
142 eprintln!("vera serve: embedding model unloaded (idle timeout reached)");
143 }
144 if reranker.evict_if_idle().await {
145 eprintln!("vera serve: reranker unloaded (idle timeout reached)");
146 }
147 }
148 });
149 }
150
151 let state = Arc::new(AppState {
152 api_key,
153 config,
154 backend,
155 model_name,
156 reranker_available,
157 embedding,
158 reranker,
159 });
160
161 let app = Router::new()
162 .route("/v1/embeddings", post(handlers::embeddings))
163 .route("/v1/rerank", post(handlers::rerank))
164 .route("/v1/health", get(handlers::health))
165 .with_state(state);
166
167 let addr = format!("{host}:{port}");
168 let listener = tokio::net::TcpListener::bind(&addr).await?;
169 eprintln!("vera serve: listening on http://{addr}");
170 eprintln!();
171 eprintln!(" Client setup:");
172 eprintln!(" vera setup --api (then set EMBEDDING_MODEL_BASE_URL=http://{addr}/v1)");
173 axum::serve(listener, app).await?;
174 Ok(())
175}
176
177fn backend_label(backend: InferenceBackend) -> &'static str {
178 use vera_core::config::OnnxExecutionProvider;
179 match backend {
180 InferenceBackend::Api => "api",
181 InferenceBackend::PotionCode => "potion-code (CPU)",
182 InferenceBackend::OnnxJina(OnnxExecutionProvider::Cpu) => "cpu",
183 InferenceBackend::OnnxJina(OnnxExecutionProvider::Cuda) => "cuda (GPU)",
184 InferenceBackend::OnnxJina(OnnxExecutionProvider::Rocm) => "rocm (AMD GPU)",
185 InferenceBackend::OnnxJina(OnnxExecutionProvider::DirectMl) => "directml (GPU)",
186 InferenceBackend::OnnxJina(OnnxExecutionProvider::CoreMl) => "coreml (Apple GPU)",
187 InferenceBackend::OnnxJina(OnnxExecutionProvider::OpenVino) => "openvino (Intel GPU)",
188 }
189}