use std::collections::HashMap;
use std::process::Child;
use std::sync::{Arc, Mutex};
use std::time::{Duration, Instant};
use axum::response::IntoResponse;
const DEFAULT_MAX_LOADED: usize = 2;
const DEFAULT_MAX_CONCURRENT: usize = 8;
const DEFAULT_UPSTREAM_TIMEOUT: Duration = Duration::from_secs(600);
const MAX_BODY_BYTES: usize = 64 * 1024 * 1024;
const READY_TIMEOUT: Duration = Duration::from_secs(300);
struct Loaded {
port: u16,
child: Child,
last_used: Instant,
}
pub struct LoaderHooks {
pub fetch: FetchFn,
pub spawn: SpawnFn,
pub wait_ready: Box<dyn Fn(u16) -> Result<(), String> + Send + Sync>,
}
pub type FetchFn = Box<dyn Fn(&str) -> Result<std::path::PathBuf, String> + Send + Sync>;
pub type SpawnFn = Box<dyn Fn(&std::path::Path, u16) -> Result<Child, String> + Send + Sync>;
pub struct GeneralLoader {
hooks: LoaderHooks,
state: Mutex<LoaderState>,
}
struct LoaderState {
loaded: HashMap<String, Loaded>,
next_port: u16,
max_loaded: usize,
}
impl GeneralLoader {
pub fn new(hooks: LoaderHooks, first_backend_port: u16, max_loaded: usize) -> Self {
GeneralLoader {
hooks,
state: Mutex::new(LoaderState {
loaded: HashMap::new(),
next_port: first_backend_port,
max_loaded: max_loaded.max(1),
}),
}
}
pub fn max_loaded_from_env() -> usize {
std::env::var("ZAKURO_GENERAL_MAX_MODELS")
.ok()
.and_then(|v| v.parse::<usize>().ok())
.filter(|n| *n >= 1)
.unwrap_or(DEFAULT_MAX_LOADED)
}
pub fn ensure_loaded(&self, model_uuid: &str) -> Result<u16, String> {
let mut st = self
.state
.lock()
.map_err(|_| "loader state poisoned".to_string())?;
if let Some(entry) = st.loaded.get_mut(model_uuid) {
let alive = entry.child.try_wait().map(|x| x.is_none()).unwrap_or(false);
if alive {
entry.last_used = Instant::now();
return Ok(entry.port);
}
st.loaded.remove(model_uuid);
}
while st.loaded.len() >= st.max_loaded {
let lru = st
.loaded
.iter()
.min_by_key(|(_, l)| l.last_used)
.map(|(k, _)| k.clone());
match lru {
Some(uuid) => {
if let Some(mut old) = st.loaded.remove(&uuid) {
let _ = old.child.kill();
let _ = old.child.wait();
}
}
None => break,
}
}
let gguf = (self.hooks.fetch)(model_uuid)?;
let port = st.next_port;
st.next_port = st.next_port.wrapping_add(1);
let mut child = (self.hooks.spawn)(&gguf, port)?;
if let Err(e) = (self.hooks.wait_ready)(port) {
let _ = child.kill();
let _ = child.wait();
return Err(format!(
"llama-server for {model_uuid} never became ready: {e}"
));
}
st.loaded.insert(
model_uuid.to_string(),
Loaded {
port,
child,
last_used: Instant::now(),
},
);
Ok(port)
}
pub fn resident(&self) -> Vec<String> {
self.state
.lock()
.map(|st| st.loaded.keys().cloned().collect())
.unwrap_or_default()
}
}
pub fn production_hooks(
agent: ureq::Agent,
api_url: String,
auth_bearer: Option<String>,
) -> LoaderHooks {
LoaderHooks {
fetch: Box::new(move |uuid| {
crate::serve::fetch_model_gguf(&agent, &api_url, uuid, auth_bearer.as_deref())
}),
spawn: Box::new(|gguf, port| {
std::process::Command::new("llama-server")
.arg("--model")
.arg(gguf)
.arg("--host")
.arg("0.0.0.0")
.arg("--port")
.arg(port.to_string())
.spawn()
.map_err(|e| format!("spawning llama-server: {e} (is it installed and on PATH?)"))
}),
wait_ready: Box::new(|port| {
let deadline = Instant::now() + READY_TIMEOUT;
let probe = ureq::Agent::new_with_config(
ureq::Agent::config_builder()
.timeout_global(Some(Duration::from_secs(2)))
.build(),
);
loop {
if probe
.get(&format!("http://127.0.0.1:{port}/health"))
.call()
.is_ok()
{
return Ok(());
}
if Instant::now() >= deadline {
return Err(format!("no /health within {}s", READY_TIMEOUT.as_secs()));
}
std::thread::sleep(Duration::from_millis(500));
}
}),
}
}
pub fn model_uuid_from_body(body: &serde_json::Value) -> Option<String> {
body.get("model")
.and_then(|m| m.as_str())
.and_then(crate::model_uri::parse_model_uri)
}
pub struct GeneralServerConfig {
pub max_concurrent: usize,
pub upstream_timeout: Duration,
}
impl Default for GeneralServerConfig {
fn default() -> Self {
GeneralServerConfig {
max_concurrent: DEFAULT_MAX_CONCURRENT,
upstream_timeout: DEFAULT_UPSTREAM_TIMEOUT,
}
}
}
impl GeneralServerConfig {
pub fn from_env() -> Self {
GeneralServerConfig {
max_concurrent: max_concurrent_from(
std::env::var("ZAKURO_GENERAL_MAX_CONCURRENT")
.ok()
.as_deref(),
),
..GeneralServerConfig::default()
}
}
}
pub fn max_concurrent_from(raw: Option<&str>) -> usize {
raw.and_then(|v| v.trim().parse::<usize>().ok())
.filter(|n| *n >= 1)
.unwrap_or(DEFAULT_MAX_CONCURRENT)
}
struct Front {
loader: GeneralLoader,
proxy: ureq::Agent,
permits: Arc<tokio::sync::Semaphore>,
max_concurrent: usize,
}
pub fn run_general_server(loader: GeneralLoader, bind_port: u16) -> Result<(), String> {
let listener = std::net::TcpListener::bind(("0.0.0.0", bind_port))
.map_err(|e| format!("binding general provider on :{bind_port}: {e}"))?;
let config = GeneralServerConfig::from_env();
println!(
" general provider listening on :{bind_port} \
(on-demand model loading, up to {} requests in flight)",
config.max_concurrent
);
serve_general_on(listener, loader, config)
}
pub fn serve_general_on(
listener: std::net::TcpListener,
loader: GeneralLoader,
config: GeneralServerConfig,
) -> Result<(), String> {
let max_concurrent = config.max_concurrent.max(1);
listener
.set_nonblocking(true)
.map_err(|e| format!("general provider listener: {e}"))?;
let front = Arc::new(Front {
loader,
proxy: ureq::Agent::new_with_config(
ureq::Agent::config_builder()
.timeout_global(Some(config.upstream_timeout))
.build(),
),
permits: Arc::new(tokio::sync::Semaphore::new(max_concurrent)),
max_concurrent,
});
let runtime = tokio::runtime::Builder::new_multi_thread()
.worker_threads(2)
.max_blocking_threads(max_concurrent)
.enable_all()
.build()
.map_err(|e| format!("general provider runtime: {e}"))?;
runtime.block_on(async move {
let listener = tokio::net::TcpListener::from_std(listener)
.map_err(|e| format!("general provider listener: {e}"))?;
let app = axum::Router::new()
.fallback(axum::routing::any(route))
.with_state(front);
axum::serve(listener, app)
.await
.map_err(|e| format!("general provider server: {e}"))
})
}
async fn route(
axum::extract::State(front): axum::extract::State<Arc<Front>>,
request: axum::extract::Request,
) -> axum::response::Response {
let (parts, body) = request.into_parts();
let path = parts.uri.path().to_string();
if path == "/health" {
return json_response(200, serde_json::json!({"status": "ok"}));
}
if !path.starts_with("/v1/chat/completions") {
return json_response(404, serde_json::json!({"error": "not found"}));
}
let Ok(bytes) = axum::body::to_bytes(body, MAX_BODY_BYTES).await else {
return json_response(400, serde_json::json!({"error": "unreadable body"}));
};
let Ok(body_str) = String::from_utf8(bytes.to_vec()) else {
return json_response(400, serde_json::json!({"error": "unreadable body"}));
};
let Ok(permit) = front.permits.clone().try_acquire_owned() else {
return busy_response(front.max_concurrent);
};
let work = front.clone();
match tokio::task::spawn_blocking(move || {
let _permit = permit;
complete(&work, &body_str)
})
.await
{
Ok((status, body)) => json_response(status, body),
Err(_) => json_response(
500,
serde_json::json!({"error": "general provider handler panicked"}),
),
}
}
fn complete(front: &Front, body_str: &str) -> (u16, serde_json::Value) {
let body: serde_json::Value = match serde_json::from_str(body_str) {
Ok(v) => v,
Err(_) => return (400, serde_json::json!({"error": "invalid json"})),
};
let uuid = match model_uuid_from_body(&body) {
Some(u) => u,
None => {
return (
400,
serde_json::json!({"error": "model field must be a model uuid (zc://<uuid> or bare)"}),
)
}
};
let port = match front.loader.ensure_loaded(&uuid) {
Ok(p) => p,
Err(e) => {
eprintln!(" [GENERAL] load failed for {uuid}: {e}");
return (
503,
serde_json::json!({"error": format!("model load failed: {e}")}),
);
}
};
match front
.proxy
.post(&format!("http://127.0.0.1:{port}/v1/chat/completions"))
.header("Content-Type", "application/json")
.send(body_str)
{
Ok(mut upstream) => {
let status = upstream.status().as_u16();
let text = upstream.body_mut().read_to_string().unwrap_or_default();
let json: serde_json::Value = serde_json::from_str(&text)
.unwrap_or_else(|_| serde_json::json!({"error": "bad upstream body"}));
(status, json)
}
Err(e) => {
eprintln!(" [GENERAL] proxy to :{port} failed: {e}");
(502, serde_json::json!({"error": "backend request failed"}))
}
}
}
fn json_response(status: u16, body: serde_json::Value) -> axum::response::Response {
let code = axum::http::StatusCode::from_u16(status)
.unwrap_or(axum::http::StatusCode::INTERNAL_SERVER_ERROR);
(
code,
[(
axum::http::header::CONTENT_TYPE,
axum::http::HeaderValue::from_static("application/json"),
)],
body.to_string(),
)
.into_response()
}
fn busy_response(max_concurrent: usize) -> axum::response::Response {
let mut response = json_response(
503,
serde_json::json!({
"error": format!(
"general provider busy: {max_concurrent} requests already in flight"
),
"code": "GENERAL_PROVIDER_BUSY",
}),
);
response.headers_mut().insert(
axum::http::header::RETRY_AFTER,
axum::http::HeaderValue::from_static("1"),
);
response
}
#[cfg(test)]
mod tests {
use super::*;
use std::sync::atomic::{AtomicUsize, Ordering};
use std::sync::Arc;
fn sleeper() -> Child {
std::process::Command::new("sleep")
.arg("300")
.spawn()
.expect("spawn sleep")
}
fn test_loader(
max: usize,
fetches: Arc<AtomicUsize>,
spawns: Arc<AtomicUsize>,
) -> GeneralLoader {
let hooks = LoaderHooks {
fetch: Box::new(move |_uuid| {
fetches.fetch_add(1, Ordering::SeqCst);
Ok(std::path::PathBuf::from("/dev/null"))
}),
spawn: Box::new(move |_path, _port| {
spawns.fetch_add(1, Ordering::SeqCst);
Ok(sleeper())
}),
wait_ready: Box::new(|_port| Ok(())),
};
GeneralLoader::new(hooks, 9500, max)
}
const A: &str = "aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa";
const B: &str = "bbbbbbbb-bbbb-bbbb-bbbb-bbbbbbbbbbbb";
const C: &str = "cccccccc-cccc-cccc-cccc-cccccccccccc";
#[test]
fn second_request_reuses_the_resident_server() {
let fetches = Arc::new(AtomicUsize::new(0));
let spawns = Arc::new(AtomicUsize::new(0));
let loader = test_loader(2, fetches.clone(), spawns.clone());
let p1 = loader.ensure_loaded(A).unwrap();
let p2 = loader.ensure_loaded(A).unwrap();
assert_eq!(p1, p2);
assert_eq!(fetches.load(Ordering::SeqCst), 1, "one fetch, then cache");
assert_eq!(spawns.load(Ordering::SeqCst), 1, "one spawn, then reuse");
}
#[test]
fn lru_eviction_at_the_cap() {
let loader = test_loader(
2,
Arc::new(AtomicUsize::new(0)),
Arc::new(AtomicUsize::new(0)),
);
loader.ensure_loaded(A).unwrap();
loader.ensure_loaded(B).unwrap();
loader.ensure_loaded(A).unwrap();
loader.ensure_loaded(C).unwrap();
let mut resident = loader.resident();
resident.sort();
assert_eq!(
resident,
vec![A.to_string(), C.to_string()],
"B evicted as LRU"
);
}
#[test]
fn dead_backend_is_respawned_not_proxied_into() {
let spawns = Arc::new(AtomicUsize::new(0));
let loader = test_loader(2, Arc::new(AtomicUsize::new(0)), spawns.clone());
loader.ensure_loaded(A).unwrap();
{
let mut st = loader.state.lock().unwrap();
let entry = st.loaded.get_mut(A).unwrap();
entry.child.kill().unwrap();
entry.child.wait().unwrap();
}
let p = loader.ensure_loaded(A).unwrap();
assert_eq!(spawns.load(Ordering::SeqCst), 2, "dead server respawned");
assert!(p >= 9500);
}
#[test]
fn failed_readiness_kills_the_spawn_and_errors() {
let hooks = LoaderHooks {
fetch: Box::new(|_| Ok(std::path::PathBuf::from("/dev/null"))),
spawn: Box::new(|_, _| Ok(sleeper())),
wait_ready: Box::new(|_| Err("never healthy".into())),
};
let loader = GeneralLoader::new(hooks, 9500, 2);
assert!(loader.ensure_loaded(A).is_err());
assert!(
loader.resident().is_empty(),
"failed spawn must not stay resident"
);
}
#[test]
fn model_uuid_accepted_in_all_spellings() {
for m in [
format!("\"{A}\""),
format!("\"zc://{A}\""),
format!("\"zc://model-{A}\""),
] {
let body: serde_json::Value =
serde_json::from_str(&format!("{{\"model\":{m}}}")).unwrap();
assert_eq!(model_uuid_from_body(&body).as_deref(), Some(A));
}
let bad: serde_json::Value = serde_json::json!({"model": "llama-3-8b"});
assert_eq!(model_uuid_from_body(&bad), None);
}
const SLOW_MODEL: &str = "11111111-1111-1111-1111-111111111111";
const FAST_MODEL: &str = "22222222-2222-2222-2222-222222222222";
const SLOW_BACKEND: Duration = Duration::from_secs(5);
fn reserve_ports(n: u16) -> u16 {
for _ in 0..200 {
let probe = std::net::TcpListener::bind(("127.0.0.1", 0)).expect("probe bind");
let base = probe.local_addr().expect("probe addr").port();
drop(probe);
if base == 0 || base > u16::MAX - n {
continue;
}
let held: Vec<_> = (0..n)
.filter_map(|i| std::net::TcpListener::bind(("127.0.0.1", base + i)).ok())
.collect();
if held.len() == n as usize {
return base;
}
}
panic!("no run of {n} consecutive free ports");
}
fn spawn_fake_backend(
port: u16,
delay: Duration,
arrived: Arc<AtomicUsize>,
) -> Result<(), String> {
let listener = std::net::TcpListener::bind(("127.0.0.1", port))
.map_err(|e| format!("fake backend bind :{port}: {e}"))?;
std::thread::spawn(move || {
for stream in listener.incoming() {
let Ok(stream) = stream else { continue };
let arrived = arrived.clone();
std::thread::spawn(move || serve_fake_backend_conn(stream, delay, arrived));
}
});
Ok(())
}
fn serve_fake_backend_conn(
stream: std::net::TcpStream,
delay: Duration,
arrived: Arc<AtomicUsize>,
) {
use std::io::{BufRead, Read, Write};
let Ok(mut writer) = stream.try_clone() else {
return;
};
let mut reader = std::io::BufReader::new(stream);
let mut length = 0usize;
loop {
let mut line = String::new();
match reader.read_line(&mut line) {
Ok(0) | Err(_) => return,
Ok(_) => {}
}
let line = line.trim_end().to_ascii_lowercase();
if line.is_empty() {
break;
}
if let Some(v) = line.strip_prefix("content-length:") {
length = v.trim().parse().unwrap_or(0);
}
}
let mut body = vec![0u8; length];
if reader.read_exact(&mut body).is_err() {
return;
}
arrived.fetch_add(1, Ordering::SeqCst);
std::thread::sleep(delay);
let payload = br#"{"choices":[{"message":{"content":"ok"}}]}"#;
let head = format!(
"HTTP/1.1 200 OK\r\nContent-Type: application/json\r\n\
Content-Length: {}\r\nConnection: close\r\n\r\n",
payload.len()
);
let _ = writer.write_all(head.as_bytes());
let _ = writer.write_all(payload);
let _ = writer.flush();
}
fn fake_backend_hooks(arrived: Arc<AtomicUsize>) -> LoaderHooks {
LoaderHooks {
fetch: Box::new(|uuid| Ok(std::path::PathBuf::from(uuid))),
spawn: Box::new(move |gguf, port| {
let delay = if gguf.to_string_lossy() == SLOW_MODEL {
SLOW_BACKEND
} else {
Duration::ZERO
};
spawn_fake_backend(port, delay, arrived.clone())?;
Ok(sleeper())
}),
wait_ready: Box::new(|_| Ok(())),
}
}
fn client() -> ureq::Agent {
ureq::Agent::new_with_config(
ureq::Agent::config_builder()
.timeout_global(Some(Duration::from_secs(30)))
.http_status_as_error(false)
.build(),
)
}
fn post_completion(front: u16, model: &str) -> (u16, String) {
let body = serde_json::json!({"model": model, "messages": []}).to_string();
let mut r = client()
.post(&format!("http://127.0.0.1:{front}/v1/chat/completions"))
.header("Content-Type", "application/json")
.send(&body[..])
.expect("the front server answered the completion");
(
r.status().as_u16(),
r.body_mut().read_to_string().unwrap_or_default(),
)
}
fn get_health(front: u16) -> u16 {
client()
.get(&format!("http://127.0.0.1:{front}/health"))
.call()
.expect("the front server answered /health")
.status()
.as_u16()
}
fn wait_until_listening(port: u16) {
let deadline = Instant::now() + Duration::from_secs(10);
while Instant::now() < deadline {
if std::net::TcpStream::connect(("127.0.0.1", port)).is_ok() {
return;
}
std::thread::sleep(Duration::from_millis(10));
}
panic!("front server never came up on :{port}");
}
fn wait_for(mut cond: impl FnMut() -> bool, within: Duration, what: &str) {
let deadline = Instant::now() + within;
while Instant::now() < deadline {
if cond() {
return;
}
std::thread::sleep(Duration::from_millis(10));
}
panic!("timed out waiting for {what}");
}
#[test]
fn a_slow_request_does_not_hold_the_next_one_at_the_socket() {
let base = reserve_ports(3);
let front = base;
let arrived = Arc::new(AtomicUsize::new(0));
let loader = GeneralLoader::new(fake_backend_hooks(arrived.clone()), base + 1, 2);
std::thread::spawn(move || {
let _ = run_general_server(loader, front);
});
wait_until_listening(front);
let slow = std::thread::spawn(move || post_completion(front, SLOW_MODEL));
wait_for(
|| arrived.load(Ordering::SeqCst) >= 1,
Duration::from_secs(10),
"the slow request to reach its backend",
);
let started = Instant::now();
let (status, _) = post_completion(front, FAST_MODEL);
let waited = started.elapsed();
assert_eq!(status, 200, "the second caller got a real answer");
assert!(
waited < Duration::from_secs(2),
"the second request waited {waited:?} behind a {SLOW_BACKEND:?} first one"
);
let (slow_status, _) = slow.join().expect("slow request thread");
assert_eq!(slow_status, 200, "and the slow one still got its answer");
}
#[test]
fn health_is_answered_while_an_inference_is_in_flight() {
let base = reserve_ports(3);
let front = base;
let arrived = Arc::new(AtomicUsize::new(0));
let loader = GeneralLoader::new(fake_backend_hooks(arrived.clone()), base + 1, 2);
std::thread::spawn(move || {
let _ = run_general_server(loader, front);
});
wait_until_listening(front);
let slow = std::thread::spawn(move || post_completion(front, SLOW_MODEL));
wait_for(
|| arrived.load(Ordering::SeqCst) >= 1,
Duration::from_secs(10),
"the slow request to reach its backend",
);
let started = Instant::now();
let status = get_health(front);
let waited = started.elapsed();
assert_eq!(status, 200);
assert!(
waited < Duration::from_secs(2),
"/health waited {waited:?} behind an inference"
);
let _ = slow.join();
}
#[test]
fn a_caller_over_the_concurrency_bound_is_refused_immediately() {
let base = reserve_ports(3);
let front = base;
let front_listener = std::net::TcpListener::bind(("127.0.0.1", front)).expect("front bind");
let arrived = Arc::new(AtomicUsize::new(0));
let loader = GeneralLoader::new(fake_backend_hooks(arrived.clone()), base + 1, 2);
std::thread::spawn(move || {
let _ = serve_general_on(
front_listener,
loader,
GeneralServerConfig {
max_concurrent: 1,
..GeneralServerConfig::default()
},
);
});
wait_until_listening(front);
let slow = std::thread::spawn(move || post_completion(front, SLOW_MODEL));
wait_for(
|| arrived.load(Ordering::SeqCst) >= 1,
Duration::from_secs(10),
"the slow request to reach its backend",
);
let started = Instant::now();
let (status, body) = post_completion(front, FAST_MODEL);
let waited = started.elapsed();
assert_eq!(status, 503, "over the bound the caller is told, not queued");
assert!(
body.contains("GENERAL_PROVIDER_BUSY"),
"the refusal names itself: {body}"
);
assert!(
waited < Duration::from_secs(2),
"the refusal took {waited:?} — that is a stall with a status code on it"
);
let (slow_status, _) = slow.join().expect("slow request thread");
assert_eq!(slow_status, 200);
let (status, _) = post_completion(front, FAST_MODEL);
assert_eq!(
status, 200,
"the permit came back when the slow request finished"
);
}
#[test]
fn the_concurrency_cap_defaults_when_unset_or_unusable() {
assert_eq!(max_concurrent_from(Some("3")), 3);
assert_eq!(max_concurrent_from(Some(" 3 ")), 3);
assert_eq!(max_concurrent_from(None), DEFAULT_MAX_CONCURRENT);
assert_eq!(max_concurrent_from(Some("")), DEFAULT_MAX_CONCURRENT);
assert_eq!(max_concurrent_from(Some("plenty")), DEFAULT_MAX_CONCURRENT);
assert_eq!(
max_concurrent_from(Some("0")),
DEFAULT_MAX_CONCURRENT,
"a server that admits nobody is not what an operator meant"
);
}
}