use std::collections::HashMap;
use axum::extract::State;
use axum::http::HeaderMap;
use axum::Json;
use futures::future::join_all;
use serde::de::DeserializeOwned;
use serde::{Deserialize, Serialize};
use tokio::time::Duration;
use super::{AppState, PeerTarget, Target};
use crate::storage::OciStore;
pub(super) const HOP: &str = "x-llmman-hop";
const PEER_TIMEOUT: Duration = Duration::from_secs(3);
#[derive(Serialize, Deserialize, Default, Debug, Clone, PartialEq)]
pub(super) struct Node {
pub(super) memory: u64,
pub(super) loaded: HashMap<String, u64>,
pub(super) stored: HashMap<String, u64>,
}
impl Node {
fn free(&self) -> u64 {
self.memory.saturating_sub(self.loaded.values().sum())
}
}
pub(super) async fn local_node(state: &AppState) -> Node {
let loaded = state
.0
.manager
.lock()
.await
.running
.iter()
.map(|(name, m)| (name.clone(), m.size))
.collect();
let stored = OciStore::open(&state.0.store_path)
.and_then(|s| s.list())
.map(|list| list.into_iter().map(|i| (i.reference, i.size)).collect())
.unwrap_or_default();
Node {
memory: state.0.memory,
loaded,
stored,
}
}
pub(super) async fn handle_node(State(state): State<AppState>) -> Json<Node> {
Json(local_node(&state).await)
}
pub(super) fn hop(req: reqwest::RequestBuilder, key: Option<&str>) -> reqwest::RequestBuilder {
let req = req.header(HOP, "1");
match key {
Some(key) => req.bearer_auth(key),
None => req,
}
}
pub(super) fn target(state: &AppState, origin: String) -> Target {
Target::Peer(std::sync::Arc::new(PeerTarget {
origin,
api_key: state.0.peer_key.clone(),
}))
}
pub(super) fn aggregates(state: &AppState, headers: &HeaderMap) -> bool {
!state.0.peers.is_empty() && !headers.contains_key(HOP)
}
pub(super) async fn poll<T: DeserializeOwned>(state: &AppState, path: &str) -> Vec<(String, T)> {
join_all(state.0.peers.iter().map(|peer| async move {
let sent = hop(
state.0.client.get(format!("{peer}{path}")),
state.0.peer_key.as_deref(),
)
.timeout(PEER_TIMEOUT)
.send()
.await
.and_then(|r| r.error_for_status());
let body = match sent {
Ok(resp) => resp.json::<T>().await,
Err(e) => Err(e),
};
match body {
Ok(body) => Some((peer.clone(), body)),
Err(e) => {
crate::debug_log!("peer {peer}{path}: {e}");
None
}
}
}))
.await
.into_iter()
.flatten()
.collect()
}
pub(super) async fn route(
state: &AppState,
model: &str,
headers: Option<&HeaderMap>,
) -> Option<String> {
let headers = headers?;
if !aggregates(state, headers) {
return None;
}
let mut origins = vec![None];
let mut nodes = vec![local_node(state).await];
for (origin, node) in poll::<Node>(state, "/llmman/node").await {
origins.push(Some(origin));
nodes.push(node);
}
let peer = origins.swap_remove(place(&nodes, model))?;
eprintln!("[llmman] aggregation: {model} -> {peer}");
Some(peer)
}
fn place(nodes: &[Node], model: &str) -> usize {
if let Some(i) = nodes.iter().position(|n| n.loaded.contains_key(model)) {
return i;
}
let size = nodes.iter().find_map(|n| n.stored.get(model).copied());
let rank = |n: &Node| {
let fits = size.is_none_or(|size| n.memory == 0 || n.free() >= size);
(fits, fits && n.stored.contains_key(model), n.free())
};
(0..nodes.len())
.rev()
.max_by_key(|&i| rank(&nodes[i]))
.unwrap_or(0)
}
pub(super) async fn unload(state: &AppState, model: &str, headers: &HeaderMap) -> bool {
if !aggregates(state, headers) {
return false;
}
let body = serde_json::json!({ "model": model, "keep_alive": 0 });
join_all(state.0.peers.iter().map(|peer| {
hop(
state.0.client.post(format!("{peer}/api/generate")),
state.0.peer_key.as_deref(),
)
.json(&body)
.timeout(PEER_TIMEOUT)
.send()
}))
.await
.into_iter()
.any(|r| r.is_ok_and(|r| r.status().is_success()))
}
#[cfg(test)]
mod tests {
use super::*;
fn node(memory: u64, loaded: &[(&str, u64)], stored: &[(&str, u64)]) -> Node {
let map = |kv: &[(&str, u64)]| {
kv.iter()
.map(|(k, v)| (k.to_string(), *v))
.collect::<HashMap<_, _>>()
};
Node {
memory,
loaded: map(loaded),
stored: map(stored),
}
}
const GIB: u64 = 1 << 30;
#[test]
fn a_node_that_has_the_model_loaded_wins_outright() {
let nodes = [
node(128 * GIB, &[], &[("m", GIB)]),
node(8 * GIB, &[("m", GIB)], &[("m", GIB)]),
];
assert_eq!(place(&nodes, "m"), 1);
}
#[test]
fn a_node_that_has_the_model_stored_beats_a_roomier_one_it_fits_on() {
let nodes = [
node(128 * GIB, &[], &[]),
node(16 * GIB, &[], &[("m", 4 * GIB)]),
];
assert_eq!(place(&nodes, "m"), 1);
}
#[test]
fn a_node_the_model_does_not_fit_on_loses_even_with_it_stored() {
let nodes = [
node(128 * GIB, &[], &[]),
node(8 * GIB, &[("other", 6 * GIB)], &[("m", 4 * GIB)]),
];
assert_eq!(place(&nodes, "m"), 0);
let nowhere = [
node(2 * GIB, &[], &[("m", 4 * GIB)]),
node(3 * GIB, &[], &[]),
];
assert_eq!(place(&nowhere, "m"), 1);
}
#[test]
fn otherwise_the_most_room_wins_and_ties_go_to_the_first() {
let nodes = [
node(64 * GIB, &[("big", 60 * GIB)], &[]),
node(128 * GIB, &[], &[]),
node(128 * GIB, &[], &[]),
];
assert_eq!(place(&nodes, "m"), 1);
let same = [node(8 * GIB, &[], &[]), node(8 * GIB, &[], &[])];
assert_eq!(place(&same, "m"), 0);
}
#[test]
fn unknown_memory_fits_anything_but_ranks_last() {
let nodes = [node(0, &[], &[("m", GIB)]), node(GIB, &[], &[("m", GIB)])];
assert_eq!(place(&nodes, "m"), 1);
let only = [node(0, &[], &[])];
assert_eq!(place(&only, "m"), 0);
}
#[test]
fn free_never_underflows() {
assert_eq!(node(GIB, &[("m", 2 * GIB)], &[]).free(), 0);
assert_eq!(
node(4 * GIB, &[("m", GIB), ("n", GIB)], &[]).free(),
2 * GIB
);
}
}