use tokio::time::{Duration, Instant};
use super::AppState;
use crate::metrics::{self, UnloadReason};
pub(super) const DEFAULT_KEEP_ALIVE: Duration = Duration::from_secs(5 * 60);
pub(super) fn default_keep_alive() -> Option<Duration> {
match std::env::var("LLMMAN_KEEP_ALIVE") {
Ok(v) => parse_keep_alive_str(&v).unwrap_or(Some(DEFAULT_KEEP_ALIVE)),
Err(_) => Some(DEFAULT_KEEP_ALIVE),
}
}
pub(super) fn resolve_keep_alive(value: &Option<serde_json::Value>) -> Option<Duration> {
value
.as_ref()
.and_then(parse_keep_alive_value)
.unwrap_or_else(default_keep_alive)
}
pub(super) fn is_explicit_unload(keep_alive: &Option<serde_json::Value>) -> bool {
keep_alive.as_ref().and_then(parse_keep_alive_value) == Some(Some(Duration::ZERO))
}
pub(super) fn parse_keep_alive_value(v: &serde_json::Value) -> Option<Option<Duration>> {
match v {
serde_json::Value::Number(n) => secs_to_keep_alive(n.as_f64()?),
serde_json::Value::String(s) => parse_keep_alive_str(s),
_ => None,
}
}
pub(super) fn secs_to_keep_alive(secs: f64) -> Option<Option<Duration>> {
if secs < 0.0 {
return Some(None);
}
Duration::try_from_secs_f64(secs).ok().map(Some)
}
pub(crate) fn parse_keep_alive_str(s: &str) -> Option<Option<Duration>> {
let s = s.trim();
if s.is_empty() {
return None;
}
if let Ok(secs) = s.parse::<f64>() {
return secs_to_keep_alive(secs);
}
if s.starts_with('-') {
return Some(None);
}
let mut total = Duration::ZERO;
let mut rest = s;
let mut matched_any = false;
while !rest.is_empty() {
let digits_end = rest
.find(|c: char| !c.is_ascii_digit() && c != '.')
.unwrap_or(rest.len());
if digits_end == 0 {
return None;
}
let (num_str, tail) = rest.split_at(digits_end);
let num: f64 = num_str.parse().ok()?;
let (secs, tail) = if let Some(t) = tail.strip_prefix("ms") {
(num / 1000.0, t)
} else if let Some(t) = tail.strip_prefix('h') {
(num * 3600.0, t)
} else if let Some(t) = tail.strip_prefix('m') {
(num * 60.0, t)
} else {
(num, tail.strip_prefix('s')?)
};
let component = Duration::try_from_secs_f64(secs).ok()?;
total = total.checked_add(component)?;
rest = tail;
matched_any = true;
}
matched_any.then_some(Some(total))
}
pub(super) struct ActivityGuard {
pub(super) state: AppState,
pub(super) model_key: String,
pub(super) keep_alive: Option<Option<Duration>>,
touch: bool,
}
impl ActivityGuard {
pub(super) fn new(state: &AppState, model_key: &str) -> Self {
Self {
state: state.clone(),
model_key: model_key.to_string(),
keep_alive: None,
touch: true,
}
}
pub(super) fn release_idle(mut self) {
self.touch = false;
}
}
impl Drop for ActivityGuard {
fn drop(&mut self) {
let state = self.state.clone();
let model_key = std::mem::take(&mut self.model_key);
let keep_alive = self.keep_alive;
let touch = self.touch;
let Ok(handle) = tokio::runtime::Handle::try_current() else {
return;
};
handle.spawn(async move {
let mut mgr = state.0.manager.lock().await;
if let Some(m) = mgr.running.get_mut(&model_key) {
m.in_flight = m.in_flight.saturating_sub(1);
if touch {
m.last_active = Instant::now();
m.last_active_wall = chrono::Utc::now();
}
if let Some(kx) = keep_alive {
m.keep_alive = kx;
}
}
});
}
}
pub(super) async fn begin_activity(
mut guard: ActivityGuard,
keep_alive: Option<Option<Duration>>,
) -> ActivityGuard {
{
let mut mgr = guard.state.0.manager.lock().await;
if let Some(m) = mgr.running.get_mut(&guard.model_key) {
m.last_active = Instant::now();
m.last_active_wall = chrono::Utc::now();
if let Some(kx) = keep_alive {
m.keep_alive = kx;
}
}
}
guard.keep_alive = keep_alive;
guard
}
pub(super) async fn refresh_activity(guard: ActivityGuard, keep_alive: Option<Duration>) {
let mut mgr = guard.state.0.manager.lock().await;
if let Some(m) = mgr.running.get_mut(&guard.model_key) {
m.last_active = Instant::now();
m.last_active_wall = chrono::Utc::now();
m.keep_alive = keep_alive;
}
}
pub(super) const IDLE_CHECK_INTERVAL: Duration = Duration::from_secs(15);
pub(super) async fn reap_idle_models(state: AppState) {
let mut ticker = tokio::time::interval(IDLE_CHECK_INTERVAL);
ticker.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay);
loop {
ticker.tick().await;
reap_idle_models_once(&state).await;
}
}
pub(super) async fn reap_idle_models_once(state: &AppState) {
let mut mgr = state.0.manager.lock().await;
let expired: Vec<String> = mgr
.running
.iter()
.filter(|(_, m)| m.in_flight == 0)
.filter_map(|(name, m)| {
let deadline = m.keep_alive?;
(m.last_active.elapsed() >= deadline).then(|| name.clone())
})
.collect();
for name in expired {
if let Some(mut running) = mgr.running.remove(&name) {
if running.process.is_alive() {
eprintln!("[llmman] unloading {name}: idle past its keep_alive deadline");
metrics::record_model_unload(&name, UnloadReason::Idle);
} else {
eprintln!("[llmman] unloading {name}: its backend had already exited");
metrics::record_model_unload(&name, UnloadReason::Crashed);
}
}
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn parse_keep_alive_str_handles_bare_seconds_units_and_negatives() {
assert_eq!(
parse_keep_alive_str("300"),
Some(Some(Duration::from_secs(300)))
);
assert_eq!(
parse_keep_alive_str("10m"),
Some(Some(Duration::from_secs(600)))
);
assert_eq!(
parse_keep_alive_str("1h30m"),
Some(Some(Duration::from_secs(5400)))
);
assert_eq!(
parse_keep_alive_str("30s"),
Some(Some(Duration::from_secs(30)))
);
assert_eq!(
parse_keep_alive_str("500ms"),
Some(Some(Duration::from_millis(500)))
);
assert_eq!(parse_keep_alive_str("-1"), Some(None));
assert_eq!(parse_keep_alive_str("-5m"), Some(None));
assert_eq!(parse_keep_alive_str("not-a-duration"), None);
assert_eq!(parse_keep_alive_str(""), None);
assert_eq!(parse_keep_alive_str("10x"), None);
}
#[test]
fn parse_keep_alive_str_never_panics_on_non_finite_or_overflowing_input() {
assert_eq!(parse_keep_alive_str("inf"), None);
assert_eq!(parse_keep_alive_str("Infinity"), None);
assert_eq!(parse_keep_alive_str("nan"), None);
assert_eq!(parse_keep_alive_str("NaN"), None);
assert_eq!(parse_keep_alive_str("-inf"), Some(None));
assert_eq!(parse_keep_alive_str("1e300"), None);
assert_eq!(parse_keep_alive_str("1e300s"), None);
assert_eq!(
parse_keep_alive_str(&format!("{}s{}s", u64::MAX, u64::MAX)),
None
);
}
#[test]
fn resolve_keep_alive_never_panics_on_an_overflowing_json_number() {
assert_eq!(
resolve_keep_alive(&Some(serde_json::json!(1e300))),
default_keep_alive()
);
}
#[test]
fn resolve_keep_alive_falls_back_to_the_default_on_absent_or_unparseable_values() {
let default = default_keep_alive();
assert_eq!(resolve_keep_alive(&None), default);
assert_eq!(
resolve_keep_alive(&Some(serde_json::json!("garbage"))),
default
);
assert_eq!(resolve_keep_alive(&Some(serde_json::json!(true))), default);
}
#[test]
fn resolve_keep_alive_accepts_a_json_number_of_seconds_or_a_duration_string() {
assert_eq!(
resolve_keep_alive(&Some(serde_json::json!(30))),
Some(Duration::from_secs(30))
);
assert_eq!(
resolve_keep_alive(&Some(serde_json::json!("10m"))),
Some(Duration::from_secs(600))
);
assert_eq!(resolve_keep_alive(&Some(serde_json::json!(-1))), None);
}
#[test]
fn only_a_keep_alive_the_request_actually_carries_means_unload() {
assert!(
!is_explicit_unload(&None),
"an absent field is not an unload"
);
assert!(is_explicit_unload(&Some(serde_json::json!(0))));
assert!(is_explicit_unload(&Some(serde_json::json!("0"))));
assert!(is_explicit_unload(&Some(serde_json::json!("0s"))));
assert!(!is_explicit_unload(&Some(serde_json::json!(300))));
assert!(!is_explicit_unload(&Some(serde_json::json!(-1))));
assert!(
!is_explicit_unload(&Some(serde_json::json!("garbage"))),
"an unparseable value leaves the daemon default deciding how long \
to keep the model, not whether to keep it"
);
}
#[test]
fn resolve_keep_alive_treats_every_zero_form_as_the_unload_sentinel() {
assert_eq!(
resolve_keep_alive(&Some(serde_json::json!(0))),
Some(Duration::ZERO)
);
assert_eq!(
resolve_keep_alive(&Some(serde_json::json!("0"))),
Some(Duration::ZERO)
);
assert_eq!(
resolve_keep_alive(&Some(serde_json::json!("0s"))),
Some(Duration::ZERO)
);
assert_eq!(
resolve_keep_alive(&Some(serde_json::json!(0.0))),
Some(Duration::ZERO)
);
}
}