use serde::{Deserialize, Serialize};
pub type Millis = u64;
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum LostReason {
Evicted,
GpuDetached,
ProcessExited,
Unhealthy,
}
impl LostReason {
pub fn is_operator_fault(self) -> bool {
!matches!(self, LostReason::GpuDetached)
}
pub fn worth_retrying_here(self) -> bool {
matches!(self, LostReason::GpuDetached | LostReason::ProcessExited)
}
}
#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
#[serde(tag = "state", rename_all = "snake_case")]
pub enum Residency {
#[default]
Unknown,
Loading { since: Millis },
Resident { since: Millis, vram_bytes: u64 },
Lost { at: Millis, reason: LostReason },
Failed { at: Millis, reason: String },
Stopped { at: Millis },
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum Observation {
LoadStarted,
ProbeOk { vram_bytes: u64 },
ProbeFailed { gpu_present: bool, detail: String },
ProcessExited { code: Option<i32> },
LoadFailed { detail: String },
StopRequested,
}
pub fn observation_from_json(kind: &str, detail: &serde_json::Value) -> Option<Observation> {
fn field<'a>(
d: &'a serde_json::Value,
camel: &str,
snake: &str,
) -> Option<&'a serde_json::Value> {
d.get(camel).or_else(|| d.get(snake))
}
fn detail_str(d: &serde_json::Value, fallback: &str) -> String {
d.get("detail")
.and_then(|v| v.as_str())
.unwrap_or(fallback)
.to_string()
}
Some(match kind {
"load_started" | "loadStarted" => Observation::LoadStarted,
"probe_ok" | "probeOk" => Observation::ProbeOk {
vram_bytes: field(detail, "vramBytes", "vram_bytes")
.and_then(crate::budget::whole_bytes)
.unwrap_or(0),
},
"probe_failed" | "probeFailed" => Observation::ProbeFailed {
gpu_present: field(detail, "gpuPresent", "gpu_present")
.and_then(|v| v.as_bool())
.unwrap_or(true),
detail: detail_str(detail, ""),
},
"process_exited" | "processExited" => Observation::ProcessExited {
code: detail
.get("code")
.and_then(|v| v.as_i64())
.map(|c| c as i32),
},
"load_failed" | "loadFailed" => Observation::LoadFailed {
detail: detail_str(detail, "unknown"),
},
"stop" | "stop_requested" | "stopRequested" => Observation::StopRequested,
_ => return None,
})
}
impl Residency {
pub fn is_ready(&self) -> bool {
matches!(self, Residency::Resident { .. })
}
pub fn is_coming(&self) -> bool {
matches!(self, Residency::Loading { .. })
}
pub fn loading_for(&self, now: Millis) -> Option<Millis> {
match self {
Residency::Loading { since } => Some(now.saturating_sub(*since)),
_ => None,
}
}
pub fn vram_bytes(&self) -> u64 {
match self {
Residency::Resident { vram_bytes, .. } => *vram_bytes,
_ => 0,
}
}
pub fn explain(&self, now: Millis) -> String {
match self {
Residency::Unknown => "never probed".into(),
Residency::Loading { .. } => {
let secs = self.loading_for(now).unwrap_or(0) / 1000;
format!("loading for {secs}s")
}
Residency::Resident { since, .. } => {
format!("resident for {}s", now.saturating_sub(*since) / 1000)
}
Residency::Lost { reason, .. } => match reason {
LostReason::Evicted => "evicted by the runtime to free VRAM".into(),
LostReason::GpuDetached => "the GPU was detached by the host".into(),
LostReason::ProcessExited => "the serving process exited".into(),
LostReason::Unhealthy => "loaded but not answering".into(),
},
Residency::Failed { reason, .. } => format!("failed to load: {reason}"),
Residency::Stopped { .. } => "stopped on request".into(),
}
}
pub fn observe(&self, obs: &Observation, now: Millis) -> Residency {
match obs {
Observation::StopRequested => Residency::Stopped { at: now },
Observation::LoadStarted => match self {
Residency::Resident { .. } => self.clone(),
Residency::Loading { .. } => self.clone(),
_ => Residency::Loading { since: now },
},
Observation::ProbeOk { vram_bytes } => match self {
Residency::Resident { since, .. } => Residency::Resident {
since: *since,
vram_bytes: *vram_bytes,
},
_ => Residency::Resident {
since: now,
vram_bytes: *vram_bytes,
},
},
Observation::ProbeFailed {
gpu_present,
detail: _,
} => {
let reason = if !*gpu_present {
LostReason::GpuDetached
} else if self.is_ready() {
LostReason::Evicted
} else {
LostReason::Unhealthy
};
match self {
Residency::Loading { .. } if *gpu_present => self.clone(),
_ => Residency::Lost { at: now, reason },
}
}
Observation::ProcessExited { .. } => match self {
Residency::Stopped { .. } => self.clone(),
_ => Residency::Lost {
at: now,
reason: LostReason::ProcessExited,
},
},
Observation::LoadFailed { detail } => Residency::Failed {
at: now,
reason: detail.clone(),
},
}
}
}
#[cfg(test)]
mod observation_json_tests {
use super::*;
use serde_json::json;
#[test]
fn both_spellings_of_gpu_present_are_read() {
for detail in [json!({"gpu_present": false}), json!({"gpuPresent": false})] {
let obs = observation_from_json("probe_failed", &detail).expect("known kind");
assert_eq!(
obs,
Observation::ProbeFailed {
gpu_present: false,
detail: String::new()
},
"spelling must not change the verdict: {detail}"
);
}
}
#[test]
fn the_bug_this_function_exists_to_delete() {
let detail = json!({"gpuPresent": false, "detail": "no CUDA device"});
let obs = observation_from_json("probe_failed", &detail).unwrap();
let state = Residency::Unknown
.observe(&Observation::LoadStarted, 0)
.observe(&Observation::ProbeOk { vram_bytes: 1 }, 1_000)
.observe(&obs, 2_000);
match state {
Residency::Lost { reason, .. } => {
assert_eq!(reason, LostReason::GpuDetached);
assert!(
!reason.is_operator_fault(),
"the host took the card — this must never count against them"
);
}
other => panic!("expected a loss, got {other:?}"),
}
}
#[test]
fn a_missing_gpu_present_still_counts_against_us() {
let obs = observation_from_json("probe_failed", &json!({"detail": "?"})).unwrap();
assert_eq!(
obs,
Observation::ProbeFailed {
gpu_present: true,
detail: "?".into()
}
);
}
#[test]
fn an_unreadable_gpu_present_is_treated_as_missing_not_as_absence() {
let obs = observation_from_json("probe_failed", &json!({"gpu_present": "nope"})).unwrap();
assert_eq!(
obs,
Observation::ProbeFailed {
gpu_present: true,
detail: String::new()
}
);
}
#[test]
fn a_js_float_vram_is_not_silently_zero() {
let gib = 1024u64 * 1024 * 1024;
let as_float = json!({"vramBytes": (21 * gib) as f64});
assert_eq!(
observation_from_json("probe_ok", &as_float).unwrap(),
Observation::ProbeOk {
vram_bytes: 21 * gib
},
);
assert_eq!(
observation_from_json("probe_ok", &json!({"vram_bytes": 21 * gib})).unwrap(),
Observation::ProbeOk {
vram_bytes: 21 * gib
},
);
}
#[test]
fn a_fractional_vram_is_refused_rather_than_truncated() {
assert_eq!(
observation_from_json("probe_ok", &json!({"vramBytes": 1.5})).unwrap(),
Observation::ProbeOk { vram_bytes: 0 },
);
}
#[test]
fn every_kind_round_trips() {
assert_eq!(
observation_from_json("load_started", &json!({})),
Some(Observation::LoadStarted)
);
assert_eq!(
observation_from_json("process_exited", &json!({"code": 1})),
Some(Observation::ProcessExited { code: Some(1) })
);
assert_eq!(
observation_from_json("process_exited", &json!({})),
Some(Observation::ProcessExited { code: None })
);
assert_eq!(
observation_from_json("load_failed", &json!({"detail": "no such file"})),
Some(Observation::LoadFailed {
detail: "no such file".into()
})
);
assert_eq!(
observation_from_json("load_failed", &json!({})),
Some(Observation::LoadFailed {
detail: "unknown".into()
})
);
assert_eq!(
observation_from_json("stop", &json!({})),
Some(Observation::StopRequested)
);
}
#[test]
fn camelcase_kinds_work_too_because_one_language_writes_them_that_way() {
assert_eq!(
observation_from_json("loadStarted", &json!({})),
Some(Observation::LoadStarted)
);
assert_eq!(
observation_from_json("probeOk", &json!({"vramBytes": 8})),
Some(Observation::ProbeOk { vram_bytes: 8 })
);
}
#[test]
fn an_unknown_kind_is_none_so_a_caller_can_say_so_out_loud() {
assert_eq!(observation_from_json("probe_okay", &json!({})), None);
assert_eq!(observation_from_json("", &json!({})), None);
}
#[test]
fn a_non_object_detail_does_not_panic() {
assert_eq!(
observation_from_json("probe_failed", &json!(null)).unwrap(),
Observation::ProbeFailed {
gpu_present: true,
detail: String::new()
}
);
}
}