use serde::{Deserialize, Serialize};
pub type Millis = u64;
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum LostReason {
Evicted,
GpuDetached,
ProcessExited,
Unhealthy,
}
impl LostReason {
pub fn is_operator_fault(self) -> bool {
!matches!(self, LostReason::GpuDetached)
}
pub fn worth_retrying_here(self) -> bool {
matches!(self, LostReason::GpuDetached | LostReason::ProcessExited)
}
}
#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
#[serde(tag = "state", rename_all = "snake_case")]
pub enum Residency {
#[default]
Unknown,
Loading { since: Millis },
Resident { since: Millis, vram_bytes: u64 },
Lost { at: Millis, reason: LostReason },
Failed { at: Millis, reason: String },
Stopped { at: Millis },
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum Observation {
LoadStarted,
ProbeOk { vram_bytes: u64 },
ProbeFailed { gpu_present: bool, detail: String },
ProcessExited { code: Option<i32> },
LoadFailed { detail: String },
StopRequested,
}
impl Residency {
pub fn is_ready(&self) -> bool {
matches!(self, Residency::Resident { .. })
}
pub fn is_coming(&self) -> bool {
matches!(self, Residency::Loading { .. })
}
pub fn loading_for(&self, now: Millis) -> Option<Millis> {
match self {
Residency::Loading { since } => Some(now.saturating_sub(*since)),
_ => None,
}
}
pub fn vram_bytes(&self) -> u64 {
match self {
Residency::Resident { vram_bytes, .. } => *vram_bytes,
_ => 0,
}
}
pub fn explain(&self, now: Millis) -> String {
match self {
Residency::Unknown => "never probed".into(),
Residency::Loading { .. } => {
let secs = self.loading_for(now).unwrap_or(0) / 1000;
format!("loading for {secs}s")
}
Residency::Resident { since, .. } => {
format!("resident for {}s", now.saturating_sub(*since) / 1000)
}
Residency::Lost { reason, .. } => match reason {
LostReason::Evicted => "evicted by the runtime to free VRAM".into(),
LostReason::GpuDetached => "the GPU was detached by the host".into(),
LostReason::ProcessExited => "the serving process exited".into(),
LostReason::Unhealthy => "loaded but not answering".into(),
},
Residency::Failed { reason, .. } => format!("failed to load: {reason}"),
Residency::Stopped { .. } => "stopped on request".into(),
}
}
pub fn observe(&self, obs: &Observation, now: Millis) -> Residency {
match obs {
Observation::StopRequested => Residency::Stopped { at: now },
Observation::LoadStarted => match self {
Residency::Resident { .. } => self.clone(),
Residency::Loading { .. } => self.clone(),
_ => Residency::Loading { since: now },
},
Observation::ProbeOk { vram_bytes } => match self {
Residency::Resident { since, .. } => Residency::Resident {
since: *since,
vram_bytes: *vram_bytes,
},
_ => Residency::Resident {
since: now,
vram_bytes: *vram_bytes,
},
},
Observation::ProbeFailed {
gpu_present,
detail: _,
} => {
let reason = if !*gpu_present {
LostReason::GpuDetached
} else if self.is_ready() {
LostReason::Evicted
} else {
LostReason::Unhealthy
};
match self {
Residency::Loading { .. } if *gpu_present => self.clone(),
_ => Residency::Lost { at: now, reason },
}
}
Observation::ProcessExited { .. } => match self {
Residency::Stopped { .. } => self.clone(),
_ => Residency::Lost {
at: now,
reason: LostReason::ProcessExited,
},
},
Observation::LoadFailed { detail } => Residency::Failed {
at: now,
reason: detail.clone(),
},
}
}
}