1pub mod probe;
21pub mod server;
22pub mod warmup;
23
24use std::collections::HashMap;
25use std::path::PathBuf;
26use std::time::{Duration, SystemTime, UNIX_EPOCH};
27
28use hearth_core::{Budget, Declared, Fleet, LostReason, Millis, Observation, Residency};
29use hearth_store::{EventRef, Spine, Transition};
30use probe::ProbeResult;
31use server::{ServerChild, ServerSpec};
32
33pub fn now_ms() -> Millis {
34 SystemTime::now()
35 .duration_since(UNIX_EPOCH)
36 .map(|d| d.as_millis() as u64)
37 .unwrap_or(0)
38}
39
40struct Managed {
43 spec: ServerSpec,
44 child: Option<ServerChild>,
45 last_event: Option<EventRef>,
46 recorded: &'static str,
49 started_at: Millis,
50}
51
52pub struct Supervisor {
54 fleet: Fleet,
55 spine: Spine,
56 managed: HashMap<String, Managed>,
57 probe_timeout: Duration,
58}
59
60impl Supervisor {
61 pub fn new(spine: Spine, budget: Budget, declared: Vec<Declared>) -> Supervisor {
66 let fleet = Fleet::declare(budget, declared);
67 let sup = Supervisor {
68 fleet,
69 spine,
70 managed: HashMap::new(),
71 probe_timeout: Duration::from_millis(750),
72 };
73 for slot in sup.fleet.slots() {
74 let _ = sup.spine.record(
75 &slot.declared.model,
76 &Transition::Declared {
77 vram_bytes: slot.declared.total_bytes(),
78 admitted: slot.admitted,
79 },
80 &[],
81 );
82 }
83 sup
84 }
85
86 pub fn fleet(&self) -> &Fleet {
87 &self.fleet
88 }
89
90 pub fn spine(&self) -> &Spine {
91 &self.spine
92 }
93
94 pub fn start(&mut self, spec: ServerSpec) -> Result<(), String> {
97 let model = spec.model.clone();
98 let slot = self
99 .fleet
100 .slot(&model)
101 .ok_or_else(|| format!("{model} was never declared"))?;
102 if !slot.admitted {
103 return Err(format!(
104 "{model} was refused by the VRAM budget at declaration; refusing to start it"
105 ));
106 }
107 let endpoint = spec.endpoint();
108 let child = ServerChild::spawn(spec.clone())?;
109 let pid = child.pid();
110 let now = now_ms();
111
112 self.fleet.set_endpoint(&model, endpoint.clone());
113 self.fleet.observe(&model, &Observation::LoadStarted, now);
114
115 let declared_ref = self
116 .spine
117 .latest(&model)
118 .map(|e| EventRef {
119 hash: e.hash,
120 seq: e.seq,
121 })
122 .into_iter()
123 .collect::<Vec<_>>();
124 let ev = self.spine.record(
125 &model,
126 &Transition::Loading { pid, endpoint },
127 &declared_ref,
128 )?;
129
130 self.managed.insert(
131 model,
132 Managed {
133 spec,
134 child: Some(child),
135 last_event: Some(ev),
136 recorded: "loading",
137 started_at: now,
138 },
139 );
140 Ok(())
141 }
142
143 pub fn tick(&mut self) -> usize {
148 let now = now_ms();
149 let mut transitions = 0;
150 let models: Vec<String> = self.managed.keys().cloned().collect();
151
152 for model in models {
153 let (observation, endpoint) = {
154 let m = self.managed.get_mut(&model).expect("just listed");
155 let endpoint = m.spec.endpoint();
156 let obs = match m.child.as_mut() {
157 None => continue, Some(child) => match child.exit_code() {
159 Some(code) => Observation::ProcessExited { code },
160 None => {
161 match probe::probe_http(&endpoint, "/health", self.probe_timeout) {
162 ProbeResult::Ok => Observation::ProbeOk {
163 vram_bytes: self
164 .fleet
165 .slot(&model)
166 .map(|s| s.declared.total_bytes())
167 .unwrap_or(0),
168 },
169 ProbeResult::Warming { .. } => continue,
171 ProbeResult::Unanswered { detail }
172 | ProbeResult::Unreachable { detail } => {
173 let gpu = probe::gpu_present().unwrap_or(true);
175 Observation::ProbeFailed {
176 gpu_present: gpu,
177 detail,
178 }
179 }
180 }
181 }
182 },
183 };
184 (obs, endpoint)
185 };
186
187 self.fleet.observe(&model, &observation, now);
188 let state = self
189 .fleet
190 .slot(&model)
191 .map(|s| s.state.clone())
192 .unwrap_or_default();
193
194 let m = self.managed.get_mut(&model).expect("just listed");
195 let transition = match &state {
196 Residency::Resident { .. } if m.recorded != "resident" => {
197 Some(Transition::Resident {
198 endpoint,
199 warmup_ms: now.saturating_sub(m.started_at),
200 })
201 }
202 Residency::Lost { reason, .. } if m.recorded != "lost" => {
203 Some(Transition::Lost { reason: *reason })
204 }
205 Residency::Failed { .. } if m.recorded != "lost" => Some(Transition::Lost {
206 reason: LostReason::ProcessExited,
207 }),
208 _ => None,
209 };
210
211 if let Some(t) = transition {
212 let causes: Vec<EventRef> = m.last_event.clone().into_iter().collect();
213 if let Ok(ev) = self.spine.record(&model, &t, &causes) {
214 m.last_event = Some(ev);
215 m.recorded = match t {
216 Transition::Resident { .. } => "resident",
217 _ => "lost",
218 };
219 transitions += 1;
220 }
221 if matches!(t, Transition::Lost { .. }) {
223 if let Some(mut c) = m.child.take() {
224 c.stop();
225 }
226 }
227 }
228 }
229 transitions
230 }
231
232 pub fn wait_ready(&mut self, model: &str, deadline: Duration) -> Result<String, String> {
235 let start = std::time::Instant::now();
236 loop {
237 self.tick();
238 match self.fleet.slot(model).map(|s| s.state.clone()) {
239 Some(Residency::Resident { .. }) => {
240 let endpoint = self
241 .managed
242 .get(model)
243 .map(|m| m.spec.endpoint())
244 .unwrap_or_default();
245 return Ok(endpoint);
246 }
247 Some(Residency::Lost { reason, .. }) => {
248 return Err(format!("{model} lost while warming: {reason:?}"));
249 }
250 Some(Residency::Failed { reason, .. }) => {
251 return Err(format!("{model} failed to load: {reason}"));
252 }
253 _ => {}
254 }
255 if start.elapsed() > deadline {
256 return Err(format!(
257 "{model} not ready after {:?} — state: {}",
258 deadline,
259 self.fleet
260 .slot(model)
261 .map(|s| s.state.explain(now_ms()))
262 .unwrap_or_else(|| "undeclared".into())
263 ));
264 }
265 std::thread::sleep(Duration::from_millis(250));
266 }
267 }
268
269 pub fn stop(&mut self, model: &str) -> Result<(), String> {
271 let m = self
272 .managed
273 .get_mut(model)
274 .ok_or_else(|| format!("{model} is not managed"))?;
275 if let Some(mut child) = m.child.take() {
276 child.stop();
277 }
278 self.fleet
279 .observe(model, &Observation::StopRequested, now_ms());
280 let causes: Vec<EventRef> = m.last_event.clone().into_iter().collect();
281 let ev = self.spine.record(model, &Transition::Unloaded, &causes)?;
282 m.last_event = Some(ev);
283 m.recorded = "unloaded";
284 Ok(())
285 }
286
287 pub fn report(&self) -> String {
289 self.fleet.report(now_ms())
290 }
291
292 pub fn stop_all(&mut self) {
295 let models: Vec<String> = self.managed.keys().cloned().collect();
296 for m in models {
297 let _ = self.stop(&m);
298 }
299 self.spine.flush();
300 }
301}
302
303pub fn hearth_home() -> PathBuf {
306 if let Ok(h) = std::env::var("HEARTH_HOME") {
307 return PathBuf::from(h);
308 }
309 let home = std::env::var("HOME").unwrap_or_else(|_| ".".into());
310 PathBuf::from(home).join(".hearth")
311}
312
313#[cfg(test)]
314mod tests {
315 use super::*;
316
317 fn budget() -> Budget {
318 Budget {
319 total_bytes: 24 * hearth_core::GIB,
320 reserve_bytes: 2 * hearth_core::GIB,
321 }
322 }
323
324 fn declared(model: &str) -> Declared {
325 Declared {
326 model: model.into(),
327 weights_bytes: 4 * hearth_core::GIB,
328 kv_bytes: hearth_core::GIB,
329 }
330 }
331
332 #[test]
333 fn declarations_are_recorded_before_anything_runs() {
334 let sup = Supervisor::new(Spine::in_memory(), budget(), vec![declared("muse")]);
335 let ev = sup.spine().latest("muse").unwrap();
336 assert_eq!(ev.kind, "declared");
337 assert_eq!(ev.facts["admitted"], serde_json::json!(true));
338 }
339
340 #[test]
341 fn budget_refusal_is_recorded_and_start_is_refused() {
342 let mut sup = Supervisor::new(
343 Spine::in_memory(),
344 budget(),
345 vec![Declared {
346 model: "whale".into(),
347 weights_bytes: 400 * hearth_core::GIB,
348 kv_bytes: 0,
349 }],
350 );
351 assert_eq!(
352 sup.spine().latest("whale").unwrap().facts["admitted"],
353 serde_json::json!(false)
354 );
355 let err = sup
356 .start(ServerSpec::new("whale", "/nope.gguf", 1))
357 .unwrap_err();
358 assert!(err.contains("refused by the VRAM budget"), "{err}");
359 }
360
361 #[test]
362 fn starting_an_undeclared_model_is_an_error() {
363 let mut sup = Supervisor::new(Spine::in_memory(), budget(), vec![]);
364 let err = sup
365 .start(ServerSpec::new("ghost", "/nope.gguf", 1))
366 .unwrap_err();
367 assert!(err.contains("never declared"), "{err}");
368 }
369
370 #[test]
371 fn a_dying_child_becomes_a_recorded_loss_with_a_causal_chain() {
372 let dir = tempfile::tempdir().unwrap();
373 let gguf = dir.path().join("fake.gguf");
374 std::fs::write(&gguf, b"bytes").unwrap();
375
376 let mut sup = Supervisor::new(Spine::in_memory(), budget(), vec![declared("fake")]);
377 let mut spec = ServerSpec::new("fake", &gguf, server::free_port().unwrap());
379 spec.binary = PathBuf::from("/bin/sh");
380 spec.extra_args = vec!["-c".into(), "exit 1".into()];
381 spec.log_dir = dir.path().to_path_buf();
382 sup.start(spec).unwrap();
383
384 let mut recorded = false;
386 for _ in 0..40 {
387 if sup.tick() > 0 {
388 recorded = true;
389 break;
390 }
391 std::thread::sleep(Duration::from_millis(25));
392 }
393 assert!(recorded, "loss never recorded");
394
395 let latest = sup.spine().latest("fake").unwrap();
396 assert_eq!(latest.kind, "lost");
397 let kinds: Vec<String> = sup
399 .spine()
400 .why("fake")
401 .into_iter()
402 .map(|e| e.kind)
403 .collect();
404 assert_eq!(kinds, vec!["lost", "loading", "declared"]);
405 }
406
407 #[test]
408 fn deliberate_stop_is_unloaded_not_lost() {
409 let dir = tempfile::tempdir().unwrap();
410 let gguf = dir.path().join("fake.gguf");
411 std::fs::write(&gguf, b"bytes").unwrap();
412
413 let mut sup = Supervisor::new(Spine::in_memory(), budget(), vec![declared("fake")]);
414 let mut spec = ServerSpec::new("fake", &gguf, server::free_port().unwrap());
415 spec.binary = PathBuf::from("/bin/sh");
416 spec.extra_args = vec!["-c".into(), "sleep 30".into()];
417 spec.log_dir = dir.path().to_path_buf();
418 sup.start(spec).unwrap();
419 sup.stop("fake").unwrap();
420
421 assert_eq!(sup.spine().latest("fake").unwrap().kind, "unloaded");
422 }
423
424 #[test]
425 fn spine_survives_what_the_supervisor_forgets() {
426 let dir = tempfile::tempdir().unwrap();
429 let spine_dir = dir.path().join("spine");
430 {
431 let mut sup = Supervisor::new(
432 Spine::open(&spine_dir).unwrap(),
433 budget(),
434 vec![declared("muse")],
435 );
436 let gguf = dir.path().join("fake.gguf");
437 std::fs::write(&gguf, b"bytes").unwrap();
438 let mut spec = ServerSpec::new("muse", &gguf, server::free_port().unwrap());
439 spec.binary = PathBuf::from("/bin/sh");
440 spec.extra_args = vec!["-c".into(), "exit 1".into()];
441 spec.log_dir = dir.path().to_path_buf();
442 sup.start(spec).unwrap();
443 for _ in 0..40 {
444 if sup.tick() > 0 {
445 break;
446 }
447 std::thread::sleep(Duration::from_millis(25));
448 }
449 sup.stop_all();
450 }
451 let spine = Spine::open(&spine_dir).unwrap();
452 let kinds: Vec<String> = spine.why("muse").into_iter().map(|e| e.kind).collect();
453 assert!(
454 kinds.starts_with(&["lost".to_string(), "loading".to_string()])
455 || kinds.starts_with(&["unloaded".to_string()]),
456 "unexpected chain: {kinds:?}"
457 );
458 let (checked, failures) = spine.verify();
459 assert!(checked >= 3);
460 assert!(failures.is_empty(), "{failures:?}");
461 }
462}