Skip to main content

hearth_serve/
lib.rs

1//! hearth-serve — the supervisor.
2//!
3//! Three layers, three jobs, wired here:
4//!
5//!   * `hearth_core::Fleet`  — the decision procedure. What state is each
6//!     model in, what should we do next, is there budget for it.
7//!   * `hearth_store::Spine` — the system of record. Every transition the
8//!     fleet decides is written as a causal, bi-temporal, verifiable event
9//!     BEFORE anything else happens. NEDB is not a log on the side: the
10//!     supervisor's memory IS the database.
11//!   * `server` / `probe`    — the mechanism. llama-server children and
12//!     the health probes that produce facts about them.
13//!
14//! The loop is `tick()`: probe every child, hand each observation to the
15//! fleet, and when the fleet's state changes, record the transition in the
16//! spine with a causal link to the event that preceded it. Restart-safe by
17//! construction — on boot the spine replays the last known state, and
18//! `hearth why <model>` answers from the chain, not from memory.
19
20pub mod probe;
21pub mod server;
22pub mod warmup;
23
24use std::collections::HashMap;
25use std::path::PathBuf;
26use std::time::{Duration, SystemTime, UNIX_EPOCH};
27
28use hearth_core::{Budget, Declared, Fleet, LostReason, Millis, Observation, Residency};
29use hearth_store::{EventRef, Spine, Transition};
30use probe::ProbeResult;
31use server::{ServerChild, ServerSpec};
32
33pub fn now_ms() -> Millis {
34    SystemTime::now()
35        .duration_since(UNIX_EPOCH)
36        .map(|d| d.as_millis() as u64)
37        .unwrap_or(0)
38}
39
40/// One model under supervision: its child, its last recorded event (the
41/// causal parent of the next one), and what we last told the spine.
42struct Managed {
43    spec: ServerSpec,
44    child: Option<ServerChild>,
45    last_event: Option<EventRef>,
46    /// The kind we last recorded, so transitions are recorded once, on
47    /// change, not once per probe.
48    recorded: &'static str,
49    started_at: Millis,
50}
51
52/// The supervisor: fleet (decisions) + spine (record) + children (mechanism).
53pub struct Supervisor {
54    fleet: Fleet,
55    spine: Spine,
56    managed: HashMap<String, Managed>,
57    probe_timeout: Duration,
58}
59
60impl Supervisor {
61    /// Declare the fleet and open the spine. Every declaration — admitted
62    /// or refused — is recorded before any process starts, because the
63    /// budget's refusals are exactly the history you need when someone asks
64    /// why a model never came up.
65    pub fn new(spine: Spine, budget: Budget, declared: Vec<Declared>) -> Supervisor {
66        let fleet = Fleet::declare(budget, declared);
67        let sup = Supervisor {
68            fleet,
69            spine,
70            managed: HashMap::new(),
71            probe_timeout: Duration::from_millis(750),
72        };
73        for slot in sup.fleet.slots() {
74            let _ = sup.spine.record(
75                &slot.declared.model,
76                &Transition::Declared {
77                    vram_bytes: slot.declared.total_bytes(),
78                    admitted: slot.admitted,
79                },
80                &[],
81            );
82        }
83        sup
84    }
85
86    pub fn fleet(&self) -> &Fleet {
87        &self.fleet
88    }
89
90    pub fn spine(&self) -> &Spine {
91        &self.spine
92    }
93
94    /// Start serving a model. Spawns the child, tells the fleet, records
95    /// `loading` in the spine — causally linked to the model's declaration.
96    pub fn start(&mut self, spec: ServerSpec) -> Result<(), String> {
97        let model = spec.model.clone();
98        let slot = self
99            .fleet
100            .slot(&model)
101            .ok_or_else(|| format!("{model} was never declared"))?;
102        if !slot.admitted {
103            return Err(format!(
104                "{model} was refused by the VRAM budget at declaration; refusing to start it"
105            ));
106        }
107        let endpoint = spec.endpoint();
108        let child = ServerChild::spawn(spec.clone())?;
109        let pid = child.pid();
110        let now = now_ms();
111
112        self.fleet.set_endpoint(&model, endpoint.clone());
113        self.fleet.observe(&model, &Observation::LoadStarted, now);
114
115        let declared_ref = self
116            .spine
117            .latest(&model)
118            .map(|e| EventRef {
119                hash: e.hash,
120                seq: e.seq,
121            })
122            .into_iter()
123            .collect::<Vec<_>>();
124        let ev = self.spine.record(
125            &model,
126            &Transition::Loading { pid, endpoint },
127            &declared_ref,
128        )?;
129
130        self.managed.insert(
131            model,
132            Managed {
133                spec,
134                child: Some(child),
135                last_event: Some(ev),
136                recorded: "loading",
137                started_at: now,
138            },
139        );
140        Ok(())
141    }
142
143    /// One supervision pass: probe every child, update the fleet, record
144    /// every state CHANGE in the spine. Returns the number of transitions
145    /// recorded, so a caller can log honestly ("2 transitions") instead of
146    /// narrating every uneventful tick.
147    pub fn tick(&mut self) -> usize {
148        let now = now_ms();
149        let mut transitions = 0;
150        let models: Vec<String> = self.managed.keys().cloned().collect();
151
152        for model in models {
153            let (observation, endpoint) = {
154                let m = self.managed.get_mut(&model).expect("just listed");
155                let endpoint = m.spec.endpoint();
156                let obs = match m.child.as_mut() {
157                    None => continue, // stopped deliberately; nothing to watch
158                    Some(child) => match child.exit_code() {
159                        Some(code) => Observation::ProcessExited { code },
160                        None => {
161                            match probe::probe_http(&endpoint, "/health", self.probe_timeout) {
162                                ProbeResult::Ok => Observation::ProbeOk {
163                                    vram_bytes: self
164                                        .fleet
165                                        .slot(&model)
166                                        .map(|s| s.declared.total_bytes())
167                                        .unwrap_or(0),
168                                },
169                                // Warming is not a failure; keep waiting.
170                                ProbeResult::Warming { .. } => continue,
171                                ProbeResult::Unanswered { detail }
172                                | ProbeResult::Unreachable { detail } => {
173                                    // The most valuable bit: is the card still there?
174                                    let gpu = probe::gpu_present().unwrap_or(true);
175                                    Observation::ProbeFailed {
176                                        gpu_present: gpu,
177                                        detail,
178                                    }
179                                }
180                            }
181                        }
182                    },
183                };
184                (obs, endpoint)
185            };
186
187            self.fleet.observe(&model, &observation, now);
188            let state = self
189                .fleet
190                .slot(&model)
191                .map(|s| s.state.clone())
192                .unwrap_or_default();
193
194            let m = self.managed.get_mut(&model).expect("just listed");
195            let transition = match &state {
196                Residency::Resident { .. } if m.recorded != "resident" => {
197                    Some(Transition::Resident {
198                        endpoint,
199                        warmup_ms: now.saturating_sub(m.started_at),
200                    })
201                }
202                Residency::Lost { reason, .. } if m.recorded != "lost" => {
203                    Some(Transition::Lost { reason: *reason })
204                }
205                Residency::Failed { .. } if m.recorded != "lost" => Some(Transition::Lost {
206                    reason: LostReason::ProcessExited,
207                }),
208                _ => None,
209            };
210
211            if let Some(t) = transition {
212                let causes: Vec<EventRef> = m.last_event.clone().into_iter().collect();
213                if let Ok(ev) = self.spine.record(&model, &t, &causes) {
214                    m.last_event = Some(ev);
215                    m.recorded = match t {
216                        Transition::Resident { .. } => "resident",
217                        _ => "lost",
218                    };
219                    transitions += 1;
220                }
221                // A lost child holds no port worth keeping.
222                if matches!(t, Transition::Lost { .. }) {
223                    if let Some(mut c) = m.child.take() {
224                        c.stop();
225                    }
226                }
227            }
228        }
229        transitions
230    }
231
232    /// Block until `model` is ready or the deadline passes. Ticks while it
233    /// waits, so every intermediate transition still lands in the spine.
234    pub fn wait_ready(&mut self, model: &str, deadline: Duration) -> Result<String, String> {
235        let start = std::time::Instant::now();
236        loop {
237            self.tick();
238            match self.fleet.slot(model).map(|s| s.state.clone()) {
239                Some(Residency::Resident { .. }) => {
240                    let endpoint = self
241                        .managed
242                        .get(model)
243                        .map(|m| m.spec.endpoint())
244                        .unwrap_or_default();
245                    return Ok(endpoint);
246                }
247                Some(Residency::Lost { reason, .. }) => {
248                    return Err(format!("{model} lost while warming: {reason:?}"));
249                }
250                Some(Residency::Failed { reason, .. }) => {
251                    return Err(format!("{model} failed to load: {reason}"));
252                }
253                _ => {}
254            }
255            if start.elapsed() > deadline {
256                return Err(format!(
257                    "{model} not ready after {:?} — state: {}",
258                    deadline,
259                    self.fleet
260                        .slot(model)
261                        .map(|s| s.state.explain(now_ms()))
262                        .unwrap_or_else(|| "undeclared".into())
263                ));
264            }
265            std::thread::sleep(Duration::from_millis(250));
266        }
267    }
268
269    /// Deliberate stop: kill the child, tell the fleet, record `unloaded`.
270    pub fn stop(&mut self, model: &str) -> Result<(), String> {
271        let m = self
272            .managed
273            .get_mut(model)
274            .ok_or_else(|| format!("{model} is not managed"))?;
275        if let Some(mut child) = m.child.take() {
276            child.stop();
277        }
278        self.fleet
279            .observe(model, &Observation::StopRequested, now_ms());
280        let causes: Vec<EventRef> = m.last_event.clone().into_iter().collect();
281        let ev = self.spine.record(model, &Transition::Unloaded, &causes)?;
282        m.last_event = Some(ev);
283        m.recorded = "unloaded";
284        Ok(())
285    }
286
287    /// The fleet's honest status report.
288    pub fn report(&self) -> String {
289        self.fleet.report(now_ms())
290    }
291
292    /// Stop everything. Called on shutdown; Drop on the children is the
293    /// backstop, this is the version that records it.
294    pub fn stop_all(&mut self) {
295        let models: Vec<String> = self.managed.keys().cloned().collect();
296        for m in models {
297            let _ = self.stop(&m);
298        }
299        self.spine.flush();
300    }
301}
302
303/// Where hearth keeps its state on this host: `$HEARTH_HOME` or
304/// `~/.hearth`. The spine lives at `<home>/spine`, blobs at `<home>/blobs`.
305pub fn hearth_home() -> PathBuf {
306    if let Ok(h) = std::env::var("HEARTH_HOME") {
307        return PathBuf::from(h);
308    }
309    let home = std::env::var("HOME").unwrap_or_else(|_| ".".into());
310    PathBuf::from(home).join(".hearth")
311}
312
313#[cfg(test)]
314mod tests {
315    use super::*;
316
317    fn budget() -> Budget {
318        Budget {
319            total_bytes: 24 * hearth_core::GIB,
320            reserve_bytes: 2 * hearth_core::GIB,
321        }
322    }
323
324    fn declared(model: &str) -> Declared {
325        Declared {
326            model: model.into(),
327            weights_bytes: 4 * hearth_core::GIB,
328            kv_bytes: hearth_core::GIB,
329        }
330    }
331
332    #[test]
333    fn declarations_are_recorded_before_anything_runs() {
334        let sup = Supervisor::new(Spine::in_memory(), budget(), vec![declared("muse")]);
335        let ev = sup.spine().latest("muse").unwrap();
336        assert_eq!(ev.kind, "declared");
337        assert_eq!(ev.facts["admitted"], serde_json::json!(true));
338    }
339
340    #[test]
341    fn budget_refusal_is_recorded_and_start_is_refused() {
342        let mut sup = Supervisor::new(
343            Spine::in_memory(),
344            budget(),
345            vec![Declared {
346                model: "whale".into(),
347                weights_bytes: 400 * hearth_core::GIB,
348                kv_bytes: 0,
349            }],
350        );
351        assert_eq!(
352            sup.spine().latest("whale").unwrap().facts["admitted"],
353            serde_json::json!(false)
354        );
355        let err = sup
356            .start(ServerSpec::new("whale", "/nope.gguf", 1))
357            .unwrap_err();
358        assert!(err.contains("refused by the VRAM budget"), "{err}");
359    }
360
361    #[test]
362    fn starting_an_undeclared_model_is_an_error() {
363        let mut sup = Supervisor::new(Spine::in_memory(), budget(), vec![]);
364        let err = sup
365            .start(ServerSpec::new("ghost", "/nope.gguf", 1))
366            .unwrap_err();
367        assert!(err.contains("never declared"), "{err}");
368    }
369
370    #[test]
371    fn a_dying_child_becomes_a_recorded_loss_with_a_causal_chain() {
372        let dir = tempfile::tempdir().unwrap();
373        let gguf = dir.path().join("fake.gguf");
374        std::fs::write(&gguf, b"bytes").unwrap();
375
376        let mut sup = Supervisor::new(Spine::in_memory(), budget(), vec![declared("fake")]);
377        // /bin/sh exits immediately — a runtime that dies on startup.
378        let mut spec = ServerSpec::new("fake", &gguf, server::free_port().unwrap());
379        spec.binary = PathBuf::from("/bin/sh");
380        spec.extra_args = vec!["-c".into(), "exit 1".into()];
381        spec.log_dir = dir.path().to_path_buf();
382        sup.start(spec).unwrap();
383
384        // Let it die, then tick until the loss is recorded (bounded).
385        let mut recorded = false;
386        for _ in 0..40 {
387            if sup.tick() > 0 {
388                recorded = true;
389                break;
390            }
391            std::thread::sleep(Duration::from_millis(25));
392        }
393        assert!(recorded, "loss never recorded");
394
395        let latest = sup.spine().latest("fake").unwrap();
396        assert_eq!(latest.kind, "lost");
397        // why() tells the whole story: lost <- loading <- declared.
398        let kinds: Vec<String> = sup
399            .spine()
400            .why("fake")
401            .into_iter()
402            .map(|e| e.kind)
403            .collect();
404        assert_eq!(kinds, vec!["lost", "loading", "declared"]);
405    }
406
407    #[test]
408    fn deliberate_stop_is_unloaded_not_lost() {
409        let dir = tempfile::tempdir().unwrap();
410        let gguf = dir.path().join("fake.gguf");
411        std::fs::write(&gguf, b"bytes").unwrap();
412
413        let mut sup = Supervisor::new(Spine::in_memory(), budget(), vec![declared("fake")]);
414        let mut spec = ServerSpec::new("fake", &gguf, server::free_port().unwrap());
415        spec.binary = PathBuf::from("/bin/sh");
416        spec.extra_args = vec!["-c".into(), "sleep 30".into()];
417        spec.log_dir = dir.path().to_path_buf();
418        sup.start(spec).unwrap();
419        sup.stop("fake").unwrap();
420
421        assert_eq!(sup.spine().latest("fake").unwrap().kind, "unloaded");
422    }
423
424    #[test]
425    fn spine_survives_what_the_supervisor_forgets() {
426        // The restart story: a fresh supervisor over the same spine dir
427        // still answers why(), because the history is on disk, not in RAM.
428        let dir = tempfile::tempdir().unwrap();
429        let spine_dir = dir.path().join("spine");
430        {
431            let mut sup = Supervisor::new(
432                Spine::open(&spine_dir).unwrap(),
433                budget(),
434                vec![declared("muse")],
435            );
436            let gguf = dir.path().join("fake.gguf");
437            std::fs::write(&gguf, b"bytes").unwrap();
438            let mut spec = ServerSpec::new("muse", &gguf, server::free_port().unwrap());
439            spec.binary = PathBuf::from("/bin/sh");
440            spec.extra_args = vec!["-c".into(), "exit 1".into()];
441            spec.log_dir = dir.path().to_path_buf();
442            sup.start(spec).unwrap();
443            for _ in 0..40 {
444                if sup.tick() > 0 {
445                    break;
446                }
447                std::thread::sleep(Duration::from_millis(25));
448            }
449            sup.stop_all();
450        }
451        let spine = Spine::open(&spine_dir).unwrap();
452        let kinds: Vec<String> = spine.why("muse").into_iter().map(|e| e.kind).collect();
453        assert!(
454            kinds.starts_with(&["lost".to_string(), "loading".to_string()])
455                || kinds.starts_with(&["unloaded".to_string()]),
456            "unexpected chain: {kinds:?}"
457        );
458        let (checked, failures) = spine.verify();
459        assert!(checked >= 3);
460        assert!(failures.is_empty(), "{failures:?}");
461    }
462}