Skip to main content

hearth_store/
lib.rs

1//! hearth-store — the NEDB spine.
2//!
3//! Every residency transition is a bi-temporal, causally-linked,
4//! tamper-evident event in an embedded NEDB database. This is not a log
5//! bolted onto the side of the supervisor: it IS the supervisor's state.
6//!
7//! Three properties no other serving stack has, inherited for free:
8//!
9//!   * `state_as_of(model, seq)` — what was resident *then*, not now.
10//!   * `why(model)` — a causal chain from the current state back through
11//!     every transition that produced it, not a guess from grep.
12//!   * `verify()` — cryptographic proof the history wasn't rewritten.
13//!
14//! Layout: one versioned document per model in the `residency` collection
15//! (id = model name). Each transition is a new version of that document,
16//! with `caused_by` linking to the event that produced it — a load links
17//! to the pull that fetched the bytes, a loss links to the load it undid.
18//! NEDB's version chain gives history; its DAG gives causality; its
19//! Merkle head gives tamper-evidence.
20
21use std::path::Path;
22
23use hearth_core::{LostReason, Residency};
24use nedb_engine::db::Db;
25use nedb_engine::store::Node;
26use serde_json::{json, Value};
27
28/// Collection holding one versioned doc per model.
29const RESIDENCY: &str = "residency";
30/// Collection holding one versioned doc per model for pulls.
31const PULLS: &str = "pulls";
32
33/// A recorded event: the node hash (for causal linking) and its seq
34/// (for AS OF reads). Everything a caller needs to chain the next event.
35#[derive(Debug, Clone, PartialEq, Eq)]
36pub struct EventRef {
37    pub hash: String,
38    pub seq: u64,
39}
40
41impl EventRef {
42    fn of(node: &Node) -> Self {
43        EventRef {
44            hash: node.hash.clone(),
45            seq: node.seq,
46        }
47    }
48}
49
50/// What kind of transition is being recorded.
51///
52/// These mirror `hearth_core::Residency` but carry the extra facts the
53/// state machine doesn't hold (endpoint, pid, timing) — the store is the
54/// system of record, the state machine is the decision procedure.
55#[derive(Debug, Clone, PartialEq)]
56pub enum Transition {
57    /// Model declared into the fleet. `admitted: false` means the budget
58    /// refused it — recorded anyway, because a refusal is a fact too.
59    Declared { vram_bytes: u64, admitted: bool },
60    /// Bytes are being fetched. Source is the resolved origin
61    /// ("ollama:library/llama3:latest", "hf:owner/repo@Q4_K_M", "file:...").
62    PullStarted { source: String },
63    /// Bytes landed and digest-verified.
64    PullCompleted { path: String, size_bytes: u64 },
65    /// A runtime child was spawned and is warming.
66    Loading { pid: u32, endpoint: String },
67    /// The runtime answered its health probe.
68    Resident { endpoint: String, warmup_ms: u64 },
69    /// The model stopped being resident, with the named reason.
70    Lost { reason: LostReason },
71    /// Deliberate operator stop. Not a loss — an instruction.
72    Unloaded,
73}
74
75impl Transition {
76    fn kind(&self) -> &'static str {
77        match self {
78            Transition::Declared { .. } => "declared",
79            Transition::PullStarted { .. } => "pull_started",
80            Transition::PullCompleted { .. } => "pull_completed",
81            Transition::Loading { .. } => "loading",
82            Transition::Resident { .. } => "resident",
83            Transition::Lost { .. } => "lost",
84            Transition::Unloaded => "unloaded",
85        }
86    }
87
88    fn facts(&self) -> Value {
89        match self {
90            Transition::Declared {
91                vram_bytes,
92                admitted,
93            } => json!({ "vram_bytes": vram_bytes, "admitted": admitted }),
94            Transition::PullStarted { source } => json!({ "source": source }),
95            Transition::PullCompleted { path, size_bytes } => {
96                json!({ "path": path, "size_bytes": size_bytes })
97            }
98            Transition::Loading { pid, endpoint } => {
99                json!({ "pid": pid, "endpoint": endpoint })
100            }
101            Transition::Resident {
102                endpoint,
103                warmup_ms,
104            } => json!({ "endpoint": endpoint, "warmup_ms": warmup_ms }),
105            Transition::Lost { reason } => json!({ "reason": lost_reason_str(*reason) }),
106            Transition::Unloaded => json!({}),
107        }
108    }
109}
110
111fn lost_reason_str(reason: LostReason) -> &'static str {
112    match reason {
113        LostReason::Evicted => "evicted",
114        LostReason::GpuDetached => "gpu_detached",
115        LostReason::ProcessExited => "process_exited",
116        LostReason::Unhealthy => "unhealthy",
117    }
118}
119
120/// One event read back out of the spine.
121#[derive(Debug, Clone)]
122pub struct Event {
123    pub model: String,
124    pub kind: String,
125    pub facts: Value,
126    pub seq: u64,
127    pub hash: String,
128    pub ts: f64,
129}
130
131impl Event {
132    fn of(node: &Node) -> Option<Event> {
133        let obj = node.data.as_object()?;
134        Some(Event {
135            model: node.id.clone(),
136            kind: obj.get("kind")?.as_str()?.to_string(),
137            facts: obj.get("facts").cloned().unwrap_or(Value::Null),
138            seq: node.seq,
139            hash: node.hash.clone(),
140            ts: node.ts,
141        })
142    }
143}
144
145/// The spine itself: an embedded NEDB database.
146pub struct Spine {
147    db: Db,
148}
149
150impl Spine {
151    /// Open (or create) an on-disk spine.
152    pub fn open(root: &Path) -> Result<Spine, String> {
153        let db = Db::open(root, None).map_err(|e| e.to_string())?;
154        Ok(Spine { db })
155    }
156
157    /// Pure in-memory spine — tests and ephemeral runs. Nothing survives drop.
158    pub fn in_memory() -> Spine {
159        Spine {
160            db: Db::in_memory(),
161        }
162    }
163
164    /// Record a transition for `model`, causally linked to the events that
165    /// produced it. Returns the ref future events should cite as cause.
166    pub fn record(
167        &self,
168        model: &str,
169        transition: &Transition,
170        caused_by: &[EventRef],
171    ) -> Result<EventRef, String> {
172        let coll = match transition {
173            Transition::PullStarted { .. } | Transition::PullCompleted { .. } => PULLS,
174            _ => RESIDENCY,
175        };
176        let doc = json!({
177            "kind": transition.kind(),
178            "facts": transition.facts(),
179        });
180        let causes: Vec<String> = caused_by.iter().map(|r| r.hash.clone()).collect();
181        let node = self
182            .db
183            .put(coll, model, doc, causes, None, None)
184            .map_err(|e| e.to_string())?;
185        // Transitions are rare and each one is a fact someone may need after
186        // a crash — durability at event granularity, not process-exit
187        // granularity. A supervisor that loses its history on SIGKILL has a
188        // spine made of RAM. (Found live: the first e2e run lost everything.)
189        self.flush();
190        Ok(EventRef::of(&node))
191    }
192
193    /// The model's current state — the latest residency event.
194    pub fn latest(&self, model: &str) -> Option<Event> {
195        self.db.get(RESIDENCY, model).as_ref().and_then(Event::of)
196    }
197
198    /// The latest pull event for `model`, if it was ever fetched.
199    ///
200    /// Separate from `latest` on purpose: a pull is not a residency state, and
201    /// letting one answer "is this model warm" would conflate having the bytes
202    /// with having them loaded. But the events were previously written to their
203    /// own collection and read by nothing, so `hearth why` reported "no history
204    /// in the spine" for a model it had just downloaded and verified — a false
205    /// statement, which is worse than an empty one.
206    pub fn latest_pull(&self, model: &str) -> Option<Event> {
207        self.db.get(PULLS, model).as_ref().and_then(Event::of)
208    }
209
210    /// Where the bytes came from, newest first: the pull chain.
211    ///
212    /// `PullCompleted` cites `PullStarted` as its cause, so this walks the same
213    /// causal links `why` does — "where did this file come from" is a query
214    /// against a verifiable chain rather than an archaeology project.
215    pub fn provenance(&self, model: &str) -> Vec<Event> {
216        let Some(latest) = self.latest_pull(model) else {
217            return Vec::new();
218        };
219        self.db
220            .trace(&latest.hash, false, 64)
221            .iter()
222            .filter_map(Event::of)
223            .collect()
224    }
225
226    /// What the model's state was at sequence `seq`. Time travel.
227    pub fn state_as_of(&self, model: &str, seq: u64) -> Option<Event> {
228        self.db
229            .get_as_of(RESIDENCY, model, seq)
230            .as_ref()
231            .and_then(Event::of)
232    }
233
234    /// The causal chain that produced the model's current state, newest
235    /// first: the answer to "why is this model (not) resident?"
236    pub fn why(&self, model: &str) -> Vec<Event> {
237        let Some(latest) = self.latest(model) else {
238            return Vec::new();
239        };
240        self.db
241            .trace(&latest.hash, false, 64)
242            .iter()
243            .filter_map(Event::of)
244            .collect()
245    }
246
247    /// Reconstruct `Residency` from the latest event, for handing back to
248    /// the `hearth_core` state machine after a restart.
249    pub fn residency(&self, model: &str, now_ms: u64) -> Option<Residency> {
250        let ev = self.latest(model)?;
251        let vram = ev
252            .facts
253            .get("vram_bytes")
254            .and_then(Value::as_u64)
255            .unwrap_or(0);
256        match ev.kind.as_str() {
257            "resident" => Some(Residency::Resident {
258                since: now_ms,
259                vram_bytes: vram,
260            }),
261            "loading" => Some(Residency::Loading { since: now_ms }),
262            "lost" => {
263                let reason = match ev.facts.get("reason").and_then(Value::as_str) {
264                    Some("gpu_detached") => LostReason::GpuDetached,
265                    Some("process_exited") => LostReason::ProcessExited,
266                    Some("unhealthy") => LostReason::Unhealthy,
267                    _ => LostReason::Evicted,
268                };
269                Some(Residency::Lost { at: now_ms, reason })
270            }
271            "unloaded" => Some(Residency::Stopped { at: now_ms }),
272            _ => None,
273        }
274    }
275
276    /// Latest residency event for every model the spine has ever seen.
277    pub fn all_latest(&self) -> Vec<Event> {
278        self.db
279            .list(RESIDENCY)
280            .iter()
281            .filter_map(Event::of)
282            .collect()
283    }
284
285    /// Current global sequence — the clock `state_as_of` reads against.
286    pub fn seq(&self) -> u64 {
287        self.db.seq.load(std::sync::atomic::Ordering::SeqCst)
288    }
289
290    /// Merkle head of the whole history.
291    pub fn head(&self) -> String {
292        self.db.head()
293    }
294
295    /// Verify every node against its hash. `(checked, failures)`.
296    pub fn verify(&self) -> (usize, Vec<String>) {
297        self.db.verify()
298    }
299
300    /// Flush everything to disk (no-op in memory).
301    pub fn flush(&self) {
302        self.db.flush_all();
303        self.db.flush_manifest_if_dirty();
304    }
305}
306
307#[cfg(test)]
308mod tests {
309    use super::*;
310
311    fn spine() -> Spine {
312        Spine::in_memory()
313    }
314
315    #[test]
316    fn declared_is_recorded_and_latest() {
317        let s = spine();
318        s.record(
319            "muse",
320            &Transition::Declared {
321                vram_bytes: 4_000_000_000,
322                admitted: true,
323            },
324            &[],
325        )
326        .unwrap();
327        let ev = s.latest("muse").unwrap();
328        assert_eq!(ev.kind, "declared");
329        assert_eq!(ev.facts["admitted"], json!(true));
330    }
331
332    #[test]
333    fn refusal_is_a_fact_too() {
334        let s = spine();
335        s.record(
336            "too-big",
337            &Transition::Declared {
338                vram_bytes: u64::MAX,
339                admitted: false,
340            },
341            &[],
342        )
343        .unwrap();
344        assert_eq!(s.latest("too-big").unwrap().facts["admitted"], json!(false));
345    }
346
347    #[test]
348    fn full_lifecycle_chains_causally() {
349        let s = spine();
350        let declared = s
351            .record(
352                "muse",
353                &Transition::Declared {
354                    vram_bytes: 4_000_000_000,
355                    admitted: true,
356                },
357                &[],
358            )
359            .unwrap();
360        let pull = s
361            .record(
362                "muse",
363                &Transition::PullStarted {
364                    source: "ollama:library/muse:latest".into(),
365                },
366                std::slice::from_ref(&declared),
367            )
368            .unwrap();
369        let pulled = s
370            .record(
371                "muse",
372                &Transition::PullCompleted {
373                    path: "/blobs/sha256-abc".into(),
374                    size_bytes: 4_000_000_000,
375                },
376                &[pull],
377            )
378            .unwrap();
379        let loading = s
380            .record(
381                "muse",
382                &Transition::Loading {
383                    pid: 4242,
384                    endpoint: "127.0.0.1:8080".into(),
385                },
386                &[pulled],
387            )
388            .unwrap();
389        let resident = s
390            .record(
391                "muse",
392                &Transition::Resident {
393                    endpoint: "127.0.0.1:8080".into(),
394                    warmup_ms: 43_000,
395                },
396                &[loading],
397            )
398            .unwrap();
399
400        // why() walks the causal chain back through the whole story.
401        let why = s.why("muse");
402        let kinds: Vec<&str> = why.iter().map(|e| e.kind.as_str()).collect();
403        assert_eq!(
404            kinds,
405            vec![
406                "resident",
407                "loading",
408                "pull_completed",
409                "pull_started",
410                "declared"
411            ]
412        );
413        assert_eq!(why[0].hash, resident.hash);
414    }
415
416    #[test]
417    fn as_of_separates_then_from_now() {
418        let s = spine();
419        let loading = s
420            .record(
421                "muse",
422                &Transition::Loading {
423                    pid: 1,
424                    endpoint: "127.0.0.1:8080".into(),
425                },
426                &[],
427            )
428            .unwrap();
429        let resident = s
430            .record(
431                "muse",
432                &Transition::Resident {
433                    endpoint: "127.0.0.1:8080".into(),
434                    warmup_ms: 100,
435                },
436                &[loading],
437            )
438            .unwrap();
439        s.record(
440            "muse",
441            &Transition::Lost {
442                reason: LostReason::GpuDetached,
443            },
444            std::slice::from_ref(&resident),
445        )
446        .unwrap();
447
448        // Now: lost. Then (at the resident event's seq): resident.
449        assert_eq!(s.latest("muse").unwrap().kind, "lost");
450        let then = s.state_as_of("muse", resident.seq).unwrap();
451        assert_eq!(then.kind, "resident");
452    }
453
454    #[test]
455    fn residency_reconstructs_for_the_state_machine() {
456        let s = spine();
457        let r = s
458            .record(
459                "muse",
460                &Transition::Resident {
461                    endpoint: "e".into(),
462                    warmup_ms: 5,
463                },
464                &[],
465            )
466            .unwrap();
467        assert!(matches!(
468            s.residency("muse", 0),
469            Some(Residency::Resident { .. })
470        ));
471        s.record(
472            "muse",
473            &Transition::Lost {
474                reason: LostReason::Evicted,
475            },
476            &[r],
477        )
478        .unwrap();
479        assert!(matches!(
480            s.residency("muse", 0),
481            Some(Residency::Lost {
482                reason: LostReason::Evicted,
483                ..
484            })
485        ));
486    }
487
488    #[test]
489    fn lost_reason_survives_the_round_trip() {
490        let s = spine();
491        s.record(
492            "muse",
493            &Transition::Lost {
494                reason: LostReason::GpuDetached,
495            },
496            &[],
497        )
498        .unwrap();
499        assert!(matches!(
500            s.residency("muse", 0),
501            Some(Residency::Lost {
502                reason: LostReason::GpuDetached,
503                ..
504            })
505        ));
506    }
507
508    #[test]
509    fn verify_is_clean_over_a_real_history() {
510        let s = spine();
511        for i in 0..5 {
512            s.record(
513                "muse",
514                &Transition::Resident {
515                    endpoint: format!("e{i}"),
516                    warmup_ms: i,
517                },
518                &[],
519            )
520            .unwrap();
521        }
522        let (checked, failures) = s.verify();
523        assert!(checked >= 5);
524        assert!(failures.is_empty(), "verify failures: {failures:?}");
525    }
526
527    #[test]
528    fn on_disk_history_survives_reopen() {
529        let dir = tempfile::tempdir().unwrap();
530        let resident_seq;
531        {
532            let s = Spine::open(dir.path()).unwrap();
533            let l = s
534                .record(
535                    "muse",
536                    &Transition::Loading {
537                        pid: 7,
538                        endpoint: "127.0.0.1:9".into(),
539                    },
540                    &[],
541                )
542                .unwrap();
543            let r = s
544                .record(
545                    "muse",
546                    &Transition::Resident {
547                        endpoint: "127.0.0.1:9".into(),
548                        warmup_ms: 12,
549                    },
550                    &[l],
551                )
552                .unwrap();
553            resident_seq = r.seq;
554            s.record(
555                "muse",
556                &Transition::Lost {
557                    reason: LostReason::Evicted,
558                },
559                &[r],
560            )
561            .unwrap();
562            s.flush();
563        }
564        // Fresh process, same disk: history intact, AS OF still answers.
565        let s = Spine::open(dir.path()).unwrap();
566        assert_eq!(s.latest("muse").unwrap().kind, "lost");
567        assert_eq!(
568            s.state_as_of("muse", resident_seq).unwrap().kind,
569            "resident"
570        );
571        let (checked, failures) = s.verify();
572        assert!(checked >= 3);
573        assert!(failures.is_empty());
574    }
575    // -- pull provenance ---------------------------------------------------
576    //
577    // Pull events are written to their own collection, and for one commit they
578    // were read by nothing — so `hearth why` reported "no history in the spine"
579    // about a model it had just downloaded and digest-verified. Recorded but
580    // unreachable is not recorded.
581
582    #[test]
583    fn a_pull_is_reachable_after_it_is_recorded() {
584        let s = Spine::in_memory();
585        let started = s
586            .record(
587                "tinyllama:latest",
588                &Transition::PullStarted {
589                    source: "ollama:library/tinyllama:latest".into(),
590                },
591                &[],
592            )
593            .unwrap();
594        s.record(
595            "tinyllama:latest",
596            &Transition::PullCompleted {
597                path: "/b/sha256-2af3".into(),
598                size_bytes: 637_699_456,
599            },
600            &[started],
601        )
602        .unwrap();
603
604        let chain = s.provenance("tinyllama:latest");
605        assert_eq!(chain.len(), 2, "completed <- started: {chain:?}");
606        assert_eq!(chain[0].kind, "pull_completed");
607        assert_eq!(chain[1].kind, "pull_started");
608        assert_eq!(
609            chain[1].facts["source"],
610            serde_json::json!("ollama:library/tinyllama:latest"),
611            "the origin is the entire point of recording it"
612        );
613        assert_eq!(
614            chain[0].facts["size_bytes"],
615            serde_json::json!(637_699_456u64)
616        );
617    }
618
619    #[test]
620    fn a_pull_is_not_a_residency_state() {
621        // Having the bytes is not having them loaded. If a pull answered
622        // `latest`, a downloaded-but-never-served model would read as resident.
623        let s = Spine::in_memory();
624        s.record(
625            "m",
626            &Transition::PullCompleted {
627                path: "/b/x".into(),
628                size_bytes: 1,
629            },
630            &[],
631        )
632        .unwrap();
633        assert!(s.latest("m").is_none(), "not a residency event");
634        assert!(s.why("m").is_empty());
635        assert!(s.latest_pull("m").is_some(), "but it IS a pull event");
636    }
637
638    #[test]
639    fn provenance_of_something_never_pulled_is_empty_not_a_panic() {
640        let s = Spine::in_memory();
641        assert!(s.provenance("never-heard-of-it").is_empty());
642    }
643}