1use std::path::Path;
22
23use hearth_core::{LostReason, Residency};
24use nedb_engine::db::Db;
25use nedb_engine::store::Node;
26use serde_json::{json, Value};
27
28const RESIDENCY: &str = "residency";
30const PULLS: &str = "pulls";
32
33#[derive(Debug, Clone, PartialEq, Eq)]
36pub struct EventRef {
37 pub hash: String,
38 pub seq: u64,
39}
40
41impl EventRef {
42 fn of(node: &Node) -> Self {
43 EventRef {
44 hash: node.hash.clone(),
45 seq: node.seq,
46 }
47 }
48}
49
50#[derive(Debug, Clone, PartialEq)]
56pub enum Transition {
57 Declared { vram_bytes: u64, admitted: bool },
60 PullStarted { source: String },
63 PullCompleted { path: String, size_bytes: u64 },
65 Loading { pid: u32, endpoint: String },
67 Resident { endpoint: String, warmup_ms: u64 },
69 Lost { reason: LostReason },
71 Unloaded,
73}
74
75impl Transition {
76 fn kind(&self) -> &'static str {
77 match self {
78 Transition::Declared { .. } => "declared",
79 Transition::PullStarted { .. } => "pull_started",
80 Transition::PullCompleted { .. } => "pull_completed",
81 Transition::Loading { .. } => "loading",
82 Transition::Resident { .. } => "resident",
83 Transition::Lost { .. } => "lost",
84 Transition::Unloaded => "unloaded",
85 }
86 }
87
88 fn facts(&self) -> Value {
89 match self {
90 Transition::Declared {
91 vram_bytes,
92 admitted,
93 } => json!({ "vram_bytes": vram_bytes, "admitted": admitted }),
94 Transition::PullStarted { source } => json!({ "source": source }),
95 Transition::PullCompleted { path, size_bytes } => {
96 json!({ "path": path, "size_bytes": size_bytes })
97 }
98 Transition::Loading { pid, endpoint } => {
99 json!({ "pid": pid, "endpoint": endpoint })
100 }
101 Transition::Resident {
102 endpoint,
103 warmup_ms,
104 } => json!({ "endpoint": endpoint, "warmup_ms": warmup_ms }),
105 Transition::Lost { reason } => json!({ "reason": lost_reason_str(*reason) }),
106 Transition::Unloaded => json!({}),
107 }
108 }
109}
110
111fn lost_reason_str(reason: LostReason) -> &'static str {
112 match reason {
113 LostReason::Evicted => "evicted",
114 LostReason::GpuDetached => "gpu_detached",
115 LostReason::ProcessExited => "process_exited",
116 LostReason::Unhealthy => "unhealthy",
117 }
118}
119
120#[derive(Debug, Clone)]
122pub struct Event {
123 pub model: String,
124 pub kind: String,
125 pub facts: Value,
126 pub seq: u64,
127 pub hash: String,
128 pub ts: f64,
129}
130
131impl Event {
132 fn of(node: &Node) -> Option<Event> {
133 let obj = node.data.as_object()?;
134 Some(Event {
135 model: node.id.clone(),
136 kind: obj.get("kind")?.as_str()?.to_string(),
137 facts: obj.get("facts").cloned().unwrap_or(Value::Null),
138 seq: node.seq,
139 hash: node.hash.clone(),
140 ts: node.ts,
141 })
142 }
143}
144
145pub struct Spine {
147 db: Db,
148}
149
150impl Spine {
151 pub fn open(root: &Path) -> Result<Spine, String> {
153 let db = Db::open(root, None).map_err(|e| e.to_string())?;
154 Ok(Spine { db })
155 }
156
157 pub fn in_memory() -> Spine {
159 Spine {
160 db: Db::in_memory(),
161 }
162 }
163
164 pub fn record(
167 &self,
168 model: &str,
169 transition: &Transition,
170 caused_by: &[EventRef],
171 ) -> Result<EventRef, String> {
172 let coll = match transition {
173 Transition::PullStarted { .. } | Transition::PullCompleted { .. } => PULLS,
174 _ => RESIDENCY,
175 };
176 let doc = json!({
177 "kind": transition.kind(),
178 "facts": transition.facts(),
179 });
180 let causes: Vec<String> = caused_by.iter().map(|r| r.hash.clone()).collect();
181 let node = self
182 .db
183 .put(coll, model, doc, causes, None, None)
184 .map_err(|e| e.to_string())?;
185 self.flush();
190 Ok(EventRef::of(&node))
191 }
192
193 pub fn latest(&self, model: &str) -> Option<Event> {
195 self.db.get(RESIDENCY, model).as_ref().and_then(Event::of)
196 }
197
198 pub fn latest_pull(&self, model: &str) -> Option<Event> {
207 self.db.get(PULLS, model).as_ref().and_then(Event::of)
208 }
209
210 pub fn provenance(&self, model: &str) -> Vec<Event> {
216 let Some(latest) = self.latest_pull(model) else {
217 return Vec::new();
218 };
219 self.db
220 .trace(&latest.hash, false, 64)
221 .iter()
222 .filter_map(Event::of)
223 .collect()
224 }
225
226 pub fn state_as_of(&self, model: &str, seq: u64) -> Option<Event> {
228 self.db
229 .get_as_of(RESIDENCY, model, seq)
230 .as_ref()
231 .and_then(Event::of)
232 }
233
234 pub fn why(&self, model: &str) -> Vec<Event> {
237 let Some(latest) = self.latest(model) else {
238 return Vec::new();
239 };
240 self.db
241 .trace(&latest.hash, false, 64)
242 .iter()
243 .filter_map(Event::of)
244 .collect()
245 }
246
247 pub fn residency(&self, model: &str, now_ms: u64) -> Option<Residency> {
250 let ev = self.latest(model)?;
251 let vram = ev
252 .facts
253 .get("vram_bytes")
254 .and_then(Value::as_u64)
255 .unwrap_or(0);
256 match ev.kind.as_str() {
257 "resident" => Some(Residency::Resident {
258 since: now_ms,
259 vram_bytes: vram,
260 }),
261 "loading" => Some(Residency::Loading { since: now_ms }),
262 "lost" => {
263 let reason = match ev.facts.get("reason").and_then(Value::as_str) {
264 Some("gpu_detached") => LostReason::GpuDetached,
265 Some("process_exited") => LostReason::ProcessExited,
266 Some("unhealthy") => LostReason::Unhealthy,
267 _ => LostReason::Evicted,
268 };
269 Some(Residency::Lost { at: now_ms, reason })
270 }
271 "unloaded" => Some(Residency::Stopped { at: now_ms }),
272 _ => None,
273 }
274 }
275
276 pub fn all_latest(&self) -> Vec<Event> {
278 self.db
279 .list(RESIDENCY)
280 .iter()
281 .filter_map(Event::of)
282 .collect()
283 }
284
285 pub fn seq(&self) -> u64 {
287 self.db.seq.load(std::sync::atomic::Ordering::SeqCst)
288 }
289
290 pub fn head(&self) -> String {
292 self.db.head()
293 }
294
295 pub fn verify(&self) -> (usize, Vec<String>) {
297 self.db.verify()
298 }
299
300 pub fn flush(&self) {
302 self.db.flush_all();
303 self.db.flush_manifest_if_dirty();
304 }
305}
306
307#[cfg(test)]
308mod tests {
309 use super::*;
310
311 fn spine() -> Spine {
312 Spine::in_memory()
313 }
314
315 #[test]
316 fn declared_is_recorded_and_latest() {
317 let s = spine();
318 s.record(
319 "muse",
320 &Transition::Declared {
321 vram_bytes: 4_000_000_000,
322 admitted: true,
323 },
324 &[],
325 )
326 .unwrap();
327 let ev = s.latest("muse").unwrap();
328 assert_eq!(ev.kind, "declared");
329 assert_eq!(ev.facts["admitted"], json!(true));
330 }
331
332 #[test]
333 fn refusal_is_a_fact_too() {
334 let s = spine();
335 s.record(
336 "too-big",
337 &Transition::Declared {
338 vram_bytes: u64::MAX,
339 admitted: false,
340 },
341 &[],
342 )
343 .unwrap();
344 assert_eq!(s.latest("too-big").unwrap().facts["admitted"], json!(false));
345 }
346
347 #[test]
348 fn full_lifecycle_chains_causally() {
349 let s = spine();
350 let declared = s
351 .record(
352 "muse",
353 &Transition::Declared {
354 vram_bytes: 4_000_000_000,
355 admitted: true,
356 },
357 &[],
358 )
359 .unwrap();
360 let pull = s
361 .record(
362 "muse",
363 &Transition::PullStarted {
364 source: "ollama:library/muse:latest".into(),
365 },
366 std::slice::from_ref(&declared),
367 )
368 .unwrap();
369 let pulled = s
370 .record(
371 "muse",
372 &Transition::PullCompleted {
373 path: "/blobs/sha256-abc".into(),
374 size_bytes: 4_000_000_000,
375 },
376 &[pull],
377 )
378 .unwrap();
379 let loading = s
380 .record(
381 "muse",
382 &Transition::Loading {
383 pid: 4242,
384 endpoint: "127.0.0.1:8080".into(),
385 },
386 &[pulled],
387 )
388 .unwrap();
389 let resident = s
390 .record(
391 "muse",
392 &Transition::Resident {
393 endpoint: "127.0.0.1:8080".into(),
394 warmup_ms: 43_000,
395 },
396 &[loading],
397 )
398 .unwrap();
399
400 let why = s.why("muse");
402 let kinds: Vec<&str> = why.iter().map(|e| e.kind.as_str()).collect();
403 assert_eq!(
404 kinds,
405 vec![
406 "resident",
407 "loading",
408 "pull_completed",
409 "pull_started",
410 "declared"
411 ]
412 );
413 assert_eq!(why[0].hash, resident.hash);
414 }
415
416 #[test]
417 fn as_of_separates_then_from_now() {
418 let s = spine();
419 let loading = s
420 .record(
421 "muse",
422 &Transition::Loading {
423 pid: 1,
424 endpoint: "127.0.0.1:8080".into(),
425 },
426 &[],
427 )
428 .unwrap();
429 let resident = s
430 .record(
431 "muse",
432 &Transition::Resident {
433 endpoint: "127.0.0.1:8080".into(),
434 warmup_ms: 100,
435 },
436 &[loading],
437 )
438 .unwrap();
439 s.record(
440 "muse",
441 &Transition::Lost {
442 reason: LostReason::GpuDetached,
443 },
444 std::slice::from_ref(&resident),
445 )
446 .unwrap();
447
448 assert_eq!(s.latest("muse").unwrap().kind, "lost");
450 let then = s.state_as_of("muse", resident.seq).unwrap();
451 assert_eq!(then.kind, "resident");
452 }
453
454 #[test]
455 fn residency_reconstructs_for_the_state_machine() {
456 let s = spine();
457 let r = s
458 .record(
459 "muse",
460 &Transition::Resident {
461 endpoint: "e".into(),
462 warmup_ms: 5,
463 },
464 &[],
465 )
466 .unwrap();
467 assert!(matches!(
468 s.residency("muse", 0),
469 Some(Residency::Resident { .. })
470 ));
471 s.record(
472 "muse",
473 &Transition::Lost {
474 reason: LostReason::Evicted,
475 },
476 &[r],
477 )
478 .unwrap();
479 assert!(matches!(
480 s.residency("muse", 0),
481 Some(Residency::Lost {
482 reason: LostReason::Evicted,
483 ..
484 })
485 ));
486 }
487
488 #[test]
489 fn lost_reason_survives_the_round_trip() {
490 let s = spine();
491 s.record(
492 "muse",
493 &Transition::Lost {
494 reason: LostReason::GpuDetached,
495 },
496 &[],
497 )
498 .unwrap();
499 assert!(matches!(
500 s.residency("muse", 0),
501 Some(Residency::Lost {
502 reason: LostReason::GpuDetached,
503 ..
504 })
505 ));
506 }
507
508 #[test]
509 fn verify_is_clean_over_a_real_history() {
510 let s = spine();
511 for i in 0..5 {
512 s.record(
513 "muse",
514 &Transition::Resident {
515 endpoint: format!("e{i}"),
516 warmup_ms: i,
517 },
518 &[],
519 )
520 .unwrap();
521 }
522 let (checked, failures) = s.verify();
523 assert!(checked >= 5);
524 assert!(failures.is_empty(), "verify failures: {failures:?}");
525 }
526
527 #[test]
528 fn on_disk_history_survives_reopen() {
529 let dir = tempfile::tempdir().unwrap();
530 let resident_seq;
531 {
532 let s = Spine::open(dir.path()).unwrap();
533 let l = s
534 .record(
535 "muse",
536 &Transition::Loading {
537 pid: 7,
538 endpoint: "127.0.0.1:9".into(),
539 },
540 &[],
541 )
542 .unwrap();
543 let r = s
544 .record(
545 "muse",
546 &Transition::Resident {
547 endpoint: "127.0.0.1:9".into(),
548 warmup_ms: 12,
549 },
550 &[l],
551 )
552 .unwrap();
553 resident_seq = r.seq;
554 s.record(
555 "muse",
556 &Transition::Lost {
557 reason: LostReason::Evicted,
558 },
559 &[r],
560 )
561 .unwrap();
562 s.flush();
563 }
564 let s = Spine::open(dir.path()).unwrap();
566 assert_eq!(s.latest("muse").unwrap().kind, "lost");
567 assert_eq!(
568 s.state_as_of("muse", resident_seq).unwrap().kind,
569 "resident"
570 );
571 let (checked, failures) = s.verify();
572 assert!(checked >= 3);
573 assert!(failures.is_empty());
574 }
575 #[test]
583 fn a_pull_is_reachable_after_it_is_recorded() {
584 let s = Spine::in_memory();
585 let started = s
586 .record(
587 "tinyllama:latest",
588 &Transition::PullStarted {
589 source: "ollama:library/tinyllama:latest".into(),
590 },
591 &[],
592 )
593 .unwrap();
594 s.record(
595 "tinyllama:latest",
596 &Transition::PullCompleted {
597 path: "/b/sha256-2af3".into(),
598 size_bytes: 637_699_456,
599 },
600 &[started],
601 )
602 .unwrap();
603
604 let chain = s.provenance("tinyllama:latest");
605 assert_eq!(chain.len(), 2, "completed <- started: {chain:?}");
606 assert_eq!(chain[0].kind, "pull_completed");
607 assert_eq!(chain[1].kind, "pull_started");
608 assert_eq!(
609 chain[1].facts["source"],
610 serde_json::json!("ollama:library/tinyllama:latest"),
611 "the origin is the entire point of recording it"
612 );
613 assert_eq!(
614 chain[0].facts["size_bytes"],
615 serde_json::json!(637_699_456u64)
616 );
617 }
618
619 #[test]
620 fn a_pull_is_not_a_residency_state() {
621 let s = Spine::in_memory();
624 s.record(
625 "m",
626 &Transition::PullCompleted {
627 path: "/b/x".into(),
628 size_bytes: 1,
629 },
630 &[],
631 )
632 .unwrap();
633 assert!(s.latest("m").is_none(), "not a residency event");
634 assert!(s.why("m").is_empty());
635 assert!(s.latest_pull("m").is_some(), "but it IS a pull event");
636 }
637
638 #[test]
639 fn provenance_of_something_never_pulled_is_empty_not_a_panic() {
640 let s = Spine::in_memory();
641 assert!(s.provenance("never-heard-of-it").is_empty());
642 }
643}