1use std::path::{Path, PathBuf};
80
81use gunnar_proto::{CapacityResponse, CapacityVerdict, SetGrowth};
82use gunnar_ssh::{headroom::human, Headroom};
83
84use crate::dashboard::probe::{self, DiskKind};
85
86#[derive(Debug, Clone, PartialEq, Eq)]
89pub struct GrowthFacts {
90 pub kind: Option<DiskKind>,
95 pub is_pid1: bool,
100 pub grow: Option<gunnar_grpc::grow_journal::Summary>,
106 pub runtime_engine: bool,
110 pub hotplug: bool,
115}
116
117impl GrowthFacts {
118 pub fn measure(root: &Path) -> GrowthFacts {
120 GrowthFacts {
121 kind: crate::dashboard::probe::disk_kind(root),
122 is_pid1: std::process::id() == 1,
123 grow: gunnar_grpc::grow_journal::Summary::measure(),
124 runtime_engine: cfg!(feature = "grow-runtime"),
125 hotplug: crate::dashboard::probe::pci_hotplug().present,
126 }
127 }
128}
129
130pub fn decide_growth(facts: &GrowthFacts, root: &Path) -> (SetGrowth, String) {
132 let root = root.display();
133 match (&facts.kind, facts.is_pid1) {
134 (None, _) => (
135 SetGrowth::GrowthUnknown,
136 format!(
137 "gunnar cannot tell what {root} sits on — it is not a mount point of its own, \
138 so whether this set can grow is UNKNOWN, which is not a yes"
139 ),
140 ),
141 (Some(kind), true) if facts.grow.is_some() && facts.runtime_engine && facts.hotplug => {
147 let journal = facts.grow.as_ref().expect("guarded");
148 let n = kind.set.as_ref().map(|s| s.members).unwrap_or(1);
149 (
150 SetGrowth::GrowthAtRuntime,
151 format!(
152 "AND THE SET GROWS WITHOUT A RESTART: {root} is a {kind} set, this gunnar-server \
153 links the grow engine, the kernel has PCI hot-plug and the journal is on this box — `Admin.GrowStart` drains \
154 the box, unmounts, waits for ALL {n} members to come back larger at one equal \
155 size, grows in place and serves again in minutes; a crash mid-way is finished \
156 at the next boot. Grow journal: {}",
157 journal.detail()
158 ),
159 )
160 }
161 (Some(kind), true) if facts.grow.is_some() => {
166 let journal = facts.grow.as_ref().expect("guarded");
167 let n = kind.set.as_ref().map(|s| s.members).unwrap_or(1);
168 (
169 SetGrowth::GrowthAtRestart,
170 format!(
171 "AND THE SET GROWS AT RESTART: {root} is a {kind} set and this box's boot stack \
172 (korp-start) grows it into enlarged member volumes at the next boot — enlarge \
173 ALL {n} members to one equal size, restart, and the next measurement reads the \
174 larger set. It does NOT grow while gunnar runs{}, and a disk ATTACHED to this \
175 machine is still invisible to it. Grow journal: {}",
176 if facts.runtime_engine && !facts.hotplug {
177 " (the grow engine is linked, but the running kernel has NO PCI hot-plug: a member detached and re-attached while gunnar runs is invisible, so a runtime growth is refused by name)"
178 } else {
179 ""
180 },
181 journal.detail()
182 ),
183 )
184 }
185 (Some(kind), true) => {
186 let laid = if kind.set.is_some() {
192 " laid once at install (korp-installer `lay_set`, InstallMode::Fresh only)"
193 } else {
194 ""
195 };
196 let engine = if facts.runtime_engine {
211 " (this server links the runtime grow engine, but there is no grow journal on this box for it to continue, so it refuses by name)"
212 } else {
213 ""
214 };
215 (
216 SetGrowth::GrowthSealed,
217 format!(
218 "AND THE SET CANNOT GROW WHILE GUNNAR RUNS: {root} is a {kind} set{laid}, \
219 this box's boot stack keeps no grow journal{engine}, and gunnar is PID 1 \
220 with no shell — so a disk attached to this machine would be UNUSED AND \
221 INVISIBLE. Serving more needs a fresh install onto a larger set, not \
222 another disk"
223 ),
224 )
225 }
226 (Some(kind), false) => (
227 SetGrowth::GrowthOperatorOnly,
228 format!(
229 "gunnar will not grow the set itself here — the grow engine runs only where \
230 gunnar is PID 1 over a set its own boot stack laid, and this process is not \
231 PID 1 — but that is also why an operator with a shell can extend {root} \
232 ({kind}) out of band, and the next measurement will see it"
233 ),
234 ),
235 }
236}
237
238#[derive(Debug, Clone, PartialEq)]
242pub struct MachineFacts {
243 pub cpus: Result<usize, String>,
249 pub memory: Result<(u64, u64), String>,
253 pub cgroup_memory_max: Option<u64>,
257 pub load1: Result<f64, String>,
259 pub disk: Option<crate::disktest::Outcome>,
266}
267
268impl MachineFacts {
269 pub fn measure() -> MachineFacts {
273 let cpus = std::thread::available_parallelism()
274 .map(|n| n.get())
275 .map_err(|err| format!("the affinity mask could not be read ({err})"));
276 let mut memory = probe::meminfo();
277 let mut cgroup_memory_max = None;
278 if let (Ok((total, available)), Some((cap, used))) = (&memory, crate::budget::cgroup_memory())
283 {
284 if cap < *total {
285 cgroup_memory_max = Some(cap);
286 memory = Ok((cap, (*available).min(cap.saturating_sub(used))));
287 }
288 }
289 MachineFacts {
290 cpus,
291 memory,
292 cgroup_memory_max,
293 load1: probe::load1_value(),
294 disk: crate::disktest::last(),
295 }
296 }
297}
298
299pub fn disk_fields(disk: &Option<crate::disktest::Outcome>) -> CapacityResponse {
313 use crate::disktest::{Outcome, CACHED_BPS};
314 use gunnar_proto::DiskTest;
315
316 let bps = |v: f64| -> Option<u64> {
317 (v.is_finite() && v > 0.0).then(|| v.round() as u64)
319 };
320 match disk {
321 None => CapacityResponse {
322 disk_test: DiskTest::DisktestNotRun as i32,
323 disk_reason: "The start-up disk test has not finished a run in this process: its \
324 gate is closed (not the appliance's PID 1 and GUNNAR_DISKTEST unset) \
325 or it is still running. Every disk_* speed field is absent — this is \
326 NOT a report of a slow disk."
327 .to_owned(),
328 ..Default::default()
329 },
330 Some(Outcome::Skipped { free, need }) => CapacityResponse {
331 disk_test: DiskTest::DisktestSkipped as i32,
332 disk_reason: format!(
333 "The start-up disk test SKIPPED: {} free on the data root, below the {} it \
334 needs, and the test must never be the thing that fills the disk. Every disk_* \
335 speed field is absent — nothing was measured.",
336 human(*free),
337 human(*need)
338 ),
339 ..Default::default()
340 },
341 Some(Outcome::Failed(why)) => CapacityResponse {
342 disk_test: DiskTest::DisktestFailed as i32,
343 disk_reason: format!(
344 "The start-up disk test FAILED ({why}). Every disk_* speed field is absent — \
345 the run did not produce numbers."
346 ),
347 ..Default::default()
348 },
349 Some(Outcome::Done(r)) => {
350 let cached = r.seq_read_bps > CACHED_BPS;
351 let mut clauses = Vec::with_capacity(3);
352 let disk_seq_write_bps = bps(r.seq_write_bps);
353 match disk_seq_write_bps {
354 Some(_) => clauses.push(format!(
355 "sequential write {}",
356 crate::disktest::fmt_rate(r.seq_write_bps)
357 )),
358 None => clauses
359 .push("sequential write UNMEASURED (the leg reported no rate)".to_owned()),
360 }
361 let disk_seq_read_bps = if cached {
362 clauses.push(format!(
363 "sequential read CACHED and therefore ABSENT: the leg came back at {} \
364 ({}), over the {} bar, so the page cache answered and the device was never \
365 touched — this is not a fast disk, it is no reading of the disk at all",
366 crate::disktest::fmt_rate(r.seq_read_bps),
367 r.read_how.as_str(),
368 crate::disktest::fmt_rate(CACHED_BPS)
369 ));
370 None
371 } else {
372 let v = bps(r.seq_read_bps);
373 match v {
374 Some(_) => clauses.push(format!(
375 "sequential read {} ({})",
376 crate::disktest::fmt_rate(r.seq_read_bps),
377 r.read_how.as_str()
378 )),
379 None => clauses
380 .push("sequential read UNMEASURED (the leg reported no rate)".to_owned()),
381 }
382 v
383 };
384 let disk_sync_milliops_per_s = (r.sync_ops_per_s.is_finite()
385 && r.sync_ops_per_s > 0.0)
386 .then(|| (r.sync_ops_per_s * 1000.0).round() as u64);
387 match disk_sync_milliops_per_s {
388 Some(_) => clauses.push(format!(
389 "4 KiB overwrite + fsync {:.0} ops/s",
390 r.sync_ops_per_s
391 )),
392 None => clauses
393 .push("4 KiB fsync UNMEASURED (the leg reported no rate)".to_owned()),
394 }
395 let us = |d: std::time::Duration| Some(u64::try_from(d.as_micros()).unwrap_or(u64::MAX));
396 CapacityResponse {
397 disk_test: DiskTest::DisktestMeasured as i32,
398 disk_seq_write_bps,
399 disk_seq_read_bps,
400 disk_sync_milliops_per_s,
401 disk_read_how: r.read_how.as_str().to_owned(),
402 disk_test_bytes: Some(r.file_bytes),
407 disk_write_us: us(r.write_elapsed),
408 disk_read_us: us(r.read_elapsed),
409 disk_sync_ops: Some(r.sync_ops),
410 disk_sync_us: us(r.sync_elapsed),
411 disk_test_us: us(r.elapsed),
412 disk_reason: format!(
413 "This disk, at start-up, on an idle box: {}. Taken in {:.1} s.",
414 clauses.join("; "),
415 r.elapsed.as_secs_f64()
416 ),
417 ..Default::default()
418 }
419 }
420 }
421}
422
423pub fn machine_fields(m: &MachineFacts) -> CapacityResponse {
431 let mut clauses = Vec::with_capacity(3);
432 let cpu_millicores_total = match &m.cpus {
433 Ok(n) => {
434 clauses.push(format!(
435 "{n} CPUs = {} millicores gunnar may run on (affinity mask, cgroup cpu.max limit \
436 applied)",
437 n * 1000
438 ));
439 Some((*n as u64) * 1000)
440 }
441 Err(why) => {
442 clauses.push(format!("cores UNMEASURED ({why}), so cpu_millicores_total is absent"));
443 None
444 }
445 };
446 let (ram_bytes_total, ram_bytes_available) = match &m.memory {
447 Ok((total, available)) => {
448 let clamp = match m.cgroup_memory_max {
449 Some(cap) => format!(
450 " — clamped to the cgroup memory.max of {}, which is what this process \
451 may use; /proc/meminfo reports the host",
452 human(cap)
453 ),
454 None => String::new(),
455 };
456 clauses.push(format!(
457 "RAM {} total, {} available (/proc/meminfo MemTotal/MemAvailable{clamp})",
458 human(*total),
459 human(*available)
460 ));
461 (Some(*total), Some(*available))
462 }
463 Err(why) => {
464 clauses.push(format!(
465 "RAM UNMEASURED ({why}), so ram_bytes_total and ram_bytes_available are absent"
466 ));
467 (None, None)
468 }
469 };
470 let cpu_millicores_busy = match &m.load1 {
471 Ok(load) => {
472 let busy = (load * 1000.0).round().max(0.0) as u64;
473 clauses.push(format!(
474 "1-min load {load:.2} ≈ {busy} millicores busy (/proc/loadavg × 1000: an \
475 APPROXIMATION of the whole box's runnable tasks, not gunnar's own CPU time)"
476 ));
477 Some(busy)
478 }
479 Err(why) => {
480 clauses.push(format!("load UNMEASURED ({why}), so cpu_millicores_busy is absent"));
481 None
482 }
483 };
484 let d = disk_fields(&m.disk);
489 CapacityResponse {
490 cpu_millicores_total,
491 ram_bytes_total,
492 ram_bytes_available,
493 cpu_millicores_busy,
494 machine_reason: format!("This box: {}.", clauses.join("; ")),
495 ..d
496 }
497}
498
499pub fn measure_disks(
509 data_root: &Path,
510 slow_root: Option<&Path>,
511 floor: gunnar_ssh::DiskFloor,
512) -> gunnar_proto::DiskSnapshot {
513 let mut volumes = vec![
514 disk_snapshot::Volume::new("system", "/"),
515 disk_snapshot::Volume::new("data", data_root),
516 ];
517 if let Some(slow) = slow_root {
518 volumes.push(disk_snapshot::Volume::new("serialdat", slow));
519 }
520 let snap = disk_snapshot::take(
521 &volumes,
522 disk_snapshot::Floor {
523 percent: floor.percent,
524 min_bytes: floor.bytes,
525 },
526 );
527 disks_to_wire(snap, gunnar_core::now_ms())
528}
529
530pub fn serialdat_breakdown(known: &[(&str, PathBuf)], used: Option<u64>) -> Vec<gunnar_proto::PathUse> {
540 let mut rows: Vec<gunnar_proto::PathUse> = known
541 .iter()
542 .map(|(label, path)| {
543 let mut row = gunnar_proto::PathUse {
544 label: (*label).to_owned(),
545 path: path.display().to_string(),
546 ..Default::default()
547 };
548 match allocated_bytes(path) {
549 Ok(bytes) => {
550 row.measured = true;
551 row.bytes = bytes;
552 }
553 Err(why) => row.error = why,
554 }
555 row
556 })
557 .collect();
558 let inner: Vec<(PathBuf, u64)> = known
561 .iter()
562 .zip(rows.iter())
563 .filter(|(_, r)| r.measured)
564 .map(|((_, p), r)| (p.clone(), r.bytes))
565 .collect();
566 for ((_, outer), row) in known.iter().zip(rows.iter_mut()) {
567 if !row.measured {
568 continue;
569 }
570 let nested: u64 = inner
571 .iter()
572 .filter(|(p, _)| p != outer && p.starts_with(outer))
573 .map(|(_, b)| *b)
574 .sum();
575 row.bytes = row.bytes.saturating_sub(nested);
576 }
577 if let Some(used) = used {
578 let known_sum: u64 = rows.iter().filter(|r| r.measured).map(|r| r.bytes).sum();
579 rows.push(gunnar_proto::PathUse {
580 label: "other".to_owned(),
581 path: String::new(),
582 measured: true,
583 bytes: used.saturating_sub(known_sum),
584 error: String::new(),
585 });
586 }
587 rows
588}
589
590fn allocated_bytes(path: &Path) -> Result<u64, String> {
593 use std::os::unix::fs::MetadataExt as _;
594 let meta = match std::fs::symlink_metadata(path) {
595 Ok(m) => m,
596 Err(e) if e.kind() == std::io::ErrorKind::NotFound => return Ok(0),
597 Err(e) => return Err(format!("{}: {e}", path.display())),
598 };
599 let mut total = meta.blocks() * 512;
600 if meta.is_dir() {
601 let entries = std::fs::read_dir(path).map_err(|e| format!("{}: {e}", path.display()))?;
602 for entry in entries {
603 let entry = entry.map_err(|e| format!("{}: {e}", path.display()))?;
604 total += allocated_bytes(&entry.path())?;
605 }
606 }
607 Ok(total)
608}
609
610pub fn disks_to_wire(snap: disk_snapshot::Snapshot, at_unix_ms: i64) -> gunnar_proto::DiskSnapshot {
614 let mounts = snap
615 .disks
616 .into_iter()
617 .map(|d| {
618 let mut row = gunnar_proto::DiskMount {
619 source: d.mount.source,
620 mount_point: d.mount.mount_point.display().to_string(),
621 fs_type: d.mount.fstype,
622 read_only: d.mount.read_only,
623 labels: d.labels,
624 floor_bytes: d.floor_bytes,
625 servable_bytes: d.servable_bytes,
626 ..Default::default()
627 };
628 match d.figures {
629 Ok(f) => {
630 row.measured = true;
631 row.total_bytes = f.total_bytes;
632 row.free_bytes = f.free_bytes;
633 row.available_bytes = f.available_bytes;
634 row.used_bytes = f.used_bytes;
635 row.reserved_bytes = f.reserved_bytes;
636 row.inodes_total = f.inodes_total;
637 row.inodes_free = f.inodes_free;
638 }
639 Err(why) => row.error = why,
640 }
641 row
642 })
643 .collect();
644 let devices = snap
645 .block_devices
646 .into_iter()
647 .map(|b| gunnar_proto::BlockDevice {
648 name: b.name,
649 size_bytes: b.size_bytes,
650 rotational: b.rotational,
651 removable: b.removable,
652 read_only: b.read_only,
653 partitions: b
654 .partitions
655 .into_iter()
656 .map(|(name, size_bytes)| gunnar_proto::BlockPartition { name, size_bytes })
657 .collect(),
658 holders: b.holders,
659 members: b.members,
660 raid_level: b.raid_level,
661 })
662 .collect();
663 let unresolved = snap
664 .unresolved
665 .into_iter()
666 .map(|u| gunnar_proto::UnresolvedDisk {
667 label: u.label,
668 path: u.path.display().to_string(),
669 why: u.why,
670 })
671 .collect();
672 gunnar_proto::DiskSnapshot {
673 measured_at_unix_ms: at_unix_ms,
674 mounts,
675 devices,
676 unresolved,
677 problems: snap.problems,
678 serialdat: Vec::new(),
680 }
681}
682
683#[derive(Debug, Clone)]
687pub struct Question {
688 pub root: PathBuf,
690 pub room: Result<Headroom, String>,
693 pub floor_spec: String,
695 pub requested_bytes: u64,
697 pub held_bytes: u64,
699 pub namespace: String,
701 pub growth: GrowthFacts,
702 pub machine: MachineFacts,
705}
706
707pub fn answer(q: &Question) -> CapacityResponse {
709 let (growth, growth_clause) = decide_growth(&q.growth, &q.root);
710 let mut machine = machine_fields(&q.machine);
713 machine.grow_state = q.growth.grow.as_ref().map(|s| s.to_wire());
716 machine.hotplug = q.growth.hotplug;
717 let data_set = q
718 .growth
719 .kind
720 .as_ref()
721 .map(|k| k.to_string())
722 .unwrap_or_default();
723 let root = q.root.display();
724
725 let additional = q.requested_bytes.saturating_sub(q.held_bytes);
729 let asked = if q.namespace.is_empty() {
730 format!("a cap of {}", human(q.requested_bytes))
731 } else {
732 format!(
733 "raising {} to {} (it holds {})",
734 q.namespace,
735 human(q.requested_bytes),
736 human(q.held_bytes)
737 )
738 };
739
740 let room = match &q.room {
741 Err(err) => {
742 return CapacityResponse {
743 verdict: CapacityVerdict::CapacityUnmeasured as i32,
744 growth: growth as i32,
745 root: q.root.display().to_string(),
746 requested_bytes: q.requested_bytes,
747 held_bytes: q.held_bytes,
748 additional_bytes: additional,
749 data_set,
750 reason: format!(
751 "{root} could not be measured ({err}), so gunnar will not answer that {asked} \
752 can be served — an unmeasured box has not said yes. {growth_clause}"
753 ),
754 total_bytes: 0,
757 free_bytes: 0,
758 floor_bytes: 0,
759 servable_bytes: 0,
760 shortfall_bytes: 0,
761 ..machine
762 };
763 }
764 Ok(room) => *room,
765 };
766
767 let servable = room.free.saturating_sub(room.floor);
772 let measured = format!(
773 "{root} has {} free of {}, {} of it reserved by the disk floor (--disk-floor {}), so {} \
774 may be served",
775 human(room.free),
776 human(room.total),
777 human(room.floor),
778 q.floor_spec,
779 human(servable),
780 );
781
782 let (verdict, shortfall, outcome) = if additional <= servable {
786 (
787 CapacityVerdict::CapacityAvailable,
788 0,
789 format!("needs {} more: YES, it can be served.", human(additional)),
790 )
791 } else {
792 let shortfall = additional - servable;
793 (
794 CapacityVerdict::CapacityExhausted,
795 shortfall,
796 format!(
797 "needs {} more, which is {} beyond that: NO.",
798 human(additional),
799 human(shortfall)
800 ),
801 )
802 };
803 CapacityResponse {
804 verdict: verdict as i32,
805 growth: growth as i32,
806 root: q.root.display().to_string(),
807 total_bytes: room.total,
808 free_bytes: room.free,
809 floor_bytes: room.floor,
810 servable_bytes: servable,
811 requested_bytes: q.requested_bytes,
812 held_bytes: q.held_bytes,
813 additional_bytes: additional,
814 shortfall_bytes: shortfall,
815 data_set,
816 reason: format!("{measured}. {asked} {outcome} {growth_clause}"),
817 ..machine
818 }
819}
820
821#[cfg(test)]
822mod tests {
823 use super::*;
824 use crate::dashboard::probe::SetFacts;
825
826 const GIB: u64 = 1024 * 1024 * 1024;
827
828 fn appliance() -> GrowthFacts {
831 GrowthFacts {
832 kind: Some(DiskKind {
833 fs: "xfs".into(),
834 set: Some(SetFacts {
835 level: "raid0".into(),
836 members: 4,
837 member_bytes: 75 * GIB,
838 chunk_bytes: 512 * 1024,
839 }),
840 }),
841 is_pid1: true,
842 grow: None,
843 runtime_engine: false,
844 hotplug: false,
845 }
846 }
847
848 fn developer_box() -> GrowthFacts {
850 GrowthFacts {
851 is_pid1: false,
852 ..appliance()
853 }
854 }
855
856 fn growing_appliance(text: &str) -> GrowthFacts {
859 GrowthFacts {
860 grow: Some(gunnar_grpc::grow_journal::Summary::of(text, "/serialdat/grow-journal")),
861 ..appliance()
862 }
863 }
864
865 #[test]
871 fn the_engine_in_the_server_with_the_journal_on_the_box_grows_at_runtime() {
872 let both = GrowthFacts { runtime_engine: true, hotplug: true, ..growing_appliance("") };
873 let (v, line) = decide_growth(&both, Path::new("/data"));
874 assert_eq!(v, SetGrowth::GrowthAtRuntime);
875 assert!(line.contains("WITHOUT A RESTART") && line.contains("Admin.GrowStart") && line.contains("ALL 4 members"), "{line}");
876 let engine_only = GrowthFacts { runtime_engine: true, hotplug: true, ..appliance() };
877 assert_eq!(decide_growth(&engine_only, Path::new("/data")).0, SetGrowth::GrowthSealed, "no journal: the engine alone proves nothing");
878 assert_eq!(decide_growth(&growing_appliance(""), Path::new("/data")).0, SetGrowth::GrowthAtRestart, "journal alone: the restart arm");
879 }
880
881 #[test]
886 fn the_engine_without_a_hotplug_kernel_grows_only_at_restart_and_says_why() {
887 let no_hp = GrowthFacts { runtime_engine: true, hotplug: false, ..growing_appliance("") };
888 let (v, line) = decide_growth(&no_hp, Path::new("/data"));
889 assert_eq!(v, SetGrowth::GrowthAtRestart);
890 assert!(line.contains("NO PCI hot-plug") && line.contains("refused by name"), "{line}");
891 let mut q = question(gunnar_rs_today(), no_hp.clone());
892 q.requested_bytes = 1;
893 assert!(!answer(&q).hotplug);
894 q.growth.hotplug = true;
895 let a = answer(&q);
896 assert!(a.hotplug && a.growth == SetGrowth::GrowthAtRuntime as i32, "{a:?}");
897 q.room = Err("EIO".into());
898 assert!(answer(&q).hotplug, "the unmeasured arm carries it too");
899 }
900
901 #[test]
909 fn a_boot_stack_with_the_grow_engine_grows_at_restart_and_reports_the_journal() {
910 let (v, line) = decide_growth(&growing_appliance(""), Path::new("/data"));
911 assert_eq!(v, SetGrowth::GrowthAtRestart);
912 assert!(line.contains("GROWS AT RESTART") && line.contains("ALL 4 members") && line.contains("invisible to it"), "{line}");
913 assert!(line.contains("no grow has run"), "{line}");
914 assert_eq!(decide_growth(&appliance(), Path::new("/data")).0, SetGrowth::GrowthSealed, "no journal: sealed as before");
915
916 let partial = "epoch=0 target_sectors=134215680 member=0 step=PARTIAL at=1\nepoch=0 target_sectors=134215680 member=1 step=PARTIAL at=1\n";
918 let (v, line) = decide_growth(&growing_appliance(partial), Path::new("/data"));
919 assert_eq!(v, SetGrowth::GrowthAtRestart);
920 assert!(line.contains("PARTIAL RESIZE: 2 member(s) [0, 1]"), "{line}");
921
922 let done = "epoch=1 target_sectors=134215680 member=- step=G1 at=1\nepoch=1 target_sectors=134215680 member=- step=G6 at=2\n";
924 let mut q = question(gunnar_rs_today(), growing_appliance(done));
925 q.requested_bytes = 1;
926 let a = answer(&q);
927 assert_eq!(a.growth, SetGrowth::GrowthAtRestart as i32);
928 let g = a.grow_state.expect("the journal is on the wire");
929 assert!(g.engine_present && g.phase == "done" && g.epoch == 1 && g.target_sectors == 134_215_680 && g.journal_path == "/serialdat/grow-journal", "{g:?}");
930 assert!(a.reason.contains("epoch 1 done"), "{}", a.reason);
931 q.requested_bytes = 1000 * GIB;
932 assert!(answer(&q).grow_state.is_some(), "the no arm carries it too");
933 q.room = Err("EIO".into());
934 assert!(answer(&q).grow_state.is_some(), "and the unmeasured arm");
935 let q = question(gunnar_rs_today(), appliance());
936 assert!(answer(&q).grow_state.is_none(), "no journal, no grow_state — an absence, never an empty one");
937 }
938
939 fn appliance_box() -> MachineFacts {
942 MachineFacts {
943 cpus: Ok(4),
944 memory: Ok((16 * GIB, 12 * GIB)),
945 cgroup_memory_max: None,
946 load1: Ok(0.42),
947 disk: Some(crate::disktest::Outcome::Done(crate::disktest::Report {
948 seq_write_bps: 812e6,
949 seq_read_bps: 1.9e9,
950 read_how: crate::disktest::ReadHow::Direct,
951 sync_ops_per_s: 3100.0,
952 elapsed: std::time::Duration::from_millis(1600),
953 file_bytes: 256 << 20,
954 write_elapsed: std::time::Duration::from_millis(330),
955 read_elapsed: std::time::Duration::from_millis(141),
956 sync_ops: 1550,
957 sync_elapsed: std::time::Duration::from_millis(500),
958 })),
959 }
960 }
961
962 fn question(room: Headroom, growth: GrowthFacts) -> Question {
963 Question {
964 root: PathBuf::from("/data"),
965 room: Ok(room),
966 floor_spec: "5%,2.0 GiB".into(),
967 requested_bytes: 0,
968 held_bytes: 0,
969 namespace: String::new(),
970 growth,
971 machine: appliance_box(),
972 }
973 }
974
975 fn gunnar_rs_today() -> Headroom {
977 let total = 300 * GIB;
978 Headroom {
979 total,
980 free: 70 * GIB,
981 floor: (total / 100) * 5,
982 }
983 }
984
985 #[test]
986 fn the_growth_verdict_separates_a_sealed_appliance_from_a_box_with_a_shell() {
987 let (sealed, sealed_line) = decide_growth(&appliance(), Path::new("/data"));
988 let (operator, operator_line) = decide_growth(&developer_box(), Path::new("/data"));
989 let (unknown, unknown_line) = decide_growth(
990 &GrowthFacts {
991 kind: None,
992 is_pid1: false,
993 grow: None,
994 runtime_engine: false,
995 hotplug: false,
996 },
997 Path::new("/tmp/x"),
998 );
999
1000 assert_eq!(sealed, SetGrowth::GrowthSealed);
1003 assert_eq!(operator, SetGrowth::GrowthOperatorOnly);
1004 assert_eq!(unknown, SetGrowth::GrowthUnknown);
1005 assert!(
1006 sealed != operator && operator != unknown && sealed != unknown,
1007 "the same facts must not collapse to one verdict"
1008 );
1009
1010 assert!(
1012 sealed_line.contains("UNUSED AND INVISIBLE"),
1013 "the sealed clause must say an attached disk is wasted: {sealed_line}"
1014 );
1015 assert!(
1016 sealed_line.contains("fresh install") && sealed_line.contains("not another disk"),
1017 "and must name what WOULD work: {sealed_line}"
1018 );
1019 assert!(
1020 sealed_line.contains("4\u{d7}75G raid0 xfs"),
1021 "and must name the measured set, not a generic one: {sealed_line}"
1022 );
1023 assert!(
1026 !operator_line.contains("UNUSED AND INVISIBLE")
1027 && operator_line.contains("an operator with a shell"),
1028 "{operator_line}"
1029 );
1030 assert!(unknown_line.contains("UNKNOWN, which is not a yes"), "{unknown_line}");
1031 }
1032
1033 #[test]
1037 fn ninety_more_gigabytes_on_the_sealed_appliance_is_a_no_that_names_the_reinstall() {
1038 let mut q = question(gunnar_rs_today(), appliance());
1039 q.namespace = "tenant".into();
1040 q.held_bytes = 10 * GIB;
1041 q.requested_bytes = 100 * GIB;
1042 let a = answer(&q);
1043
1044 assert_eq!(a.verdict, CapacityVerdict::CapacityExhausted as i32, "{}", a.reason);
1045 assert_eq!(a.growth, SetGrowth::GrowthSealed as i32);
1046 assert_eq!(a.free_bytes, 70 * GIB);
1048 assert_eq!(a.floor_bytes, 15 * GIB, "5 % of 300 GiB");
1049 assert_eq!(a.servable_bytes, 55 * GIB, "free minus the floor, not free");
1050 assert_eq!(a.additional_bytes, 90 * GIB, "the raise needs the DELTA, not the cap");
1051 assert_eq!(a.shortfall_bytes, 35 * GIB);
1052 assert_eq!(a.data_set, "4\u{d7}75G raid0 xfs");
1053 assert!(a.reason.contains("NO."), "{}", a.reason);
1055 assert!(a.reason.contains("UNUSED AND INVISIBLE"), "{}", a.reason);
1056 }
1057
1058 #[test]
1063 fn the_same_shortfall_on_a_box_with_a_shell_does_not_say_a_disk_is_wasted() {
1064 let mut sealed = question(gunnar_rs_today(), appliance());
1065 sealed.requested_bytes = 90 * GIB;
1066 let mut shelled = sealed.clone();
1067 shelled.growth = developer_box();
1068
1069 let a = answer(&sealed);
1070 let b = answer(&shelled);
1071 assert_eq!(a.verdict, b.verdict, "the shortfall is identical");
1072 assert_eq!(a.shortfall_bytes, b.shortfall_bytes);
1073 assert_ne!(a.growth, b.growth, "the growth answer is NOT the verdict");
1074 assert_ne!(a.reason, b.reason, "and it changes what the operator is told");
1075 assert!(b.reason.contains("NO.") && !b.reason.contains("UNUSED AND INVISIBLE"));
1076 }
1077
1078 #[test]
1081 fn a_request_that_fits_is_a_yes_that_still_says_the_set_is_sealed() {
1082 let mut q = question(gunnar_rs_today(), appliance());
1083 q.requested_bytes = 20 * GIB;
1084 let a = answer(&q);
1085 assert_eq!(a.verdict, CapacityVerdict::CapacityAvailable as i32);
1086 assert_eq!(a.shortfall_bytes, 0);
1087 assert!(a.reason.contains("YES"), "{}", a.reason);
1088 assert_eq!(
1089 a.growth,
1090 SetGrowth::GrowthSealed as i32,
1091 "the growth fact is reported on the YES arm too"
1092 );
1093 assert!(a.reason.contains("UNUSED AND INVISIBLE"), "{}", a.reason);
1094
1095 q.requested_bytes = 60 * GIB;
1098 assert_eq!(
1099 answer(&q).verdict,
1100 CapacityVerdict::CapacityExhausted as i32,
1101 "a request under free but over free-minus-floor must be refused: the floor is not \
1102 for sale"
1103 );
1104 q.requested_bytes = 55 * GIB;
1105 assert_eq!(answer(&q).verdict, CapacityVerdict::CapacityAvailable as i32);
1106 }
1107
1108 #[test]
1109 fn a_filesystem_that_cannot_be_measured_is_not_a_yes_and_reports_no_numbers() {
1110 let mut q = question(gunnar_rs_today(), appliance());
1111 q.room = Err("No such file or directory (os error 2)".into());
1112 q.requested_bytes = 1;
1113 let a = answer(&q);
1114 assert_eq!(a.verdict, CapacityVerdict::CapacityUnmeasured as i32);
1115 assert_ne!(
1116 a.verdict,
1117 CapacityVerdict::CapacityAvailable as i32,
1118 "one byte would fit on any real disk; an unmeasured one must still not say yes"
1119 );
1120 assert_eq!(
1121 (a.total_bytes, a.free_bytes, a.floor_bytes, a.servable_bytes),
1122 (0, 0, 0, 0),
1123 "nothing was measured, so nothing may be reported"
1124 );
1125 assert!(a.reason.contains("os error 2"), "{}", a.reason);
1126 assert!(a.reason.contains("has not said yes"), "{}", a.reason);
1127 }
1128
1129 #[test]
1134 fn the_appliance_box_is_reported_in_millicores_and_bytes_with_its_sources_named() {
1135 let q = question(gunnar_rs_today(), appliance());
1136 let a = answer(&q);
1137 assert_eq!(a.cpu_millicores_total, Some(4000));
1138 assert_eq!(a.ram_bytes_total, Some(16 * GIB));
1139 assert_eq!(a.ram_bytes_available, Some(12 * GIB));
1140 assert_eq!(a.cpu_millicores_busy, Some(420), "0.42 × 1000, rounded");
1141 for word in ["affinity mask", "/proc/meminfo", "/proc/loadavg", "APPROXIMATION"] {
1142 assert!(a.machine_reason.contains(word), "{word}: {}", a.machine_reason);
1143 }
1144 assert!(
1145 !a.machine_reason.contains("clamped"),
1146 "no cgroup clamped the appliance: {}",
1147 a.machine_reason
1148 );
1149
1150 let mut unmeasured = q.clone();
1153 unmeasured.room = Err("No such file or directory (os error 2)".into());
1154 let u = answer(&unmeasured);
1155 assert_eq!(
1156 (u.cpu_millicores_total, u.ram_bytes_total, u.ram_bytes_available, u.cpu_millicores_busy),
1157 (a.cpu_millicores_total, a.ram_bytes_total, a.ram_bytes_available, a.cpu_millicores_busy)
1158 );
1159 assert_eq!(u.machine_reason, a.machine_reason);
1160 }
1161
1162 #[test]
1166 fn an_unreadable_meminfo_leaves_the_ram_fields_absent_and_names_why() {
1167 let mut q = question(gunnar_rs_today(), appliance());
1168 q.machine.memory = Err("/proc/meminfo could not be read (No such file or directory (os error 2))".into());
1169 q.machine.load1 = Err("/proc/loadavg could not be read (Permission denied (os error 13))".into());
1170 let a = answer(&q);
1171 assert_eq!(a.ram_bytes_total, None, "absent, not 0: {a:?}");
1172 assert_eq!(a.ram_bytes_available, None);
1173 assert_eq!(a.cpu_millicores_busy, None);
1174 assert_ne!(a.ram_bytes_total, Some(0), "a zero here would be a box with no RAM");
1175 assert_eq!(a.cpu_millicores_total, Some(4000), "the cores were measured and stay");
1176 assert!(a.machine_reason.contains("RAM UNMEASURED"), "{}", a.machine_reason);
1177 assert!(a.machine_reason.contains("os error 2"), "{}", a.machine_reason);
1178 assert!(a.machine_reason.contains("load UNMEASURED"), "{}", a.machine_reason);
1179 assert!(a.machine_reason.contains("os error 13"), "{}", a.machine_reason);
1180
1181 q.machine.cpus = Err("the affinity mask could not be read (os error 22)".into());
1183 let none = answer(&q);
1184 assert_eq!(
1185 (none.cpu_millicores_total, none.ram_bytes_total, none.ram_bytes_available, none.cpu_millicores_busy),
1186 (None, None, None, None)
1187 );
1188 assert!(none.machine_reason.contains("cores UNMEASURED"), "{}", none.machine_reason);
1189 assert_eq!(none.verdict, a.verdict);
1191 assert_eq!(none.reason, a.reason);
1192 }
1193
1194 #[test]
1197 fn a_cgroup_memory_clamp_is_reported_as_the_ram_and_named_in_the_reason() {
1198 let mut q = question(gunnar_rs_today(), appliance());
1199 q.machine.memory = Ok((8 * GIB, 6 * GIB));
1200 q.machine.cgroup_memory_max = Some(8 * GIB);
1201 let a = answer(&q);
1202 assert_eq!(a.ram_bytes_total, Some(8 * GIB));
1203 assert!(a.machine_reason.contains("clamped to the cgroup memory.max of 8.0 GiB"), "{}", a.machine_reason);
1204 }
1205
1206 #[test]
1211 fn the_disk_answer_does_not_change_with_the_box() {
1212 let q = question(gunnar_rs_today(), appliance());
1213 let mut blind = q.clone();
1214 blind.machine = MachineFacts {
1215 cpus: Err("x".into()),
1216 memory: Err("y".into()),
1217 cgroup_memory_max: None,
1218 load1: Err("z".into()),
1219 disk: None,
1220 };
1221 let (a, b) = (answer(&q), answer(&blind));
1222 assert_ne!(a.machine_reason, b.machine_reason, "the fixtures differ");
1223 assert_ne!(a.ram_bytes_total, b.ram_bytes_total);
1224 let disk = |r: &CapacityResponse| {
1231 CapacityResponse {
1232 cpu_millicores_total: None,
1233 ram_bytes_total: None,
1234 ram_bytes_available: None,
1235 cpu_millicores_busy: None,
1236 machine_reason: String::new(),
1237 disk_test: 0,
1238 disk_seq_write_bps: None,
1239 disk_seq_read_bps: None,
1240 disk_sync_milliops_per_s: None,
1241 disk_read_how: String::new(),
1242 disk_reason: String::new(),
1243 disk_test_bytes: None,
1246 disk_write_us: None,
1247 disk_read_us: None,
1248 disk_sync_ops: None,
1249 disk_sync_us: None,
1250 disk_test_us: None,
1251 ..r.clone()
1252 }
1253 };
1254 assert_eq!(disk(&a), disk(&b));
1255 assert_ne!(a.disk_test, b.disk_test);
1258 assert_ne!(a.disk_reason, b.disk_reason);
1259 }
1260}
1261
1262#[cfg(test)]
1270mod disk_on_the_wire {
1271 use super::*;
1272 use crate::disktest::{Outcome, ReadHow, Report};
1273 use gunnar_proto::DiskTest;
1274 use std::time::Duration;
1275
1276 fn done(seq_read_bps: f64) -> Outcome {
1277 Outcome::Done(Report {
1278 seq_write_bps: 812e6,
1279 seq_read_bps,
1280 read_how: ReadHow::Direct,
1281 sync_ops_per_s: 3100.4,
1282 elapsed: Duration::from_millis(1600),
1283 file_bytes: 256 << 20,
1284 write_elapsed: Duration::from_millis(330),
1285 read_elapsed: Duration::from_millis(141),
1286 sync_ops: 1550,
1287 sync_elapsed: Duration::from_millis(500),
1288 })
1289 }
1290
1291 #[test]
1293 fn a_measured_disk_carries_its_three_numbers_and_says_how_it_read() {
1294 let a = disk_fields(&Some(done(1.9e9)));
1295 assert_eq!(a.disk_test, DiskTest::DisktestMeasured as i32);
1296 assert_eq!(a.disk_seq_write_bps, Some(812_000_000));
1297 assert_eq!(a.disk_seq_read_bps, Some(1_900_000_000));
1298 assert_eq!(a.disk_sync_milliops_per_s, Some(3_100_400));
1300 assert_eq!(a.disk_read_how, "direct");
1301 assert!(a.disk_reason.contains("812 MB/s"), "{}", a.disk_reason);
1302 assert!(a.disk_reason.contains("1.9 GB/s"), "{}", a.disk_reason);
1303 assert_eq!(a.disk_test_bytes, Some(256 << 20));
1305 assert_eq!((a.disk_write_us, a.disk_read_us), (Some(330_000), Some(141_000)));
1306 assert_eq!((a.disk_sync_ops, a.disk_sync_us), (Some(1550), Some(500_000)));
1307 assert_eq!(a.disk_test_us, Some(1_600_000));
1308 }
1309
1310 #[test]
1315 fn a_cached_read_is_absent_from_the_wire_and_is_never_reported_as_a_fast_disk() {
1316 let a = disk_fields(&Some(done(7.2e9)));
1317 assert_eq!(a.disk_test, DiskTest::DisktestMeasured as i32, "the run DID measure");
1318 assert_eq!(a.disk_seq_read_bps, None, "not 7.2 GB/s and not 0 — absent");
1319 assert_eq!(a.disk_seq_write_bps, Some(812_000_000), "the other legs stand");
1320 assert_eq!(a.disk_sync_milliops_per_s, Some(3_100_400));
1321 assert!(a.disk_reason.contains("CACHED"), "{}", a.disk_reason);
1322 assert!(
1323 a.disk_reason.contains("not a fast disk"),
1324 "the reason says what the absence MEANS: {}",
1325 a.disk_reason
1326 );
1327 }
1328
1329 #[test]
1332 fn a_disk_that_was_not_measured_is_named_and_carries_no_number_at_all() {
1333 let cases = [
1334 (None, DiskTest::DisktestNotRun, "gate"),
1335 (
1336 Some(Outcome::Skipped { free: 800 << 20, need: 1 << 30 }),
1337 DiskTest::DisktestSkipped,
1338 "SKIPPED",
1339 ),
1340 (
1341 Some(Outcome::Failed("write: No space left on device".into())),
1342 DiskTest::DisktestFailed,
1343 "FAILED",
1344 ),
1345 ];
1346 for (outcome, want, word) in cases {
1347 let a = disk_fields(&outcome);
1348 assert_eq!(a.disk_test, want as i32, "{outcome:?}");
1349 assert_eq!(a.disk_seq_write_bps, None, "{outcome:?}");
1350 assert_eq!(a.disk_seq_read_bps, None, "{outcome:?}");
1351 assert_eq!(a.disk_sync_milliops_per_s, None, "{outcome:?}");
1352 assert!(a.disk_read_how.is_empty(), "{outcome:?}");
1353 assert_eq!(
1354 (a.disk_test_bytes, a.disk_write_us, a.disk_read_us, a.disk_sync_ops, a.disk_sync_us, a.disk_test_us),
1355 (None, None, None, None, None, None),
1356 "no run, no bytes and no times: {outcome:?}"
1357 );
1358 assert!(!a.disk_reason.is_empty(), "never silent: {outcome:?}");
1359 assert!(a.disk_reason.contains(word), "{}", a.disk_reason);
1360 }
1361 let s = disk_fields(&Some(Outcome::Skipped { free: 800 << 20, need: 1 << 30 }));
1363 assert!(s.disk_reason.contains("800") && s.disk_reason.contains("1.0"), "{}", s.disk_reason);
1364 let reasons: std::collections::BTreeSet<String> = [
1366 disk_fields(&None).disk_reason,
1367 disk_fields(&Some(Outcome::Skipped { free: 0, need: 1 })).disk_reason,
1368 disk_fields(&Some(Outcome::Failed("x".into()))).disk_reason,
1369 ]
1370 .into_iter()
1371 .collect();
1372 assert_eq!(reasons.len(), 3);
1373 }
1374
1375 #[test]
1378 fn the_box_fields_and_the_disk_fields_come_out_of_one_call() {
1379 let m = MachineFacts {
1380 cpus: Ok(4),
1381 memory: Ok((16 << 30, 12 << 30)),
1382 cgroup_memory_max: None,
1383 load1: Ok(0.42),
1384 disk: Some(done(1.9e9)),
1385 };
1386 let a = machine_fields(&m);
1387 assert_eq!(a.cpu_millicores_total, Some(4000));
1388 assert_eq!(a.disk_seq_write_bps, Some(812_000_000));
1389 assert!(!a.machine_reason.is_empty() && !a.disk_reason.is_empty());
1390 let blind = machine_fields(&MachineFacts { disk: None, ..m });
1392 assert_eq!(blind.cpu_millicores_total, Some(4000));
1393 assert_eq!(blind.disk_test, DiskTest::DisktestNotRun as i32);
1394 }
1395
1396 #[test]
1399 fn the_latch_hands_back_what_was_recorded_and_runs_nothing() {
1400 let before = crate::disktest::last();
1401 crate::disktest::record(&done(1.9e9));
1402 assert_eq!(crate::disktest::last(), Some(done(1.9e9)));
1403 crate::disktest::record(&Outcome::Failed("boom".into()));
1404 assert_eq!(crate::disktest::last(), Some(Outcome::Failed("boom".into())));
1405 if let Some(b) = before {
1407 crate::disktest::record(&b);
1408 }
1409 }
1410}
1411
1412#[cfg(test)]
1413mod disk_tests {
1414 use super::*;
1415
1416 #[test]
1420 fn a_snapshot_reaches_the_wire_and_an_unmeasured_mount_is_not_a_number() {
1421 let mount = |src: &str, at: &str| disk_snapshot::Mount {
1422 source: src.to_owned(),
1423 mount_point: PathBuf::from(at),
1424 fstype: "xfs".to_owned(),
1425 read_only: false,
1426 };
1427 let snap = disk_snapshot::Snapshot {
1428 disks: vec![
1429 disk_snapshot::DiskEntry {
1430 mount: mount("/dev/md0", "/data"),
1431 labels: vec!["data".to_owned()],
1432 paths: vec![PathBuf::from("/data")],
1433 figures: disk_snapshot::Figures::from_counts(1000, 400, 350, 4096, 10, 5),
1434 floor_bytes: 4096 * 50,
1435 servable_bytes: 4096 * 300,
1436 },
1437 disk_snapshot::DiskEntry {
1438 mount: mount("/dev/vda2", "/"),
1439 labels: vec!["system".to_owned()],
1440 paths: vec![PathBuf::from("/")],
1441 figures: Err("statvfs(/): EIO".to_owned()),
1442 floor_bytes: 0,
1443 servable_bytes: 0,
1444 },
1445 ],
1446 unresolved: vec![],
1447 block_devices: vec![disk_snapshot::BlockDevice {
1448 name: "md0".to_owned(),
1449 size_bytes: 4096 * 1000,
1450 members: vec!["vdb1".to_owned(), "vdc1".to_owned()],
1451 raid_level: "raid0".to_owned(),
1452 ..Default::default()
1453 }],
1454 problems: vec![],
1455 };
1456 let wire = disks_to_wire(snap, 42);
1457 assert_eq!(wire.measured_at_unix_ms, 42);
1458 let data = &wire.mounts[0];
1459 assert!(data.measured && data.labels == ["data"]);
1460 assert_eq!(
1461 (data.total_bytes, data.free_bytes, data.available_bytes, data.used_bytes, data.reserved_bytes),
1462 (4096 * 1000, 4096 * 400, 4096 * 350, 4096 * 600, 4096 * 50)
1463 );
1464 assert_eq!((data.floor_bytes, data.servable_bytes), (4096 * 50, 4096 * 300));
1465 let system = &wire.mounts[1];
1466 assert!(!system.measured && system.error.contains("EIO") && system.total_bytes == 0);
1467 assert_eq!(wire.devices[0].members, ["vdb1", "vdc1"]);
1468 assert_eq!(wire.devices[0].raid_level, "raid0");
1469 }
1470
1471 #[test]
1476 fn the_serialdat_breakdown_sums_to_the_mounts_use() {
1477 use std::io::Write as _;
1478 let dir = tempfile::tempdir().expect("a temp mount");
1479 let root = dir.path();
1480 let write = |rel: &str, n: usize| {
1481 let p = root.join(rel);
1482 std::fs::create_dir_all(p.parent().unwrap()).unwrap();
1483 let mut f = std::fs::File::create(&p).unwrap();
1484 f.write_all(&vec![7u8; n]).unwrap();
1485 f.sync_all().unwrap();
1486 };
1487 write("meta/principals.json", 10_000);
1488 write("log/audit-chain.jsonl", 70_000);
1489 write("log/console.log", 30_000);
1490 write("crash/last.txt", 5_000);
1491 let known = vec![
1492 ("metadata copy", root.join("meta")),
1493 ("crash record", root.join("crash")),
1494 ("audit trail", root.join("log/audit-chain.jsonl")),
1495 ("logs", root.join("log")),
1496 ("grow journal", root.join("grow-journal")),
1497 ];
1498 let used = 10 << 20;
1499 let rows = serialdat_breakdown(&known, Some(used));
1500 let get = |label: &str| rows.iter().find(|r| r.label == label).unwrap_or_else(|| panic!("{label}: {rows:?}"));
1501 assert!(rows.iter().all(|r| r.measured), "{rows:?}");
1502 assert!(get("audit trail").bytes >= 70_000, "{rows:?}");
1503 let logs = get("logs").bytes;
1504 assert!((30_000..70_000).contains(&logs), "the audit trail was counted inside the logs too: {rows:?}");
1505 assert_eq!(get("grow journal").bytes, 0, "nothing there is 0, not an error");
1506 let sum: u64 = rows.iter().map(|r| r.bytes).sum();
1507 assert_eq!(sum, used, "the rows must add up to the mount's used bytes: {rows:?}");
1508 assert!(serialdat_breakdown(&known, None).iter().all(|r| r.label != "other"));
1510 }
1511
1512 #[test]
1515 fn this_machines_disks_carry_gunnars_labels() {
1516 let dir = tempfile::tempdir().expect("a temp dir");
1517 let wire = measure_disks(dir.path(), None, gunnar_ssh::DiskFloor::default());
1518 assert!(wire.problems.is_empty(), "{:?}", wire.problems);
1519 let data = wire
1520 .mounts
1521 .iter()
1522 .find(|m| m.labels.iter().any(|l| l == "data"))
1523 .expect("the data root is on some mount");
1524 assert!(data.measured && data.total_bytes > 0, "{data:?}");
1525 assert!(
1526 wire.mounts.iter().any(|m| m.labels.iter().any(|l| l == "system")),
1527 "{:?}",
1528 wire.mounts
1529 );
1530 }
1531}