helixforge_engine/
reliability.rs

1//! How far each entry has been tested: real installs in distribution
2//! containers (recorded by the Distributions workflow and carried by release
3//! builds), installer tests with stand-in commands, and reports from virtual
4//! and real machines (`catalog/validation.toml`), with this machine's own
5//! runs beside them. Nothing here claims more than its record shows.
6
7use crate::history;
8use helixforge_core::{Action, Entry, Risk};
9use serde::Deserialize;
10use std::{
11    collections::BTreeMap,
12    path::{Path, PathBuf},
13};
14
15/// How far an entry has been validated, best first.
16#[derive(Clone, Copy, Debug, Eq, Ord, PartialEq, PartialOrd, serde::Serialize)]
17#[serde(rename_all = "kebab-case")]
18pub enum Level {
19    /// Run on a real machine, from a tester's report.
20    HardwareTested,
21    /// Run in a virtual machine, from a report.
22    VmTested,
23    /// Installed (or run) and verified in a distribution container by the
24    /// Distributions workflow.
25    ContainerTested,
26    /// Its installer was run against stand-in commands and files.
27    StandInTested,
28    NotYetValidated,
29}
30
31impl Level {
32    pub fn label(self) -> &'static str {
33        match self {
34            Level::HardwareTested => "Hardware Tested",
35            Level::VmTested => "VM Tested",
36            Level::ContainerTested => "Container Tested",
37            Level::StandInTested => "Stand-in Tested",
38            Level::NotYetValidated => "Not Yet Validated",
39        }
40    }
41
42    pub fn explain(self) -> &'static str {
43        match self {
44            Level::HardwareTested => "A tester ran it on a real machine and reported the result.",
45            Level::VmTested => {
46                "It ran for real in a virtual machine: the Distributions workflow boots the distribution's cloud image, runs the entry, reboots into the result and checks it (the configuration, the kernel command line, the boot images, the services, a graphical boot menu or login screen). Real graphics hardware is not covered."
47            }
48            Level::ContainerTested => {
49                "The Distributions workflow ran it in distribution containers and HelixForge verified the result there. Containers have no boot loader, login screen or desktop, so those parts are not covered."
50            }
51            Level::StandInTested => {
52                "Installer tests run its script, or the helpers it calls, against stand-in commands and files, on every change and in each distribution's container. They show what it would change, without changing a system."
53            }
54            Level::NotYetValidated => {
55                "No automated test runs it and nobody has reported running it. Its plan is built and checked, like every entry's."
56            }
57        }
58    }
59}
60
61/// `catalog/validation.toml`.
62#[derive(Clone, Debug, Default, Deserialize)]
63pub struct Claims {
64    #[serde(default)]
65    pub critical: Critical,
66    #[serde(default, rename = "stand-ins")]
67    pub stand_ins: Vec<StandIns>,
68    #[serde(default)]
69    pub hardware: Vec<Report>,
70    #[serde(default)]
71    pub vm: Vec<Report>,
72}
73
74#[derive(Clone, Debug, Default, Deserialize)]
75pub struct Critical {
76    #[serde(default)]
77    pub entries: Vec<String>,
78}
79
80/// Installer tests that cover entries with stand-ins.
81#[derive(Clone, Debug, Default, Deserialize)]
82pub struct StandIns {
83    pub entries: Vec<String>,
84    pub tests: Vec<String>,
85    #[serde(default)]
86    pub covers: String,
87}
88
89/// A run reported from a virtual or real machine.
90#[derive(Clone, Debug, Default, Deserialize)]
91pub struct Report {
92    pub entry: String,
93    pub system: String,
94    #[serde(default)]
95    pub version: String,
96    pub date: String,
97    #[serde(default)]
98    pub note: String,
99}
100
101/// What a Distributions run verified in each container, as release builds
102/// carry it (`catalog/validation-results.json`).
103#[derive(Clone, Debug, Default, Deserialize)]
104pub struct Results {
105    #[serde(default)]
106    pub commit: String,
107    #[serde(default)]
108    pub run: u64,
109    #[serde(default)]
110    pub url: String,
111    /// When the run finished, in Unix seconds.
112    #[serde(default)]
113    pub finished: u64,
114    /// By container image: each entry and what was shown (`verified`, or
115    /// `ran` for an entry with nothing to verify).
116    #[serde(default)]
117    pub images: BTreeMap<String, Image>,
118    /// By virtual machine (arch, debian): the entries run there, rebooted
119    /// into and checked by scripts/vm-tests.py.
120    #[serde(default)]
121    pub vms: BTreeMap<String, Image>,
122}
123
124#[derive(Clone, Debug, Default, Deserialize)]
125pub struct Image {
126    #[serde(default)]
127    pub entries: BTreeMap<String, String>,
128}
129
130/// Everything known about testing, from this build's catalog.
131#[derive(Clone, Debug, Default)]
132pub struct Evidence {
133    pub claims: Claims,
134    pub results: Option<Results>,
135    /// Why a file could not be read, if one could not.
136    pub problems: Vec<String>,
137}
138
139/// One entry's standing.
140#[derive(Clone, Debug, Eq, PartialEq, serde::Serialize)]
141pub struct Assessment {
142    pub level: Level,
143    /// Where the tests ran: container images or reported systems.
144    pub systems: Vec<String>,
145    /// When it was last verified, as a date, when known.
146    pub verified: Option<String>,
147    /// What shows it, line by line.
148    pub evidence: Vec<String>,
149    /// Its last successful run on this machine, in Unix seconds.
150    pub here: Option<u64>,
151    /// A boot, login-screen, driver or whole-system entry.
152    pub critical: bool,
153}
154
155/// A virtual machine as people name it: "Arch VM".
156pub fn vm_label(name: &str) -> String {
157    match name {
158        "arch" => "Arch VM".to_string(),
159        "debian" => "Debian VM".to_string(),
160        "fedora" => "Fedora VM".to_string(),
161        other => format!("{} VM", helixforge_core::printable(other)),
162    }
163}
164
165/// A container image as people name the system.
166pub fn image_label(image: &str) -> String {
167    match image {
168        "archlinux:latest" => "Arch".to_string(),
169        "debian:stable" => "Debian".to_string(),
170        "fedora:latest" => "Fedora".to_string(),
171        "opensuse/tumbleweed:latest" => "openSUSE Tumbleweed".to_string(),
172        other => match other.split_once(':') {
173            Some(("ubuntu", version)) => format!("Ubuntu {version}"),
174            _ => helixforge_core::printable(other),
175        },
176    }
177}
178
179impl Evidence {
180    /// Read this build's record: `catalog/validation.toml` and, in a
181    /// release build, `catalog/validation-results.json`.
182    pub fn load(root: &Path) -> Self {
183        let mut evidence = Evidence::default();
184        let catalog = root.join("catalog");
185        let claims = catalog.join("validation.toml");
186        match std::fs::read_to_string(&claims) {
187            Ok(text) => match toml::from_str(&text) {
188                Ok(claims) => evidence.claims = claims,
189                Err(error) => evidence.problems.push(format!(
190                    "{}: {}",
191                    claims.display(),
192                    helixforge_core::printable(error.message())
193                )),
194            },
195            Err(error) if error.kind() == std::io::ErrorKind::NotFound => {}
196            Err(error) => evidence
197                .problems
198                .push(format!("{}: {error}", claims.display())),
199        }
200        let results: PathBuf = catalog.join("validation-results.json");
201        if let Ok(text) = std::fs::read_to_string(&results) {
202            match serde_json::from_str(&text) {
203                Ok(results) => evidence.results = Some(results),
204                Err(error) => evidence
205                    .problems
206                    .push(format!("{}: {error}", results.display())),
207            }
208        }
209        evidence
210    }
211
212    /// Whether failing would leave the machine without a working boot,
213    /// login screen or display, or the entry changes disks or the whole
214    /// system.
215    pub fn critical(&self, entry: &Entry) -> bool {
216        Risk::from_flags(&entry.flags) == Risk::High
217            || self.claims.critical.entries.contains(&entry.id)
218    }
219
220    /// How an entry stands; `history` is this machine's, newest first.
221    pub fn assess(&self, entry: &Entry, history: &[history::Record]) -> Assessment {
222        let here = history
223            .iter()
224            .find(|record| record.success && record.entries.contains(&entry.id))
225            .map(|record| record.unix_timestamp);
226        let mut assessment = Assessment {
227            level: Level::NotYetValidated,
228            systems: Vec::new(),
229            verified: None,
230            evidence: Vec::new(),
231            here,
232            critical: self.critical(entry),
233        };
234        let better = |assessment: &mut Assessment, level: Level| {
235            if level < assessment.level {
236                assessment.level = level;
237            }
238        };
239        for (kind, reports, level) in [
240            (
241                "a real machine",
242                &self.claims.hardware,
243                Level::HardwareTested,
244            ),
245            ("a virtual machine", &self.claims.vm, Level::VmTested),
246        ] {
247            for report in reports.iter().filter(|report| report.entry == entry.id) {
248                better(&mut assessment, level);
249                let system = helixforge_core::printable(&report.system);
250                if !assessment.systems.contains(&system) {
251                    assessment.systems.push(system.clone());
252                }
253                let date = helixforge_core::printable(&report.date);
254                if assessment
255                    .verified
256                    .as_ref()
257                    .is_none_or(|known| *known < date)
258                {
259                    assessment.verified = Some(date.clone());
260                }
261                let mut line = format!("Ran on {kind}: {system}, {date}");
262                if !report.version.is_empty() {
263                    line.push_str(&format!(
264                        ", HelixForge {}",
265                        helixforge_core::printable(&report.version)
266                    ));
267                }
268                if !report.note.is_empty() {
269                    line.push_str(&format!(" ({})", helixforge_core::printable(&report.note)));
270                }
271                assessment.evidence.push(line);
272            }
273        }
274        if let Some(results) = &self.results {
275            let date = crate::date(results.finished);
276            let machines: Vec<String> = results
277                .vms
278                .iter()
279                .filter(|(_, found)| {
280                    found
281                        .entries
282                        .get(&entry.id)
283                        .is_some_and(|outcome| outcome == "verified")
284                })
285                .map(|(name, _)| vm_label(name))
286                .collect();
287            if !machines.is_empty() {
288                better(&mut assessment, Level::VmTested);
289                for machine in &machines {
290                    if !assessment.systems.contains(machine) {
291                        assessment.systems.push(machine.clone());
292                    }
293                }
294                if assessment
295                    .verified
296                    .as_ref()
297                    .is_none_or(|known| *known < date)
298                {
299                    assessment.verified = Some(date.clone());
300                }
301                assessment.evidence.push(format!(
302                    "In virtual machines, Distributions run {} on {date}: {}; run for real, rebooted into and checked",
303                    results.run,
304                    machines.join(", ")
305                ));
306            }
307            let mut shown: Vec<String> = Vec::new();
308            for (image, found) in &results.images {
309                if let Some(outcome) = found.entries.get(&entry.id) {
310                    shown.push(format!(
311                        "{} ({})",
312                        image_label(image),
313                        if outcome == "verified" {
314                            "installed and verified"
315                        } else {
316                            "ran"
317                        }
318                    ));
319                    let label = image_label(image);
320                    if !assessment.systems.contains(&label) {
321                        assessment.systems.push(label);
322                    }
323                }
324            }
325            if !shown.is_empty() {
326                better(&mut assessment, Level::ContainerTested);
327                if assessment
328                    .verified
329                    .as_ref()
330                    .is_none_or(|known| *known < date)
331                {
332                    assessment.verified = Some(date.clone());
333                }
334                assessment.evidence.push(format!(
335                    "In containers, Distributions run {} on {date}: {}",
336                    results.run,
337                    shown.join(", ")
338                ));
339            }
340        }
341        for stand_ins in self
342            .claims
343            .stand_ins
344            .iter()
345            .filter(|group| group.entries.contains(&entry.id))
346        {
347            better(&mut assessment, Level::StandInTested);
348            let mut line = format!(
349                "Installer tests with stand-ins ({}): {}",
350                stand_ins.tests.join(", "),
351                helixforge_core::printable(&stand_ins.covers)
352            );
353            if let Some(results) = &self.results {
354                let date = crate::date(results.finished);
355                line.push_str(&format!(
356                    "; they ran in all {} containers on {date}",
357                    results.images.len()
358                ));
359                if assessment.level == Level::StandInTested {
360                    assessment.verified = Some(date);
361                    for image in results.images.keys() {
362                        let label = image_label(image);
363                        if !assessment.systems.contains(&label) {
364                            assessment.systems.push(label);
365                        }
366                    }
367                }
368            }
369            assessment.evidence.push(line);
370        }
371        if let Action::Package { .. } = entry.action {
372            assessment.evidence.push(
373                "Its commands are built by HelixForge from package names, and tested against harmless stand-ins on every change.".to_string(),
374            );
375        }
376        if let Some(when) = here {
377            assessment.evidence.push(format!(
378                "Ran successfully on this machine, {}",
379                crate::date(when)
380            ));
381        }
382        assessment
383    }
384
385    /// A line on the record this build carries, for the screen's header.
386    pub fn summary(&self) -> String {
387        match &self.results {
388            Some(results) => format!(
389                "Container record: Distributions run {} on commit {}, {}, {} distributions{}.",
390                results.run,
391                results.commit.chars().take(12).collect::<String>(),
392                crate::date(results.finished),
393                results.images.len(),
394                match results.vms.len() {
395                    0 => String::new(),
396                    count => format!(" and {count} virtual machines"),
397                }
398            ),
399            None => "No container record in this build; release builds carry one.".to_string(),
400        }
401    }
402}
403
404/// Warnings for a plan's entries that are critical and not validated past
405/// stand-ins, shown before it runs.
406pub fn warnings<'a>(
407    ids: impl IntoIterator<Item = &'a str>,
408    lookup: &dyn Fn(&str) -> Option<&'a Entry>,
409    evidence: &Evidence,
410) -> Vec<String> {
411    let mut warnings = Vec::new();
412    for id in ids {
413        let Some(entry) = lookup(id) else { continue };
414        if !evidence.critical(entry) {
415            continue;
416        }
417        let assessment = evidence.assess(entry, &[]);
418        if assessment.level >= Level::ContainerTested {
419            warnings.push(format!(
420                "{} is {}: {}",
421                entry.name,
422                assessment.level.label(),
423                match assessment.level {
424                    Level::NotYetValidated => "nothing has tested it yet",
425                    Level::StandInTested => "only tested with stand-ins, never on a real system",
426                    _ => "containers cannot test boot, login or drivers",
427                }
428            ));
429        }
430    }
431    warnings
432}
433
434#[cfg(test)]
435mod tests {
436    use super::*;
437    use helixforge_catalog::Catalog;
438
439    fn evidence(claims: &str, results: Option<&str>) -> Evidence {
440        Evidence {
441            claims: toml::from_str(claims).unwrap(),
442            results: results.map(|text| serde_json::from_str(text).unwrap()),
443            problems: Vec::new(),
444        }
445    }
446
447    const RESULTS: &str = r#"{"format": 1, "commit": "0123456789abcdef0123", "run": 42,
448        "finished": 1792172552,
449        "images": {"archlinux:latest": {"entries": {"htop": "verified"}},
450                   "ubuntu:24.04": {"entries": {"htop": "verified", "system-report": "ran"}}},
451        "vms": {"arch": {"entries": {"sddm": "verified"}}}}"#;
452
453    #[test]
454    fn levels_come_from_the_record_never_beyond_it() {
455        let catalog = Catalog::builtin(false).unwrap();
456        let found = evidence(
457            r#"
458            [critical]
459            entries = ["sddm", "grub"]
460            [[stand-ins]]
461            entries = ["sddm"]
462            tests = ["test_sddm"]
463            covers = "the theme"
464            [[hardware]]
465            entry = "grub"
466            system = "Garuda Linux"
467            date = "2026-10-20"
468            "#,
469            Some(RESULTS),
470        );
471        let assess = |id: &str| found.assess(catalog.find(id).unwrap(), &[]);
472        let htop = assess("htop");
473        assert_eq!(htop.level, Level::ContainerTested);
474        assert_eq!(htop.systems, ["Arch", "Ubuntu 24.04"]);
475        assert_eq!(htop.verified.as_deref(), Some("2026-10-16"));
476        assert!(!htop.critical);
477        assert_eq!(assess("system-report").level, Level::ContainerTested);
478        // Verified in a virtual machine: above its stand-in tests.
479        let sddm = assess("sddm");
480        assert_eq!(sddm.level, Level::VmTested);
481        assert!(sddm.critical);
482        assert!(
483            sddm.systems.contains(&"Arch VM".to_string()),
484            "{:?}",
485            sddm.systems
486        );
487        assert!(
488            sddm.evidence[0].contains("virtual machines"),
489            "{:?}",
490            sddm.evidence
491        );
492        assert!(sddm.evidence.iter().any(|line| line.contains("test_sddm")));
493        assert!(
494            found.summary().contains("1 virtual machines"),
495            "{}",
496            found.summary()
497        );
498        let grub = assess("grub");
499        assert_eq!(grub.level, Level::HardwareTested);
500        assert_eq!(grub.verified.as_deref(), Some("2026-10-20"));
501        assert_eq!(assess("ripgrep").level, Level::NotYetValidated);
502        // This machine's runs show beside the level, never raise it.
503        let ran = [history::Record {
504            unix_timestamp: 1_700_000_000,
505            entries: vec!["ripgrep".into()],
506            names: Vec::new(),
507            success: true,
508            result: String::new(),
509            log: None,
510        }];
511        let ripgrep = found.assess(catalog.find("ripgrep").unwrap(), &ran);
512        assert_eq!(ripgrep.level, Level::NotYetValidated);
513        assert_eq!(ripgrep.here, Some(1_700_000_000));
514
515        // Before a run, critical entries short of a real test are named.
516        // SDDM, tested in a VM, and GRUB, on hardware, are not.
517        let warned = warnings(
518            ["sddm", "grub", "htop", "arch-server-setup"],
519            &|id| catalog.find(id),
520            &found,
521        );
522        assert_eq!(warned.len(), 1, "{warned:?}");
523        assert!(warned[0].starts_with("Arch Server Setup"), "{warned:?}");
524        // Whole-system entries count as critical by their risk.
525        assert!(found.critical(catalog.find("arch-server-setup").unwrap()));
526    }
527
528    #[test]
529    fn a_build_without_a_record_says_so() {
530        let found = evidence("", None);
531        assert!(found.summary().contains("No container record"));
532        let catalog = Catalog::builtin(false).unwrap();
533        assert_eq!(
534            found.assess(catalog.find("htop").unwrap(), &[]).level,
535            Level::NotYetValidated
536        );
537    }
538
539    #[test]
540    fn the_catalogs_own_record_reads_and_names_real_entries() {
541        let catalog = Catalog::builtin(false).unwrap();
542        let found = Evidence::load(catalog.root());
543        assert!(found.problems.is_empty(), "{:?}", found.problems);
544        assert!(!found.claims.stand_ins.is_empty());
545        for id in found
546            .claims
547            .stand_ins
548            .iter()
549            .flat_map(|group| group.entries.iter())
550            .chain(found.claims.critical.entries.iter())
551            .chain(found.claims.hardware.iter().map(|report| &report.entry))
552            .chain(found.claims.vm.iter().map(|report| &report.entry))
553        {
554            assert!(catalog.find(id).is_some(), "validation.toml names {id}");
555        }
556    }
557}
helixforge_engine/reliability.rs

helixforge_engine/
reliability.rs
