helixforge_engine/
reliability.rs

1//! How far each entry has been tested: real installs in distribution
2//! containers (recorded by the Distributions workflow and carried by release
3//! builds), installer tests with stand-in commands, and reports from virtual
4//! and real machines (`catalog/validation.toml`), with this machine's own
5//! runs beside them. Nothing here claims more than its record shows.
6
7use crate::history;
8use helixforge_core::{Action, Entry, Risk};
9use serde::Deserialize;
10use std::{
11    collections::BTreeMap,
12    path::{Path, PathBuf},
13};
14
15/// How far an entry has been validated, best first.
16#[derive(Clone, Copy, Debug, Eq, Ord, PartialEq, PartialOrd, serde::Serialize)]
17#[serde(rename_all = "kebab-case")]
18pub enum Level {
19    /// Run on a real machine, from a tester's report.
20    HardwareTested,
21    /// Run in a virtual machine, from a report.
22    VmTested,
23    /// Installed (or run) and verified in a distribution container by the
24    /// Distributions workflow.
25    ContainerTested,
26    /// Its installer was run against stand-in commands and files.
27    StandInTested,
28    NotYetValidated,
29}
30
31impl Level {
32    pub fn label(self) -> &'static str {
33        match self {
34            Level::HardwareTested => "Hardware Tested",
35            Level::VmTested => "VM Tested",
36            Level::ContainerTested => "Container Tested",
37            Level::StandInTested => "Stand-in Tested",
38            Level::NotYetValidated => "Not Yet Validated",
39        }
40    }
41
42    pub fn explain(self) -> &'static str {
43        match self {
44            Level::HardwareTested => "A tester ran it on a real machine and reported the result.",
45            Level::VmTested => "It was run in a virtual machine and the result reported.",
46            Level::ContainerTested => {
47                "The Distributions workflow ran it in distribution containers and HelixForge verified the result there. Containers have no boot loader, login screen or desktop, so those parts are not covered."
48            }
49            Level::StandInTested => {
50                "Installer tests run its script, or the helpers it calls, against stand-in commands and files, on every change and in each distribution's container. They show what it would change, without changing a system."
51            }
52            Level::NotYetValidated => {
53                "No automated test runs it and nobody has reported running it. Its plan is built and checked, like every entry's."
54            }
55        }
56    }
57}
58
59/// `catalog/validation.toml`.
60#[derive(Clone, Debug, Default, Deserialize)]
61pub struct Claims {
62    #[serde(default)]
63    pub critical: Critical,
64    #[serde(default, rename = "stand-ins")]
65    pub stand_ins: Vec<StandIns>,
66    #[serde(default)]
67    pub hardware: Vec<Report>,
68    #[serde(default)]
69    pub vm: Vec<Report>,
70}
71
72#[derive(Clone, Debug, Default, Deserialize)]
73pub struct Critical {
74    #[serde(default)]
75    pub entries: Vec<String>,
76}
77
78/// Installer tests that cover entries with stand-ins.
79#[derive(Clone, Debug, Default, Deserialize)]
80pub struct StandIns {
81    pub entries: Vec<String>,
82    pub tests: Vec<String>,
83    #[serde(default)]
84    pub covers: String,
85}
86
87/// A run reported from a virtual or real machine.
88#[derive(Clone, Debug, Default, Deserialize)]
89pub struct Report {
90    pub entry: String,
91    pub system: String,
92    #[serde(default)]
93    pub version: String,
94    pub date: String,
95    #[serde(default)]
96    pub note: String,
97}
98
99/// What a Distributions run verified in each container, as release builds
100/// carry it (`catalog/validation-results.json`).
101#[derive(Clone, Debug, Default, Deserialize)]
102pub struct Results {
103    #[serde(default)]
104    pub commit: String,
105    #[serde(default)]
106    pub run: u64,
107    #[serde(default)]
108    pub url: String,
109    /// When the run finished, in Unix seconds.
110    #[serde(default)]
111    pub finished: u64,
112    /// By container image: each entry and what was shown (`verified`, or
113    /// `ran` for an entry with nothing to verify).
114    #[serde(default)]
115    pub images: BTreeMap<String, Image>,
116}
117
118#[derive(Clone, Debug, Default, Deserialize)]
119pub struct Image {
120    #[serde(default)]
121    pub entries: BTreeMap<String, String>,
122}
123
124/// Everything known about testing, from this build's catalog.
125#[derive(Clone, Debug, Default)]
126pub struct Evidence {
127    pub claims: Claims,
128    pub results: Option<Results>,
129    /// Why a file could not be read, if one could not.
130    pub problems: Vec<String>,
131}
132
133/// One entry's standing.
134#[derive(Clone, Debug, Eq, PartialEq, serde::Serialize)]
135pub struct Assessment {
136    pub level: Level,
137    /// Where the tests ran: container images or reported systems.
138    pub systems: Vec<String>,
139    /// When it was last verified, as a date, when known.
140    pub verified: Option<String>,
141    /// What shows it, line by line.
142    pub evidence: Vec<String>,
143    /// Its last successful run on this machine, in Unix seconds.
144    pub here: Option<u64>,
145    /// A boot, login-screen, driver or whole-system entry.
146    pub critical: bool,
147}
148
149/// A container image as people name the system.
150pub fn image_label(image: &str) -> String {
151    match image {
152        "archlinux:latest" => "Arch".to_string(),
153        "debian:stable" => "Debian".to_string(),
154        "fedora:latest" => "Fedora".to_string(),
155        "opensuse/tumbleweed:latest" => "openSUSE Tumbleweed".to_string(),
156        other => match other.split_once(':') {
157            Some(("ubuntu", version)) => format!("Ubuntu {version}"),
158            _ => helixforge_core::printable(other),
159        },
160    }
161}
162
163impl Evidence {
164    /// Read this build's record: `catalog/validation.toml` and, in a
165    /// release build, `catalog/validation-results.json`.
166    pub fn load(root: &Path) -> Self {
167        let mut evidence = Evidence::default();
168        let catalog = root.join("catalog");
169        let claims = catalog.join("validation.toml");
170        match std::fs::read_to_string(&claims) {
171            Ok(text) => match toml::from_str(&text) {
172                Ok(claims) => evidence.claims = claims,
173                Err(error) => evidence.problems.push(format!(
174                    "{}: {}",
175                    claims.display(),
176                    helixforge_core::printable(error.message())
177                )),
178            },
179            Err(error) if error.kind() == std::io::ErrorKind::NotFound => {}
180            Err(error) => evidence
181                .problems
182                .push(format!("{}: {error}", claims.display())),
183        }
184        let results: PathBuf = catalog.join("validation-results.json");
185        if let Ok(text) = std::fs::read_to_string(&results) {
186            match serde_json::from_str(&text) {
187                Ok(results) => evidence.results = Some(results),
188                Err(error) => evidence
189                    .problems
190                    .push(format!("{}: {error}", results.display())),
191            }
192        }
193        evidence
194    }
195
196    /// Whether failing would leave the machine without a working boot,
197    /// login screen or display, or the entry changes disks or the whole
198    /// system.
199    pub fn critical(&self, entry: &Entry) -> bool {
200        Risk::from_flags(&entry.flags) == Risk::High
201            || self.claims.critical.entries.contains(&entry.id)
202    }
203
204    /// How an entry stands; `history` is this machine's, newest first.
205    pub fn assess(&self, entry: &Entry, history: &[history::Record]) -> Assessment {
206        let here = history
207            .iter()
208            .find(|record| record.success && record.entries.contains(&entry.id))
209            .map(|record| record.unix_timestamp);
210        let mut assessment = Assessment {
211            level: Level::NotYetValidated,
212            systems: Vec::new(),
213            verified: None,
214            evidence: Vec::new(),
215            here,
216            critical: self.critical(entry),
217        };
218        let better = |assessment: &mut Assessment, level: Level| {
219            if level < assessment.level {
220                assessment.level = level;
221            }
222        };
223        for (kind, reports, level) in [
224            (
225                "a real machine",
226                &self.claims.hardware,
227                Level::HardwareTested,
228            ),
229            ("a virtual machine", &self.claims.vm, Level::VmTested),
230        ] {
231            for report in reports.iter().filter(|report| report.entry == entry.id) {
232                better(&mut assessment, level);
233                let system = helixforge_core::printable(&report.system);
234                if !assessment.systems.contains(&system) {
235                    assessment.systems.push(system.clone());
236                }
237                let date = helixforge_core::printable(&report.date);
238                if assessment
239                    .verified
240                    .as_ref()
241                    .is_none_or(|known| *known < date)
242                {
243                    assessment.verified = Some(date.clone());
244                }
245                let mut line = format!("Ran on {kind}: {system}, {date}");
246                if !report.version.is_empty() {
247                    line.push_str(&format!(
248                        ", HelixForge {}",
249                        helixforge_core::printable(&report.version)
250                    ));
251                }
252                if !report.note.is_empty() {
253                    line.push_str(&format!(" ({})", helixforge_core::printable(&report.note)));
254                }
255                assessment.evidence.push(line);
256            }
257        }
258        if let Some(results) = &self.results {
259            let date = crate::date(results.finished);
260            let mut shown: Vec<String> = Vec::new();
261            for (image, found) in &results.images {
262                if let Some(outcome) = found.entries.get(&entry.id) {
263                    shown.push(format!(
264                        "{} ({})",
265                        image_label(image),
266                        if outcome == "verified" {
267                            "installed and verified"
268                        } else {
269                            "ran"
270                        }
271                    ));
272                    let label = image_label(image);
273                    if !assessment.systems.contains(&label) {
274                        assessment.systems.push(label);
275                    }
276                }
277            }
278            if !shown.is_empty() {
279                better(&mut assessment, Level::ContainerTested);
280                if assessment
281                    .verified
282                    .as_ref()
283                    .is_none_or(|known| *known < date)
284                {
285                    assessment.verified = Some(date.clone());
286                }
287                assessment.evidence.push(format!(
288                    "In containers, Distributions run {} on {date}: {}",
289                    results.run,
290                    shown.join(", ")
291                ));
292            }
293        }
294        for stand_ins in self
295            .claims
296            .stand_ins
297            .iter()
298            .filter(|group| group.entries.contains(&entry.id))
299        {
300            better(&mut assessment, Level::StandInTested);
301            let mut line = format!(
302                "Installer tests with stand-ins ({}): {}",
303                stand_ins.tests.join(", "),
304                helixforge_core::printable(&stand_ins.covers)
305            );
306            if let Some(results) = &self.results {
307                let date = crate::date(results.finished);
308                line.push_str(&format!(
309                    "; they ran in all {} containers on {date}",
310                    results.images.len()
311                ));
312                if assessment.level == Level::StandInTested {
313                    assessment.verified = Some(date);
314                    for image in results.images.keys() {
315                        let label = image_label(image);
316                        if !assessment.systems.contains(&label) {
317                            assessment.systems.push(label);
318                        }
319                    }
320                }
321            }
322            assessment.evidence.push(line);
323        }
324        if let Action::Package { .. } = entry.action {
325            assessment.evidence.push(
326                "Its commands are built by HelixForge from package names, and tested against harmless stand-ins on every change.".to_string(),
327            );
328        }
329        if let Some(when) = here {
330            assessment.evidence.push(format!(
331                "Ran successfully on this machine, {}",
332                crate::date(when)
333            ));
334        }
335        assessment
336    }
337
338    /// A line on the record this build carries, for the screen's header.
339    pub fn summary(&self) -> String {
340        match &self.results {
341            Some(results) => format!(
342                "Container record: Distributions run {} on commit {}, {}, {} distributions.",
343                results.run,
344                results.commit.chars().take(12).collect::<String>(),
345                crate::date(results.finished),
346                results.images.len()
347            ),
348            None => "No container record in this build; release builds carry one.".to_string(),
349        }
350    }
351}
352
353/// Warnings for a plan's entries that are critical and not validated past
354/// stand-ins, shown before it runs.
355pub fn warnings<'a>(
356    ids: impl IntoIterator<Item = &'a str>,
357    lookup: &dyn Fn(&str) -> Option<&'a Entry>,
358    evidence: &Evidence,
359) -> Vec<String> {
360    let mut warnings = Vec::new();
361    for id in ids {
362        let Some(entry) = lookup(id) else { continue };
363        if !evidence.critical(entry) {
364            continue;
365        }
366        let assessment = evidence.assess(entry, &[]);
367        if assessment.level >= Level::ContainerTested {
368            warnings.push(format!(
369                "{} is {}: {}",
370                entry.name,
371                assessment.level.label(),
372                match assessment.level {
373                    Level::NotYetValidated => "nothing has tested it yet",
374                    Level::StandInTested => "only tested with stand-ins, never on a real system",
375                    _ => "containers cannot test boot, login or drivers",
376                }
377            ));
378        }
379    }
380    warnings
381}
382
383#[cfg(test)]
384mod tests {
385    use super::*;
386    use helixforge_catalog::Catalog;
387
388    fn evidence(claims: &str, results: Option<&str>) -> Evidence {
389        Evidence {
390            claims: toml::from_str(claims).unwrap(),
391            results: results.map(|text| serde_json::from_str(text).unwrap()),
392            problems: Vec::new(),
393        }
394    }
395
396    const RESULTS: &str = r#"{"format": 1, "commit": "0123456789abcdef0123", "run": 42,
397        "finished": 1792172552,
398        "images": {"archlinux:latest": {"entries": {"htop": "verified"}},
399                   "ubuntu:24.04": {"entries": {"htop": "verified", "system-report": "ran"}}}}"#;
400
401    #[test]
402    fn levels_come_from_the_record_never_beyond_it() {
403        let catalog = Catalog::builtin(false).unwrap();
404        let found = evidence(
405            r#"
406            [critical]
407            entries = ["sddm", "grub"]
408            [[stand-ins]]
409            entries = ["sddm"]
410            tests = ["test_sddm"]
411            covers = "the theme"
412            [[hardware]]
413            entry = "grub"
414            system = "Garuda Linux"
415            date = "2026-10-20"
416            "#,
417            Some(RESULTS),
418        );
419        let assess = |id: &str| found.assess(catalog.find(id).unwrap(), &[]);
420        let htop = assess("htop");
421        assert_eq!(htop.level, Level::ContainerTested);
422        assert_eq!(htop.systems, ["Arch", "Ubuntu 24.04"]);
423        assert_eq!(htop.verified.as_deref(), Some("2026-10-16"));
424        assert!(!htop.critical);
425        assert_eq!(assess("system-report").level, Level::ContainerTested);
426        let sddm = assess("sddm");
427        assert_eq!(sddm.level, Level::StandInTested);
428        assert!(sddm.critical);
429        assert!(sddm.evidence[0].contains("test_sddm"));
430        assert!(sddm.evidence[0].contains("all 2 containers"));
431        let grub = assess("grub");
432        assert_eq!(grub.level, Level::HardwareTested);
433        assert_eq!(grub.verified.as_deref(), Some("2026-10-20"));
434        assert_eq!(assess("ripgrep").level, Level::NotYetValidated);
435        // This machine's runs show beside the level, never raise it.
436        let ran = [history::Record {
437            unix_timestamp: 1_700_000_000,
438            entries: vec!["ripgrep".into()],
439            names: Vec::new(),
440            success: true,
441            result: String::new(),
442            log: None,
443        }];
444        let ripgrep = found.assess(catalog.find("ripgrep").unwrap(), &ran);
445        assert_eq!(ripgrep.level, Level::NotYetValidated);
446        assert_eq!(ripgrep.here, Some(1_700_000_000));
447
448        // Before a run, critical entries short of a real test are named.
449        let warned = warnings(["sddm", "grub", "htop"], &|id| catalog.find(id), &found);
450        assert_eq!(warned.len(), 1, "{warned:?}");
451        assert!(warned[0].starts_with("SDDM"), "{warned:?}");
452        // Whole-system entries count as critical by their risk.
453        assert!(found.critical(catalog.find("arch-server-setup").unwrap()));
454    }
455
456    #[test]
457    fn a_build_without_a_record_says_so() {
458        let found = evidence("", None);
459        assert!(found.summary().contains("No container record"));
460        let catalog = Catalog::builtin(false).unwrap();
461        assert_eq!(
462            found.assess(catalog.find("htop").unwrap(), &[]).level,
463            Level::NotYetValidated
464        );
465    }
466
467    #[test]
468    fn the_catalogs_own_record_reads_and_names_real_entries() {
469        let catalog = Catalog::builtin(false).unwrap();
470        let found = Evidence::load(catalog.root());
471        assert!(found.problems.is_empty(), "{:?}", found.problems);
472        assert!(!found.claims.stand_ins.is_empty());
473        for id in found
474            .claims
475            .stand_ins
476            .iter()
477            .flat_map(|group| group.entries.iter())
478            .chain(found.claims.critical.entries.iter())
479            .chain(found.claims.hardware.iter().map(|report| &report.entry))
480            .chain(found.claims.vm.iter().map(|report| &report.entry))
481        {
482            assert!(catalog.find(id).is_some(), "validation.toml names {id}");
483        }
484    }
485}
helixforge_engine/reliability.rs

helixforge_engine/
reliability.rs
