rightkit_qa/
evidence.rs

1//! A receipt that names a file without hashing it is a claim, not evidence.
2//! Every artifact is hashed when recorded, a check may only reference hashed
3//! artifacts, and the bundle can be re-verified against the filesystem.
4use crate::util::{err, now_iso, require_text, sha256_hex, Result};
5use serde::{Deserialize, Serialize};
6use serde_json::Value;
7use std::fs;
8use std::path::{Path, PathBuf};
9
10#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
11#[serde(rename_all = "lowercase")]
12pub enum CheckStatus {
13    Passed,
14    Failed,
15    Skipped,
16}
17
18#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
19#[serde(rename_all = "lowercase")]
20pub enum RunStatus {
21    Running,
22    Passed,
23    Failed,
24}
25
26#[derive(Debug, Clone, Serialize, Deserialize)]
27pub struct Artifact {
28    pub name: String,
29    pub path: PathBuf,
30    pub sha256: String,
31    pub size_bytes: u64,
32    pub recorded_at: String,
33}
34
35#[derive(Debug, Clone, Serialize, Deserialize)]
36pub struct Check {
37    pub name: String,
38    pub status: CheckStatus,
39    #[serde(default, skip_serializing_if = "Value::is_null")]
40    pub details: Value,
41    #[serde(default, skip_serializing_if = "Vec::is_empty")]
42    pub artifacts: Vec<String>,
43}
44
45#[derive(Debug, Clone, Serialize, Deserialize)]
46#[serde(rename_all = "camelCase")]
47pub struct Bundle {
48    pub schema_version: u32,
49    pub app: String,
50    pub platform: String,
51    pub run_id: String,
52    pub started_at: String,
53    #[serde(skip_serializing_if = "Option::is_none")]
54    pub finished_at: Option<String>,
55    pub status: RunStatus,
56    /// What was tested: config, scenario, binary and source-revision digests.
57    #[serde(default)]
58    pub identity: Value,
59    pub checks: Vec<Check>,
60    pub artifacts: Vec<Artifact>,
61}
62
63impl Bundle {
64    pub fn new(app: &str, platform: &str, run_id: &str) -> Result<Self> {
65        Ok(Self {
66            schema_version: 4,
67            app: require_text(app, "app")?,
68            platform: require_text(platform, "platform")?,
69            run_id: require_text(run_id, "runId")?,
70            started_at: now_iso(),
71            finished_at: None,
72            status: RunStatus::Running,
73            identity: Value::Null,
74            checks: vec![],
75            artifacts: vec![],
76        })
77    }
78
79    fn require_running(&self) -> Result<()> {
80        if self.status != RunStatus::Running {
81            return err("cannot modify a QA evidence bundle after the run has finished");
82        }
83        Ok(())
84    }
85
86    /// Hash a file into the bundle now, so a later replacement cannot masquerade.
87    pub fn add_artifact(&mut self, name: &str, path: &Path) -> Result<Artifact> {
88        self.require_running()?;
89        let name = require_text(name, "artifact name")?;
90        if self.artifacts.iter().any(|a| a.name == name) {
91            return err(format!(
92                "QA evidence already records an artifact named {name}"
93            ));
94        }
95        let bytes = fs::read(path).map_err(|e| {
96            crate::util::Error(format!(
97                "cannot record QA evidence artifact {name}: {} is unreadable ({e})",
98                path.display()
99            ))
100        })?;
101        let rec = Artifact {
102            name,
103            path: fs::canonicalize(path).unwrap_or_else(|_| path.to_path_buf()),
104            sha256: sha256_hex(&bytes),
105            size_bytes: bytes.len() as u64,
106            recorded_at: now_iso(),
107        };
108        self.artifacts.push(rec.clone());
109        Ok(rec)
110    }
111
112    pub fn add_check(
113        &mut self,
114        name: &str,
115        status: CheckStatus,
116        details: Value,
117        artifacts: &[String],
118    ) -> Result<()> {
119        self.require_running()?;
120        let name = require_text(name, "check name")?;
121        let unknown: Vec<&String> = artifacts
122            .iter()
123            .filter(|r| !self.artifacts.iter().any(|a| &a.name == *r))
124            .collect();
125        if !unknown.is_empty() {
126            return err(format!(
127                "QA check {name} references artifacts that were never hashed: {}",
128                unknown
129                    .iter()
130                    .map(|s| s.as_str())
131                    .collect::<Vec<_>>()
132                    .join(", ")
133            ));
134        }
135        self.checks.push(Check {
136            name,
137            status,
138            details,
139            artifacts: artifacts.to_vec(),
140        });
141        Ok(())
142    }
143
144    /// Bind this receipt to the exact inputs it exercised. A passing receipt without
145    /// an identity is refused at write time.
146    pub fn set_identity(&mut self, identity: Value) -> Result<()> {
147        self.require_running()?;
148        self.identity = identity;
149        Ok(())
150    }
151
152    pub fn finish(&mut self, status: RunStatus) {
153        self.status = status;
154        self.finished_at = Some(now_iso());
155    }
156
157    /// Re-hash every artifact; empty means the receipt still describes the disk.
158    pub fn verify(&self) -> Vec<(String, String)> {
159        let mut drift = vec![];
160        for a in &self.artifacts {
161            match fs::read(&a.path) {
162                Err(_) => drift.push((a.name.clone(), format!("missing at {}", a.path.display()))),
163                Ok(b) => {
164                    let h = sha256_hex(&b);
165                    if h != a.sha256 {
166                        drift.push((
167                            a.name.clone(),
168                            format!("sha256 {h} does not match recorded {}", a.sha256),
169                        ));
170                    }
171                }
172            }
173        }
174        drift
175    }
176
177    pub fn write(&self, path: &Path) -> Result<()> {
178        if self.status == RunStatus::Running || self.finished_at.is_none() {
179            return err("refusing to write unfinished QA evidence");
180        }
181        if self.status == RunStatus::Passed {
182            let drift = self.verify();
183            if !drift.is_empty() {
184                return err(format!(
185                    "refusing to write a passing QA receipt whose evidence changed: {}",
186                    drift
187                        .iter()
188                        .map(|(n, r)| format!("{n} ({r})"))
189                        .collect::<Vec<_>>()
190                        .join("; ")
191                ));
192            }
193            let ok = self
194                .identity
195                .get("config")
196                .and_then(|c| c.get("sha256"))
197                .is_some()
198                && self
199                    .identity
200                    .get("scenarios")
201                    .and_then(Value::as_array)
202                    .map(|a| !a.is_empty())
203                    .unwrap_or(false);
204            if !ok {
205                return err("refusing to write a passing QA receipt that is not bound to config and scenario identity");
206            }
207        }
208        if let Some(p) = path.parent() {
209            fs::create_dir_all(p)?;
210        }
211        let tmp = path.with_extension(format!("tmp-{}", std::process::id()));
212        fs::write(&tmp, serde_json::to_vec_pretty(self)?)?;
213        let r = fs::rename(&tmp, path);
214        let _ = fs::remove_file(&tmp);
215        Ok(r?)
216    }
217}
218
219#[cfg(test)]
220mod tests {
221    use super::*;
222    use serde_json::json;
223
224    fn tmpfile(body: &str) -> PathBuf {
225        let p = std::env::temp_dir().join(format!("rkqa-ev-{}", crate::util::new_id()));
226        fs::write(&p, body).unwrap();
227        p
228    }
229
230    #[test]
231    fn check_must_reference_hashed_artifacts() {
232        let mut b = Bundle::new("a", "darwin", "r").unwrap();
233        let e = b
234            .add_check("c", CheckStatus::Passed, json!(null), &["ghost".into()])
235            .unwrap_err();
236        assert!(e.0.contains("never hashed"));
237        let f = tmpfile("x");
238        b.add_artifact("shot", &f).unwrap();
239        assert!(b.add_artifact("shot", &f).is_err());
240        b.add_check("c", CheckStatus::Passed, json!({"k":1}), &["shot".into()])
241            .unwrap();
242    }
243
244    #[test]
245    fn passing_receipt_refused_when_bytes_drift() {
246        let mut b = Bundle::new("a", "darwin", "r").unwrap();
247        let f = tmpfile("one");
248        b.add_artifact("shot", &f).unwrap();
249        b.finish(RunStatus::Passed);
250        fs::write(&f, "two").unwrap();
251        let out = std::env::temp_dir().join(format!("rkqa-ev-out-{}.json", crate::util::new_id()));
252        assert!(b.write(&out).unwrap_err().0.contains("evidence changed"));
253        b.status = RunStatus::Failed;
254        b.write(&out).unwrap();
255        assert!(Bundle::new("a", "d", "r").unwrap().write(&out).is_err());
256    }
257}
rightkit_qa/evidence.rs

rightkit_qa/
evidence.rs
