Skip to main content

visual_rubric/
qa.rs

1//! Portable, versioned contracts for autonomous visual QA evidence.
2
3use std::collections::{BTreeMap, BTreeSet};
4use std::path::{Component, Path};
5
6use serde::{Deserialize, Serialize};
7use sha2::{Digest, Sha256};
8
9use crate::manifest::{ArtifactDigest, CAPTURE_MANIFEST_SCHEMA_VERSION, CaptureManifest};
10
11/// Current observation schema version.
12pub const OBSERVATION_SCHEMA_VERSION: u32 = 1;
13/// Current finding schema version.
14pub const FINDING_SCHEMA_VERSION: u32 = 1;
15/// Current visual-run schema version.
16pub const VISUAL_RUN_REPORT_SCHEMA_VERSION: u32 = 1;
17/// Current calibration-sentinel schema version.
18pub const CALIBRATION_SENTINEL_SCHEMA_VERSION: u32 = 1;
19
20/// A normalized observation produced by deterministic or semantic inspection.
21#[derive(Clone, Debug, Deserialize, Serialize, PartialEq, Eq)]
22pub struct ObservationV1 {
23    /// Schema discriminator.
24    pub schema_version: u32,
25    /// Stable observation identifier within the run.
26    pub id: String,
27    /// Capture cell that was inspected.
28    pub capture_id: String,
29    /// Producer-defined observation class.
30    pub kind: String,
31    /// Concise factual description.
32    pub summary: String,
33    /// Evidence artifacts supporting the observation.
34    #[serde(default)]
35    pub evidence: Vec<ArtifactDigest>,
36}
37
38/// A deduplicatable issue derived from one or more observations.
39#[derive(Clone, Debug, Deserialize, Serialize, PartialEq, Eq)]
40pub struct FindingV1 {
41    /// Schema discriminator.
42    pub schema_version: u32,
43    /// Stable content-derived fingerprint used by issue trackers.
44    pub fingerprint: String,
45    /// Visual target identifier.
46    pub target: String,
47    /// Exact source revision that was inspected.
48    pub revision: String,
49    /// Human-readable issue title.
50    pub title: String,
51    /// Observation identifiers supporting this finding.
52    pub observation_ids: Vec<String>,
53    /// Reproduction steps or command.
54    pub reproduction: Vec<String>,
55}
56
57/// Result of a complete producer/evaluator visual QA run.
58#[derive(Clone, Debug, Deserialize, Serialize, PartialEq, Eq)]
59pub struct VisualRunReportV1 {
60    /// Schema discriminator.
61    pub schema_version: u32,
62    /// Unique run identifier.
63    pub run_id: String,
64    /// Visual target identifier.
65    pub target: String,
66    /// Exact source revision under test.
67    pub revision: String,
68    /// Whether the producer checkout contained uncommitted changes.
69    pub dirty: bool,
70    /// Content-addressed capture manifest used by this run.
71    pub capture_manifest: ArtifactDigest,
72    /// Capture cells declared by the producer as part of this run.
73    #[serde(default)]
74    pub planned_capture_ids: Vec<String>,
75    /// Capture cells for which deterministic and semantic evaluation completed.
76    #[serde(default)]
77    pub evaluated_capture_ids: Vec<String>,
78    /// Semantic result of the run.
79    pub status: VisualRunStatus,
80    /// Normalized observations.
81    #[serde(default)]
82    pub observations: Vec<ObservationV1>,
83    /// Deduplicatable findings.
84    #[serde(default)]
85    pub findings: Vec<FindingV1>,
86}
87
88/// Semantic visual-run result.
89#[derive(Clone, Debug, Deserialize, Serialize, PartialEq, Eq)]
90#[serde(rename_all = "snake_case")]
91pub enum VisualRunStatus {
92    /// Every planned cell was evaluated and passed.
93    Pass,
94    /// At least one complete evaluation produced a finding.
95    Fail,
96    /// Required inputs or infrastructure were unavailable.
97    Blocked,
98}
99
100/// A known-good or known-bad capture used to detect evaluator drift.
101#[derive(Clone, Debug, Deserialize, Serialize, PartialEq, Eq)]
102pub struct CalibrationSentinelV1 {
103    /// Schema discriminator.
104    pub schema_version: u32,
105    /// Stable sentinel identifier.
106    pub id: String,
107    /// Capture cell containing the calibrated fixture.
108    pub capture_id: String,
109    /// Result the evaluator must reproduce.
110    pub expectation: CalibrationExpectation,
111}
112
113/// Expected evaluator behavior for a calibration fixture.
114#[derive(Clone, Debug, Deserialize, Serialize, PartialEq, Eq)]
115#[serde(tag = "kind", rename_all = "snake_case")]
116pub enum CalibrationExpectation {
117    /// The known-good fixture must not support a finding.
118    Clear,
119    /// The known-bad fixture must produce a finding backed by this observation kind.
120    Finding {
121        /// Expected observation class, such as `layout_overlap`.
122        observation_kind: String,
123    },
124}
125
126impl VisualRunReportV1 {
127    /// Serializes this run report into deterministic compact JSON.
128    ///
129    /// Set-like record collections are normalized while ordered reproduction
130    /// steps retain their producer-defined order.
131    pub fn to_canonical_json(&self) -> Result<Vec<u8>, serde_json::Error> {
132        let mut normalized = self.clone();
133        normalized.planned_capture_ids.sort();
134        normalized.evaluated_capture_ids.sort();
135        normalized
136            .observations
137            .sort_by(|left, right| left.id.cmp(&right.id));
138        for observation in &mut normalized.observations {
139            observation
140                .evidence
141                .sort_by(|left, right| left.path.cmp(&right.path));
142        }
143        normalized
144            .findings
145            .sort_by(|left, right| left.fingerprint.cmp(&right.fingerprint));
146        for finding in &mut normalized.findings {
147            finding.observation_ids.sort();
148        }
149        serde_json::to_vec(&normalized)
150    }
151}
152
153/// Computes the revision-independent fingerprint for one finding.
154///
155/// The identity is derived from the target plus the sorted set of capture
156/// cells and observation kinds. Titles, summaries, and source revisions are
157/// deliberately excluded so the same defect deduplicates across runs.
158pub fn finding_fingerprint(target: &str, observations: &[ObservationV1]) -> String {
159    let mut identities = observations
160        .iter()
161        .map(|observation| (observation.capture_id.trim(), observation.kind.trim()))
162        .collect::<Vec<_>>();
163    identities.sort_unstable();
164    identities.dedup();
165
166    let mut hasher = Sha256::new();
167    hash_field(&mut hasher, b"visual-rubric:finding:v1");
168    hash_field(&mut hasher, target.trim().as_bytes());
169    for (capture_id, kind) in identities {
170        hash_field(&mut hasher, capture_id.as_bytes());
171        hash_field(&mut hasher, kind.as_bytes());
172    }
173    format!("{:x}", hasher.finalize())
174}
175
176fn hash_field(hasher: &mut Sha256, value: &[u8]) {
177    hasher.update((value.len() as u64).to_be_bytes());
178    hasher.update(value);
179}
180
181/// Validates cross-record invariants that do not require filesystem access.
182pub fn validate_visual_run(report: &VisualRunReportV1) -> Result<(), Vec<String>> {
183    let mut issues = Vec::new();
184    if report.schema_version != VISUAL_RUN_REPORT_SCHEMA_VERSION {
185        issues.push("unsupported visual run schema_version".to_owned());
186    }
187    required("run_id", &report.run_id, &mut issues);
188    required("target", &report.target, &mut issues);
189    required("revision", &report.revision, &mut issues);
190    validate_digest(&report.capture_manifest, &mut issues);
191    if matches!(report.status, VisualRunStatus::Pass) && report.dirty {
192        issues.push("passing visual run must not be dirty".to_owned());
193    }
194    validate_cell_coverage(
195        report,
196        &report.planned_capture_ids,
197        &report.evaluated_capture_ids,
198        &mut issues,
199    );
200
201    let mut observation_ids = BTreeSet::new();
202    let mut observations_by_id = BTreeMap::new();
203    for observation in &report.observations {
204        if observation.schema_version != OBSERVATION_SCHEMA_VERSION {
205            issues.push(format!(
206                "observation {:?} has unsupported schema_version",
207                observation.id
208            ));
209        }
210        required("observation.id", &observation.id, &mut issues);
211        required(
212            "observation.capture_id",
213            &observation.capture_id,
214            &mut issues,
215        );
216        required("observation.kind", &observation.kind, &mut issues);
217        required("observation.summary", &observation.summary, &mut issues);
218        if !observation_ids.insert(observation.id.as_str()) {
219            issues.push(format!("duplicate observation id {:?}", observation.id));
220        } else {
221            observations_by_id.insert(observation.id.as_str(), observation);
222        }
223        for artifact in &observation.evidence {
224            validate_digest(artifact, &mut issues);
225        }
226    }
227
228    let mut fingerprints = BTreeSet::new();
229    for finding in &report.findings {
230        if finding.schema_version != FINDING_SCHEMA_VERSION {
231            issues.push(format!(
232                "finding {:?} has unsupported schema_version",
233                finding.fingerprint
234            ));
235        }
236        required("finding.fingerprint", &finding.fingerprint, &mut issues);
237        if finding.fingerprint.len() != 64
238            || !finding
239                .fingerprint
240                .bytes()
241                .all(|byte| byte.is_ascii_digit() || (b'a'..=b'f').contains(&byte))
242        {
243            issues.push(format!(
244                "finding {:?} fingerprint must be lowercase SHA-256 hex",
245                finding.fingerprint
246            ));
247        }
248        required("finding.target", &finding.target, &mut issues);
249        required("finding.revision", &finding.revision, &mut issues);
250        required("finding.title", &finding.title, &mut issues);
251        if finding.target != report.target || finding.revision != report.revision {
252            issues.push(format!(
253                "finding {:?} target/revision does not match its run",
254                finding.fingerprint
255            ));
256        }
257        if !fingerprints.insert(finding.fingerprint.as_str()) {
258            issues.push(format!(
259                "duplicate finding fingerprint {:?}",
260                finding.fingerprint
261            ));
262        }
263        let unique_observation_ids = finding
264            .observation_ids
265            .iter()
266            .map(String::as_str)
267            .collect::<BTreeSet<_>>();
268        let references_missing = finding.observation_ids.is_empty()
269            || unique_observation_ids.len() != finding.observation_ids.len()
270            || finding
271                .observation_ids
272                .iter()
273                .any(|id| !observation_ids.contains(id.as_str()));
274        if references_missing {
275            issues.push(format!(
276                "finding {:?} references missing or duplicate observations",
277                finding.fingerprint
278            ));
279        } else {
280            let referenced = finding
281                .observation_ids
282                .iter()
283                .filter_map(|id| observations_by_id.get(id.as_str()).copied())
284                .cloned()
285                .collect::<Vec<_>>();
286            let expected = finding_fingerprint(&finding.target, &referenced);
287            if finding.fingerprint != expected {
288                issues.push(format!(
289                    "finding {:?} fingerprint does not match its observation identity; expected {expected}",
290                    finding.fingerprint
291                ));
292            }
293        }
294        if finding.reproduction.is_empty() {
295            issues.push(format!(
296                "finding {:?} requires reproduction steps",
297                finding.fingerprint
298            ));
299        }
300    }
301
302    if matches!(report.status, VisualRunStatus::Pass) && !report.findings.is_empty() {
303        issues.push("passing visual run cannot contain findings".to_owned());
304    }
305    if matches!(report.status, VisualRunStatus::Fail) && report.findings.is_empty() {
306        issues.push("failed visual run requires at least one finding".to_owned());
307    }
308
309    if issues.is_empty() {
310        Ok(())
311    } else {
312        Err(issues)
313    }
314}
315
316/// Validates a report and proves that it covers exactly the cells in the
317/// referenced capture manifest. Callers should use this at the controller
318/// boundary after loading the manifest with [`CaptureManifest::load`].
319pub fn validate_visual_run_against_manifest(
320    report: &VisualRunReportV1,
321    manifest: &CaptureManifest,
322) -> Result<(), Vec<String>> {
323    let mut issues = validate_visual_run(report).err().unwrap_or_default();
324    if manifest.schema_version != CAPTURE_MANIFEST_SCHEMA_VERSION {
325        issues.push("capture manifest has unsupported schema_version".to_owned());
326    }
327    if report.target != manifest.target {
328        issues.push("visual run target does not match capture manifest".to_owned());
329    }
330    if report.revision != manifest.revision {
331        issues.push("visual run revision does not match capture manifest".to_owned());
332    }
333    if report.dirty != manifest.dirty {
334        issues.push("visual run dirty state does not match capture manifest".to_owned());
335    }
336    let manifest_ids: BTreeSet<_> = manifest
337        .captures
338        .iter()
339        .map(|capture| capture.id.as_str())
340        .collect();
341    let planned_ids: BTreeSet<_> = report
342        .planned_capture_ids
343        .iter()
344        .map(String::as_str)
345        .collect();
346    let evaluated_ids: BTreeSet<_> = report
347        .evaluated_capture_ids
348        .iter()
349        .map(String::as_str)
350        .collect();
351    if planned_ids != manifest_ids {
352        issues.push("visual run planned cells do not match capture manifest".to_owned());
353    }
354    if evaluated_ids != manifest_ids {
355        issues.push("visual run evaluated cells do not match capture manifest".to_owned());
356    }
357    let observed_ids = report
358        .observations
359        .iter()
360        .map(|observation| observation.capture_id.as_str())
361        .collect::<BTreeSet<_>>();
362    for unknown in observed_ids.difference(&manifest_ids) {
363        issues.push(format!(
364            "visual run observation references unknown capture {unknown:?}"
365        ));
366    }
367    if issues.is_empty() {
368        Ok(())
369    } else {
370        Err(issues)
371    }
372}
373
374/// Validates known-good and known-bad calibration fixtures against a run.
375pub fn validate_calibration_sentinels(
376    report: &VisualRunReportV1,
377    sentinels: &[CalibrationSentinelV1],
378) -> Result<(), Vec<String>> {
379    let mut issues = Vec::new();
380    if sentinels.is_empty() {
381        issues.push("calibration requires at least one sentinel".to_owned());
382    }
383    let planned = report
384        .planned_capture_ids
385        .iter()
386        .map(String::as_str)
387        .collect::<BTreeSet<_>>();
388    let evaluated = report
389        .evaluated_capture_ids
390        .iter()
391        .map(String::as_str)
392        .collect::<BTreeSet<_>>();
393    let observations = report
394        .observations
395        .iter()
396        .map(|observation| (observation.id.as_str(), observation))
397        .collect::<BTreeMap<_, _>>();
398    let finding_observations = report
399        .findings
400        .iter()
401        .flat_map(|finding| finding.observation_ids.iter().map(String::as_str))
402        .collect::<BTreeSet<_>>();
403
404    let mut ids = BTreeSet::new();
405    for sentinel in sentinels {
406        if sentinel.schema_version != CALIBRATION_SENTINEL_SCHEMA_VERSION {
407            issues.push(format!(
408                "calibration sentinel {:?} has unsupported schema_version",
409                sentinel.id
410            ));
411        }
412        required("calibration sentinel id", &sentinel.id, &mut issues);
413        required(
414            "calibration sentinel capture_id",
415            &sentinel.capture_id,
416            &mut issues,
417        );
418        if !ids.insert(sentinel.id.as_str()) {
419            issues.push(format!(
420                "duplicate calibration sentinel id {:?}",
421                sentinel.id
422            ));
423        }
424        if !planned.contains(sentinel.capture_id.as_str())
425            || !evaluated.contains(sentinel.capture_id.as_str())
426        {
427            issues.push(format!(
428                "calibration sentinel {:?} capture was not planned and evaluated",
429                sentinel.id
430            ));
431            continue;
432        }
433
434        let supporting = observations.values().filter(|observation| {
435            observation.capture_id == sentinel.capture_id
436                && finding_observations.contains(observation.id.as_str())
437        });
438        match &sentinel.expectation {
439            CalibrationExpectation::Clear => {
440                if supporting.count() != 0 {
441                    issues.push(format!(
442                        "known-good calibration sentinel {:?} produced a finding",
443                        sentinel.id
444                    ));
445                }
446            }
447            CalibrationExpectation::Finding { observation_kind } => {
448                required(
449                    "calibration observation_kind",
450                    observation_kind,
451                    &mut issues,
452                );
453                if !supporting
454                    .into_iter()
455                    .any(|observation| observation.kind == *observation_kind)
456                {
457                    issues.push(format!(
458                        "known-bad calibration sentinel {:?} did not produce finding kind {:?}",
459                        sentinel.id, observation_kind
460                    ));
461                }
462            }
463        }
464    }
465
466    if issues.is_empty() {
467        Ok(())
468    } else {
469        Err(issues)
470    }
471}
472
473fn validate_cell_coverage(
474    report: &VisualRunReportV1,
475    planned: &[String],
476    evaluated: &[String],
477    issues: &mut Vec<String>,
478) {
479    let planned_set: BTreeSet<_> = planned.iter().map(String::as_str).collect();
480    let evaluated_set: BTreeSet<_> = evaluated.iter().map(String::as_str).collect();
481    if planned.is_empty() {
482        issues.push("visual run must declare at least one planned capture cell".to_owned());
483    }
484    if planned_set.len() != planned.len() {
485        issues.push("visual run planned capture ids must be unique".to_owned());
486    }
487    if evaluated_set.len() != evaluated.len() {
488        issues.push("visual run evaluated capture ids must be unique".to_owned());
489    }
490    if !evaluated_set.is_subset(&planned_set) {
491        issues.push("visual run evaluated cells must be planned cells".to_owned());
492    }
493    if matches!(report.status, VisualRunStatus::Pass) && planned_set != evaluated_set {
494        issues.push("passing visual run must evaluate every planned capture cell".to_owned());
495    }
496}
497
498fn required(name: &str, value: &str, issues: &mut Vec<String>) {
499    if value.trim().is_empty() {
500        issues.push(format!("{name} must not be empty"));
501    }
502}
503
504fn validate_digest(artifact: &ArtifactDigest, issues: &mut Vec<String>) {
505    validate_relative_path("artifact", &artifact.path, issues);
506    if artifact.bytes == 0 {
507        issues.push(format!("artifact {} is empty", artifact.path.display()));
508    }
509    if artifact.sha256.len() != 64
510        || !artifact
511            .sha256
512            .bytes()
513            .all(|byte| byte.is_ascii_hexdigit() && !byte.is_ascii_uppercase())
514    {
515        issues.push(format!(
516            "artifact {} has invalid sha256",
517            artifact.path.display()
518        ));
519    }
520}
521
522fn validate_relative_path(name: &str, path: &Path, issues: &mut Vec<String>) {
523    if path.as_os_str().is_empty()
524        || path.is_absolute()
525        || path
526            .components()
527            .any(|component| matches!(component, Component::ParentDir))
528    {
529        issues.push(format!(
530            "{name} path {} must be relative and contained",
531            path.display()
532        ));
533    }
534}
535
536#[cfg(test)]
537mod tests {
538    use super::*;
539
540    #[test]
541    fn rejects_a_pass_with_findings_or_dirty_source() {
542        let report = VisualRunReportV1 {
543            schema_version: VISUAL_RUN_REPORT_SCHEMA_VERSION,
544            run_id: "run-1".to_owned(),
545            target: "regicide/game".to_owned(),
546            revision: "abc123".to_owned(),
547            dirty: true,
548            capture_manifest: ArtifactDigest {
549                path: "capture_manifest.json".into(),
550                sha256: "0".repeat(64),
551                bytes: 1,
552            },
553            planned_capture_ids: vec!["menu/desktop".to_owned()],
554            evaluated_capture_ids: vec!["menu/desktop".to_owned()],
555            status: VisualRunStatus::Pass,
556            observations: vec![],
557            findings: vec![FindingV1 {
558                schema_version: FINDING_SCHEMA_VERSION,
559                fingerprint: "layout/main-menu".to_owned(),
560                target: "regicide/game".to_owned(),
561                revision: "abc123".to_owned(),
562                title: "Menu overlaps".to_owned(),
563                observation_ids: vec!["missing".to_owned()],
564                reproduction: vec!["run visual suite".to_owned()],
565            }],
566        };
567        let issues = validate_visual_run(&report).expect_err("invalid pass");
568        assert!(
569            issues
570                .iter()
571                .any(|issue| issue.contains("must not be dirty"))
572        );
573        assert!(
574            issues
575                .iter()
576                .any(|issue| issue.contains("cannot contain findings"))
577        );
578    }
579
580    #[test]
581    fn rejects_empty_passing_coverage() {
582        let report = VisualRunReportV1 {
583            schema_version: VISUAL_RUN_REPORT_SCHEMA_VERSION,
584            run_id: "run-1".to_owned(),
585            target: "regicide/game".to_owned(),
586            revision: "abc123".to_owned(),
587            dirty: false,
588            capture_manifest: ArtifactDigest {
589                path: "capture_manifest.json".into(),
590                sha256: "0".repeat(64),
591                bytes: 1,
592            },
593            planned_capture_ids: Vec::new(),
594            evaluated_capture_ids: Vec::new(),
595            status: VisualRunStatus::Pass,
596            observations: Vec::new(),
597            findings: Vec::new(),
598        };
599        let issues = validate_visual_run(&report).expect_err("empty pass must fail");
600        assert!(issues.iter().any(|issue| issue.contains("planned capture")));
601    }
602
603    #[test]
604    fn finding_fingerprint_is_stable_and_verified() {
605        let observations = vec![
606            ObservationV1 {
607                schema_version: OBSERVATION_SCHEMA_VERSION,
608                id: "obs-b".to_owned(),
609                capture_id: "menu/mobile".to_owned(),
610                kind: "layout-overlap".to_owned(),
611                summary: "buttons overlap".to_owned(),
612                evidence: Vec::new(),
613            },
614            ObservationV1 {
615                schema_version: OBSERVATION_SCHEMA_VERSION,
616                id: "obs-a".to_owned(),
617                capture_id: "menu/desktop".to_owned(),
618                kind: "layout-overlap".to_owned(),
619                summary: "buttons overlap".to_owned(),
620                evidence: Vec::new(),
621            },
622        ];
623        let mut reversed = observations.clone();
624        reversed.reverse();
625        let fingerprint = finding_fingerprint("regicide/game", &observations);
626        assert_eq!(fingerprint, finding_fingerprint("regicide/game", &reversed));
627
628        let report = VisualRunReportV1 {
629            schema_version: VISUAL_RUN_REPORT_SCHEMA_VERSION,
630            run_id: "run-1".to_owned(),
631            target: "regicide/game".to_owned(),
632            revision: "abc123".to_owned(),
633            dirty: false,
634            capture_manifest: ArtifactDigest {
635                path: "capture_manifest.json".into(),
636                sha256: "0".repeat(64),
637                bytes: 1,
638            },
639            planned_capture_ids: vec!["menu/desktop".to_owned(), "menu/mobile".to_owned()],
640            evaluated_capture_ids: vec!["menu/desktop".to_owned(), "menu/mobile".to_owned()],
641            status: VisualRunStatus::Fail,
642            observations,
643            findings: vec![FindingV1 {
644                schema_version: FINDING_SCHEMA_VERSION,
645                fingerprint,
646                target: "regicide/game".to_owned(),
647                revision: "abc123".to_owned(),
648                title: "Menu buttons overlap".to_owned(),
649                observation_ids: vec!["obs-a".to_owned(), "obs-b".to_owned()],
650                reproduction: vec!["run the visual matrix".to_owned()],
651            }],
652        };
653        assert!(validate_visual_run(&report).is_ok());
654
655        let mut reordered = report.clone();
656        reordered.planned_capture_ids.reverse();
657        reordered.evaluated_capture_ids.reverse();
658        reordered.observations.reverse();
659        reordered.findings[0].observation_ids.reverse();
660        assert_eq!(
661            report.to_canonical_json().expect("canonical report"),
662            reordered.to_canonical_json().expect("canonical report")
663        );
664
665        let sentinel = CalibrationSentinelV1 {
666            schema_version: CALIBRATION_SENTINEL_SCHEMA_VERSION,
667            id: "known-overlap".to_owned(),
668            capture_id: "menu/desktop".to_owned(),
669            expectation: CalibrationExpectation::Finding {
670                observation_kind: "layout-overlap".to_owned(),
671            },
672        };
673        assert!(validate_calibration_sentinels(&report, &[sentinel]).is_ok());
674    }
675
676    #[test]
677    fn calibration_fails_when_a_negative_sentinel_is_missed() {
678        let report = VisualRunReportV1 {
679            schema_version: VISUAL_RUN_REPORT_SCHEMA_VERSION,
680            run_id: "run-1".to_owned(),
681            target: "regicide/game".to_owned(),
682            revision: "abc123".to_owned(),
683            dirty: false,
684            capture_manifest: ArtifactDigest {
685                path: "capture_manifest.json".into(),
686                sha256: "0".repeat(64),
687                bytes: 1,
688            },
689            planned_capture_ids: vec!["menu/desktop".to_owned()],
690            evaluated_capture_ids: vec!["menu/desktop".to_owned()],
691            status: VisualRunStatus::Pass,
692            observations: Vec::new(),
693            findings: Vec::new(),
694        };
695        let sentinel = CalibrationSentinelV1 {
696            schema_version: CALIBRATION_SENTINEL_SCHEMA_VERSION,
697            id: "known-overlap".to_owned(),
698            capture_id: "menu/desktop".to_owned(),
699            expectation: CalibrationExpectation::Finding {
700                observation_kind: "layout-overlap".to_owned(),
701            },
702        };
703        let issues = validate_calibration_sentinels(&report, &[sentinel])
704            .expect_err("missed negative sentinel must fail calibration");
705        assert!(issues.iter().any(|issue| issue.contains("did not produce")));
706    }
707}