1use std::collections::{BTreeMap, BTreeSet};
4use std::path::{Component, Path};
5
6use serde::{Deserialize, Serialize};
7use sha2::{Digest, Sha256};
8
9use crate::manifest::{ArtifactDigest, CAPTURE_MANIFEST_SCHEMA_VERSION, CaptureManifest};
10
11pub const OBSERVATION_SCHEMA_VERSION: u32 = 1;
13pub const FINDING_SCHEMA_VERSION: u32 = 1;
15pub const VISUAL_RUN_REPORT_SCHEMA_VERSION: u32 = 1;
17pub const CALIBRATION_SENTINEL_SCHEMA_VERSION: u32 = 1;
19
20#[derive(Clone, Debug, Deserialize, Serialize, PartialEq, Eq)]
22pub struct ObservationV1 {
23 pub schema_version: u32,
25 pub id: String,
27 pub capture_id: String,
29 pub kind: String,
31 pub summary: String,
33 #[serde(default)]
35 pub evidence: Vec<ArtifactDigest>,
36}
37
38#[derive(Clone, Debug, Deserialize, Serialize, PartialEq, Eq)]
40pub struct FindingV1 {
41 pub schema_version: u32,
43 pub fingerprint: String,
45 pub target: String,
47 pub revision: String,
49 pub title: String,
51 pub observation_ids: Vec<String>,
53 pub reproduction: Vec<String>,
55}
56
57#[derive(Clone, Debug, Deserialize, Serialize, PartialEq, Eq)]
59pub struct VisualRunReportV1 {
60 pub schema_version: u32,
62 pub run_id: String,
64 pub target: String,
66 pub revision: String,
68 pub dirty: bool,
70 pub capture_manifest: ArtifactDigest,
72 #[serde(default)]
74 pub planned_capture_ids: Vec<String>,
75 #[serde(default)]
77 pub evaluated_capture_ids: Vec<String>,
78 pub status: VisualRunStatus,
80 #[serde(default)]
82 pub observations: Vec<ObservationV1>,
83 #[serde(default)]
85 pub findings: Vec<FindingV1>,
86}
87
88#[derive(Clone, Debug, Deserialize, Serialize, PartialEq, Eq)]
90#[serde(rename_all = "snake_case")]
91pub enum VisualRunStatus {
92 Pass,
94 Fail,
96 Blocked,
98}
99
100#[derive(Clone, Debug, Deserialize, Serialize, PartialEq, Eq)]
102pub struct CalibrationSentinelV1 {
103 pub schema_version: u32,
105 pub id: String,
107 pub capture_id: String,
109 pub expectation: CalibrationExpectation,
111}
112
113#[derive(Clone, Debug, Deserialize, Serialize, PartialEq, Eq)]
115#[serde(tag = "kind", rename_all = "snake_case")]
116pub enum CalibrationExpectation {
117 Clear,
119 Finding {
121 observation_kind: String,
123 },
124}
125
126impl VisualRunReportV1 {
127 pub fn to_canonical_json(&self) -> Result<Vec<u8>, serde_json::Error> {
132 let mut normalized = self.clone();
133 normalized.planned_capture_ids.sort();
134 normalized.evaluated_capture_ids.sort();
135 normalized
136 .observations
137 .sort_by(|left, right| left.id.cmp(&right.id));
138 for observation in &mut normalized.observations {
139 observation
140 .evidence
141 .sort_by(|left, right| left.path.cmp(&right.path));
142 }
143 normalized
144 .findings
145 .sort_by(|left, right| left.fingerprint.cmp(&right.fingerprint));
146 for finding in &mut normalized.findings {
147 finding.observation_ids.sort();
148 }
149 serde_json::to_vec(&normalized)
150 }
151}
152
153pub fn finding_fingerprint(target: &str, observations: &[ObservationV1]) -> String {
159 let mut identities = observations
160 .iter()
161 .map(|observation| (observation.capture_id.trim(), observation.kind.trim()))
162 .collect::<Vec<_>>();
163 identities.sort_unstable();
164 identities.dedup();
165
166 let mut hasher = Sha256::new();
167 hash_field(&mut hasher, b"visual-rubric:finding:v1");
168 hash_field(&mut hasher, target.trim().as_bytes());
169 for (capture_id, kind) in identities {
170 hash_field(&mut hasher, capture_id.as_bytes());
171 hash_field(&mut hasher, kind.as_bytes());
172 }
173 format!("{:x}", hasher.finalize())
174}
175
176fn hash_field(hasher: &mut Sha256, value: &[u8]) {
177 hasher.update((value.len() as u64).to_be_bytes());
178 hasher.update(value);
179}
180
181pub fn validate_visual_run(report: &VisualRunReportV1) -> Result<(), Vec<String>> {
183 let mut issues = Vec::new();
184 if report.schema_version != VISUAL_RUN_REPORT_SCHEMA_VERSION {
185 issues.push("unsupported visual run schema_version".to_owned());
186 }
187 required("run_id", &report.run_id, &mut issues);
188 required("target", &report.target, &mut issues);
189 required("revision", &report.revision, &mut issues);
190 validate_digest(&report.capture_manifest, &mut issues);
191 if matches!(report.status, VisualRunStatus::Pass) && report.dirty {
192 issues.push("passing visual run must not be dirty".to_owned());
193 }
194 validate_cell_coverage(
195 report,
196 &report.planned_capture_ids,
197 &report.evaluated_capture_ids,
198 &mut issues,
199 );
200
201 let mut observation_ids = BTreeSet::new();
202 let mut observations_by_id = BTreeMap::new();
203 for observation in &report.observations {
204 if observation.schema_version != OBSERVATION_SCHEMA_VERSION {
205 issues.push(format!(
206 "observation {:?} has unsupported schema_version",
207 observation.id
208 ));
209 }
210 required("observation.id", &observation.id, &mut issues);
211 required(
212 "observation.capture_id",
213 &observation.capture_id,
214 &mut issues,
215 );
216 required("observation.kind", &observation.kind, &mut issues);
217 required("observation.summary", &observation.summary, &mut issues);
218 if !observation_ids.insert(observation.id.as_str()) {
219 issues.push(format!("duplicate observation id {:?}", observation.id));
220 } else {
221 observations_by_id.insert(observation.id.as_str(), observation);
222 }
223 for artifact in &observation.evidence {
224 validate_digest(artifact, &mut issues);
225 }
226 }
227
228 let mut fingerprints = BTreeSet::new();
229 for finding in &report.findings {
230 if finding.schema_version != FINDING_SCHEMA_VERSION {
231 issues.push(format!(
232 "finding {:?} has unsupported schema_version",
233 finding.fingerprint
234 ));
235 }
236 required("finding.fingerprint", &finding.fingerprint, &mut issues);
237 if finding.fingerprint.len() != 64
238 || !finding
239 .fingerprint
240 .bytes()
241 .all(|byte| byte.is_ascii_digit() || (b'a'..=b'f').contains(&byte))
242 {
243 issues.push(format!(
244 "finding {:?} fingerprint must be lowercase SHA-256 hex",
245 finding.fingerprint
246 ));
247 }
248 required("finding.target", &finding.target, &mut issues);
249 required("finding.revision", &finding.revision, &mut issues);
250 required("finding.title", &finding.title, &mut issues);
251 if finding.target != report.target || finding.revision != report.revision {
252 issues.push(format!(
253 "finding {:?} target/revision does not match its run",
254 finding.fingerprint
255 ));
256 }
257 if !fingerprints.insert(finding.fingerprint.as_str()) {
258 issues.push(format!(
259 "duplicate finding fingerprint {:?}",
260 finding.fingerprint
261 ));
262 }
263 let unique_observation_ids = finding
264 .observation_ids
265 .iter()
266 .map(String::as_str)
267 .collect::<BTreeSet<_>>();
268 let references_missing = finding.observation_ids.is_empty()
269 || unique_observation_ids.len() != finding.observation_ids.len()
270 || finding
271 .observation_ids
272 .iter()
273 .any(|id| !observation_ids.contains(id.as_str()));
274 if references_missing {
275 issues.push(format!(
276 "finding {:?} references missing or duplicate observations",
277 finding.fingerprint
278 ));
279 } else {
280 let referenced = finding
281 .observation_ids
282 .iter()
283 .filter_map(|id| observations_by_id.get(id.as_str()).copied())
284 .cloned()
285 .collect::<Vec<_>>();
286 let expected = finding_fingerprint(&finding.target, &referenced);
287 if finding.fingerprint != expected {
288 issues.push(format!(
289 "finding {:?} fingerprint does not match its observation identity; expected {expected}",
290 finding.fingerprint
291 ));
292 }
293 }
294 if finding.reproduction.is_empty() {
295 issues.push(format!(
296 "finding {:?} requires reproduction steps",
297 finding.fingerprint
298 ));
299 }
300 }
301
302 if matches!(report.status, VisualRunStatus::Pass) && !report.findings.is_empty() {
303 issues.push("passing visual run cannot contain findings".to_owned());
304 }
305 if matches!(report.status, VisualRunStatus::Fail) && report.findings.is_empty() {
306 issues.push("failed visual run requires at least one finding".to_owned());
307 }
308
309 if issues.is_empty() {
310 Ok(())
311 } else {
312 Err(issues)
313 }
314}
315
316pub fn validate_visual_run_against_manifest(
320 report: &VisualRunReportV1,
321 manifest: &CaptureManifest,
322) -> Result<(), Vec<String>> {
323 let mut issues = validate_visual_run(report).err().unwrap_or_default();
324 if manifest.schema_version != CAPTURE_MANIFEST_SCHEMA_VERSION {
325 issues.push("capture manifest has unsupported schema_version".to_owned());
326 }
327 if report.target != manifest.target {
328 issues.push("visual run target does not match capture manifest".to_owned());
329 }
330 if report.revision != manifest.revision {
331 issues.push("visual run revision does not match capture manifest".to_owned());
332 }
333 if report.dirty != manifest.dirty {
334 issues.push("visual run dirty state does not match capture manifest".to_owned());
335 }
336 let manifest_ids: BTreeSet<_> = manifest
337 .captures
338 .iter()
339 .map(|capture| capture.id.as_str())
340 .collect();
341 let planned_ids: BTreeSet<_> = report
342 .planned_capture_ids
343 .iter()
344 .map(String::as_str)
345 .collect();
346 let evaluated_ids: BTreeSet<_> = report
347 .evaluated_capture_ids
348 .iter()
349 .map(String::as_str)
350 .collect();
351 if planned_ids != manifest_ids {
352 issues.push("visual run planned cells do not match capture manifest".to_owned());
353 }
354 if evaluated_ids != manifest_ids {
355 issues.push("visual run evaluated cells do not match capture manifest".to_owned());
356 }
357 let observed_ids = report
358 .observations
359 .iter()
360 .map(|observation| observation.capture_id.as_str())
361 .collect::<BTreeSet<_>>();
362 for unknown in observed_ids.difference(&manifest_ids) {
363 issues.push(format!(
364 "visual run observation references unknown capture {unknown:?}"
365 ));
366 }
367 if issues.is_empty() {
368 Ok(())
369 } else {
370 Err(issues)
371 }
372}
373
374pub fn validate_calibration_sentinels(
376 report: &VisualRunReportV1,
377 sentinels: &[CalibrationSentinelV1],
378) -> Result<(), Vec<String>> {
379 let mut issues = Vec::new();
380 if sentinels.is_empty() {
381 issues.push("calibration requires at least one sentinel".to_owned());
382 }
383 let planned = report
384 .planned_capture_ids
385 .iter()
386 .map(String::as_str)
387 .collect::<BTreeSet<_>>();
388 let evaluated = report
389 .evaluated_capture_ids
390 .iter()
391 .map(String::as_str)
392 .collect::<BTreeSet<_>>();
393 let observations = report
394 .observations
395 .iter()
396 .map(|observation| (observation.id.as_str(), observation))
397 .collect::<BTreeMap<_, _>>();
398 let finding_observations = report
399 .findings
400 .iter()
401 .flat_map(|finding| finding.observation_ids.iter().map(String::as_str))
402 .collect::<BTreeSet<_>>();
403
404 let mut ids = BTreeSet::new();
405 for sentinel in sentinels {
406 if sentinel.schema_version != CALIBRATION_SENTINEL_SCHEMA_VERSION {
407 issues.push(format!(
408 "calibration sentinel {:?} has unsupported schema_version",
409 sentinel.id
410 ));
411 }
412 required("calibration sentinel id", &sentinel.id, &mut issues);
413 required(
414 "calibration sentinel capture_id",
415 &sentinel.capture_id,
416 &mut issues,
417 );
418 if !ids.insert(sentinel.id.as_str()) {
419 issues.push(format!(
420 "duplicate calibration sentinel id {:?}",
421 sentinel.id
422 ));
423 }
424 if !planned.contains(sentinel.capture_id.as_str())
425 || !evaluated.contains(sentinel.capture_id.as_str())
426 {
427 issues.push(format!(
428 "calibration sentinel {:?} capture was not planned and evaluated",
429 sentinel.id
430 ));
431 continue;
432 }
433
434 let supporting = observations.values().filter(|observation| {
435 observation.capture_id == sentinel.capture_id
436 && finding_observations.contains(observation.id.as_str())
437 });
438 match &sentinel.expectation {
439 CalibrationExpectation::Clear => {
440 if supporting.count() != 0 {
441 issues.push(format!(
442 "known-good calibration sentinel {:?} produced a finding",
443 sentinel.id
444 ));
445 }
446 }
447 CalibrationExpectation::Finding { observation_kind } => {
448 required(
449 "calibration observation_kind",
450 observation_kind,
451 &mut issues,
452 );
453 if !supporting
454 .into_iter()
455 .any(|observation| observation.kind == *observation_kind)
456 {
457 issues.push(format!(
458 "known-bad calibration sentinel {:?} did not produce finding kind {:?}",
459 sentinel.id, observation_kind
460 ));
461 }
462 }
463 }
464 }
465
466 if issues.is_empty() {
467 Ok(())
468 } else {
469 Err(issues)
470 }
471}
472
473fn validate_cell_coverage(
474 report: &VisualRunReportV1,
475 planned: &[String],
476 evaluated: &[String],
477 issues: &mut Vec<String>,
478) {
479 let planned_set: BTreeSet<_> = planned.iter().map(String::as_str).collect();
480 let evaluated_set: BTreeSet<_> = evaluated.iter().map(String::as_str).collect();
481 if planned.is_empty() {
482 issues.push("visual run must declare at least one planned capture cell".to_owned());
483 }
484 if planned_set.len() != planned.len() {
485 issues.push("visual run planned capture ids must be unique".to_owned());
486 }
487 if evaluated_set.len() != evaluated.len() {
488 issues.push("visual run evaluated capture ids must be unique".to_owned());
489 }
490 if !evaluated_set.is_subset(&planned_set) {
491 issues.push("visual run evaluated cells must be planned cells".to_owned());
492 }
493 if matches!(report.status, VisualRunStatus::Pass) && planned_set != evaluated_set {
494 issues.push("passing visual run must evaluate every planned capture cell".to_owned());
495 }
496}
497
498fn required(name: &str, value: &str, issues: &mut Vec<String>) {
499 if value.trim().is_empty() {
500 issues.push(format!("{name} must not be empty"));
501 }
502}
503
504fn validate_digest(artifact: &ArtifactDigest, issues: &mut Vec<String>) {
505 validate_relative_path("artifact", &artifact.path, issues);
506 if artifact.bytes == 0 {
507 issues.push(format!("artifact {} is empty", artifact.path.display()));
508 }
509 if artifact.sha256.len() != 64
510 || !artifact
511 .sha256
512 .bytes()
513 .all(|byte| byte.is_ascii_hexdigit() && !byte.is_ascii_uppercase())
514 {
515 issues.push(format!(
516 "artifact {} has invalid sha256",
517 artifact.path.display()
518 ));
519 }
520}
521
522fn validate_relative_path(name: &str, path: &Path, issues: &mut Vec<String>) {
523 if path.as_os_str().is_empty()
524 || path.is_absolute()
525 || path
526 .components()
527 .any(|component| matches!(component, Component::ParentDir))
528 {
529 issues.push(format!(
530 "{name} path {} must be relative and contained",
531 path.display()
532 ));
533 }
534}
535
536#[cfg(test)]
537mod tests {
538 use super::*;
539
540 #[test]
541 fn rejects_a_pass_with_findings_or_dirty_source() {
542 let report = VisualRunReportV1 {
543 schema_version: VISUAL_RUN_REPORT_SCHEMA_VERSION,
544 run_id: "run-1".to_owned(),
545 target: "regicide/game".to_owned(),
546 revision: "abc123".to_owned(),
547 dirty: true,
548 capture_manifest: ArtifactDigest {
549 path: "capture_manifest.json".into(),
550 sha256: "0".repeat(64),
551 bytes: 1,
552 },
553 planned_capture_ids: vec!["menu/desktop".to_owned()],
554 evaluated_capture_ids: vec!["menu/desktop".to_owned()],
555 status: VisualRunStatus::Pass,
556 observations: vec![],
557 findings: vec![FindingV1 {
558 schema_version: FINDING_SCHEMA_VERSION,
559 fingerprint: "layout/main-menu".to_owned(),
560 target: "regicide/game".to_owned(),
561 revision: "abc123".to_owned(),
562 title: "Menu overlaps".to_owned(),
563 observation_ids: vec!["missing".to_owned()],
564 reproduction: vec!["run visual suite".to_owned()],
565 }],
566 };
567 let issues = validate_visual_run(&report).expect_err("invalid pass");
568 assert!(
569 issues
570 .iter()
571 .any(|issue| issue.contains("must not be dirty"))
572 );
573 assert!(
574 issues
575 .iter()
576 .any(|issue| issue.contains("cannot contain findings"))
577 );
578 }
579
580 #[test]
581 fn rejects_empty_passing_coverage() {
582 let report = VisualRunReportV1 {
583 schema_version: VISUAL_RUN_REPORT_SCHEMA_VERSION,
584 run_id: "run-1".to_owned(),
585 target: "regicide/game".to_owned(),
586 revision: "abc123".to_owned(),
587 dirty: false,
588 capture_manifest: ArtifactDigest {
589 path: "capture_manifest.json".into(),
590 sha256: "0".repeat(64),
591 bytes: 1,
592 },
593 planned_capture_ids: Vec::new(),
594 evaluated_capture_ids: Vec::new(),
595 status: VisualRunStatus::Pass,
596 observations: Vec::new(),
597 findings: Vec::new(),
598 };
599 let issues = validate_visual_run(&report).expect_err("empty pass must fail");
600 assert!(issues.iter().any(|issue| issue.contains("planned capture")));
601 }
602
603 #[test]
604 fn finding_fingerprint_is_stable_and_verified() {
605 let observations = vec![
606 ObservationV1 {
607 schema_version: OBSERVATION_SCHEMA_VERSION,
608 id: "obs-b".to_owned(),
609 capture_id: "menu/mobile".to_owned(),
610 kind: "layout-overlap".to_owned(),
611 summary: "buttons overlap".to_owned(),
612 evidence: Vec::new(),
613 },
614 ObservationV1 {
615 schema_version: OBSERVATION_SCHEMA_VERSION,
616 id: "obs-a".to_owned(),
617 capture_id: "menu/desktop".to_owned(),
618 kind: "layout-overlap".to_owned(),
619 summary: "buttons overlap".to_owned(),
620 evidence: Vec::new(),
621 },
622 ];
623 let mut reversed = observations.clone();
624 reversed.reverse();
625 let fingerprint = finding_fingerprint("regicide/game", &observations);
626 assert_eq!(fingerprint, finding_fingerprint("regicide/game", &reversed));
627
628 let report = VisualRunReportV1 {
629 schema_version: VISUAL_RUN_REPORT_SCHEMA_VERSION,
630 run_id: "run-1".to_owned(),
631 target: "regicide/game".to_owned(),
632 revision: "abc123".to_owned(),
633 dirty: false,
634 capture_manifest: ArtifactDigest {
635 path: "capture_manifest.json".into(),
636 sha256: "0".repeat(64),
637 bytes: 1,
638 },
639 planned_capture_ids: vec!["menu/desktop".to_owned(), "menu/mobile".to_owned()],
640 evaluated_capture_ids: vec!["menu/desktop".to_owned(), "menu/mobile".to_owned()],
641 status: VisualRunStatus::Fail,
642 observations,
643 findings: vec![FindingV1 {
644 schema_version: FINDING_SCHEMA_VERSION,
645 fingerprint,
646 target: "regicide/game".to_owned(),
647 revision: "abc123".to_owned(),
648 title: "Menu buttons overlap".to_owned(),
649 observation_ids: vec!["obs-a".to_owned(), "obs-b".to_owned()],
650 reproduction: vec!["run the visual matrix".to_owned()],
651 }],
652 };
653 assert!(validate_visual_run(&report).is_ok());
654
655 let mut reordered = report.clone();
656 reordered.planned_capture_ids.reverse();
657 reordered.evaluated_capture_ids.reverse();
658 reordered.observations.reverse();
659 reordered.findings[0].observation_ids.reverse();
660 assert_eq!(
661 report.to_canonical_json().expect("canonical report"),
662 reordered.to_canonical_json().expect("canonical report")
663 );
664
665 let sentinel = CalibrationSentinelV1 {
666 schema_version: CALIBRATION_SENTINEL_SCHEMA_VERSION,
667 id: "known-overlap".to_owned(),
668 capture_id: "menu/desktop".to_owned(),
669 expectation: CalibrationExpectation::Finding {
670 observation_kind: "layout-overlap".to_owned(),
671 },
672 };
673 assert!(validate_calibration_sentinels(&report, &[sentinel]).is_ok());
674 }
675
676 #[test]
677 fn calibration_fails_when_a_negative_sentinel_is_missed() {
678 let report = VisualRunReportV1 {
679 schema_version: VISUAL_RUN_REPORT_SCHEMA_VERSION,
680 run_id: "run-1".to_owned(),
681 target: "regicide/game".to_owned(),
682 revision: "abc123".to_owned(),
683 dirty: false,
684 capture_manifest: ArtifactDigest {
685 path: "capture_manifest.json".into(),
686 sha256: "0".repeat(64),
687 bytes: 1,
688 },
689 planned_capture_ids: vec!["menu/desktop".to_owned()],
690 evaluated_capture_ids: vec!["menu/desktop".to_owned()],
691 status: VisualRunStatus::Pass,
692 observations: Vec::new(),
693 findings: Vec::new(),
694 };
695 let sentinel = CalibrationSentinelV1 {
696 schema_version: CALIBRATION_SENTINEL_SCHEMA_VERSION,
697 id: "known-overlap".to_owned(),
698 capture_id: "menu/desktop".to_owned(),
699 expectation: CalibrationExpectation::Finding {
700 observation_kind: "layout-overlap".to_owned(),
701 },
702 };
703 let issues = validate_calibration_sentinels(&report, &[sentinel])
704 .expect_err("missed negative sentinel must fail calibration");
705 assert!(issues.iter().any(|issue| issue.contains("did not produce")));
706 }
707}