use std::collections::BTreeSet;
use std::fs;
use std::fs::File;
use std::fs::OpenOptions;
use std::io;
use std::os::fd::AsRawFd;
use std::os::unix::fs::MetadataExt;
use std::os::unix::fs::OpenOptionsExt;
use std::path::Path;
use std::path::PathBuf;
use colored::Colorize;
use detcore::logdiff;
use detcore::logdiff::ComparisonSideLabels;
use detcore::logdiff::LogComparisonMode;
use detcore_model::summary::RunSummary;
use hermit::Context;
use hermit::Error;
use hermit::HERMIT_VERIFICATION_DIVERGENCE_EXIT;
use hermit::canonical_verdict::ComparedLogMessages;
use hermit::canonical_verdict::ComparedLogScope;
use hermit::canonical_verdict::ComparedOutput;
use hermit::canonical_verdict::ComparedOutputs;
use hermit::canonical_verdict::ComparisonReport;
use hermit::canonical_verdict::ContainerDisposition;
use hermit::canonical_verdict::ContainerFailure;
pub(crate) use hermit::canonical_verdict::DbtCountedBranchComparison;
use hermit::canonical_verdict::InfrastructureError;
pub(crate) use hermit::canonical_verdict::LogCompareStrictness;
pub(crate) use hermit::canonical_verdict::NoResultReason;
use hermit::canonical_verdict::RecordEnvelopeReport;
use hermit::canonical_verdict::RuntimeStats;
pub(crate) use hermit::canonical_verdict::Verdict;
pub(crate) use hermit::canonical_verdict::VerificationReport;
pub(crate) use hermit::canonical_verdict::VerificationRun;
pub(crate) use hermit::canonical_verdict::VerificationRuntime;
use pretty_assertions::Comparison;
use reverie::process::ExitStatus;
use reverie::process::Output;
use tempfile::NamedTempFile;
use tempfile::TempPath;
use tracing::metadata::LevelFilter;
use super::global_opts::GlobalOpts;
use super::record_envelope::RecordEnvelope;
use super::record_envelope::RecordEnvelopePolicy;
const FAILED_VERIFY_LOG_DIR_PREFIX: &str = "comparison-";
const FAILED_VERIFY_LOG_PENDING_PREFIX: &str = ".pending-comparison-";
const FAILED_VERIFY_LOG_RETIRING_PREFIX: &str = ".retiring-comparison-";
const FAILED_VERIFY_LOG_LOCK: &str = ".retention.lock";
const FAILED_VERIFY_LOG_ROOT_LOCK: &str = ".retirement.lock";
const FAILED_VERIFY_LOG_COMPARISONS_TO_KEEP: usize = 64;
#[derive(Clone, Debug)]
pub(crate) struct FailedVerifyLogRetention {
root: PathBuf,
keep: usize,
}
impl FailedVerifyLogRetention {
#[cfg(test)]
fn new(root: PathBuf, keep: usize) -> Self {
Self { root, keep }
}
}
pub(crate) fn default_failed_verify_log_retention() -> FailedVerifyLogRetention {
let root = dirs::state_dir()
.map(|path| path.join("hermit").join("verify-failures"))
.unwrap_or_else(|| std::env::temp_dir().join("hermit-verify-failures"));
FailedVerifyLogRetention {
root,
keep: FAILED_VERIFY_LOG_COMPARISONS_TO_KEEP,
}
}
pub(crate) struct ComparedRun<'a> {
pub output: &'a Output,
pub log: TempPath,
pub label: &'a str,
}
pub(crate) struct ComparisonOptions {
pub verbose: bool,
pub strictness: LogCompareStrictness,
pub compare_logs: bool,
pub diagnostic_full_trace: bool,
pub compare_io_buffers: bool,
pub keep_logs: bool,
pub match_overridden: bool,
pub failed_log_retention: Option<FailedVerifyLogRetention>,
pub record_envelope: RecordEnvelope,
pub virtualize_time: bool,
}
pub use detcore::logdiff::CANON_ADDRESS_ORDINAL_V1;
pub use detcore::logdiff::STRIP_WALL_CLOCK_PREFIX_V1;
pub const STRIP_UNSAFE_NORMALIZATION_V1: &str = "unsafe-numeric-address-and-path-normalization/v1";
const PARITY_STRIPPED_PREFIXES: &[&str] = &[STRIP_WALL_CLOCK_PREFIX_V1];
const PARITY_CANONICALIZATIONS: &[&str] = &[CANON_ADDRESS_ORDINAL_V1];
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub struct ComparisonSpec {
pub strictness: LogCompareStrictness,
pub display_name: &'static str,
pub compare_logs: bool,
pub compare_io_buffers: bool,
pub log_scope: ComparedLogScope,
pub record_envelope: RecordEnvelopePolicy,
pub virtualize_time: bool,
pub strip_lines: bool,
pub canonicalize_addresses: bool,
pub full_trace: bool,
pub exact_remainder: bool,
pub stripped_prefixes: &'static [&'static str],
pub canonicalizations: &'static [&'static str],
pub ignore_lines: bool,
pub skip_commit: bool,
pub skip_detlog: bool,
}
impl ComparisonSpec {
pub fn new(
strictness: LogCompareStrictness,
compare_logs: bool,
diagnostic_full_trace: bool,
compare_io_buffers: bool,
record_envelope: RecordEnvelopePolicy,
virtualize_time: bool,
) -> Self {
let (
strip_lines,
canonicalize_addresses,
full_trace,
exact_remainder,
log_scope,
stripped_prefixes,
canonicalizations,
display_name,
) = match strictness {
LogCompareStrictness::Stripped => {
debug_assert!(!diagnostic_full_trace);
(
true,
false,
false,
false,
ComparedLogScope::Deterministic,
&[STRIP_WALL_CLOCK_PREFIX_V1, STRIP_UNSAFE_NORMALIZATION_V1][..],
&[][..],
"Stripped",
)
}
LogCompareStrictness::Canonical => (
false,
true,
true,
true,
if diagnostic_full_trace {
ComparedLogScope::FullTrace
} else {
ComparedLogScope::Info
},
PARITY_STRIPPED_PREFIXES,
PARITY_CANONICALIZATIONS,
if diagnostic_full_trace {
"BitwiseFullTraceV1"
} else {
"BitwiseInfoV1"
},
),
};
ComparisonSpec {
strictness,
display_name,
compare_logs,
compare_io_buffers,
log_scope,
record_envelope,
virtualize_time,
strip_lines,
canonicalize_addresses,
full_trace,
exact_remainder,
stripped_prefixes,
canonicalizations,
ignore_lines: false,
skip_commit: false,
skip_detlog: false,
}
}
fn log_comparison_mode(&self) -> LogComparisonMode {
match self.log_scope {
ComparedLogScope::Deterministic => LogComparisonMode::Deterministic,
ComparedLogScope::Info => LogComparisonMode::Info,
ComparedLogScope::FullTrace => LogComparisonMode::FullTrace,
}
}
fn uses_bitwise_info_v1(&self) -> bool {
self.strictness == LogCompareStrictness::Canonical
&& self.compare_logs
&& self.log_scope == ComparedLogScope::Info
&& self.record_envelope == RecordEnvelopePolicy::AllRecordsV1
&& !self.strip_lines
&& self.canonicalize_addresses
&& self.full_trace
&& self.exact_remainder
&& self.stripped_prefixes == PARITY_STRIPPED_PREFIXES
&& self.canonicalizations == PARITY_CANONICALIZATIONS
&& !self.ignore_lines
&& !self.skip_commit
&& !self.skip_detlog
}
pub fn is_bitwise_parity(&self) -> bool {
self.compare_io_buffers
&& self.compare_logs
&& self.full_trace
&& matches!(
self.log_scope,
ComparedLogScope::Info | ComparedLogScope::FullTrace
)
&& !self.strip_lines
&& self.canonicalize_addresses
&& self.exact_remainder
&& self.stripped_prefixes == PARITY_STRIPPED_PREFIXES
&& self.canonicalizations == PARITY_CANONICALIZATIONS
&& self.record_envelope.is_canonical()
&& !self.ignore_lines
&& !self.skip_commit
&& !self.skip_detlog
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub struct ComparedLogCounts {
pub left: usize,
pub right: usize,
}
impl ComparedLogCounts {
pub fn is_nonzero(&self) -> bool {
self.left > 0 && self.right > 0
}
}
fn runtime_stats(summary: &RunSummary) -> RuntimeStats {
RuntimeStats {
scheduler_turns: summary.sched_turns,
virtual_nanoseconds: summary.virttime_elapsed,
syscalls: summary.syscalls,
dispatch: summary.dispatch_stats.clone(),
}
}
pub(crate) fn verification_runtime_from_summaries(
run1: Option<&RunSummary>,
run2: Option<&RunSummary>,
) -> Option<VerificationRuntime> {
(run1.is_some() || run2.is_some()).then(|| VerificationRuntime {
run1: run1.map(runtime_stats),
run2: run2.map(runtime_stats),
})
}
#[derive(Debug, Clone)]
pub struct VerificationOutcome {
pub verdict: Verdict,
pub no_result_reason: Option<NoResultReason>,
pub guest_status: ExitStatus,
pub comparison: ComparisonSpec,
pub compared_log_messages: Option<ComparedLogCounts>,
pub compared_outputs: ComparedOutputs,
pub dbt_counted_branches: Option<DbtCountedBranchComparison>,
pub runtime: Option<VerificationRuntime>,
pub first_divergent_scheduler_turn: Option<u64>,
pub first_divergent_virtual_nanoseconds: Option<u64>,
pub first_divergent_record: Option<usize>,
pub first_divergent_syscall: Option<u64>,
pub first_divergent_left_message: Option<String>,
pub first_divergent_right_message: Option<String>,
}
impl VerificationOutcome {
pub fn verified(&self) -> bool {
self.verdict == Verdict::Matched
}
pub fn into_exit_status(self) -> Result<ExitStatus, Error> {
match self.verdict {
Verdict::Matched => Ok(self.guest_status),
Verdict::Diverged => Ok(ExitStatus::Exited(HERMIT_VERIFICATION_DIVERGENCE_EXIT)),
Verdict::NoResult => match self.no_result_reason {
Some(NoResultReason::ComparisonRefused { detail }) => Err(Error::msg(format!(
"Verification did not reach a verdict: {detail}"
))),
Some(reason) => Err(Error::msg(format!(
"Verification did not reach a verdict: {reason:?}"
))),
None => Err(Error::msg("Verification did not reach a verdict.")),
},
Verdict::InfrastructureError => {
Err(Error::msg("Verification recorded an infrastructure error."))
}
}
}
}
fn comparison_report(comparison: &ComparisonSpec) -> ComparisonReport {
ComparisonReport {
strictness: comparison.strictness,
display_name: Some(comparison.display_name.to_string()),
compare_logs: comparison.compare_logs,
compare_io_buffers: Some(comparison.compare_io_buffers),
log_scope: Some(comparison.log_scope),
record_envelope: match comparison.record_envelope {
RecordEnvelopePolicy::AllRecordsV1 => RecordEnvelopeReport::AllRecordsV1,
RecordEnvelopePolicy::DbtEvidenceTransportV1 => {
RecordEnvelopeReport::DbtEvidenceTransportV1
}
RecordEnvelopePolicy::CrossBackendDetcoreV1 => {
RecordEnvelopeReport::CrossBackendDetcoreV1
}
RecordEnvelopePolicy::CallerDefined => RecordEnvelopeReport::CallerDefined,
},
virtualize_time: Some(comparison.virtualize_time),
strip_lines: Some(comparison.strip_lines),
canonicalize_addresses: Some(comparison.canonicalize_addresses),
full_trace: Some(comparison.full_trace),
exact_remainder: Some(comparison.exact_remainder),
stripped_prefixes: Some(
comparison
.stripped_prefixes
.iter()
.map(|value| (*value).to_string())
.collect(),
),
canonicalizations: Some(
comparison
.canonicalizations
.iter()
.map(|value| (*value).to_string())
.collect(),
),
ignore_lines: Some(comparison.ignore_lines),
skip_commit: Some(comparison.skip_commit),
skip_detlog: Some(comparison.skip_detlog),
}
}
pub(crate) fn verification_report(outcome: &VerificationOutcome) -> VerificationReport {
VerificationReport {
verified: outcome.verified(),
bitwise_parity: outcome.verified()
&& outcome.comparison.is_bitwise_parity()
&& outcome
.compared_log_messages
.is_some_and(|counts| counts.is_nonzero()),
verdict: outcome.verdict,
no_result_reason: outcome.no_result_reason.clone(),
infrastructure_error: None,
comparison: (outcome.verdict != Verdict::NoResult)
.then(|| comparison_report(&outcome.comparison)),
compared_log_messages: if outcome.verdict == Verdict::NoResult {
None
} else {
outcome
.compared_log_messages
.map(|counts| ComparedLogMessages {
left: u64::try_from(counts.left).expect("compared log count fits u64"),
right: u64::try_from(counts.right).expect("compared log count fits u64"),
})
},
compared_outputs: (outcome.verdict != Verdict::NoResult)
.then(|| outcome.compared_outputs.clone()),
dbt_counted_branches: if outcome.verdict == Verdict::NoResult {
None
} else {
outcome.dbt_counted_branches
},
runtime: outcome.runtime.clone(),
guest_exit_code: outcome.guest_status.code(),
guest_signal: outcome.guest_status.signal(),
first_divergent_scheduler_turn: outcome.first_divergent_scheduler_turn,
first_divergent_virtual_nanoseconds: outcome.first_divergent_virtual_nanoseconds,
first_divergent_record: outcome
.first_divergent_record
.map(|record| u64::try_from(record).expect("divergent record index fits u64")),
first_divergent_syscall: outcome.first_divergent_syscall,
first_divergent_left_message: outcome.first_divergent_left_message.clone(),
first_divergent_right_message: outcome.first_divergent_right_message.clone(),
}
}
pub fn write_verification_json(path: &Path, outcome: &VerificationOutcome) -> Result<(), Error> {
write_report_json(path, &verification_report(outcome))
}
pub fn write_skid_overshoot_verification_json(
path: &Path,
outcome: &VerificationOutcome,
count: u64,
) -> Result<(), Error> {
assert!(
count > 0,
"a skid-overshoot report requires a positive count"
);
let mut report = verification_report(outcome);
report.verified = false;
report.bitwise_parity = false;
report.verdict = Verdict::InfrastructureError;
report.no_result_reason = None;
report.infrastructure_error = Some(InfrastructureError::SkidOvershoot { count });
report.dbt_counted_branches = None;
write_report_json(path, &report)
}
pub fn write_skid_overshoot_without_comparison_json(
path: &Path,
count: u64,
runtime: Option<VerificationRuntime>,
guest_status: Option<ExitStatus>,
) -> Result<(), Error> {
assert!(
count > 0,
"a skid-overshoot report requires a positive count"
);
let mut report = VerificationReport::no_result();
report.verdict = Verdict::InfrastructureError;
report.no_result_reason = None;
report.infrastructure_error = Some(InfrastructureError::SkidOvershoot { count });
report.runtime = runtime;
report.guest_exit_code = guest_status.as_ref().and_then(ExitStatus::code);
report.guest_signal = guest_status.as_ref().and_then(ExitStatus::signal);
write_report_json(path, &report)
}
pub fn write_pending_verification_json(path: &Path) -> Result<(), Error> {
write_report_json(path, &VerificationReport::no_result())
}
pub(crate) fn emit_compared_guest_output(
verified: bool,
first: &Output,
second: &Output,
) -> io::Result<()> {
use std::io::Write;
if verified || (first.stdout == second.stdout && first.stderr == second.stderr) {
std::io::stdout().write_all(&first.stdout)?;
std::io::stderr().write_all(&first.stderr)?;
}
Ok(())
}
pub(crate) fn run_verification_execution<T>(
path: Option<&Path>,
run: VerificationRun,
execute: impl FnOnce() -> Result<T, Error>,
) -> Result<T, Error> {
execute().map_err(|error| {
let Some(super::container::ContainerChildExit(status)) =
error.downcast_ref::<super::container::ContainerChildExit>()
else {
return error;
};
let Some(path) = path else { return error };
let disposition = match *status {
ExitStatus::Exited(code) => ContainerDisposition::Exited { code },
ExitStatus::Signaled(signal, core_dumped) => ContainerDisposition::Signaled {
signal: signal as i32,
core_dumped,
},
};
let mut report = VerificationReport::no_result();
report.no_result_reason = Some(NoResultReason::ContainerFailed(ContainerFailure {
run,
disposition,
}));
match write_report_json(path, &report) {
Ok(()) => error,
Err(secondary) => {
retain_verification_error(error, "publishing container failure", secondary)
}
}
})
}
#[derive(Debug)]
pub(crate) struct VerificationFailureContext {
errors: Vec<(&'static str, Error)>,
}
impl std::fmt::Display for VerificationFailureContext {
fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
for (index, (operation, error)) in self.errors.iter().enumerate() {
if index > 0 {
formatter.write_str("; ")?;
}
write!(formatter, "{operation} also failed: {error:#}")?;
}
Ok(())
}
}
pub(crate) fn retain_verification_error(
mut primary: Error,
operation: &'static str,
error: Error,
) -> Error {
if let Some(context) = primary.downcast_mut::<VerificationFailureContext>() {
context.errors.push((operation, error));
primary
} else {
primary.context(VerificationFailureContext {
errors: vec![(operation, error)],
})
}
}
pub(crate) fn retain_logs_after_verification_error<const N: usize>(
mut primary: Error,
logs: [(&str, TempPath); N],
) -> Error {
eprintln!(":: Verification logs retained:");
for (label, log) in logs {
if let Err(secondary) = retain_verification_log(label, log) {
primary = retain_verification_error(primary, "retaining verification log", secondary);
}
}
primary
}
fn staging_directory(path: &Path) -> &Path {
match path.parent() {
Some(parent) if !parent.as_os_str().is_empty() => parent,
_ => Path::new("."),
}
}
pub fn write_report_json(path: &Path, report: &VerificationReport) -> Result<(), Error> {
use std::io::Write as _;
let json = serde_json::to_string(report)?;
let mut temp = NamedTempFile::new_in(staging_directory(path))
.with_context(|| format!("creating a temporary file beside {}", path.display()))?;
writeln!(temp, "{json}")
.with_context(|| format!("writing verification verdict for {}", path.display()))?;
temp.flush()
.with_context(|| format!("flushing verification verdict for {}", path.display()))?;
temp.persist(path)
.map_err(|e| e.error)
.with_context(|| format!("publishing verification verdict to {}", path.display()))?;
Ok(())
}
pub(crate) fn validate_log_level(global: &GlobalOpts) -> Result<(), Error> {
if let Some(level) = global.log
&& level < LevelFilter::INFO
{
anyhow::bail!(
"--verify requires --log=info or a more verbose level; received --log={}",
level.to_string().to_ascii_lowercase()
);
}
Ok(())
}
pub(crate) fn verification_log_level(
requested: Option<LevelFilter>,
strictness: LogCompareStrictness,
diagnostic_full_trace: bool,
) -> LevelFilter {
if diagnostic_full_trace {
requested
.unwrap_or(LevelFilter::TRACE)
.max(LevelFilter::TRACE)
} else {
requested.unwrap_or(match strictness {
LogCompareStrictness::Stripped => LevelFilter::DEBUG,
LogCompareStrictness::Canonical => LevelFilter::INFO,
})
}
}
pub fn temp_log_files(name1: &str, name2: &str) -> io::Result<(NamedTempFile, NamedTempFile)> {
temp_log_files_in(name1, name2, None)
}
pub fn temp_log_files_in(
name1: &str,
name2: &str,
directory: Option<&Path>,
) -> io::Result<(NamedTempFile, NamedTempFile)> {
let create = |name: &str| {
let prefix = format!("{}_log_", name);
let mut builder = tempfile::Builder::new();
builder.prefix(&prefix).rand_bytes(5);
match directory {
Some(directory) => builder.tempfile_in(directory),
None => builder.tempfile(),
}
};
let file1 = create(name1)?;
let file2 = create(name2)?;
Ok((file1, file2))
}
pub fn setup_double_run(
global: &GlobalOpts,
name1: &str,
name2: &str,
strictness: LogCompareStrictness,
) -> ((GlobalOpts, NamedTempFile), (GlobalOpts, NamedTempFile)) {
let (file1, file2) = temp_log_files(name1, name2).unwrap();
let path1 = PathBuf::from(file1.path());
let path2 = PathBuf::from(file2.path());
let mut global = global.clone();
global.log_file = Some(path1);
global.log = Some(verification_log_level(global.log, strictness, false));
let mut global2 = global.clone();
global2.log_file = Some(path2);
((global, file1), (global2, file2))
}
fn unsupported_syscalls_from_log(path: &Path) -> io::Result<BTreeSet<String>> {
let mut syscalls = BTreeSet::new();
for line in std::fs::read_to_string(path)?.lines() {
let Some((_, remainder)) = line.split_once("syscalls ") else {
continue;
};
let Some((names, _)) = remainder.split_once(" used but not yet supported") else {
continue;
};
for name in names.split(',') {
if !name.is_empty()
&& name
.bytes()
.all(|byte| byte.is_ascii_alphanumeric() || byte == b'_')
{
syscalls.insert(name.to_owned());
}
}
}
Ok(syscalls)
}
pub(crate) fn retain_verification_logs<const N: usize>(
logs: [(&str, TempPath); N],
) -> Result<Vec<PathBuf>, Error> {
let mut retained = Vec::with_capacity(N);
eprintln!(":: Verification logs retained:");
for (label, log) in logs {
retained.push(retain_verification_log(label, log)?);
}
Ok(retained)
}
fn retain_verification_log(label: &str, log: TempPath) -> Result<PathBuf, Error> {
let path = log.keep()?;
eprintln!(":: {label}: {}", path.display());
Ok(path)
}
fn flock(file: &File, operation: i32) -> io::Result<()> {
if unsafe { libc::flock(file.as_raw_fd(), operation) } == 0 {
Ok(())
} else {
Err(io::Error::last_os_error())
}
}
fn open_retention_lock(path: &Path) -> io::Result<File> {
OpenOptions::new()
.read(true)
.write(true)
.create(true)
.truncate(false)
.mode(0o600)
.open(path)
}
fn persist_log(log: TempPath, destination: &Path) -> io::Result<()> {
match log.persist_noclobber(destination) {
Ok(()) => Ok(()),
Err(error) if error.error.raw_os_error() == Some(libc::EXDEV) => {
fs::copy(&*error.path, destination)?;
drop(error.path);
Ok(())
}
Err(error) => Err(error.error),
}
}
fn retained_comparison_dirs(root: &Path) -> io::Result<Vec<(PathBuf, std::time::SystemTime)>> {
let mut directories = Vec::new();
for entry in fs::read_dir(root)? {
let entry = entry?;
let name = entry.file_name();
let Some(name) = name.to_str() else {
continue;
};
if !name.starts_with(FAILED_VERIFY_LOG_DIR_PREFIX) {
continue;
}
let metadata = fs::symlink_metadata(entry.path())?;
if !metadata.file_type().is_dir() || metadata.uid() != unsafe { libc::geteuid() } {
continue;
}
directories.push((entry.path(), metadata.modified()?));
}
directories.sort_by(|(left_path, left_time), (right_path, right_time)| {
right_time
.cmp(left_time)
.then_with(|| right_path.cmp(left_path))
});
Ok(directories)
}
fn finish_interrupted_retirements(root: &Path) -> io::Result<()> {
for entry in fs::read_dir(root)? {
let entry = entry?;
let name = entry.file_name();
let Some(name) = name.to_str() else {
continue;
};
if !name.starts_with(FAILED_VERIFY_LOG_RETIRING_PREFIX)
&& !name.starts_with(FAILED_VERIFY_LOG_PENDING_PREFIX)
{
continue;
}
let metadata = fs::symlink_metadata(entry.path())?;
if metadata.file_type().is_dir() && metadata.uid() == unsafe { libc::geteuid() } {
fs::remove_dir_all(entry.path())?;
}
}
Ok(())
}
fn retire_failed_verification_logs(root: &Path, current: &Path, keep: usize) -> io::Result<usize> {
let directories = retained_comparison_dirs(root)?;
let mut kept = BTreeSet::from([current.to_path_buf()]);
for (path, _) in &directories {
if kept.len() >= keep.max(1) {
break;
}
kept.insert(path.clone());
}
let mut retired = 0;
for (path, _) in directories {
if kept.contains(&path) {
continue;
}
let lock = match open_retention_lock(&path.join(FAILED_VERIFY_LOG_LOCK)) {
Ok(lock) => lock,
Err(error) => {
eprintln!(
"WARNING: could not inspect retained verification logs {}: {error}",
path.display()
);
continue;
}
};
match flock(&lock, libc::LOCK_EX | libc::LOCK_NB) {
Ok(()) => {}
Err(error) if error.kind() == io::ErrorKind::WouldBlock => continue,
Err(error) => {
eprintln!(
"WARNING: could not lock retained verification logs {}: {error}",
path.display()
);
continue;
}
}
let name = path
.file_name()
.and_then(|name| name.to_str())
.ok_or_else(|| io::Error::other("retained verification directory has no name"))?;
let retiring = tempfile::Builder::new()
.prefix(&format!("{FAILED_VERIFY_LOG_RETIRING_PREFIX}{name}-"))
.rand_bytes(12)
.tempdir_in(root)?
.keep();
fs::remove_dir(&retiring)?;
fs::rename(&path, &retiring)?;
fs::remove_dir_all(&retiring)?;
retired += 1;
}
Ok(retired)
}
fn retain_failed_verification_logs(
logs: [(&str, TempPath); 2],
retention: &FailedVerifyLogRetention,
) -> Result<Vec<PathBuf>, Error> {
fs::create_dir_all(&retention.root).with_context(|| {
format!(
"could not create failed verification log directory {}",
retention.root.display()
)
})?;
let root = fs::canonicalize(&retention.root).with_context(|| {
format!(
"could not resolve failed verification log directory {}",
retention.root.display()
)
})?;
let root_lock = open_retention_lock(&root.join(FAILED_VERIFY_LOG_ROOT_LOCK))?;
flock(&root_lock, libc::LOCK_EX)?;
finish_interrupted_retirements(&root)?;
let pending = tempfile::Builder::new()
.prefix(FAILED_VERIFY_LOG_PENDING_PREFIX)
.rand_bytes(12)
.tempdir_in(&root)?
.keep();
let pending_name = pending
.file_name()
.and_then(|name| name.to_str())
.ok_or_else(|| io::Error::other("failed verification directory has no name"))?;
let final_name = pending_name
.strip_prefix(FAILED_VERIFY_LOG_PENDING_PREFIX)
.ok_or_else(|| io::Error::other("failed verification directory has an invalid name"))?;
let completed = root.join(format!("{FAILED_VERIFY_LOG_DIR_PREFIX}{final_name}"));
let pair_lock = open_retention_lock(&pending.join(FAILED_VERIFY_LOG_LOCK))?;
flock(&pair_lock, libc::LOCK_EX)?;
let mut retained = Vec::with_capacity(logs.len());
for (index, (label, log)) in logs.into_iter().enumerate() {
let original_name = log
.file_name()
.and_then(|name| name.to_str())
.unwrap_or(if index == 0 { "run1.log" } else { "run2.log" });
let destination = pending.join(original_name);
persist_log(log, &destination)?;
retained.push((label, destination));
}
fs::rename(&pending, &completed)?;
for (label, path) in &mut retained {
*path = completed.join(path.file_name().expect("retained log has a name"));
eprintln!(":: {label}: {}", path.display());
}
drop(pair_lock);
let retired = retire_failed_verification_logs(&root, &completed, retention.keep)?;
if retired > 0 {
eprintln!(
":: Retired {retired} older failed verification comparison(s); keeping the newest {} plus comparisons with active readers",
retention.keep.max(1)
);
}
Ok(retained.into_iter().map(|(_, path)| path).collect())
}
pub(crate) fn lock_failed_verification_logs_for_read(
paths: impl IntoIterator<Item = PathBuf>,
) -> io::Result<Vec<File>> {
let mut directories = BTreeSet::new();
for path in paths {
let path = match fs::canonicalize(path) {
Ok(path) => path,
Err(error) if error.kind() == io::ErrorKind::NotFound => continue,
Err(error) => return Err(error),
};
let Some(directory) = path.parent() else {
continue;
};
let Some(name) = directory.file_name().and_then(|name| name.to_str()) else {
continue;
};
if name.starts_with(FAILED_VERIFY_LOG_DIR_PREFIX)
&& directory.join(FAILED_VERIFY_LOG_LOCK).is_file()
{
directories.insert(directory.to_path_buf());
}
}
let mut locks = Vec::with_capacity(directories.len());
for directory in directories {
let lock = open_retention_lock(&directory.join(FAILED_VERIFY_LOG_LOCK))?;
flock(&lock, libc::LOCK_SH)?;
locks.push(lock);
}
Ok(locks)
}
pub fn compare_two_runs(
first: ComparedRun<'_>,
second: ComparedRun<'_>,
options: ComparisonOptions,
) -> Result<VerificationOutcome, Error> {
compare_two_runs_with_unsupported_scan(first, second, options, unsupported_syscalls_from_log)
}
fn compare_two_runs_with_unsupported_scan(
first: ComparedRun<'_>,
second: ComparedRun<'_>,
options: ComparisonOptions,
scan_unsupported_syscalls: impl Fn(&Path) -> io::Result<BTreeSet<String>>,
) -> Result<VerificationOutcome, Error> {
let ComparedRun {
output: out1,
log: log1,
label: label1,
} = first;
let ComparedRun {
output: out2,
log: log2,
label: label2,
} = second;
let output_evidence = |output: &Output| ComparedOutput {
exit_code: output.status.code(),
signal: output.status.signal(),
stdout_sha256: detcore::Digest::new(&output.stdout).to_string(),
stdout_bytes: u64::try_from(output.stdout.len()).expect("guest stdout length fits u64"),
stderr_sha256: detcore::Digest::new(&output.stderr).to_string(),
stderr_bytes: u64::try_from(output.stderr.len()).expect("guest stderr length fits u64"),
};
let compared_outputs = ComparedOutputs {
left: output_evidence(out1),
right: output_evidence(out2),
};
let compared_labels = ComparisonSideLabels::new(label1, label2);
let mut failed = false;
let mut observed_divergence = false;
let mut comparison_refusal_reason = None;
let mut compared_log_messages: Option<ComparedLogCounts> = None;
let mut first_divergent_scheduler_turn = None;
let mut first_divergent_virtual_nanoseconds = None;
let mut first_divergent_record = None;
let mut first_divergent_syscall = None;
let mut first_divergent_left_message = None;
let mut first_divergent_right_message = None;
let spec = ComparisonSpec::new(
options.strictness,
options.compare_logs,
options.diagnostic_full_trace,
options.compare_io_buffers,
options.record_envelope.policy(),
options.virtualize_time,
);
if out1.stdout != out2.stdout {
failed = true;
observed_divergence = true;
eprintln!("Mismatch in stdout between {label1} and {label2}:");
let str1 = String::from_utf8_lossy(&out1.stdout);
let str2 = String::from_utf8_lossy(&out2.stdout);
if str1.lines().count() > 1 {
display_diff(&str1, &str2);
} else {
eprintln!("{}", Comparison::new(&str1, &str2));
}
}
if out1.stderr != out2.stderr {
failed = true;
observed_divergence = true;
eprintln!("Mismatch in stderr between {label1} and {label2}:");
let str1 = String::from_utf8_lossy(&out1.stderr);
let str2 = String::from_utf8_lossy(&out2.stderr);
if str1.lines().count() > 1 {
display_diff(&str1, &str2);
} else {
eprintln!("{}", Comparison::new(&str1, &str2));
}
}
let log_processing_result = (|| -> Result<(), Error> {
if options.compare_logs {
eprintln!(
":: {}",
"Comparing captured verification logs...".yellow().bold()
);
let diff_options = logdiff::LogDiffOpts {
strip_lines: spec.strip_lines,
canonicalize_addresses: spec.canonicalize_addresses,
comparison: spec.log_comparison_mode(),
side_labels: compared_labels.clone(),
require_structured_events: true,
syscall_history: if options.verbose { 10 } else { 5 },
skip_commit: spec.skip_commit,
skip_detlog: spec.skip_detlog,
..Default::default()
};
debug_assert!(
diff_options.ignore_lines.is_empty() != spec.ignore_lines,
"ComparisonSpec.ignore_lines must match the diff engine's ignore_lines"
);
let summary = if spec.uses_bitwise_info_v1() {
let mut diagnostic_output = std::io::stderr();
logdiff::try_compare_bitwise_info_v1_with_diagnostics(
log1.as_ref(),
log2.as_ref(),
compared_labels.clone(),
logdiff::BitwiseInfoV1Diagnostics {
difference_limit: diff_options.limit,
syscall_history: diff_options.syscall_history,
no_color: diff_options.no_color,
print_logs: diff_options.print_logs,
},
&mut diagnostic_output,
)?
} else {
logdiff::try_log_diff_detailed_with_filter(
log1.as_ref(),
log2.as_ref(),
&diff_options,
options.record_envelope.predicate(),
)?
};
compared_log_messages = Some(ComparedLogCounts {
left: summary.compared_left,
right: summary.compared_right,
});
if summary.diff_found {
failed = true;
if let Some(refusal_reason) = summary.refusal_reason.clone() {
comparison_refusal_reason = Some(refusal_reason);
} else {
observed_divergence = true;
}
first_divergent_scheduler_turn = summary.first_divergent_scheduler_turn;
first_divergent_virtual_nanoseconds = summary.first_divergent_virtual_nanoseconds;
first_divergent_record = summary.first_divergent_record;
first_divergent_syscall = summary.first_divergent_syscall;
first_divergent_left_message = summary.first_divergent_left_message.clone();
first_divergent_right_message = summary.first_divergent_right_message.clone();
if summary.refusal_reason.is_none() {
eprintln!(
":: {}",
format!("Log differences found between {label1} and {label2}.")
.red()
.bold()
);
}
}
} else {
eprintln!(
":: Comparing guest output and exit status only; internal logs were not compared"
);
}
if out1.status != out2.status {
failed = true;
observed_divergence = true;
eprintln!(
"Mismatch in exit status between {label1} and {label2}: {}",
Comparison::new(&out1.status, &out2.status)
);
}
if !failed {
let mut unsupported = scan_unsupported_syscalls(log1.as_ref())?;
unsupported.extend(scan_unsupported_syscalls(log2.as_ref())?);
if let Some(message) = detcore::format_unsupported_syscall_warning(&unsupported) {
eprintln!("WARNING: {message}");
}
}
Ok(())
})();
if let Err(error) = log_processing_result {
if options.keep_logs || failed {
if let Some(retention) = &options.failed_log_retention {
eprintln!(":: Verification logs retained:");
retain_failed_verification_logs([(label1, log1), (label2, log2)], retention)?;
} else {
retain_verification_logs([(label1, log1), (label2, log2)])?;
}
}
return Err(error);
}
let keep_golden_log_only = options.keep_logs
&& !failed
&& !options.match_overridden
&& compared_log_messages.is_some();
if keep_golden_log_only {
retain_verification_logs([(label1, log1)])?;
let matched_log = log2.to_path_buf();
log2.close().with_context(|| {
format!(
"could not delete {label2}'s log {}, which matched {label1}'s",
matched_log.display()
)
})?;
eprintln!(
":: Deleted {label2}'s log: it matched {label1}'s, which is kept as the golden log"
);
} else if options.keep_logs || failed {
if let Some(retention) = &options.failed_log_retention {
eprintln!(":: Verification logs retained:");
retain_failed_verification_logs([(label1, log1), (label2, log2)], retention)?;
} else {
retain_verification_logs([(label1, log1), (label2, log2)])?;
}
}
if failed {
let verdict = if comparison_refusal_reason.is_some() && !observed_divergence {
Verdict::NoResult
} else {
Verdict::Diverged
};
let no_result_reason =
(verdict == Verdict::NoResult).then(|| NoResultReason::ComparisonRefused {
detail: comparison_refusal_reason
.expect("a refused comparison retains its producer reason"),
});
Ok(VerificationOutcome {
verdict,
no_result_reason,
guest_status: out2.status,
comparison: spec,
compared_log_messages,
compared_outputs,
dbt_counted_branches: None,
runtime: None,
first_divergent_scheduler_turn,
first_divergent_virtual_nanoseconds,
first_divergent_record,
first_divergent_syscall,
first_divergent_left_message,
first_divergent_right_message,
})
} else {
Ok(VerificationOutcome {
verdict: Verdict::Matched,
no_result_reason: None,
guest_status: out2.status,
comparison: spec,
compared_log_messages,
compared_outputs,
dbt_counted_branches: None,
runtime: None,
first_divergent_scheduler_turn,
first_divergent_virtual_nanoseconds,
first_divergent_record,
first_divergent_syscall,
first_divergent_left_message,
first_divergent_right_message,
})
}
}
fn comparison_relaxations(comparison: &ComparisonSpec) -> Option<String> {
let policy_shape_matches = match comparison.strictness {
LogCompareStrictness::Stripped => {
comparison.display_name == "Stripped"
&& comparison.strip_lines
&& !comparison.canonicalize_addresses
&& !comparison.full_trace
&& !comparison.exact_remainder
&& comparison.log_scope == ComparedLogScope::Deterministic
&& comparison.stripped_prefixes
== [STRIP_WALL_CLOCK_PREFIX_V1, STRIP_UNSAFE_NORMALIZATION_V1]
&& comparison.canonicalizations.is_empty()
}
LogCompareStrictness::Canonical => {
!comparison.strip_lines
&& comparison.canonicalize_addresses
&& comparison.full_trace
&& comparison.exact_remainder
&& matches!(
(comparison.log_scope, comparison.display_name),
(ComparedLogScope::Info, "BitwiseInfoV1")
| (ComparedLogScope::FullTrace, "BitwiseFullTraceV1")
)
&& comparison.stripped_prefixes == PARITY_STRIPPED_PREFIXES
&& comparison.canonicalizations == PARITY_CANONICALIZATIONS
}
};
if !policy_shape_matches
|| !comparison.record_envelope.is_canonical()
|| comparison.ignore_lines
|| comparison.skip_commit
|| comparison.skip_detlog
{
return None;
}
let mut relaxations = Vec::new();
if comparison.strictness == LogCompareStrictness::Stripped {
relaxations.push(STRIP_UNSAFE_NORMALIZATION_V1);
}
if !comparison.compare_io_buffers {
relaxations.push("--no-detlog-io-buffers");
}
Some(if relaxations.is_empty() {
"none".to_owned()
} else {
relaxations.join(",")
})
}
fn comparison_evidence_line(comparison: &ComparisonSpec) -> Option<String> {
if !comparison.compare_logs {
return None;
}
Some(match comparison_relaxations(comparison) {
Some(relaxations) => format!(
":: comparison={} relaxations={relaxations}",
comparison.display_name
),
None => ":: comparison=unknown relaxations=unknown".to_owned(),
})
}
pub fn announce_verification_outcome(
outcome: &VerificationOutcome,
second_run: SecondRun,
success_message: &str,
failure_message: &str,
) {
let _ = write_verification_announcement(
&mut io::stderr().lock(),
outcome,
second_run,
success_message,
failure_message,
);
}
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub enum SecondRun {
Rerun,
Replay,
}
pub(crate) const BITWISE_PARITY_CLAIM: &str = "bitwise parity established";
fn write_verification_announcement(
out: &mut impl io::Write,
outcome: &VerificationOutcome,
second_run: SecondRun,
success_message: &str,
failure_message: &str,
) -> io::Result<()> {
if let Some(evidence) = comparison_evidence_line(&outcome.comparison) {
writeln!(out, "{evidence}")?;
}
match outcome.verdict {
Verdict::Matched => {
writeln!(out, ":: {}", success_message.green().bold())?;
writeln!(
out,
":: {}",
matched_comparison_qualifier(outcome, second_run)
)?;
}
Verdict::Diverged => writeln!(out, ":: {}", failure_message.red().bold())?,
Verdict::NoResult => writeln!(
out,
":: {}",
"No result: the comparison was refused, so this is neither a match nor a difference."
.red()
.bold()
)?,
Verdict::InfrastructureError => writeln!(
out,
":: {}",
"Infrastructure error: the result is neither a match nor a product difference."
.red()
.bold()
)?,
}
Ok(())
}
fn matched_comparison_qualifier(outcome: &VerificationOutcome, second_run: SecondRun) -> String {
let comparison = &outcome.comparison;
let scope = match comparison.log_scope {
ComparedLogScope::Deterministic => {
"only the DETLOG and scheduler COMMIT records of the guest's log"
}
ComparedLogScope::Info => "the INFO records of the guest's log",
ComparedLogScope::FullTrace => "every record of the guest's log",
};
let records = outcome.compared_log_messages.map_or_else(
|| scope.to_owned(),
|counts| format!("{scope} ({} vs {})", counts.left, counts.right),
);
let compared = if verification_report(outcome).bitwise_parity {
format!(
"Compared: stdout and stderr byte for byte, and {records} exactly apart from \
the wall-clock prefix and canonicalized addresses, under {}: \
{BITWISE_PARITY_CLAIM}.",
comparison.display_name
)
} else if !comparison.compare_logs {
"Compared: stdout and stderr only. The guest's log records were not compared, \
so this is not a bitwise comparison."
.to_owned()
} else if comparison.strictness == LogCompareStrictness::Stripped {
format!(
"Compared: stdout and stderr byte for byte, and {records} under the lossy \
Stripped comparison, which erases numbers, addresses, paths and timestamps \
before comparing. This is not a bitwise comparison; add --verify-strict \
for one."
)
} else {
format!(
"Compared: stdout and stderr byte for byte, and {records} under {}, but \
without all the evidence the parity contract requires (see \
`bitwise_parity` in --verify-json). This is not a bitwise comparison.",
comparison.display_name
)
};
match (comparison.virtualize_time, second_run) {
(true, _) => compared,
(false, SecondRun::Replay) => format!("{compared} {TIME_NOT_VIRTUALIZED_NOTE}"),
(false, SecondRun::Rerun) => format!("{compared} {RERUN_TIME_NOT_VIRTUALIZED_NOTE}"),
}
}
pub(crate) const TIME_NOT_VIRTUALIZED_NOTE: &str = "Time was not virtualized, so this shows \
that the second run reproduced the first, not that separate runs of the guest agree.";
pub(crate) const RERUN_TIME_NOT_VIRTUALIZED_NOTE: &str = "Time was not virtualized: the guest \
read the host's clock, so these two runs matched, but a guest whose output depends on the \
time can differ from one run to the next.";
fn display_diff(left: &str, right: &str) {
for result in diff::lines(left, right) {
match result {
diff::Result::Left(s) => {
eprintln!("- {}", s.red());
}
diff::Result::Right(s) => {
eprintln!("+ {}", s.green());
}
diff::Result::Both(s, _) => {
eprintln!(" {}", s);
}
}
}
}
#[cfg(test)]
mod tests {
use std::fs;
use super::*;
fn classified_container<T: serde::Serialize + serde::de::DeserializeOwned>(
mut child: impl FnMut() -> Result<T, Error>,
) -> Result<T, Error> {
super::super::container::classify_container_result(
reverie::process::Container::new()
.run(|| child().map_err(hermit::SerializableError::from)),
)
}
fn real_container_exit(path: &Path, run: VerificationRun, code: Option<i32>) -> Error {
run_verification_execution::<()>(Some(path), run, || {
classified_container(|| {
unsafe {
if let Some(code) = code {
libc::_exit(code);
}
let mut mask = std::mem::zeroed();
assert_eq!(libc::sigemptyset(&mut mask), 0);
assert_eq!(libc::sigaddset(&mut mask, libc::SIGALRM), 0);
assert_eq!(
libc::sigprocmask(libc::SIG_UNBLOCK, &mask, std::ptr::null_mut()),
0
);
assert_ne!(libc::signal(libc::SIGALRM, libc::SIG_DFL), libc::SIG_ERR);
libc::raise(libc::SIGALRM);
libc::_exit(99);
}
})
})
.unwrap_err()
}
fn check_real_container_failure(run: VerificationRun) {
let dir = tempfile::tempdir().unwrap();
let path = dir.path().join("verify.json");
write_pending_verification_json(&path).unwrap();
if run == VerificationRun::Run2 {
let first = run_verification_execution(Some(&path), VerificationRun::Run1, || {
classified_container(|| Ok(output(0, b"first run", b"")))
})
.unwrap();
assert_eq!(first.stdout, b"first run");
assert_eq!(first.status, ExitStatus::Exited(0));
assert_eq!(
VerificationReport::from_current_json_slice(&fs::read(&path).unwrap()).unwrap(),
VerificationReport::no_result()
);
}
let error = real_container_exit(&path, run, None);
let actual = error
.downcast_ref::<super::super::container::ContainerChildExit>()
.expect("real child must die by SIGALRM, not refuse setup");
assert_eq!(
actual.0,
ExitStatus::Signaled(reverie::process::Signal::SIGALRM, false)
);
assert_eq!(super::super::failure_exit_code(&error), 125);
let report = VerificationReport::from_current_json_slice(&fs::read(path).unwrap()).unwrap();
assert_eq!(
report.no_result_reason,
Some(NoResultReason::ContainerFailed(ContainerFailure {
run,
disposition: ContainerDisposition::Signaled {
signal: libc::SIGALRM,
core_dumped: false
}
}))
);
assert!(report.guest_exit_code.is_none() && report.guest_signal.is_none());
assert!(report.comparison.is_none() && report.compared_outputs.is_none());
assert!(report.require_canonical_comparison().is_err());
}
#[test]
fn real_container_failure_replaces_run1_stamp() {
check_real_container_failure(VerificationRun::Run1);
}
#[test]
fn real_container_failure_replaces_run2_stamp_without_claiming_run1_status() {
check_real_container_failure(VerificationRun::Run2);
}
#[test]
fn real_container_zero_exit_and_deliberate_stops_remain_distinct() {
let dir = tempfile::tempdir().unwrap();
let path = dir.path().join("verify.json");
for code in [
0,
101,
detcore_model::HERMIT_POLICY_REFUSAL_EXIT,
hermit::HERMIT_DEADLINE_EXIT,
detcore_model::signal_exit_status(libc::SIGTERM),
] {
write_pending_verification_json(&path).unwrap();
let error = real_container_exit(&path, VerificationRun::Run1, Some(code));
let report =
VerificationReport::from_current_json_slice(&fs::read(&path).unwrap()).unwrap();
match code {
0 | 101 => {
assert!(
error
.downcast_ref::<super::super::container::ContainerChildExit>()
.is_some()
);
assert_eq!(
report.no_result_reason,
Some(NoResultReason::ContainerFailed(ContainerFailure {
run: VerificationRun::Run1,
disposition: ContainerDisposition::Exited { code }
}))
);
assert_eq!(super::super::failure_exit_code(&error), 125);
}
122 => {
assert!(
error
.downcast_ref::<super::super::container::PolicyRefusal>()
.is_some()
);
assert_eq!(report, VerificationReport::no_result());
}
124 => {
assert!(
error
.downcast_ref::<super::super::container::RunTimeoutMarker>()
.is_some()
);
assert_eq!(report, VerificationReport::no_result());
}
_ => {
assert!(
error
.downcast_ref::<super::super::container::SignalDeath>()
.is_some()
);
assert_eq!(report, VerificationReport::no_result());
}
}
}
}
#[test]
fn returned_guest_signal_and_unknown_error_do_not_become_container_failure() {
let dir = tempfile::tempdir().unwrap();
let path = dir.path().join("verify.json");
write_pending_verification_json(&path).unwrap();
let expected = Output {
status: ExitStatus::Signaled(reverie::process::Signal::SIGALRM, false),
stdout: Vec::new(),
stderr: b"guest".to_vec(),
};
let returned = run_verification_execution(Some(&path), VerificationRun::Run1, || {
classified_container(|| Ok(expected.clone()))
})
.unwrap();
assert_eq!(returned.status, expected.status);
assert_eq!(returned.stderr, expected.stderr);
let error = run_verification_execution::<()>(Some(&path), VerificationRun::Run1, || {
classified_container(|| Err(Error::msg("ordinary unknown error")))
})
.unwrap_err();
assert_eq!(error.to_string(), "ordinary unknown error");
let panic = run_verification_execution::<()>(Some(&path), VerificationRun::Run1, || {
super::super::container::classify_container_result(Ok(Err(
hermit::SerializableError::from(Error::msg("caught panic control")).into_panic(),
)))
})
.unwrap_err();
assert!(
panic
.downcast_ref::<super::super::container::ContainerChildPanic>()
.is_some()
);
assert_eq!(
VerificationReport::from_current_json_slice(&fs::read(path).unwrap()).unwrap(),
VerificationReport::no_result()
);
}
#[test]
fn container_failure_keeps_primary_type_and_secondary_errors() {
let dir = tempfile::tempdir().unwrap();
let error = real_container_exit(
&dir.path().join("absent/verify.json"),
VerificationRun::Run1,
Some(101),
);
assert!(
error
.downcast_ref::<super::super::container::ContainerChildExit>()
.is_some()
);
let context = error.downcast_ref::<VerificationFailureContext>().unwrap();
assert_eq!(context.errors.len(), 1);
assert_eq!(context.errors[0].0, "publishing container failure");
assert!(
context.errors[0]
.1
.chain()
.any(|cause| cause.downcast_ref::<std::io::Error>().is_some())
);
let log = tempfile::NamedTempFile::new_in(dir.path())
.unwrap()
.into_temp_path();
fs::write(&log, b"retained failure log").unwrap();
let retained = log.to_path_buf();
let error = retain_logs_after_verification_error(error, [("run 1", log)]);
assert_eq!(fs::read(retained).unwrap(), b"retained failure log");
let secondary = Error::new(std::io::Error::from_raw_os_error(libc::EACCES));
let error = retain_verification_error(error, "retaining verification log", secondary);
assert!(
error
.downcast_ref::<super::super::container::ContainerChildExit>()
.is_some()
);
assert_eq!(super::super::failure_exit_code(&error), 125);
assert!(format!("{error:#}").contains("publishing container failure"));
let context = error.downcast_ref::<VerificationFailureContext>().unwrap();
assert_eq!(context.errors.len(), 2);
assert_eq!(context.errors[0].0, "publishing container failure");
assert_eq!(context.errors[1].0, "retaining verification log");
assert_eq!(
context.errors[1]
.1
.downcast_ref::<std::io::Error>()
.unwrap()
.raw_os_error(),
Some(libc::EACCES)
);
for (primary, expected_exit) in [
(
Error::new(super::super::container::PolicyRefusal),
detcore_model::HERMIT_POLICY_REFUSAL_EXIT,
),
(
Error::new(super::super::container::RunTimeoutMarker),
hermit::HERMIT_DEADLINE_EXIT,
),
] {
let error = retain_verification_error(
primary,
"retaining verification log",
Error::new(std::io::Error::from_raw_os_error(libc::EACCES)),
);
assert_eq!(super::super::failure_exit_code(&error), expected_exit);
let context = error.downcast_ref::<VerificationFailureContext>().unwrap();
assert_eq!(context.errors.len(), 1);
assert_eq!(context.errors[0].0, "retaining verification log");
assert_eq!(
context.errors[0]
.1
.downcast_ref::<std::io::Error>()
.unwrap()
.raw_os_error(),
Some(libc::EACCES)
);
}
}
fn output(status: i32, stdout: &[u8], stderr: &[u8]) -> Output {
Output {
status: ExitStatus::Exited(status),
stdout: stdout.to_vec(),
stderr: stderr.to_vec(),
}
}
fn empty_logs() -> (TempPath, TempPath) {
let (left, right) = temp_log_files("verify_left", "verify_right").unwrap();
(left.into_temp_path(), right.into_temp_path())
}
fn logs_with_identical_detlog() -> (TempPath, TempPath) {
let (left, right) = temp_log_files("verify_left", "verify_right").unwrap();
let left_path = left.into_temp_path();
let right_path = right.into_temp_path();
let body = format!("{}{}", detlog_with_value(1), detlog_with_value(2));
fs::write(&left_path, body.as_bytes()).unwrap();
fs::write(&right_path, body.as_bytes()).unwrap();
(left_path, right_path)
}
fn global_with_log(log: Option<LevelFilter>) -> GlobalOpts {
GlobalOpts {
log,
log_file: None,
log_file_handle: None,
run_evidence_log_handle: None,
run_evidence_write_error: None,
backend: None,
}
}
#[test]
fn verify_rejects_explicit_log_levels_below_info() {
for level in [LevelFilter::OFF, LevelFilter::ERROR, LevelFilter::WARN] {
let error = validate_log_level(&global_with_log(Some(level))).unwrap_err();
assert!(
error.to_string().contains("requires --log=info"),
"unexpected error for {level}: {error}"
);
}
}
#[test]
fn verify_accepts_default_and_info_or_more_verbose_logs() {
for level in [
None,
Some(LevelFilter::INFO),
Some(LevelFilter::DEBUG),
Some(LevelFilter::TRACE),
] {
validate_log_level(&global_with_log(level)).unwrap();
}
}
#[test]
fn verification_capture_level_honors_info_and_preserves_explicit_debug() {
assert_eq!(
verification_log_level(None, LogCompareStrictness::Canonical, false),
LevelFilter::INFO
);
assert_eq!(
verification_log_level(
Some(LevelFilter::INFO),
LogCompareStrictness::Canonical,
false,
),
LevelFilter::INFO
);
assert_eq!(
verification_log_level(
Some(LevelFilter::DEBUG),
LogCompareStrictness::Canonical,
false,
),
LevelFilter::DEBUG,
"explicit DEBUG remains captured for diagnostics"
);
assert_eq!(
verification_log_level(None, LogCompareStrictness::Stripped, false),
LevelFilter::DEBUG,
"legacy stripped verification keeps its default"
);
assert_eq!(
verification_log_level(
Some(LevelFilter::INFO),
LogCompareStrictness::Stripped,
false,
),
LevelFilter::INFO,
"an explicit INFO request must not be promoted"
);
assert_eq!(
verification_log_level(
Some(LevelFilter::INFO),
LogCompareStrictness::Canonical,
true,
),
LevelFilter::TRACE,
"explicit full-trace diagnostics require TRACE capture"
);
}
fn compare_with(
left: &Output,
left_log: TempPath,
right: &Output,
right_log: TempPath,
strictness: LogCompareStrictness,
) -> Result<VerificationOutcome, Error> {
compare_with_envelope(
left,
left_log,
right,
right_log,
strictness,
RecordEnvelope::all_records_v1(),
)
}
fn compare_with_envelope(
left: &Output,
left_log: TempPath,
right: &Output,
right_log: TempPath,
strictness: LogCompareStrictness,
record_envelope: RecordEnvelope,
) -> Result<VerificationOutcome, Error> {
compare_two_runs(
ComparedRun {
output: left,
log: left_log,
label: "run 1",
},
ComparedRun {
output: right,
log: right_log,
label: "run 2",
},
ComparisonOptions {
verbose: false,
strictness,
compare_logs: true,
diagnostic_full_trace: false,
compare_io_buffers: true,
keep_logs: false,
match_overridden: false,
failed_log_retention: None,
record_envelope,
virtualize_time: true,
},
)
}
fn compare(
left: &Output,
left_log: TempPath,
right: &Output,
right_log: TempPath,
) -> Result<VerificationOutcome, Error> {
compare_with(
left,
left_log,
right,
right_log,
LogCompareStrictness::Stripped,
)
}
fn detlog_with_value(value: u64) -> String {
let record_suffix = detcore::detlog::record_suffix(detcore::detlog::DetLogEvent::Syscall);
format!(
"2026-08-06T01:00:00.000000Z INFO detcore: [dtid 2] DETLOG [syscall] write(fd=1, count={value}){record_suffix}\n"
)
}
#[test]
fn extracts_unsupported_syscall_warning_union_from_logs() {
let file = NamedTempFile::new().unwrap();
fs::write(
file.path(),
b"2026 WARN syscalls vmsplice,getppid used but not yet supported\ninvalid\n",
)
.unwrap();
assert_eq!(
unsupported_syscalls_from_log(file.path()).unwrap(),
BTreeSet::from(["getppid".to_owned(), "vmsplice".to_owned()])
);
}
fn logs_truncated_at_the_bound() -> (TempPath, TempPath) {
let (left, right) = temp_log_files("verify_left", "verify_right").unwrap();
let left_path = left.into_temp_path();
let right_path = right.into_temp_path();
let body = format!(
"{}{}{}\n",
detlog_with_value(1),
detlog_with_value(2),
detcore::logdiff::TRUNCATION_MARKER
);
fs::write(&left_path, body.as_bytes()).unwrap();
fs::write(&right_path, body.as_bytes()).unwrap();
(left_path, right_path)
}
#[test]
fn a_refused_comparison_is_a_no_result_not_a_divergence() {
let left = output(0, b"hello\n", b"");
let right = left.clone();
let (log1, log2) = logs_truncated_at_the_bound();
let outcome = compare(&left, log1, &right, log2).unwrap();
assert_eq!(
outcome.verdict,
Verdict::NoResult,
"a refused comparison must not be reported as a divergence"
);
assert!(!outcome.verified(), "a no-result is never verified");
assert_eq!(
outcome.compared_log_messages,
Some(ComparedLogCounts { left: 0, right: 0 })
);
let report = verification_report(&outcome);
assert!(!report.verified);
assert!(!report.bitwise_parity);
assert!(
matches!(
report.no_result_reason.as_ref(),
Some(NoResultReason::ComparisonRefused { detail })
if detail.contains("truncated at the configured size bound")
),
"the typed report must retain the exact comparator refusal: {report:?}"
);
assert_eq!(report.comparison, None);
assert_eq!(report.compared_log_messages, None);
assert!(
outcome.into_exit_status().is_err(),
"a no-result must still exit nonzero; it is not a pass"
);
}
#[test]
fn an_observed_difference_outranks_a_refused_comparison() {
for (label, left, right) in [
(
"stdout",
output(0, b"hello\n", b""),
output(0, b"goodbye\n", b""),
),
(
"stderr",
output(0, b"hello\n", b""),
output(0, b"hello\n", b"oops\n"),
),
(
"exit status",
output(0, b"hello\n", b""),
output(3, b"hello\n", b""),
),
] {
let (log1, log2) = logs_truncated_at_the_bound();
let outcome = compare(&left, log1, &right, log2).unwrap();
assert_eq!(
outcome.verdict,
Verdict::Diverged,
"{label} differed, so the verdict must be Diverged even though the log \
comparison was refused"
);
assert!(!outcome.verified());
}
}
#[test]
fn identical_outputs_verify_successfully() {
let left = output(0, b"hello\n", b"");
let right = left.clone();
let (log1, log2) = empty_logs();
let outcome = compare(&left, log1, &right, log2).unwrap();
assert_eq!(outcome.verdict, Verdict::Matched);
assert!(outcome.verified());
assert_eq!(outcome.guest_status, ExitStatus::Exited(0));
assert_eq!(
outcome.comparison.strictness,
LogCompareStrictness::Stripped
);
assert!(outcome.comparison.strip_lines);
assert!(!outcome.comparison.full_trace);
}
#[test]
fn nonzero_exit_with_matching_outputs_reports_verified() {
let left = output(3, b"hello\n", b"oops\n");
let right = left.clone();
let (log1, log2) = empty_logs();
let outcome = compare(&left, log1, &right, log2).unwrap();
assert_eq!(outcome.verdict, Verdict::Matched);
assert!(outcome.verified());
assert_eq!(outcome.guest_status, ExitStatus::Exited(3));
let report = verification_report(&outcome);
assert!(report.verified);
assert_eq!(report.guest_exit_code, Some(3));
assert_eq!(report.guest_signal, None);
assert_eq!(
report.comparison.unwrap().strictness,
LogCompareStrictness::Stripped
);
assert_eq!(outcome.into_exit_status().unwrap(), ExitStatus::Exited(3));
}
#[test]
fn production_comparison_callers_bind_truthful_side_labels() {
for (name, source, left, right) in [
("generic run", include_str!("run.rs"), "run 1", "run 2"),
("DBT run", include_str!("backends.rs"), "run 1", "run 2"),
(
"record/replay",
include_str!("record_start.rs"),
"the recording",
"the replay",
),
] {
let production = source
.rsplit_once("#[cfg(test)]")
.expect("production/test boundary")
.0;
for label in [left, right] {
let binding = format!("label: \"{label}\"");
assert_eq!(
production.matches(&binding).count(),
1,
"{name} must bind {label:?} exactly once"
);
}
}
let production = include_str!("verify.rs")
.rsplit_once("#[cfg(test)]")
.expect("production/test boundary")
.0;
assert!(production.contains("side_labels: compared_labels.clone()"));
assert_eq!(
production
.matches("retain_verification_logs([(label1, log1), (label2, log2)])")
.count(),
2,
"both retained-log paths must use the caller-bound labels"
);
assert_eq!(
production
.matches("retain_verification_logs([(label1, log1)])")
.count(),
1,
"the golden-log path after a match must use the caller-bound label"
);
}
#[test]
fn output_only_mode_ignores_internal_log_order() {
let left = output(0, b"console", b"warning");
let right = output(0, b"console", b"warning");
let (left_log, right_log) = empty_logs();
fs::write(&left_log, "DETLOG root event A\n").unwrap();
fs::write(&right_log, "DETLOG root event B\n").unwrap();
let outcome = compare_two_runs(
ComparedRun {
output: &left,
log: left_log,
label: "run 1",
},
ComparedRun {
output: &right,
log: right_log,
label: "run 2",
},
ComparisonOptions {
verbose: false,
strictness: LogCompareStrictness::Stripped,
compare_logs: false,
diagnostic_full_trace: false,
compare_io_buffers: false,
keep_logs: false,
match_overridden: false,
failed_log_retention: None,
record_envelope: RecordEnvelope::all_records_v1(),
virtualize_time: true,
},
)
.unwrap();
assert_eq!(outcome.verdict, Verdict::Matched);
assert_eq!(outcome.guest_status, ExitStatus::Exited(0));
assert!(!outcome.comparison.compare_logs);
}
#[test]
fn stdout_stderr_and_status_mismatches_fail_verification() {
let baseline = output(0, b"hello\n", b"");
let mismatches = [
output(0, b"different\n", b""),
output(0, b"hello\n", b"different\n"),
output(1, b"hello\n", b""),
];
for mismatch in mismatches {
let (log1, log2) = empty_logs();
let path1 = log1.to_path_buf();
let path2 = log2.to_path_buf();
let outcome = compare(&baseline, log1, &mismatch, log2).unwrap();
assert_eq!(outcome.verdict, Verdict::Diverged);
assert!(!outcome.verified());
assert_eq!(
outcome.into_exit_status().unwrap(),
ExitStatus::Exited(HERMIT_VERIFICATION_DIVERGENCE_EXIT)
);
let _ = fs::remove_file(path1);
let _ = fs::remove_file(path2);
}
}
#[test]
fn comparison_spec_maps_strictness_to_concrete_flags() {
let stripped = ComparisonSpec::new(
LogCompareStrictness::Stripped,
true,
false,
true,
RecordEnvelopePolicy::AllRecordsV1,
true,
);
assert!(stripped.strip_lines);
assert!(!stripped.full_trace);
assert_eq!(stripped.display_name, "Stripped");
assert_eq!(
stripped.stripped_prefixes,
[STRIP_WALL_CLOCK_PREFIX_V1, STRIP_UNSAFE_NORMALIZATION_V1]
);
assert_eq!(
comparison_evidence_line(&stripped).as_deref(),
Some(
":: comparison=Stripped relaxations=unsafe-numeric-address-and-path-normalization/v1"
)
);
assert_eq!(
stripped.log_comparison_mode(),
LogComparisonMode::Deterministic
);
let canonical = ComparisonSpec::new(
LogCompareStrictness::Canonical,
true,
false,
true,
RecordEnvelopePolicy::AllRecordsV1,
true,
);
assert!(!canonical.strip_lines);
assert!(canonical.canonicalize_addresses);
assert!(canonical.exact_remainder);
assert!(canonical.full_trace);
assert_eq!(canonical.display_name, "BitwiseInfoV1");
assert_eq!(canonical.stripped_prefixes, PARITY_STRIPPED_PREFIXES);
assert_eq!(canonical.canonicalizations, PARITY_CANONICALIZATIONS);
assert_eq!(
comparison_evidence_line(&canonical).as_deref(),
Some(":: comparison=BitwiseInfoV1 relaxations=none")
);
assert_eq!(canonical.log_scope, ComparedLogScope::Info);
assert_eq!(canonical.log_comparison_mode(), LogComparisonMode::Info);
let no_io_buffers = ComparisonSpec {
compare_io_buffers: false,
..canonical
};
assert_eq!(
comparison_evidence_line(&no_io_buffers).as_deref(),
Some(":: comparison=BitwiseInfoV1 relaxations=--no-detlog-io-buffers")
);
let invalid_policy_shape = ComparisonSpec {
skip_commit: true,
..canonical
};
assert_eq!(
comparison_evidence_line(&invalid_policy_shape).as_deref(),
Some(":: comparison=unknown relaxations=unknown")
);
let invalid_display_name = ComparisonSpec {
display_name: "Stripped",
..canonical
};
assert_eq!(
comparison_evidence_line(&invalid_display_name).as_deref(),
Some(":: comparison=unknown relaxations=unknown")
);
let diagnostic = ComparisonSpec::new(
LogCompareStrictness::Canonical,
true,
true,
true,
RecordEnvelopePolicy::AllRecordsV1,
true,
);
assert_eq!(diagnostic.log_scope, ComparedLogScope::FullTrace);
assert_eq!(diagnostic.display_name, "BitwiseFullTraceV1");
assert_eq!(
comparison_evidence_line(&diagnostic).as_deref(),
Some(":: comparison=BitwiseFullTraceV1 relaxations=none")
);
let output_only = ComparisonSpec {
compare_logs: false,
..canonical
};
assert_eq!(comparison_evidence_line(&output_only), None);
assert_eq!(
diagnostic.log_comparison_mode(),
LogComparisonMode::FullTrace
);
}
#[test]
fn bitwise_info_ignores_debug_diagnostics_but_rejects_real_info_divergence() {
let out = output(0, b"hello\n", b"");
let make_logs = |right_info: u64| {
let (left, right) = empty_logs();
fs::write(
&left,
format!(
"{}2026-08-06T01:00:00.000001Z DEBUG detcore: diagnostic host timing=100\n",
detlog_with_value(7)
),
)
.unwrap();
fs::write(
&right,
format!(
"{}2026-08-06T01:00:00.000002Z DEBUG detcore: diagnostic host timing=200\n",
detlog_with_value(right_info)
),
)
.unwrap();
(left, right)
};
let (left, right) = make_logs(7);
let matched =
compare_with(&out, left, &out, right, LogCompareStrictness::Canonical).unwrap();
assert_eq!(matched.verdict, Verdict::Matched);
assert_eq!(matched.comparison.log_scope, ComparedLogScope::Info);
assert_eq!(
matched.compared_log_messages,
Some(ComparedLogCounts { left: 1, right: 1 })
);
assert!(verification_report(&matched).bitwise_parity);
let (left, right) = make_logs(8);
let left_path = left.to_path_buf();
let right_path = right.to_path_buf();
let info_diverged =
compare_with(&out, left, &out, right, LogCompareStrictness::Canonical).unwrap();
assert_eq!(info_diverged.verdict, Verdict::Diverged);
let _ = fs::remove_file(left_path);
let _ = fs::remove_file(right_path);
let (left, right) = make_logs(7);
let left_path = left.to_path_buf();
let right_path = right.to_path_buf();
let debug_diverged = compare_two_runs(
ComparedRun {
output: &out,
log: left,
label: "run 1",
},
ComparedRun {
output: &out,
log: right,
label: "run 2",
},
ComparisonOptions {
verbose: true,
strictness: LogCompareStrictness::Canonical,
compare_logs: true,
diagnostic_full_trace: true,
compare_io_buffers: false,
keep_logs: false,
match_overridden: false,
failed_log_retention: None,
record_envelope: RecordEnvelope::all_records_v1(),
virtualize_time: true,
},
)
.unwrap();
assert_eq!(debug_diverged.verdict, Verdict::Diverged);
assert_eq!(
debug_diverged.comparison.log_scope,
ComparedLogScope::FullTrace
);
let _ = fs::remove_file(left_path);
let _ = fs::remove_file(right_path);
}
#[test]
fn stripped_matches_but_bitwise_diverges_on_numeric_only_log_difference() {
let out = output(0, b"hello\n", b"");
let (log1, log2) = empty_logs();
fs::write(&log1, detlog_with_value(100)).unwrap();
fs::write(&log2, detlog_with_value(200)).unwrap();
let stripped =
compare_with(&out, log1, &out, log2, LogCompareStrictness::Stripped).unwrap();
assert_eq!(stripped.verdict, Verdict::Matched);
assert!(stripped.verified());
assert!(stripped.comparison.strip_lines);
assert!(!stripped.comparison.full_trace);
let (log1, log2) = empty_logs();
let path1 = log1.to_path_buf();
let path2 = log2.to_path_buf();
fs::write(&path1, detlog_with_value(100)).unwrap();
fs::write(&path2, detlog_with_value(200)).unwrap();
let canonical =
compare_with(&out, log1, &out, log2, LogCompareStrictness::Canonical).unwrap();
assert_eq!(canonical.verdict, Verdict::Diverged);
assert!(!canonical.verified());
assert_eq!(
canonical.comparison.strictness,
LogCompareStrictness::Canonical
);
assert!(!canonical.comparison.strip_lines);
assert!(canonical.comparison.full_trace);
let report = verification_report(&canonical);
assert!(!report.verified);
assert_eq!(report.comparison.unwrap().strip_lines, Some(false));
assert!(path1.exists(), "divergent run-1 log must be retained");
assert!(path2.exists(), "divergent run-2 log must be retained");
fs::remove_file(path1).unwrap();
fs::remove_file(path2).unwrap();
}
#[test]
fn stripped_success_does_not_claim_bitwise_parity_but_strict_success_does() {
const SUCCESS: &str = "Success: deterministic. Determinism verified.";
let announce_as = |outcome: &VerificationOutcome, second_run: SecondRun| {
let mut bytes = Vec::new();
write_verification_announcement(
&mut bytes,
outcome,
second_run,
SUCCESS,
"Failure: nondeterministic.",
)
.unwrap();
String::from_utf8(bytes).unwrap()
};
let announce = |outcome: &VerificationOutcome| announce_as(outcome, SecondRun::Rerun);
let out = output(0, b"hello\n", b"");
let (log1, log2) = empty_logs();
fs::write(&log1, detlog_with_value(100)).unwrap();
fs::write(&log2, detlog_with_value(100)).unwrap();
let stripped =
compare_with(&out, log1, &out, log2, LogCompareStrictness::Stripped).unwrap();
assert_eq!(stripped.verdict, Verdict::Matched);
assert!(!verification_report(&stripped).bitwise_parity);
let text = announce(&stripped);
assert!(text.contains(SUCCESS), "{text}");
assert!(!text.contains(BITWISE_PARITY_CLAIM), "{text}");
assert!(
text.contains(
"only the DETLOG and scheduler COMMIT records of the guest's log (1 vs 1) \
under the lossy Stripped comparison"
),
"{text}"
);
assert!(
text.contains("This is not a bitwise comparison; add --verify-strict for one."),
"{text}"
);
assert!(!text.contains(TIME_NOT_VIRTUALIZED_NOTE), "{text}");
assert!(!text.contains(RERUN_TIME_NOT_VIRTUALIZED_NOTE), "{text}");
let (log1, log2) = empty_logs();
fs::write(&log1, detlog_with_value(100)).unwrap();
fs::write(&log2, detlog_with_value(100)).unwrap();
let canonical =
compare_with(&out, log1, &out, log2, LogCompareStrictness::Canonical).unwrap();
assert_eq!(canonical.verdict, Verdict::Matched);
assert!(verification_report(&canonical).bitwise_parity);
let text = announce(&canonical);
assert!(text.contains(SUCCESS), "{text}");
assert!(
text.contains(&format!("under BitwiseInfoV1: {BITWISE_PARITY_CLAIM}.")),
"{text}"
);
assert!(
text.contains("the INFO records of the guest's log (1 vs 1) exactly"),
"{text}"
);
assert!(!text.contains("not a bitwise comparison"), "{text}");
assert!(!text.contains(TIME_NOT_VIRTUALIZED_NOTE), "{text}");
assert!(!text.contains(RERUN_TIME_NOT_VIRTUALIZED_NOTE), "{text}");
let text = announce_as(&canonical, SecondRun::Replay);
assert!(!text.contains(TIME_NOT_VIRTUALIZED_NOTE), "{text}");
assert!(!text.contains(RERUN_TIME_NOT_VIRTUALIZED_NOTE), "{text}");
let time_off = VerificationOutcome {
comparison: ComparisonSpec {
virtualize_time: false,
..canonical.comparison
},
..canonical.clone()
};
assert!(verification_report(&time_off).bitwise_parity);
let text = announce_as(&time_off, SecondRun::Replay);
assert!(
text.contains(&format!(
"{BITWISE_PARITY_CLAIM}. {TIME_NOT_VIRTUALIZED_NOTE}"
)),
"{text}"
);
assert!(!text.contains(RERUN_TIME_NOT_VIRTUALIZED_NOTE), "{text}");
let text = announce_as(&time_off, SecondRun::Rerun);
assert!(
text.contains(&format!(
"{BITWISE_PARITY_CLAIM}. {RERUN_TIME_NOT_VIRTUALIZED_NOTE}"
)),
"{text}"
);
assert!(!text.contains(TIME_NOT_VIRTUALIZED_NOTE), "{text}");
assert!(!text.contains("reproduced the first"), "{text}");
let full_trace = VerificationOutcome {
comparison: ComparisonSpec {
log_scope: ComparedLogScope::FullTrace,
..canonical.comparison
},
..canonical.clone()
};
let text = announce(&full_trace);
assert!(
text.contains("every record of the guest's log (1 vs 1)"),
"{text}"
);
let no_evidence = VerificationOutcome {
compared_log_messages: Some(ComparedLogCounts { left: 0, right: 0 }),
..canonical.clone()
};
let text = announce(&no_evidence);
assert!(!text.contains(BITWISE_PARITY_CLAIM), "{text}");
assert!(text.contains("This is not a bitwise comparison."), "{text}");
let output_only = VerificationOutcome {
comparison: ComparisonSpec {
compare_logs: false,
..canonical.comparison
},
compared_log_messages: None,
..canonical.clone()
};
let text = announce(&output_only);
assert!(!text.contains(BITWISE_PARITY_CLAIM), "{text}");
assert!(text.contains("Compared: stdout and stderr only."), "{text}");
let diverged = VerificationOutcome {
verdict: Verdict::Diverged,
..canonical
};
let text = announce(&diverged);
assert!(!text.contains("Compared:"), "{text}");
assert!(!text.contains(BITWISE_PARITY_CLAIM), "{text}");
}
fn compare_for_retention(
directory: &Path,
left_log: &str,
right_log: &str,
compare_logs: bool,
keep_logs: bool,
match_overridden: bool,
) -> (Verdict, PathBuf, PathBuf) {
let output = output(0, b"hello\n", b"");
let (left, right) = temp_log_files_in("run1", "run2", Some(directory)).unwrap();
fs::write(left.path(), left_log).unwrap();
fs::write(right.path(), right_log).unwrap();
let left_path = left.path().to_path_buf();
let right_path = right.path().to_path_buf();
let outcome = compare_two_runs(
ComparedRun {
output: &output,
log: left.into_temp_path(),
label: "run 1",
},
ComparedRun {
output: &output,
log: right.into_temp_path(),
label: "run 2",
},
ComparisonOptions {
verbose: false,
strictness: LogCompareStrictness::Canonical,
compare_logs,
diagnostic_full_trace: false,
compare_io_buffers: false,
keep_logs,
match_overridden,
failed_log_retention: None,
record_envelope: RecordEnvelope::all_records_v1(),
virtualize_time: true,
},
)
.unwrap();
(outcome.verdict, left_path, right_path)
}
fn assert_retained_log(path: &Path, expected: &str, side: &str) {
let metadata = fs::symlink_metadata(path)
.unwrap_or_else(|error| panic!("{side} log {} is missing: {error}", path.display()));
assert!(metadata.file_type().is_file(), "{side} log is not a file");
assert_eq!(
fs::read_to_string(path).unwrap(),
expected,
"{side} log was not retained byte for byte"
);
}
#[test]
fn verification_log_retention_matches_the_cli_contract() {
let directory = tempfile::tempdir().unwrap();
for (
left_value,
right_value,
keep_logs,
match_overridden,
expected,
run1_retained,
run2_retained,
) in [
(7, 7, false, false, Verdict::Matched, false, false),
(7, 8, false, false, Verdict::Diverged, true, true),
(7, 7, true, false, Verdict::Matched, true, false),
(7, 8, true, false, Verdict::Diverged, true, true),
(7, 7, false, true, Verdict::Matched, false, false),
(7, 7, true, true, Verdict::Matched, true, true),
(7, 8, true, true, Verdict::Diverged, true, true),
] {
let case = format!(
"values {left_value}/{right_value}, keep_logs={keep_logs}, \
match_overridden={match_overridden}"
);
let (verdict, left_path, right_path) = compare_for_retention(
directory.path(),
&detlog_with_value(left_value),
&detlog_with_value(right_value),
true,
keep_logs,
match_overridden,
);
assert_eq!(verdict, expected, "verdict mismatch for {case}");
if run1_retained {
assert_retained_log(&left_path, &detlog_with_value(left_value), "run-1");
} else {
assert!(!left_path.exists(), "run-1 log must be deleted for {case}");
}
if run2_retained {
assert_retained_log(&right_path, &detlog_with_value(right_value), "run-2");
} else {
assert!(!right_path.exists(), "run-2 log must be deleted for {case}");
}
for (path, retained) in [(left_path, run1_retained), (right_path, run2_retained)] {
if retained {
fs::remove_file(path).unwrap();
}
}
}
assert_eq!(
fs::read_dir(directory.path()).unwrap().count(),
0,
"every retained log was accounted for"
);
}
#[test]
fn requested_logs_survive_a_match_that_did_not_compare_them() {
let directory = tempfile::tempdir().unwrap();
let (verdict, left_path, right_path) = compare_for_retention(
directory.path(),
&detlog_with_value(7),
&detlog_with_value(8),
false,
true,
false,
);
assert_eq!(verdict, Verdict::Matched);
assert_retained_log(&left_path, &detlog_with_value(7), "run-1");
assert_retained_log(&right_path, &detlog_with_value(8), "run-2");
}
#[test]
fn requested_logs_survive_a_refused_comparison() {
let directory = tempfile::tempdir().unwrap();
let body = format!(
"{}{}{}\n",
detlog_with_value(1),
detlog_with_value(2),
detcore::logdiff::TRUNCATION_MARKER
);
let (verdict, left_path, right_path) =
compare_for_retention(directory.path(), &body, &body, true, true, false);
assert_eq!(verdict, Verdict::NoResult);
assert_retained_log(&left_path, &body, "run-1");
assert_retained_log(&right_path, &body, "run-2");
}
#[test]
fn an_overridden_match_keeps_both_logs_in_the_failure_directory() {
let temporary = tempfile::tempdir().unwrap();
let source = temporary.path().join("source");
let root = temporary.path().join("verify-failures");
fs::create_dir(&source).unwrap();
let output = output(0, b"same output\n", b"");
let (left, right) = temp_log_files_in("run1", "run2", Some(&source)).unwrap();
fs::write(left.path(), detlog_with_value(7)).unwrap();
fs::write(right.path(), detlog_with_value(7)).unwrap();
let left_name = left.path().file_name().unwrap().to_owned();
let right_name = right.path().file_name().unwrap().to_owned();
let outcome = compare_two_runs(
ComparedRun {
output: &output,
log: left.into_temp_path(),
label: "run 1",
},
ComparedRun {
output: &output,
log: right.into_temp_path(),
label: "run 2",
},
ComparisonOptions {
verbose: false,
strictness: LogCompareStrictness::Canonical,
compare_logs: true,
diagnostic_full_trace: false,
compare_io_buffers: true,
keep_logs: true,
match_overridden: true,
failed_log_retention: Some(FailedVerifyLogRetention::new(root.clone(), 2)),
record_envelope: RecordEnvelope::all_records_v1(),
virtualize_time: true,
},
)
.unwrap();
assert_eq!(outcome.verdict, Verdict::Matched);
assert_eq!(fs::read_dir(&source).unwrap().count(), 0);
let directories = retained_comparison_dirs(&root).unwrap();
assert_eq!(directories.len(), 1, "one retained comparison");
let retained = &directories[0].0;
assert_retained_log(&retained.join(&left_name), &detlog_with_value(7), "run-1");
assert_retained_log(&retained.join(&right_name), &detlog_with_value(7), "run-2");
}
fn run_failed_comparison(source: &Path, retention: FailedVerifyLogRetention, value: u64) {
let output = output(0, b"same output\n", b"");
let (left, right) = temp_log_files_in("run1", "run2", Some(source)).unwrap();
fs::write(left.path(), detlog_with_value(value)).unwrap();
fs::write(right.path(), detlog_with_value(value + 1)).unwrap();
let outcome = compare_two_runs(
ComparedRun {
output: &output,
log: left.into_temp_path(),
label: "run 1",
},
ComparedRun {
output: &output,
log: right.into_temp_path(),
label: "run 2",
},
ComparisonOptions {
verbose: false,
strictness: LogCompareStrictness::Canonical,
compare_logs: true,
diagnostic_full_trace: false,
compare_io_buffers: true,
keep_logs: false,
match_overridden: false,
failed_log_retention: Some(retention),
record_envelope: RecordEnvelope::all_records_v1(),
virtualize_time: true,
},
)
.unwrap();
assert_eq!(outcome.verdict, Verdict::Diverged);
}
#[test]
fn failed_comparison_materialization_holds_the_retained_count_at_the_bound() {
let temporary = tempfile::tempdir().unwrap();
let source = temporary.path().join("source");
let root = temporary.path().join("verify-failures");
fs::create_dir(&source).unwrap();
let retention = FailedVerifyLogRetention::new(root.clone(), 2);
for value in 0..3 {
run_failed_comparison(&source, retention.clone(), value);
}
let directories = retained_comparison_dirs(&root).unwrap();
assert_eq!(
directories.len(),
2,
"the third failure must retire the oldest"
);
for (directory, _) in directories {
let logs = fs::read_dir(directory)
.unwrap()
.filter_map(Result::ok)
.filter(|entry| entry.file_name() != FAILED_VERIFY_LOG_LOCK)
.collect::<Vec<_>>();
assert_eq!(logs.len(), 2, "each retained failure keeps both logs");
}
}
#[test]
fn failed_comparison_retirement_refuses_a_log_diff_reader() {
let temporary = tempfile::tempdir().unwrap();
let source = temporary.path().join("source");
let root = temporary.path().join("verify-failures");
fs::create_dir(&source).unwrap();
let retention = FailedVerifyLogRetention::new(root.clone(), 2);
run_failed_comparison(&source, retention.clone(), 0);
run_failed_comparison(&source, retention.clone(), 2);
let oldest = retained_comparison_dirs(&root).unwrap().pop().unwrap().0;
let protected_log = fs::read_dir(&oldest)
.unwrap()
.filter_map(Result::ok)
.find(|entry| entry.file_name() != FAILED_VERIFY_LOG_LOCK)
.unwrap()
.path();
let read_locks = lock_failed_verification_logs_for_read([protected_log]).unwrap();
run_failed_comparison(&source, retention.clone(), 4);
assert!(oldest.is_dir(), "an active reader must prevent retirement");
assert_eq!(retained_comparison_dirs(&root).unwrap().len(), 3);
drop(read_locks);
run_failed_comparison(&source, retention, 6);
assert!(
!oldest.exists(),
"the unlocked old evidence is retired later"
);
assert_eq!(retained_comparison_dirs(&root).unwrap().len(), 2);
}
#[test]
fn failed_comparison_retirement_protects_symlinked_readers() {
for file_alias in [false, true] {
let temporary = tempfile::tempdir().unwrap();
let source = temporary.path().join("source");
let root = temporary.path().join("verify-failures");
fs::create_dir(&source).unwrap();
let retention = FailedVerifyLogRetention::new(root.clone(), 2);
run_failed_comparison(&source, retention.clone(), 0);
run_failed_comparison(&source, retention.clone(), 2);
let oldest = retained_comparison_dirs(&root).unwrap().pop().unwrap().0;
let protected_log = fs::read_dir(&oldest)
.unwrap()
.filter_map(Result::ok)
.find(|entry| entry.file_name() != FAILED_VERIFY_LOG_LOCK)
.unwrap()
.path();
let alias = temporary.path().join("reader-path");
let input = if file_alias {
std::os::unix::fs::symlink(&protected_log, &alias).unwrap();
alias
} else {
std::os::unix::fs::symlink(&oldest, &alias).unwrap();
alias.join(protected_log.file_name().unwrap())
};
let read_locks = lock_failed_verification_logs_for_read([input]).unwrap();
run_failed_comparison(&source, retention.clone(), 4);
assert!(
oldest.is_dir(),
"a reader through a symlink must prevent retirement (file_alias={file_alias})"
);
assert_eq!(retained_comparison_dirs(&root).unwrap().len(), 3);
drop(read_locks);
run_failed_comparison(&source, retention, 6);
assert!(
!oldest.exists(),
"the aliased evidence must be retired after its reader releases the lock"
);
assert_eq!(retained_comparison_dirs(&root).unwrap().len(), 2);
}
}
#[test]
fn requested_logs_survive_unsupported_syscall_scan_io_error() {
let directory = tempfile::tempdir().unwrap();
let output = output(0, b"hello\n", b"");
let (left, right) = temp_log_files_in("left", "right", Some(directory.path())).unwrap();
fs::write(left.path(), detlog_with_value(7)).unwrap();
fs::write(right.path(), detlog_with_value(7)).unwrap();
let left_path = left.path().to_path_buf();
let right_path = right.path().to_path_buf();
let error = compare_two_runs_with_unsupported_scan(
ComparedRun {
output: &output,
log: left.into_temp_path(),
label: "run 1",
},
ComparedRun {
output: &output,
log: right.into_temp_path(),
label: "run 2",
},
ComparisonOptions {
verbose: false,
strictness: LogCompareStrictness::Canonical,
compare_logs: true,
diagnostic_full_trace: false,
compare_io_buffers: false,
keep_logs: true,
match_overridden: false,
failed_log_retention: None,
record_envelope: RecordEnvelope::all_records_v1(),
virtualize_time: true,
},
|_| Err(io::Error::other("injected unsupported-syscall scan error")),
)
.unwrap_err();
assert!(
error
.to_string()
.contains("injected unsupported-syscall scan error")
);
assert!(left_path.exists(), "run-1 log must survive the scan error");
assert!(right_path.exists(), "run-2 log must survive the scan error");
assert_eq!(
fs::read_to_string(&left_path).unwrap(),
detlog_with_value(7)
);
assert_eq!(
fs::read_to_string(&right_path).unwrap(),
detlog_with_value(7)
);
fs::remove_file(left_path).unwrap();
fs::remove_file(right_path).unwrap();
}
#[cfg(unix)]
#[test]
fn requested_logs_survive_log_comparison_io_error() {
let directory = tempfile::tempdir().unwrap();
let output = output(0, b"hello\n", b"");
let (left, right) = temp_log_files_in("left", "right", Some(directory.path())).unwrap();
let mut left_bytes = detlog_with_value(7).into_bytes();
left_bytes.push(0xff);
let right_bytes = detlog_with_value(7).into_bytes();
fs::write(left.path(), &left_bytes).unwrap();
fs::write(right.path(), &right_bytes).unwrap();
let left_path = left.path().to_path_buf();
let right_path = right.path().to_path_buf();
assert!(
std::str::from_utf8(&fs::read(&left_path).unwrap()).is_err(),
"the run-1 log must contain invalid UTF-8"
);
let result = compare_two_runs(
ComparedRun {
output: &output,
log: left.into_temp_path(),
label: "run 1",
},
ComparedRun {
output: &output,
log: right.into_temp_path(),
label: "run 2",
},
ComparisonOptions {
verbose: false,
strictness: LogCompareStrictness::Canonical,
compare_logs: true,
diagnostic_full_trace: false,
compare_io_buffers: false,
keep_logs: true,
match_overridden: false,
failed_log_retention: None,
record_envelope: RecordEnvelope::all_records_v1(),
virtualize_time: true,
},
);
let error = result.expect_err("invalid UTF-8 must fail comparison");
assert_eq!(
error
.downcast_ref::<io::Error>()
.expect("the real comparator must return an I/O error")
.kind(),
io::ErrorKind::InvalidData
);
assert!(
left_path.exists(),
"run-1 log must survive the comparison error"
);
assert!(
right_path.exists(),
"run-2 log must survive the comparison error"
);
for (path, expected) in [
(&left_path, left_bytes.as_slice()),
(&right_path, right_bytes.as_slice()),
] {
assert!(fs::symlink_metadata(path).unwrap().file_type().is_file());
assert_eq!(fs::read(path).unwrap(), expected);
}
fs::remove_file(left_path).unwrap();
fs::remove_file(right_path).unwrap();
}
#[test]
fn verification_report_carries_first_log_divergence_position() {
let output = output(0, b"hello\n", b"");
let (left, right) = empty_logs();
let left_path = left.to_path_buf();
let right_path = right.to_path_buf();
let commit =
detcore::detlog::record_suffix(detcore::detlog::DetLogEvent::SchedulerCommit {
scheduler_turn: 23,
virtual_nanoseconds: 4_567_890_123,
internal_io_poll: false,
runtime_maps_read: false,
});
let other = detcore::detlog::record_suffix(detcore::detlog::DetLogEvent::Other);
fs::write(
&left_path,
format!(
"2026-08-13T01:02:03.000000Z INFO detcore::scheduler: COMMIT turn 23, dettid 2, on previously committed 4.567_890_123s{commit}\n\
2026-08-13T01:02:03.000001Z INFO detcore: DETLOG value=1{other}\n"
),
)
.unwrap();
fs::write(
&right_path,
format!(
"2026-08-13T01:02:04.000000Z INFO detcore::scheduler: COMMIT turn 23, dettid 2, on previously committed 4.567_890_123s{commit}\n\
2026-08-13T01:02:04.000001Z INFO detcore: DETLOG value=2{other}\n"
),
)
.unwrap();
let outcome = compare_with(
&output,
left,
&output,
right,
LogCompareStrictness::Canonical,
)
.unwrap();
let report = verification_report(&outcome);
assert_eq!(report.verdict, Verdict::Diverged);
assert_eq!(report.first_divergent_scheduler_turn, Some(23));
assert_eq!(
report.first_divergent_virtual_nanoseconds,
Some(4_567_890_123)
);
assert_eq!(
report.first_divergent_left_message.as_deref(),
Some("INFO detcore: DETLOG value=1")
);
assert_eq!(
report.first_divergent_right_message.as_deref(),
Some("INFO detcore: DETLOG value=2")
);
let _ = fs::remove_file(left_path);
let _ = fs::remove_file(right_path);
}
#[test]
fn output_divergence_does_not_discard_the_log_divergence_position() {
let left_output = output(0, b"left\n", b"");
let right_output = output(0, b"right\n", b"");
let (left, right) = empty_logs();
let left_path = left.to_path_buf();
let right_path = right.to_path_buf();
let commit =
detcore::detlog::record_suffix(detcore::detlog::DetLogEvent::SchedulerCommit {
scheduler_turn: 23,
virtual_nanoseconds: 4_567_890_123,
internal_io_poll: false,
runtime_maps_read: false,
});
fs::write(
&left_path,
format!(
"INFO detcore::scheduler: COMMIT turn 23 at time 4567890123\n\
INFO detcore: DETLOG value=1{commit}\n"
),
)
.unwrap();
fs::write(
&right_path,
format!(
"INFO detcore::scheduler: COMMIT turn 23 at time 4567890123\n\
INFO detcore: DETLOG value=2{commit}\n"
),
)
.unwrap();
let outcome = compare_with(
&left_output,
left,
&right_output,
right,
LogCompareStrictness::Canonical,
)
.unwrap();
let report = verification_report(&outcome);
assert_eq!(report.verdict, Verdict::Diverged);
assert_eq!(report.first_divergent_scheduler_turn, Some(23));
assert_eq!(
report.first_divergent_virtual_nanoseconds,
Some(4_567_890_123)
);
assert_eq!(
report.first_divergent_left_message.as_deref(),
Some(
"INFO detcore::scheduler: COMMIT turn <NUM> at time <NANOSECONDS>\nINFO detcore: DETLOG value=1"
)
);
assert_eq!(
report.first_divergent_right_message.as_deref(),
Some(
"INFO detcore::scheduler: COMMIT turn <NUM> at time <NANOSECONDS>\nINFO detcore: DETLOG value=2"
)
);
let _ = fs::remove_file(left_path);
let _ = fs::remove_file(right_path);
}
#[test]
fn empty_log_comparison_matches_but_is_never_parity() {
let out = output(0, b"hello\n", b"");
let (log1, log2) = empty_logs();
let outcome =
compare_with(&out, log1, &out, log2, LogCompareStrictness::Canonical).unwrap();
assert_eq!(outcome.verdict, Verdict::Matched);
assert_eq!(
outcome.compared_log_messages,
Some(ComparedLogCounts { left: 0, right: 0 })
);
assert!(outcome.comparison.is_bitwise_parity());
let report = verification_report(&outcome);
assert!(report.verified);
assert!(
!report.bitwise_parity,
"zero compared log messages must never certify bitwise parity"
);
}
fn plant_previous_green(path: &Path) {
fs::write(
path,
"{\"verified\":true,\"bitwise_parity\":true,\"verdict\":\"matched\"}\n",
)
.unwrap();
}
fn read_verdict(path: &Path) -> serde_json::Value {
serde_json::from_str(&fs::read_to_string(path).unwrap()).unwrap()
}
#[test]
fn a_bare_filename_stages_beside_its_target_not_in_the_system_temp_dir() {
assert_eq!(
staging_directory(Path::new("verdict.json")),
Path::new("."),
"a bare filename must stage in the working directory; staging in \
TMPDIR makes persist() a cross-filesystem rename"
);
assert_eq!(
staging_directory(Path::new("./verdict.json")),
Path::new(".")
);
assert_eq!(
staging_directory(Path::new("/tmp/run/verdict.json")),
Path::new("/tmp/run")
);
assert_eq!(
staging_directory(Path::new("sub/verdict.json")),
Path::new("sub")
);
for candidate in ["verdict.json", "./v.json", "/tmp/run/v.json", "sub/v.json"] {
assert!(
!staging_directory(Path::new(candidate))
.as_os_str()
.is_empty(),
"{candidate}: staging directory must be usable"
);
}
}
#[test]
fn a_bare_filename_target_is_overwritten_not_left_stale() {
static CWD: std::sync::Mutex<()> = std::sync::Mutex::new(());
let _guard = CWD.lock().unwrap_or_else(|e| e.into_inner());
let dir = tempfile::Builder::new()
.prefix("verify-json-bare-")
.tempdir_in(env!("CARGO_MANIFEST_DIR"))
.unwrap();
let previous = std::env::current_dir().unwrap();
std::env::set_current_dir(dir.path()).unwrap();
let outcome = (|| {
let bare = Path::new("verdict.json");
plant_previous_green(bare);
write_pending_verification_json(bare)?;
Ok::<_, Error>(read_verdict(bare))
})();
std::env::set_current_dir(previous).unwrap();
let now = outcome.expect("staging beside a bare filename must succeed");
assert_eq!(now["verdict"], serde_json::json!("no_result"));
assert_eq!(now["verified"], serde_json::json!(false));
}
#[test]
fn pending_stamp_overwrites_a_previous_green_verdict() {
let file = NamedTempFile::new().unwrap();
let path = file.path().to_path_buf();
let out = output(0, b"hello\n", b"");
let (log1, log2) = logs_with_identical_detlog();
let good = compare_with(&out, log1, &out, log2, LogCompareStrictness::Canonical).unwrap();
write_verification_json(&path, &good).unwrap();
let previous: serde_json::Value =
serde_json::from_str(&fs::read_to_string(&path).unwrap()).unwrap();
assert_eq!(previous["verified"], serde_json::json!(true));
assert_eq!(previous["bitwise_parity"], serde_json::json!(true));
write_pending_verification_json(&path).unwrap();
let now: serde_json::Value =
serde_json::from_str(&fs::read_to_string(&path).unwrap()).unwrap();
assert_eq!(now["verdict"], serde_json::json!("no_result"));
assert_eq!(now["verified"], serde_json::json!(false));
assert_eq!(now["bitwise_parity"], serde_json::json!(false));
assert_eq!(now["comparison"], serde_json::Value::Null);
assert_eq!(now["compared_log_messages"], serde_json::Value::Null);
assert!(
now.get("dbt_counted_branches").is_none(),
"no-result must not claim that a DBT branch-clock comparison completed"
);
}
#[test]
fn shared_report_preserves_the_current_no_result_bytes() {
let encoded = serde_json::to_string(&VerificationReport::no_result()).unwrap();
assert_eq!(
encoded,
concat!(
r#"{"verified":false,"bitwise_parity":false,"verdict":"no_result","#,
r#""no_result_reason":{"kind":"not_run"},"infrastructure_error":null,"comparison":null,"#,
r#""compared_log_messages":null,"guest_exit_code":null,"guest_signal":null,"#,
r#""first_divergent_scheduler_turn":null,"#,
r#""first_divergent_virtual_nanoseconds":null,"#,
r#""first_divergent_record":null,"first_divergent_syscall":null,"#,
r#""first_divergent_left_message":null,"first_divergent_right_message":null}"#,
)
);
}
#[test]
fn terminal_verdict_replaces_the_pending_stamp() {
let file = NamedTempFile::new().unwrap();
let path = file.path().to_path_buf();
write_pending_verification_json(&path).unwrap();
let out = output(0, b"hello\n", b"");
let (log1, log2) = logs_with_identical_detlog();
let outcome =
compare_with(&out, log1, &out, log2, LogCompareStrictness::Canonical).unwrap();
write_verification_json(&path, &outcome).unwrap();
let published: serde_json::Value =
serde_json::from_str(&fs::read_to_string(&path).unwrap()).unwrap();
assert_eq!(published["verdict"], serde_json::json!("matched"));
assert_eq!(published["verified"], serde_json::json!(true));
assert_eq!(published["bitwise_parity"], serde_json::json!(true));
}
#[test]
fn skid_overshoot_refusal_retains_the_completed_comparison() {
let file = NamedTempFile::new().unwrap();
let path = file.path().to_path_buf();
let out = output(0, b"hello\n", b"");
let (log1, log2) = logs_with_identical_detlog();
let outcome =
compare_with(&out, log1, &out, log2, LogCompareStrictness::Canonical).unwrap();
write_skid_overshoot_verification_json(&path, &outcome, 2).unwrap();
let report = VerificationReport::from_current_json_value(
serde_json::from_str(&fs::read_to_string(&path).unwrap()).unwrap(),
)
.unwrap();
assert_eq!(report.verdict, Verdict::InfrastructureError);
assert_eq!(
report.infrastructure_error,
Some(InfrastructureError::SkidOvershoot { count: 2 })
);
assert!(!report.verified);
assert!(!report.bitwise_parity);
assert!(report.comparison.is_some());
assert!(report.compared_log_messages.is_some());
assert_eq!(report.guest_exit_code, Some(0));
}
#[test]
fn skid_overshoot_before_comparison_is_not_reported_as_no_result() {
let file = NamedTempFile::new().unwrap();
write_skid_overshoot_without_comparison_json(
file.path(),
1,
None,
Some(ExitStatus::Exited(7)),
)
.unwrap();
let report = VerificationReport::from_current_json_value(
serde_json::from_str(&fs::read_to_string(file.path()).unwrap()).unwrap(),
)
.unwrap();
assert_eq!(report.verdict, Verdict::InfrastructureError);
assert_eq!(
report.infrastructure_error,
Some(InfrastructureError::SkidOvershoot { count: 1 })
);
assert!(report.no_result_reason.is_none());
assert!(report.comparison.is_none());
assert_eq!(report.guest_exit_code, Some(7));
}
#[test]
fn verification_report_json_carries_the_comparison() {
let out = output(0, b"hello\n", b"");
let (log1, log2) = logs_with_identical_detlog();
let outcome =
compare_with(&out, log1, &out, log2, LogCompareStrictness::Canonical).unwrap();
let json = serde_json::to_string(&verification_report(&outcome)).unwrap();
let parsed: serde_json::Value = serde_json::from_str(&json).unwrap();
assert_eq!(parsed["verified"], serde_json::json!(true));
assert_eq!(parsed["verdict"], serde_json::json!("matched"));
assert!(
parsed.get("dbt_counted_branches").is_none(),
"non-DBT reports must preserve the pre-field JSON shape"
);
assert_eq!(
parsed["first_divergent_scheduler_turn"],
serde_json::Value::Null
);
assert_eq!(
parsed["first_divergent_virtual_nanoseconds"],
serde_json::Value::Null
);
assert!(parsed["compared_log_messages"]["left"].as_u64().unwrap() > 0);
assert!(parsed["compared_log_messages"]["right"].as_u64().unwrap() > 0);
assert_eq!(parsed["bitwise_parity"], serde_json::json!(true));
assert_eq!(
parsed["comparison"]["strictness"],
serde_json::json!("canonical")
);
assert_eq!(
parsed["comparison"]["display_name"],
serde_json::json!("BitwiseInfoV1")
);
assert_eq!(
parsed["comparison"]["strip_lines"],
serde_json::json!(false)
);
assert_eq!(parsed["comparison"]["full_trace"], serde_json::json!(true));
assert_eq!(parsed["comparison"]["log_scope"], serde_json::json!("info"));
assert_eq!(
parsed["comparison"]["record_envelope"],
serde_json::json!("all_records_v1")
);
assert_eq!(
parsed["comparison"]["compare_logs"],
serde_json::json!(true)
);
assert_eq!(
parsed["comparison"]["ignore_lines"],
serde_json::json!(false)
);
assert_eq!(
parsed["comparison"]["skip_commit"],
serde_json::json!(false)
);
assert_eq!(
parsed["comparison"]["skip_detlog"],
serde_json::json!(false)
);
}
#[test]
fn a_green_report_says_what_it_compared_and_whether_time_was_virtual() {
for virtualize_time in [true, false] {
let spec = ComparisonSpec::new(
LogCompareStrictness::Canonical,
true,
false,
true,
RecordEnvelopePolicy::AllRecordsV1,
virtualize_time,
);
assert!(
spec.is_bitwise_parity(),
"parity must not depend on the time policy: the record/replay path \
qualifies for parity while running with virtualize_time false"
);
let json = serde_json::to_value(comparison_report(&spec)).unwrap();
assert_eq!(
json["virtualize_time"],
serde_json::json!(virtualize_time),
"a comparison that ran must publish the time policy it ran under"
);
}
let nothing_compared = serde_json::to_value(VerificationReport::no_result()).unwrap();
assert!(
nothing_compared["comparison"].is_null(),
"no verdict was reached, so no comparison may be described"
);
assert!(
nothing_compared.get("virtualize_time").is_none(),
"a run that compared nothing must not publish a time policy at all"
);
}
#[test]
fn bitwise_parity_contract_accepts_only_named_canonical_envelopes() {
let full = ComparisonSpec::new(
LogCompareStrictness::Canonical,
true,
false,
true,
RecordEnvelopePolicy::AllRecordsV1,
true,
);
assert!(
full.is_bitwise_parity(),
"a full-INFO unstripped all-records comparison must qualify"
);
assert!(
ComparisonSpec {
record_envelope: RecordEnvelopePolicy::DbtEvidenceTransportV1,
..full
}
.is_bitwise_parity(),
"the named DBT transport envelope is a disclosed canonical policy"
);
let stripped = ComparisonSpec::new(
LogCompareStrictness::Stripped,
true,
false,
false,
RecordEnvelopePolicy::AllRecordsV1,
true,
);
assert!(
!stripped.is_bitwise_parity(),
"a stripped comparison normalizes away the parity-relevant data"
);
let missing_io_buffers = ComparisonSpec {
compare_io_buffers: false,
..full
};
assert!(
!missing_io_buffers.is_bitwise_parity(),
"a comparison without syscall output-buffer content is not bitwise parity"
);
let output_only = ComparisonSpec {
compare_logs: false,
..full
};
assert!(
!output_only.is_bitwise_parity(),
"an output-only fallback never compared the log stream"
);
for weakened in [
ComparisonSpec {
ignore_lines: true,
..full
},
ComparisonSpec {
skip_commit: true,
..full
},
ComparisonSpec {
skip_detlog: true,
..full
},
ComparisonSpec {
full_trace: false,
..full
},
ComparisonSpec {
log_scope: ComparedLogScope::Deterministic,
..full
},
ComparisonSpec {
record_envelope: RecordEnvelopePolicy::CallerDefined,
..full
},
] {
assert!(
!weakened.is_bitwise_parity(),
"a filtered/subset comparison must not pass as bitwise parity: {weakened:?}"
);
}
let diverged = VerificationOutcome {
verdict: Verdict::Diverged,
no_result_reason: None,
guest_status: ExitStatus::Exited(0),
comparison: full,
compared_log_messages: Some(ComparedLogCounts { left: 9, right: 9 }),
compared_outputs: ComparedOutputs {
left: ComparedOutput {
exit_code: Some(0),
signal: None,
stdout_sha256: detcore::Digest::new(b"same stdout").to_string(),
stdout_bytes: 11,
stderr_sha256: detcore::Digest::new(b"").to_string(),
stderr_bytes: 0,
},
right: ComparedOutput {
exit_code: Some(0),
signal: None,
stdout_sha256: detcore::Digest::new(b"different stdout").to_string(),
stdout_bytes: 16,
stderr_sha256: detcore::Digest::new(b"").to_string(),
stderr_bytes: 0,
},
},
dbt_counted_branches: None,
runtime: None,
first_divergent_scheduler_turn: None,
first_divergent_virtual_nanoseconds: None,
first_divergent_record: None,
first_divergent_syscall: None,
first_divergent_left_message: None,
first_divergent_right_message: None,
};
assert!(!verification_report(&diverged).bitwise_parity);
}
#[test]
fn report_carries_both_run_runtime_totals_without_changing_the_verdict() {
let first = RunSummary {
sched_turns: 12,
virttime_elapsed: 34,
syscalls: Some(5),
..Default::default()
};
let second = RunSummary {
sched_turns: 13,
virttime_elapsed: 35,
syscalls: Some(6),
..Default::default()
};
let runtime = verification_runtime_from_summaries(Some(&first), Some(&second))
.expect("two summaries produce runtime totals");
assert_eq!(runtime.run1.as_ref().unwrap().scheduler_turns, 12);
assert_eq!(runtime.run2.as_ref().unwrap().syscalls, Some(6));
let mut report = VerificationReport::no_result();
report.runtime = Some(runtime);
let json = serde_json::to_value(report).unwrap();
assert_eq!(json["verdict"], "no_result");
assert_eq!(json["runtime"]["run1"]["virtual_nanoseconds"], 34);
assert_eq!(json["runtime"]["run2"]["syscalls"], 6);
assert!(json["runtime"]["run1"].get("dispatch").is_none());
}
#[test]
fn report_carries_each_run_dispatch_record() {
let record = reverie::DispatchStats::new(
"ptrace",
reverie::DispatchCounters {
ptrace_seccomp_stops: Some(9),
..reverie::DispatchCounters::ZERO
},
reverie::SiteCounters::NONE_PATCHED,
);
let first = RunSummary {
dispatch_stats: Some(record.clone()),
..Default::default()
};
let runtime = verification_runtime_from_summaries(Some(&first), None)
.expect("one summary produces runtime totals");
assert_eq!(runtime.run1.unwrap().dispatch, Some(record.clone()));
let mut report = VerificationReport::no_result();
report.runtime = verification_runtime_from_summaries(Some(&first), Some(&first));
let json = serde_json::to_value(report).unwrap();
assert_eq!(json["runtime"]["run2"]["dispatch"]["backend"], "ptrace");
assert_eq!(
json["runtime"]["run2"]["dispatch"]["schema_version"],
reverie::DISPATCH_STATS_SCHEMA_VERSION
);
}
#[test]
fn dbt_transport_only_is_disclosed_and_cannot_qualify_as_parity() {
let output = output(0, b"hello\n", b"");
let (left, right) = empty_logs();
let transport = "1970-01-01T00:00:00.000000Z INFO reverie_dbt::evidence: protected evidence initialized\n";
fs::write(&left, transport).unwrap();
fs::write(&right, transport).unwrap();
let outcome = compare_with_envelope(
&output,
left,
&output,
right,
LogCompareStrictness::Canonical,
RecordEnvelope::dbt_evidence_transport_v1(),
)
.unwrap();
let report = verification_report(&outcome);
assert_eq!(outcome.verdict, Verdict::Matched);
assert_eq!(
outcome.compared_log_messages,
Some(ComparedLogCounts { left: 0, right: 0 })
);
assert_eq!(
outcome.comparison.record_envelope,
RecordEnvelopePolicy::DbtEvidenceTransportV1
);
assert!(!report.bitwise_parity);
}
#[test]
fn dbt_real_detcore_record_matches_under_the_disclosed_envelope() {
let output = output(0, b"hello\n", b"");
let (left, right) = empty_logs();
let record = detcore::detlog::record_suffix(detcore::detlog::DetLogEvent::Syscall);
let log = format!(
"1970-01-01T00:00:00.000000Z INFO reverie_dbt::evidence: protected evidence initialized\n\
2026-08-21T10:00:00.000000Z INFO detcore: DETLOG [syscall] getpid() = Ok(3){record}\n"
);
fs::write(&left, &log).unwrap();
fs::write(&right, &log).unwrap();
let outcome = compare_with_envelope(
&output,
left,
&output,
right,
LogCompareStrictness::Canonical,
RecordEnvelope::dbt_evidence_transport_v1(),
)
.unwrap();
let report = verification_report(&outcome);
let json = serde_json::to_value(&report).unwrap();
assert_eq!(
outcome.compared_log_messages,
Some(ComparedLogCounts { left: 1, right: 1 })
);
assert!(report.bitwise_parity);
assert_eq!(
json["comparison"]["record_envelope"],
"dbt_evidence_transport_v1"
);
}
#[test]
fn caller_defined_record_filter_is_disclosed_but_never_parity() {
fn keep_everything(_record: &str) -> bool {
true
}
let output = output(0, b"hello\n", b"");
let (left, right) = logs_with_identical_detlog();
let outcome = compare_with_envelope(
&output,
left,
&output,
right,
LogCompareStrictness::Canonical,
RecordEnvelope::caller_defined(keep_everything),
)
.unwrap();
let report = verification_report(&outcome);
let json = serde_json::to_value(&report).unwrap();
assert!(outcome.verified());
assert_eq!(
outcome.comparison.record_envelope,
RecordEnvelopePolicy::CallerDefined
);
assert!(!report.bitwise_parity);
assert_eq!(json["comparison"]["record_envelope"], "caller_defined");
}
#[test]
fn default_log_diff_opts_apply_no_line_filters() {
let default = logdiff::LogDiffOpts::default();
assert!(default.ignore_lines.is_empty());
assert!(!default.skip_commit);
assert!(!default.skip_detlog);
}
}