use super::fork_recovery;
use super::import_recovery;
use super::recovery::{run, RecoveryOutcome};
use super::recovery_matrix::{self, Boundary, Classification, FaultMode};
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub(crate) enum CorpusOracle {
StoreRecovery,
ForkIsolation,
ImportReapply,
}
impl CorpusOracle {
pub(crate) fn parse(raw: &str) -> Option<Self> {
match raw {
"StoreRecovery" => Some(Self::StoreRecovery),
"ForkIsolation" => Some(Self::ForkIsolation),
"ImportReapply" => Some(Self::ImportReapply),
_ => None,
}
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub(crate) enum ImportReapplyKind {
UnderFault,
SameStoreCeiling,
}
pub(crate) const IMPORT_CEILING_BOUNDARY: &str = "SameStoreCeiling";
fn parse_import_kind(boundary: Option<&str>) -> ImportReapplyKind {
match boundary {
Some(IMPORT_CEILING_BOUNDARY) => ImportReapplyKind::SameStoreCeiling,
_ => ImportReapplyKind::UnderFault,
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub(crate) enum CorpusOutcome {
CommittedPrefix,
RolledBack,
CanonicalRefusal,
}
impl CorpusOutcome {
pub(crate) fn as_str(self) -> &'static str {
match self {
Self::CommittedPrefix => "CommittedPrefix",
Self::RolledBack => "RolledBack",
Self::CanonicalRefusal => "CanonicalRefusal",
}
}
pub(crate) fn parse(raw: &str) -> Option<Self> {
match raw {
"CommittedPrefix" => Some(Self::CommittedPrefix),
"RolledBack" => Some(Self::RolledBack),
"CanonicalRefusal" => Some(Self::CanonicalRefusal),
_ => None,
}
}
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub(crate) struct CorpusEntry {
pub seed: u64,
pub oracle: CorpusOracle,
pub fault_mode: FaultModeLabel,
pub boundary: Option<Boundary>,
pub import_kind: Option<ImportReapplyKind>,
pub seam_touched: String,
pub assurance_level: String,
pub steps: usize,
pub op_trace_digest: u64,
pub outcome: CorpusOutcome,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub(crate) enum FaultModeLabel {
HonestDiskCrash,
LyingDiskFsyncDrop {
one_in: u32,
},
CrashBeforeFsync,
}
impl FaultModeLabel {
pub(crate) fn as_str(self) -> &'static str {
match self {
Self::HonestDiskCrash => "HonestDiskCrash",
Self::LyingDiskFsyncDrop { .. } => "LyingDiskFsyncDrop",
Self::CrashBeforeFsync => "CrashBeforeFsync",
}
}
pub(crate) fn parse(raw: &str) -> Option<Self> {
match raw {
"HonestDiskCrash" => Some(Self::HonestDiskCrash),
"LyingDiskFsyncDrop" => Some(Self::LyingDiskFsyncDrop { one_in: 0 }),
"CrashBeforeFsync" => Some(Self::CrashBeforeFsync),
_ => None,
}
}
pub(crate) fn parse_with_rate(raw: &str, one_in: Option<u32>) -> Option<Self> {
match Self::parse(raw)? {
Self::LyingDiskFsyncDrop { .. } => Some(Self::LyingDiskFsyncDrop {
one_in: one_in?.max(1),
}),
mode @ (Self::HonestDiskCrash | Self::CrashBeforeFsync) => Some(mode),
}
}
}
fn resolve_matrix_mode(label: FaultModeLabel, boundary: Option<Boundary>) -> Option<FaultMode> {
match label {
FaultModeLabel::HonestDiskCrash => None,
FaultModeLabel::LyingDiskFsyncDrop { one_in } => Some(FaultMode::LyingDiskFsyncDrop {
one_in: one_in.max(1),
}),
FaultModeLabel::CrashBeforeFsync => {
boundary.map(|boundary| FaultMode::CrashBeforeFsync { boundary })
}
}
}
fn classification_to_outcome(class: Classification) -> CorpusOutcome {
match class {
Classification::CommittedPrefix => CorpusOutcome::CommittedPrefix,
Classification::RolledBack => CorpusOutcome::RolledBack,
Classification::CanonicalRefusal => CorpusOutcome::CanonicalRefusal,
}
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub(crate) struct GraduationCandidate {
pub entry: CorpusEntry,
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub(crate) enum GraduationRefusal {
NonDeterministic { seed: u64, first: u64, second: u64 },
IllegalRecovery { seed: u64, reason: String },
EmptySeam { seed: u64 },
}
impl std::fmt::Display for GraduationRefusal {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
match self {
Self::NonDeterministic {
seed,
first,
second,
} => write!(
f,
"seed 0x{seed:X} is non-deterministic (digests 0x{first:X} != 0x{second:X})"
),
Self::IllegalRecovery { seed, reason } => {
write!(f, "seed 0x{seed:X} failed legality oracle: {reason}")
}
Self::EmptySeam { seed } => {
write!(f, "seed 0x{seed:X} refused: seam_touched must be non-empty")
}
}
}
}
fn classify_honest_recovery(outcome: &RecoveryOutcome) -> CorpusOutcome {
if outcome.recovered_visible == 0 {
CorpusOutcome::CanonicalRefusal
} else {
CorpusOutcome::CommittedPrefix
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub(crate) struct CorpusReplay {
pub digest: u64,
pub outcome: CorpusOutcome,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub struct CorpusReplayPublic {
pub digest: u64,
pub outcome: &'static str,
}
impl From<CorpusReplay> for CorpusReplayPublic {
fn from(replay: CorpusReplay) -> Self {
Self {
digest: replay.digest,
outcome: replay.outcome.as_str(),
}
}
}
fn replay_cell(
seed: u64,
steps: usize,
fault_mode: FaultModeLabel,
boundary: Option<Boundary>,
) -> Result<CorpusReplay, String> {
match fault_mode {
FaultModeLabel::HonestDiskCrash => {
if boundary.is_some() {
return Err(format!(
"corpus replay (seed=0x{seed:X}): HonestDiskCrash must not declare a boundary"
));
}
run(seed, steps).map(|o| CorpusReplay {
digest: o.digest,
outcome: classify_honest_recovery(&o),
})
}
FaultModeLabel::LyingDiskFsyncDrop { .. } | FaultModeLabel::CrashBeforeFsync => {
let mode = resolve_matrix_mode(fault_mode, boundary).ok_or_else(|| {
format!(
"corpus replay (seed=0x{seed:X}): {} requires a boundary cell",
fault_mode.as_str()
)
})?;
recovery_matrix::run(seed, steps, mode).map(|o| CorpusReplay {
digest: o.digest,
outcome: classification_to_outcome(o.classification),
})
}
}
}
fn replay_fork_isolation_cell(seed: u64) -> Result<CorpusReplay, String> {
let outcome = fork_recovery::run_seeded_fork_fault(seed)?;
Ok(CorpusReplay {
digest: outcome.digest,
outcome: classification_to_outcome(outcome.classification),
})
}
fn replay_import_reapply_cell(seed: u64, kind: ImportReapplyKind) -> Result<CorpusReplay, String> {
match kind {
ImportReapplyKind::UnderFault => {
let outcome = import_recovery::run_seeded_import_fault(seed)?;
Ok(CorpusReplay {
digest: outcome.digest,
outcome: CorpusOutcome::CommittedPrefix,
})
}
ImportReapplyKind::SameStoreCeiling => {
let outcome = import_recovery::run_seeded_import_same_store_ceiling(seed)?;
Ok(CorpusReplay {
digest: outcome.digest,
outcome: CorpusOutcome::CommittedPrefix,
})
}
}
}
fn replay_entry(entry: &CorpusEntry) -> Result<CorpusReplay, String> {
match entry.oracle {
CorpusOracle::StoreRecovery => {
replay_cell(entry.seed, entry.steps, entry.fault_mode, entry.boundary)
}
CorpusOracle::ForkIsolation => replay_fork_isolation_cell(entry.seed),
CorpusOracle::ImportReapply => {
let kind = entry.import_kind.ok_or_else(|| {
format!(
"corpus replay (seed=0x{:X}): ImportReapply row missing import_kind",
entry.seed
)
})?;
replay_import_reapply_cell(entry.seed, kind)
}
}
}
pub(crate) fn replay_corpus_entry(entry: &CorpusEntry) -> Result<CorpusReplay, String> {
replay_entry(entry)
}
fn check_graduation_for(
cell: GraduationCell<'_>,
) -> Result<GraduationCandidate, GraduationRefusal> {
if cell.seam_touched.is_empty() {
return Err(GraduationRefusal::EmptySeam { seed: cell.seed });
}
let entry = CorpusEntry {
seed: cell.seed,
oracle: cell.oracle,
fault_mode: cell.fault_mode,
boundary: cell.boundary,
import_kind: cell.import_kind,
seam_touched: cell.seam_touched.to_owned(),
assurance_level: cell.assurance_level.to_owned(),
steps: cell.steps,
op_trace_digest: 0,
outcome: CorpusOutcome::CommittedPrefix,
};
let first = replay_entry(&entry).map_err(|reason| GraduationRefusal::IllegalRecovery {
seed: cell.seed,
reason,
})?;
let second = replay_entry(&entry).map_err(|reason| GraduationRefusal::IllegalRecovery {
seed: cell.seed,
reason,
})?;
if first.digest != second.digest {
return Err(GraduationRefusal::NonDeterministic {
seed: cell.seed,
first: first.digest,
second: second.digest,
});
}
Ok(GraduationCandidate {
entry: CorpusEntry {
op_trace_digest: first.digest,
outcome: first.outcome,
..entry
},
})
}
#[derive(Clone, Copy)]
struct GraduationCell<'a> {
seed: u64,
steps: usize,
fault_mode: FaultModeLabel,
boundary: Option<Boundary>,
seam_touched: &'a str,
assurance_level: &'a str,
oracle: CorpusOracle,
import_kind: Option<ImportReapplyKind>,
}
pub fn verify_corpus_row(
seed: u64,
steps: usize,
fault_mode: &str,
expected_digest: u64,
) -> Result<(), String> {
verify_corpus_row_cell(
seed,
steps,
fault_mode,
None,
None,
"StoreRecovery",
expected_digest,
)
}
pub fn verify_corpus_row_cell(
seed: u64,
steps: usize,
fault_mode: &str,
boundary: Option<&str>,
one_in: Option<u32>,
oracle: &str,
expected_digest: u64,
) -> Result<(), String> {
let oracle = CorpusOracle::parse(oracle)
.ok_or_else(|| format!("corpus row (seed=0x{seed:X}): unknown oracle `{oracle}`"))?;
let mode = FaultModeLabel::parse_with_rate(fault_mode, one_in).ok_or_else(|| {
format!("corpus row (seed=0x{seed:X}): unknown fault_mode `{fault_mode}`")
})?;
let (boundary, import_kind) = parse_row_labels(seed, oracle, boundary)?;
let entry = CorpusEntry {
seed,
oracle,
fault_mode: mode,
boundary,
import_kind,
seam_touched: String::new(),
assurance_level: String::new(),
steps,
op_trace_digest: expected_digest,
outcome: CorpusOutcome::CommittedPrefix,
};
let live = replay_entry(&entry)?;
if live.digest != expected_digest {
return Err(format!(
"corpus currency (seed=0x{seed:X}): expected digest 0x{expected_digest:X}, replay 0x{:X}",
live.digest
));
}
Ok(())
}
pub fn run_fork_isolation_corpus_cell(seed: u64) -> Result<CorpusReplayPublic, String> {
replay_fork_isolation_cell(seed).map(Into::into)
}
pub fn run_import_reapply_corpus_cell(
seed: u64,
boundary: Option<&str>,
) -> Result<CorpusReplayPublic, String> {
let kind = parse_import_kind(boundary);
replay_import_reapply_cell(seed, kind).map(Into::into)
}
fn parse_boundary(seed: u64, boundary: Option<&str>) -> Result<Option<Boundary>, String> {
boundary
.map(|raw| {
Boundary::parse(raw)
.ok_or_else(|| format!("corpus row (seed=0x{seed:X}): unknown boundary `{raw}`"))
})
.transpose()
}
fn parse_row_labels(
seed: u64,
oracle: CorpusOracle,
boundary: Option<&str>,
) -> Result<(Option<Boundary>, Option<ImportReapplyKind>), String> {
match oracle {
CorpusOracle::StoreRecovery => Ok((parse_boundary(seed, boundary)?, None)),
CorpusOracle::ForkIsolation => {
if boundary.is_some() {
return Err(format!(
"corpus row (seed=0x{seed:X}): ForkIsolation must leave boundary null"
));
}
Ok((None, None))
}
CorpusOracle::ImportReapply => {
let kind = parse_import_kind(boundary);
if let (ImportReapplyKind::UnderFault, Some(boundary)) = (kind, boundary) {
return Err(format!(
"corpus row (seed=0x{seed:X}): unknown import boundary `{}`",
boundary
));
}
Ok((None, Some(kind)))
}
}
}
pub fn graduate_corpus_seed(
seed: u64,
steps: usize,
seam_touched: &str,
assurance_level: &str,
) -> Result<u64, String> {
graduate_corpus_cell(&GraduationRequest {
seed,
steps,
fault_mode: "HonestDiskCrash",
boundary: None,
one_in: None,
seam_touched,
assurance_level,
oracle: "StoreRecovery",
})
}
#[derive(Debug, Clone, Copy)]
pub struct GraduationRequest<'a> {
pub seed: u64,
pub steps: usize,
pub fault_mode: &'a str,
pub boundary: Option<&'a str>,
pub one_in: Option<u32>,
pub seam_touched: &'a str,
pub assurance_level: &'a str,
pub oracle: &'a str,
}
pub fn graduate_corpus_cell(req: &GraduationRequest<'_>) -> Result<u64, String> {
let oracle = CorpusOracle::parse(req.oracle).ok_or_else(|| {
format!(
"corpus row (seed=0x{:X}): unknown oracle `{}`",
req.seed, req.oracle
)
})?;
let mode = FaultModeLabel::parse_with_rate(req.fault_mode, req.one_in).ok_or_else(|| {
format!(
"corpus row (seed=0x{:X}): unknown fault_mode `{}`",
req.seed, req.fault_mode
)
})?;
let (boundary, import_kind) = parse_row_labels(req.seed, oracle, req.boundary)?;
let candidate = check_graduation_for(GraduationCell {
seed: req.seed,
steps: req.steps,
fault_mode: mode,
boundary,
seam_touched: req.seam_touched,
assurance_level: req.assurance_level,
oracle,
import_kind,
})
.map_err(|r| r.to_string())?;
Ok(candidate.entry.op_trace_digest)
}
pub fn assert_corpus_rows_current(rows: &[CorpusRowDescriptor<'_>]) -> Result<(), String> {
let mut entries = Vec::with_capacity(rows.len());
for row in rows {
let oracle = CorpusOracle::parse(row.oracle).ok_or_else(|| {
format!(
"corpus row (seed=0x{:X}): unknown oracle `{}`",
row.seed, row.oracle
)
})?;
let fault_mode =
FaultModeLabel::parse_with_rate(row.fault_mode, row.one_in).ok_or_else(|| {
format!(
"corpus row (seed=0x{:X}): unknown fault_mode `{}`",
row.seed, row.fault_mode
)
})?;
let (boundary, import_kind) = parse_row_labels(row.seed, oracle, row.boundary)?;
let outcome = CorpusOutcome::parse(row.outcome).ok_or_else(|| {
format!(
"corpus row (seed=0x{:X}): unknown outcome `{}`",
row.seed, row.outcome
)
})?;
let steps = usize::try_from(row.steps).map_err(|_| {
format!(
"corpus row (seed=0x{:X}): steps {} must fit usize",
row.seed, row.steps
)
})?;
entries.push(CorpusEntry {
seed: row.seed,
oracle,
fault_mode,
boundary,
import_kind,
seam_touched: String::new(),
assurance_level: String::new(),
steps,
op_trace_digest: row.op_trace_digest,
outcome,
});
}
assert_corpus_currency(&entries)
}
#[derive(Debug, Clone, Copy)]
pub struct CorpusRowDescriptor<'a> {
pub seed: u64,
pub steps: u32,
pub oracle: &'a str,
pub fault_mode: &'a str,
pub boundary: Option<&'a str>,
pub one_in: Option<u32>,
pub outcome: &'a str,
pub op_trace_digest: u64,
}
#[cfg(test)]
pub(crate) fn check_graduation(
seed: u64,
steps: usize,
seam_touched: &str,
assurance_level: &str,
) -> Result<GraduationCandidate, GraduationRefusal> {
check_graduation_for(GraduationCell {
seed,
steps,
fault_mode: FaultModeLabel::HonestDiskCrash,
boundary: None,
seam_touched,
assurance_level,
oracle: CorpusOracle::StoreRecovery,
import_kind: None,
})
}
#[cfg(test)]
pub(crate) fn run_corpus_sweep(
seeds: &[u64],
steps: usize,
seam_touched: &str,
assurance_level: &str,
) -> (Vec<GraduationCandidate>, Vec<GraduationRefusal>) {
let mut graduated = Vec::new();
let mut refused = Vec::new();
for &seed in seeds {
match check_graduation(seed, steps, seam_touched, assurance_level) {
Ok(candidate) => graduated.push(candidate),
Err(reason) => refused.push(reason),
}
}
(graduated, refused)
}
pub(crate) fn assert_corpus_currency(entries: &[CorpusEntry]) -> Result<(), String> {
if entries.is_empty() {
return Err("dst corpus must be non-empty".to_owned());
}
for entry in entries {
let live = replay_corpus_entry(entry)?;
if live.digest != entry.op_trace_digest {
return Err(format!(
"corpus currency (seed=0x{:X}): stored digest 0x{:X} != replay 0x{:X}",
entry.seed, entry.op_trace_digest, live.digest
));
}
if live.outcome != entry.outcome {
return Err(format!(
"corpus currency (seed=0x{:X}): stored outcome {} != replay {}",
entry.seed,
entry.outcome.as_str(),
live.outcome.as_str()
));
}
}
Ok(())
}
#[cfg(test)]
#[path = "corpus_tests.rs"]
mod tests;