mod ancestry;
mod progress;
mod stage;
use ancestry::{
acquire_stage, read_admitted_binding, read_binding, stage_path, validate_predecessors,
};
pub use progress::{ExecutionWorkflowProgressError, read_execution_workflow_progress};
pub use stage::ExecutionStageGuard;
#[cfg(test)]
use stage::StagePreparationBarrier;
use super::{
AttemptJournalError, AttemptJournalGuard, BackupLayoutGuard, ExecutionProgressPersistenceError,
ExecutionSettlementCheckpointError, ExecutionSettlementPersistenceError, JournalLock,
JournalLockError, OperationPlanPersistenceError, PersistenceError, read_execution_progress,
read_execution_settlement, read_json, read_operation_plan, reconcile_execution_settlement,
replay_execution_settlement,
};
use crate::model::{
artifacts::ArtifactChecksumRecord,
execution_workflow::{
ExecutionStageBindingRecord, ExecutionStagePredecessorRecord, ExecutionWorkflowError,
ExecutionWorkflowRecord, MAX_EXECUTION_STAGE_BYTES, MAX_EXECUTION_WORKFLOW_BYTES,
},
operation_plan::{OperationPlanError, OperationPlanRecord},
};
use crate::policy::execution_progress::ExecutionProgressView;
use std::{
collections::{BTreeMap, BTreeSet, btree_map::Entry},
fs,
path::PathBuf,
};
use thiserror::Error;
const WORKFLOW_FILE: &str = "execution-workflow.json";
const BINDING_FILE: &str = "stage-binding.json";
#[derive(Clone, Debug, Eq, PartialEq)]
pub struct ExecutionWorkflowSettlementView {
workflow: ArtifactChecksumRecord,
stages: Vec<ExecutionStageSettlementView>,
}
impl ExecutionWorkflowSettlementView {
#[must_use]
pub const fn workflow(&self) -> &ArtifactChecksumRecord {
&self.workflow
}
#[must_use]
pub fn stages(&self) -> &[ExecutionStageSettlementView] {
&self.stages
}
}
#[derive(Clone, Debug, Eq, PartialEq)]
pub struct ExecutionStageSettlementView {
stage_sequence: u64,
binding: ArtifactChecksumRecord,
settlement: ArtifactChecksumRecord,
}
impl ExecutionStageSettlementView {
#[must_use]
pub const fn stage_sequence(&self) -> u64 {
self.stage_sequence
}
#[must_use]
pub const fn binding(&self) -> &ArtifactChecksumRecord {
&self.binding
}
#[must_use]
pub const fn settlement(&self) -> &ArtifactChecksumRecord {
&self.settlement
}
}
pub fn create_execution_workflow(
layout: &BackupLayoutGuard,
record: &ExecutionWorkflowRecord,
) -> Result<(), ExecutionWorkflowPersistenceError> {
layout.check_root()?;
let bytes = super::json::encode_json(record, MAX_EXECUTION_WORKFLOW_BYTES)?;
super::immutable_record::create_encoded_record(layout, WORKFLOW_FILE, &bytes)
}
pub fn read_execution_workflow(
layout: &BackupLayoutGuard,
expected: &ArtifactChecksumRecord,
) -> Result<ExecutionWorkflowRecord, ExecutionWorkflowPersistenceError> {
super::immutable_record::read(
layout,
WORKFLOW_FILE,
MAX_EXECUTION_WORKFLOW_BYTES,
|record: &ExecutionWorkflowRecord| &record.digest() == expected,
|| ExecutionWorkflowPersistenceError::DigestMismatch,
)
}
pub(crate) fn read_execution_stage_occupancy(
layout: &BackupLayoutGuard,
expected: &ArtifactChecksumRecord,
) -> Result<BTreeSet<u64>, ExecutionWorkflowPersistenceError> {
let workflow = read_execution_workflow(layout, expected)?;
let mut occupied = BTreeSet::new();
for stage in workflow.stages() {
let sequence = stage.operation_sequence();
match fs::symlink_metadata(stage_path(layout, sequence)) {
Ok(metadata) if metadata.is_dir() => {
occupied.insert(sequence);
}
Ok(_) => return Err(ExecutionWorkflowPersistenceError::UnsafeStage),
Err(error) if error.kind() == std::io::ErrorKind::NotFound => {}
Err(error) => return Err(PersistenceError::from(error).into()),
}
}
read_execution_workflow(layout, expected)?;
Ok(occupied)
}
pub fn read_complete_execution_workflow(
layout: &BackupLayoutGuard,
expected: &ArtifactChecksumRecord,
) -> Result<ExecutionWorkflowSettlementView, ExecutionStageCheckpointError> {
read_complete_with(layout, expected, || {
read_execution_workflow(layout, expected)
})
}
pub fn read_complete_execution_workflow_locked(
layout: &BackupLayoutGuard,
journal: &JournalLock,
expected: &ArtifactChecksumRecord,
) -> Result<ExecutionWorkflowSettlementView, ExecutionStageCheckpointError> {
if journal.journal() != layout.root().join(WORKFLOW_FILE) {
return Err(ExecutionWorkflowPersistenceError::DigestMismatch.into());
}
read_complete_with(layout, expected, || {
layout.check_root()?;
let workflow: ExecutionWorkflowRecord =
read_json(journal.journal(), MAX_EXECUTION_WORKFLOW_BYTES)?;
super::json::check_json_size(&workflow, MAX_EXECUTION_WORKFLOW_BYTES)?;
if &workflow.digest() != expected {
return Err(ExecutionWorkflowPersistenceError::DigestMismatch);
}
Ok(workflow)
})
}
fn read_complete_with(
layout: &BackupLayoutGuard,
expected: &ArtifactChecksumRecord,
read_original: impl Fn() -> Result<ExecutionWorkflowRecord, ExecutionWorkflowPersistenceError>,
) -> Result<ExecutionWorkflowSettlementView, ExecutionStageCheckpointError> {
let workflow = read_original()?;
let mut stages = Vec::with_capacity(workflow.stages().len());
for allocated in workflow.stages() {
let sequence = allocated.operation_sequence();
let stage = acquire_stage(layout, sequence)?;
let binding: ExecutionStageBindingRecord =
read_json(&stage.root().join(BINDING_FILE), MAX_EXECUTION_STAGE_BYTES)
.map_err(ExecutionWorkflowPersistenceError::from)?;
let (binding, plan) = read_binding(&stage, &workflow, sequence, &binding.digest())?;
let settlement = replay_execution_settlement(&stage, &plan.digest())?;
read_binding(&stage, &workflow, sequence, &binding.digest())?;
let view = ExecutionStageSettlementView {
stage_sequence: sequence,
binding: binding.digest(),
settlement: settlement.digest(),
};
stages.push((binding, view));
}
for (binding, _) in &stages {
for predecessor in binding.predecessors() {
let index = stages
.binary_search_by_key(&predecessor.stage_sequence(), |(_, view)| {
view.stage_sequence()
})
.map_err(|_| ExecutionWorkflowPersistenceError::DigestMismatch)?;
let actual = &stages[index].1;
if predecessor.binding() != actual.binding()
|| predecessor.settlement() != actual.settlement()
{
return Err(ExecutionWorkflowPersistenceError::DigestMismatch.into());
}
}
}
read_original()?;
Ok(ExecutionWorkflowSettlementView {
workflow: expected.clone(),
stages: stages.into_iter().map(|(_, view)| view).collect(),
})
}
#[derive(Debug, Error)]
pub enum ExecutionStageCheckpointError {
#[error(transparent)]
Stage(#[from] ExecutionWorkflowPersistenceError),
#[error(transparent)]
Settlement(#[from] ExecutionSettlementCheckpointError),
}
#[derive(Debug, Error)]
pub enum ExecutionStagePreparationError {
#[error(transparent)]
Stage(#[from] ExecutionWorkflowPersistenceError),
#[error(transparent)]
Authority(#[from] OperationPlanError),
#[error(transparent)]
Journal(#[from] AttemptJournalError),
}
#[derive(Debug, Error)]
pub enum ExecutionStageResumeError {
#[error(transparent)]
Stage(#[from] ExecutionWorkflowPersistenceError),
#[error(transparent)]
Progress(#[from] ExecutionProgressPersistenceError),
}
#[derive(Debug, Error)]
pub enum ExecutionWorkflowPersistenceError {
#[error("execution workflow/stage digest mismatch")]
DigestMismatch,
#[error("unsafe execution stage directory")]
UnsafeStage,
#[error(transparent)]
Model(#[from] ExecutionWorkflowError),
#[error(transparent)]
Plan(#[from] OperationPlanPersistenceError),
#[error(transparent)]
Settlement(#[from] ExecutionSettlementPersistenceError),
#[error(transparent)]
Lock(#[from] JournalLockError),
#[error(transparent)]
Persistence(#[from] PersistenceError),
}
#[cfg(all(test, unix))]
mod tests;