use crate::{
model::{
artifacts::ArtifactChecksumRecord,
execution_workflow::{
ExecutionStageBindingRecord, ExecutionStagePredecessorRecord, ExecutionWorkflowError,
ExecutionWorkflowRecord,
},
operation_plan::OperationPlanRecord,
},
ops::persistence::{
BackupLayoutGuard, ExecutionStageCheckpointError, ExecutionStageGuard,
ExecutionStagePreparationError, ExecutionWorkflowPersistenceError,
ExecutionWorkflowSettlementView, create_execution_workflow,
read_complete_execution_workflow, read_execution_stage_occupancy, read_execution_workflow,
},
};
use std::{collections::BTreeMap, future::Future};
use thiserror::Error;
pub trait ExecutionWorkflowStageDriver {
type Error: std::error::Error + 'static;
fn bind_stage(
&mut self,
workflow: &ExecutionWorkflowRecord,
stage_sequence: u64,
predecessors: &[ExecutionStagePredecessorRecord],
) -> Result<OperationPlanRecord, Self::Error>;
fn execute_stage(
&mut self,
stage: &ExecutionStageGuard<'_>,
) -> impl Future<Output = Result<ArtifactChecksumRecord, Self::Error>>;
}
#[derive(Clone, Debug, Eq, PartialEq)]
pub struct RetainedStageEvidence {
binding: ArtifactChecksumRecord,
learned_evidence: ArtifactChecksumRecord,
}
impl RetainedStageEvidence {
#[must_use]
pub const fn new(
binding: ArtifactChecksumRecord,
learned_evidence: ArtifactChecksumRecord,
) -> Self {
Self {
binding,
learned_evidence,
}
}
}
pub async fn run_new_execution_workflow<D: ExecutionWorkflowStageDriver>(
layout: &BackupLayoutGuard,
workflow: &ExecutionWorkflowRecord,
driver: &mut D,
) -> Result<ExecutionWorkflowSettlementView, ExecutionWorkflowRunError<D::Error>> {
create_execution_workflow(layout, workflow)?;
let mut completed: BTreeMap<u64, ExecutionStagePredecessorRecord> = BTreeMap::new();
for node in workflow.allocation().graph().ordered_nodes() {
let sequence = node.operation_sequence();
let predecessors = node
.depends_on()
.iter()
.map(|dependency| {
completed
.get(dependency)
.cloned()
.ok_or(ExecutionWorkflowRunError::MissingPredecessor(*dependency))
})
.collect::<Result<Vec<_>, _>>()?;
let plan = driver
.bind_stage(workflow, sequence, &predecessors)
.map_err(ExecutionWorkflowRunError::Driver)?;
let binding = ExecutionStageBindingRecord::new(workflow, sequence, &plan, predecessors)?;
let stage = ExecutionStageGuard::prepare(layout, binding, plan)?;
let learned = driver
.execute_stage(&stage)
.await
.map_err(ExecutionWorkflowRunError::Driver)?;
let checkpoint = stage.reconcile_checkpoint(learned)?;
completed.insert(sequence, checkpoint);
}
Ok(read_complete_execution_workflow(
layout,
&workflow.digest(),
)?)
}
pub async fn run_retained_execution_workflow<D: ExecutionWorkflowStageDriver>(
layout: &BackupLayoutGuard,
expected_workflow: &ArtifactChecksumRecord,
retained: &BTreeMap<u64, RetainedStageEvidence>,
driver: &mut D,
) -> Result<ExecutionWorkflowSettlementView, ExecutionWorkflowRunError<D::Error>> {
let workflow = read_execution_workflow(layout, expected_workflow)?;
let occupied = read_execution_stage_occupancy(layout, expected_workflow)?;
if !retained.keys().copied().eq(occupied.iter().copied()) {
return Err(ExecutionWorkflowRunError::RetainedStageSetMismatch);
}
let mut completed: BTreeMap<u64, ExecutionStagePredecessorRecord> = BTreeMap::new();
for (&sequence, evidence) in retained {
let stage =
ExecutionStageGuard::open(layout, expected_workflow, sequence, &evidence.binding)?;
for predecessor in stage.binding().predecessors() {
let Some(original) = retained.get(&predecessor.stage_sequence()) else {
return Err(ExecutionWorkflowRunError::RetainedStageSetMismatch);
};
if &original.learned_evidence != predecessor.learned_evidence() {
return Err(ExecutionWorkflowRunError::RetainedLearnedMismatch(sequence));
}
}
let checkpoint = stage.replay_checkpoint(evidence.learned_evidence.clone())?;
completed.insert(sequence, checkpoint);
}
if read_execution_stage_occupancy(layout, expected_workflow)? != occupied {
return Err(ExecutionWorkflowRunError::RetainedStageSetMismatch);
}
for node in workflow.allocation().graph().ordered_nodes() {
let sequence = node.operation_sequence();
if completed.contains_key(&sequence) {
continue;
}
let predecessors = node
.depends_on()
.iter()
.map(|dependency| {
completed
.get(dependency)
.cloned()
.ok_or(ExecutionWorkflowRunError::MissingPredecessor(*dependency))
})
.collect::<Result<Vec<_>, _>>()?;
let plan = driver
.bind_stage(&workflow, sequence, &predecessors)
.map_err(ExecutionWorkflowRunError::Driver)?;
let binding = ExecutionStageBindingRecord::new(&workflow, sequence, &plan, predecessors)?;
let stage = ExecutionStageGuard::prepare(layout, binding, plan)?;
let learned = driver
.execute_stage(&stage)
.await
.map_err(ExecutionWorkflowRunError::Driver)?;
let checkpoint = stage.reconcile_checkpoint(learned)?;
completed.insert(sequence, checkpoint);
}
Ok(read_complete_execution_workflow(layout, expected_workflow)?)
}
#[derive(Debug, Error)]
pub enum ExecutionWorkflowRunError<E: std::error::Error + 'static> {
#[error("original workflow predecessor {0} did not complete")]
MissingPredecessor(u64),
#[error("retained stage identities do not match occupied original stages")]
RetainedStageSetMismatch,
#[error("retained stage {0} has a different learned predecessor commitment")]
RetainedLearnedMismatch(u64),
#[error(transparent)]
Workflow(#[from] ExecutionWorkflowPersistenceError),
#[error(transparent)]
Binding(#[from] ExecutionWorkflowError),
#[error(transparent)]
Preparation(#[from] ExecutionStagePreparationError),
#[error("original workflow stage driver failed: {0}")]
Driver(#[source] E),
#[error(transparent)]
Checkpoint(#[from] ExecutionStageCheckpointError),
}
#[cfg(test)]
mod tests;