use crate::{
model::{
artifacts::ArtifactChecksumRecord,
execution_workflow::{
ExecutionStageBindingRecord, ExecutionStagePredecessorRecord, ExecutionWorkflowError,
ExecutionWorkflowRecord,
},
operation_plan::OperationPlanRecord,
},
ops::persistence::{
BackupLayoutGuard, ExecutionStageCheckpointError, ExecutionStageGuard,
ExecutionStagePreparationError, ExecutionWorkflowPersistenceError,
ExecutionWorkflowSettlementView, create_execution_workflow,
read_complete_execution_workflow, read_execution_stage_occupancy, read_execution_workflow,
},
};
use std::{collections::BTreeMap, future::Future};
use thiserror::Error;
pub trait ExecutionWorkflowStageDriver {
type Error: std::error::Error + 'static;
fn bind_stage(
&mut self,
workflow: &ExecutionWorkflowRecord,
stage_sequence: u64,
predecessors: &[ExecutionStagePredecessorRecord],
) -> Result<OperationPlanRecord, Self::Error>;
fn execute_stage(
&mut self,
stage: &ExecutionStageGuard<'_>,
) -> impl Future<Output = Result<ArtifactChecksumRecord, Self::Error>>;
fn learned_for_successor(
&mut self,
predecessor: &ExecutionStagePredecessorRecord,
_successor_sequence: u64,
) -> Result<ArtifactChecksumRecord, Self::Error> {
Ok(predecessor.learned_evidence().clone())
}
}
#[derive(Clone, Debug, Eq, PartialEq)]
pub struct RetainedStageEvidence {
binding: ArtifactChecksumRecord,
learned_evidence: ArtifactChecksumRecord,
successor_evidence: BTreeMap<u64, ArtifactChecksumRecord>,
}
impl RetainedStageEvidence {
#[must_use]
pub const fn new(
binding: ArtifactChecksumRecord,
learned_evidence: ArtifactChecksumRecord,
) -> Self {
Self {
binding,
learned_evidence,
successor_evidence: BTreeMap::new(),
}
}
#[must_use]
pub fn with_successor_evidence(
mut self,
successor_evidence: BTreeMap<u64, ArtifactChecksumRecord>,
) -> Self {
self.successor_evidence = successor_evidence;
self
}
fn learned_for_successor(&self, sequence: u64) -> &ArtifactChecksumRecord {
self.successor_evidence
.get(&sequence)
.unwrap_or(&self.learned_evidence)
}
}
pub async fn run_new_execution_workflow<D: ExecutionWorkflowStageDriver>(
layout: &BackupLayoutGuard,
workflow: &ExecutionWorkflowRecord,
driver: &mut D,
) -> Result<ExecutionWorkflowSettlementView, ExecutionWorkflowRunError<D::Error>> {
create_execution_workflow(layout, workflow)?;
let mut completed: BTreeMap<u64, ExecutionStagePredecessorRecord> = BTreeMap::new();
for node in workflow.allocation().graph().ordered_nodes() {
let sequence = node.operation_sequence();
let predecessors = node
.depends_on()
.iter()
.map(|dependency| {
let original =
completed.get(dependency).ok_or(
ExecutionWorkflowRunError::<D::Error>::MissingPredecessor(*dependency),
)?;
let learned = driver
.learned_for_successor(original, sequence)
.map_err(ExecutionWorkflowRunError::Driver)?;
Ok(ExecutionStagePredecessorRecord::new(
*dependency,
original.binding().clone(),
original.settlement().clone(),
learned,
))
})
.collect::<Result<Vec<_>, ExecutionWorkflowRunError<D::Error>>>()?;
let plan = driver
.bind_stage(workflow, sequence, &predecessors)
.map_err(ExecutionWorkflowRunError::Driver)?;
let binding = ExecutionStageBindingRecord::new(workflow, sequence, &plan, predecessors)?;
let stage = ExecutionStageGuard::prepare(layout, binding, plan)?;
let learned = driver
.execute_stage(&stage)
.await
.map_err(ExecutionWorkflowRunError::Driver)?;
let checkpoint = stage.reconcile_checkpoint(learned)?;
completed.insert(sequence, checkpoint);
}
Ok(read_complete_execution_workflow(
layout,
&workflow.digest(),
)?)
}
pub async fn run_retained_execution_workflow<D: ExecutionWorkflowStageDriver>(
layout: &BackupLayoutGuard,
expected_workflow: &ArtifactChecksumRecord,
retained: &BTreeMap<u64, RetainedStageEvidence>,
driver: &mut D,
) -> Result<ExecutionWorkflowSettlementView, ExecutionWorkflowRunError<D::Error>> {
let workflow = read_execution_workflow(layout, expected_workflow)?;
let occupied = read_execution_stage_occupancy(layout, expected_workflow)?;
if !retained.keys().copied().eq(occupied.iter().copied()) {
return Err(ExecutionWorkflowRunError::RetainedStageSetMismatch);
}
let mut completed: BTreeMap<u64, ExecutionStagePredecessorRecord> = BTreeMap::new();
for (&sequence, evidence) in retained {
for successor in evidence.successor_evidence.keys() {
let Ok(node) = workflow.allocation().graph().node(*successor) else {
return Err(ExecutionWorkflowRunError::RetainedLearnedMismatch(sequence));
};
if !node.depends_on().contains(&sequence) {
return Err(ExecutionWorkflowRunError::RetainedLearnedMismatch(sequence));
}
}
let stage =
ExecutionStageGuard::open(layout, expected_workflow, sequence, &evidence.binding)?;
for predecessor in stage.binding().predecessors() {
let Some(original) = retained.get(&predecessor.stage_sequence()) else {
return Err(ExecutionWorkflowRunError::RetainedStageSetMismatch);
};
if original.learned_for_successor(sequence) != predecessor.learned_evidence() {
return Err(ExecutionWorkflowRunError::RetainedLearnedMismatch(sequence));
}
}
let checkpoint = stage.replay_checkpoint(evidence.learned_evidence.clone())?;
completed.insert(sequence, checkpoint);
}
if read_execution_stage_occupancy(layout, expected_workflow)? != occupied {
return Err(ExecutionWorkflowRunError::RetainedStageSetMismatch);
}
for node in workflow.allocation().graph().ordered_nodes() {
let sequence = node.operation_sequence();
if completed.contains_key(&sequence) {
continue;
}
let predecessors = node
.depends_on()
.iter()
.map(|dependency| {
let original =
completed.get(dependency).ok_or(
ExecutionWorkflowRunError::<D::Error>::MissingPredecessor(*dependency),
)?;
let learned = if let Some(evidence) = retained.get(dependency) {
evidence.learned_for_successor(sequence).clone()
} else {
driver
.learned_for_successor(original, sequence)
.map_err(ExecutionWorkflowRunError::Driver)?
};
Ok(ExecutionStagePredecessorRecord::new(
*dependency,
original.binding().clone(),
original.settlement().clone(),
learned,
))
})
.collect::<Result<Vec<_>, ExecutionWorkflowRunError<D::Error>>>()?;
let plan = driver
.bind_stage(&workflow, sequence, &predecessors)
.map_err(ExecutionWorkflowRunError::Driver)?;
let binding = ExecutionStageBindingRecord::new(&workflow, sequence, &plan, predecessors)?;
let stage = ExecutionStageGuard::prepare(layout, binding, plan)?;
let learned = driver
.execute_stage(&stage)
.await
.map_err(ExecutionWorkflowRunError::Driver)?;
let checkpoint = stage.reconcile_checkpoint(learned)?;
completed.insert(sequence, checkpoint);
}
Ok(read_complete_execution_workflow(layout, expected_workflow)?)
}
#[derive(Debug, Error)]
pub enum ExecutionWorkflowRunError<E: std::error::Error + 'static> {
#[error("original workflow predecessor {0} did not complete")]
MissingPredecessor(u64),
#[error("retained stage identities do not match occupied original stages")]
RetainedStageSetMismatch,
#[error("retained stage {0} has a different learned predecessor commitment")]
RetainedLearnedMismatch(u64),
#[error(transparent)]
Workflow(#[from] ExecutionWorkflowPersistenceError),
#[error(transparent)]
Binding(#[from] ExecutionWorkflowError),
#[error(transparent)]
Preparation(#[from] ExecutionStagePreparationError),
#[error("original workflow stage driver failed: {0}")]
Driver(#[source] E),
#[error(transparent)]
Checkpoint(#[from] ExecutionStageCheckpointError),
}
#[cfg(test)]
mod tests;