#![forbid(unsafe_code)]
use std::collections::{BTreeMap, BTreeSet};
use std::error::Error;
use std::fmt;
use std::num::NonZeroU32;
use serde_json::{Value, json};
use oxide_batch_core::{
ChunkComponentRevisions, ChunkSize, ComponentRevision, DefinitionError, DefinitionIdentity,
DefinitionRevision, DefinitionTokenKind, ExitCode, FaultPolicy, FlowTarget, InFlightPolicy,
JobName, MAX_NODES, MAX_PARTITIONS, MAX_TRANSITIONS, NodeId, StartControls, StepName,
TerminalKind, definition_token, validate_token,
};
pub const MAX_OUTGOING_TRANSITIONS: usize = 64;
pub const MAX_PATTERN_BYTES: usize = 64;
pub const MAX_SPLIT_BRANCHES: usize = 8;
pub const MAX_BRANCH_STEPS: usize = 8;
pub const MAX_PARTITION_WORKERS: u8 = 64;
#[derive(Clone, Copy, Debug, Default, Eq, Hash, Ord, PartialEq, PartialOrd)]
#[non_exhaustive]
pub enum LocalFailurePolicy {
#[default]
CancelSiblings,
DrainSiblings,
}
impl LocalFailurePolicy {
const fn as_str(self) -> &'static str {
match self {
Self::CancelSiblings => "cancel_siblings",
Self::DrainSiblings => "drain_siblings",
}
}
}
#[derive(Clone, Copy, Debug, Eq, Hash, Ord, PartialEq, PartialOrd)]
pub struct SplitBudget {
max_parallel_branches: u8,
repository_pool_size: u32,
}
impl SplitBudget {
pub fn new(max_parallel_branches: u8, repository_pool_size: u32) -> Result<Self, PlanError> {
if max_parallel_branches == 0 || usize::from(max_parallel_branches) > MAX_SPLIT_BRANCHES {
return Err(PlanError::InvalidParallelBranchBudget {
max: MAX_SPLIT_BRANCHES,
});
}
let required = u32::from(max_parallel_branches).saturating_add(1);
if repository_pool_size < required {
return Err(PlanError::InsufficientPoolCapacity {
required,
configured: repository_pool_size,
});
}
Ok(Self {
max_parallel_branches,
repository_pool_size,
})
}
#[must_use]
pub const fn max_parallel_branches(self) -> u8 {
self.max_parallel_branches
}
#[must_use]
pub const fn repository_pool_size(self) -> u32 {
self.repository_pool_size
}
}
impl Default for SplitBudget {
fn default() -> Self {
Self {
max_parallel_branches: 1,
repository_pool_size: 2,
}
}
}
#[derive(Clone, Copy, Debug, Eq, Hash, Ord, PartialEq, PartialOrd)]
pub struct PartitionBudget {
max_partition_workers: u8,
repository_pool_size: u32,
}
impl PartitionBudget {
pub fn new(max_partition_workers: u8, repository_pool_size: u32) -> Result<Self, PlanError> {
if !(1..=MAX_PARTITION_WORKERS).contains(&max_partition_workers) {
return Err(PlanError::InvalidPartitionWorkerBudget {
max: MAX_PARTITION_WORKERS,
});
}
let required = u32::from(max_partition_workers).saturating_add(1);
if repository_pool_size < required {
return Err(PlanError::InsufficientPoolCapacity {
required,
configured: repository_pool_size,
});
}
Ok(Self {
max_partition_workers,
repository_pool_size,
})
}
#[must_use]
pub const fn max_partition_workers(self) -> u8 {
self.max_partition_workers
}
#[must_use]
pub const fn repository_pool_size(self) -> u32 {
self.repository_pool_size
}
}
impl Default for PartitionBudget {
fn default() -> Self {
Self {
max_partition_workers: 4,
repository_pool_size: 5,
}
}
}
definition_token!(
DeciderRevision,
DefinitionTokenKind::Decider,
"An application-owned revision token for one deterministic decider."
);
#[derive(Clone, Copy, Debug, Eq, Hash, Ord, PartialEq, PartialOrd)]
pub struct DecisionInputVersion(NonZeroU32);
impl DecisionInputVersion {
pub fn new(value: u32) -> Result<Self, PlanError> {
NonZeroU32::new(value)
.map(Self)
.ok_or(PlanError::ZeroDecisionInputVersion)
}
#[must_use]
pub const fn get(self) -> u32 {
self.0.get()
}
}
#[derive(Clone, Debug, Eq, Hash, Ord, PartialEq, PartialOrd)]
pub struct ExitPattern(String);
impl ExitPattern {
pub fn new(value: impl Into<String>) -> Result<Self, PlanError> {
let value = value.into();
if value.is_empty()
|| value.len() > MAX_PATTERN_BYTES
|| value.trim() != value
|| value.chars().any(char::is_control)
{
return Err(PlanError::InvalidPattern {
max_bytes: MAX_PATTERN_BYTES,
});
}
Ok(Self(value))
}
#[must_use]
pub fn as_str(&self) -> &str {
&self.0
}
#[must_use]
pub fn matches(&self, code: &ExitCode) -> bool {
let pattern: Vec<char> = self.0.chars().collect();
let value: Vec<char> = code.as_str().chars().collect();
matches_from(&pattern, &value)
}
#[must_use]
pub fn specificity(&self) -> PatternSpecificity {
let wildcards = self
.0
.chars()
.filter(|character| matches!(character, '*' | '?'))
.count();
let literals = self.0.chars().count() - wildcards;
PatternSpecificity {
literals,
wildcards,
bytes: self.0.len(),
}
}
#[must_use]
pub fn intersects(&self, other: &Self) -> bool {
let left: Vec<char> = self.0.chars().collect();
let right: Vec<char> = other.0.chars().collect();
let mut memo = vec![None; (left.len() + 1) * (right.len() + 1)];
intersects_from(&left, &right, 0, 0, &mut memo)
}
}
impl fmt::Display for ExitPattern {
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
formatter.write_str(&self.0)
}
}
fn matches_from(pattern: &[char], value: &[char]) -> bool {
let mut pattern_index = 0_usize;
let mut value_index = 0_usize;
let mut star: Option<(usize, usize)> = None;
while value_index < value.len() {
match pattern.get(pattern_index) {
Some('*') => {
star = Some((pattern_index, value_index));
pattern_index += 1;
}
Some('?') => {
pattern_index += 1;
value_index += 1;
}
Some(literal) if *literal == value[value_index] => {
pattern_index += 1;
value_index += 1;
}
_ => match star {
Some((star_index, resume)) => {
pattern_index = star_index + 1;
value_index = resume + 1;
star = Some((star_index, resume + 1));
}
None => return false,
},
}
}
pattern[pattern_index..]
.iter()
.all(|character| *character == '*')
}
fn intersects_from(
left: &[char],
right: &[char],
left_index: usize,
right_index: usize,
memo: &mut [Option<bool>],
) -> bool {
let key = left_index * (right.len() + 1) + right_index;
if let Some(cached) = memo[key] {
return cached;
}
let answer = match (left.get(left_index), right.get(right_index)) {
(None, None) => true,
(None, Some(_)) => right[right_index..].iter().all(|value| *value == '*'),
(Some(_), None) => left[left_index..].iter().all(|value| *value == '*'),
(Some('*'), _) => {
intersects_from(left, right, left_index + 1, right_index, memo)
|| intersects_from(left, right, left_index, right_index + 1, memo)
}
(_, Some('*')) => {
intersects_from(left, right, left_index, right_index + 1, memo)
|| intersects_from(left, right, left_index + 1, right_index, memo)
}
(Some(left_character), Some(right_character)) => {
(*left_character == '?' || *right_character == '?' || left_character == right_character)
&& intersects_from(left, right, left_index + 1, right_index + 1, memo)
}
};
memo[key] = Some(answer);
answer
}
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
pub struct PatternSpecificity {
literals: usize,
wildcards: usize,
bytes: usize,
}
impl PatternSpecificity {
#[must_use]
pub const fn literals(self) -> usize {
self.literals
}
#[must_use]
pub const fn wildcards(self) -> usize {
self.wildcards
}
#[must_use]
pub const fn bytes(self) -> usize {
self.bytes
}
}
impl Ord for PatternSpecificity {
fn cmp(&self, other: &Self) -> std::cmp::Ordering {
self.literals
.cmp(&other.literals)
.then_with(|| other.wildcards.cmp(&self.wildcards))
.then_with(|| self.bytes.cmp(&other.bytes))
}
}
impl PartialOrd for PatternSpecificity {
fn partial_cmp(&self, other: &Self) -> Option<std::cmp::Ordering> {
Some(self.cmp(other))
}
}
#[derive(Clone, Debug, Eq, PartialEq)]
#[non_exhaustive]
pub enum StepComponents {
Tasklet(ComponentRevision),
Chunk {
size: ChunkSize,
revisions: Box<ChunkComponentRevisions>,
},
}
impl StepComponents {
fn kind_name(&self) -> &'static str {
match self {
Self::Tasklet(_) => "tasklet",
Self::Chunk { .. } => "chunk",
}
}
fn manifest_value(&self) -> Value {
match self {
Self::Tasklet(revision) => json!({
"component": revision.as_str(),
"delivery_mode": "best_effort",
"transaction_boundary": "tasklet_completion"
}),
Self::Chunk { size, revisions } => {
let mut chunk = chunk_declaration_manifest(revisions);
if let Some(members) = chunk.as_object_mut() {
members.insert("size".to_owned(), json!(size.get()));
members.insert(
"transaction_boundary".to_owned(),
Value::String("chunk".to_owned()),
);
}
chunk
}
}
}
}
#[derive(Clone, Debug, Eq, PartialEq)]
pub struct StepNode {
id: NodeId,
step_name: StepName,
components: StepComponents,
start: StartControls,
fault: Option<FaultPolicy>,
listeners: Vec<ComponentRevision>,
}
impl StepNode {
#[must_use]
pub fn new(id: NodeId, step_name: StepName, components: StepComponents) -> Self {
Self {
id,
step_name,
components,
start: StartControls::default(),
fault: None,
listeners: Vec::new(),
}
}
#[must_use]
pub const fn with_start_controls(mut self, start: StartControls) -> Self {
self.start = start;
self
}
#[must_use]
pub fn with_fault_policy(mut self, policy: FaultPolicy) -> Self {
self.fault = Some(policy);
self
}
#[must_use]
pub fn with_listener_revision(mut self, revision: ComponentRevision) -> Self {
self.listeners.push(revision);
self
}
#[must_use]
pub const fn id(&self) -> &NodeId {
&self.id
}
#[must_use]
pub const fn step_name(&self) -> &StepName {
&self.step_name
}
#[must_use]
pub const fn components(&self) -> &StepComponents {
&self.components
}
#[must_use]
pub const fn start_controls(&self) -> StartControls {
self.start
}
#[must_use]
pub const fn fault_policy(&self) -> Option<&FaultPolicy> {
self.fault.as_ref()
}
#[must_use]
pub fn listener_revisions(&self) -> &[ComponentRevision] {
&self.listeners
}
fn manifest_value(&self) -> Value {
json!({
"id": self.id.as_str(),
"kind": "step",
"listeners": self
.listeners
.iter()
.map(|revision| Value::String(revision.as_str().to_owned()))
.collect::<Vec<_>>(),
"policy": self.fault.as_ref().map_or(Value::Null, fault_manifest_value),
"start": start_controls_manifest(self.start),
"step": {
"declaration": self.components.manifest_value(),
"kind": self.components.kind_name(),
"name": self.step_name.as_str()
}
})
}
}
#[derive(Clone, Debug, Eq, PartialEq)]
pub struct DecisionNode {
id: NodeId,
revision: DeciderRevision,
input_version: DecisionInputVersion,
}
impl DecisionNode {
#[must_use]
pub const fn new(
id: NodeId,
revision: DeciderRevision,
input_version: DecisionInputVersion,
) -> Self {
Self {
id,
revision,
input_version,
}
}
#[must_use]
pub const fn id(&self) -> &NodeId {
&self.id
}
#[must_use]
pub const fn revision(&self) -> &DeciderRevision {
&self.revision
}
#[must_use]
pub const fn input_version(&self) -> DecisionInputVersion {
self.input_version
}
fn manifest_value(&self) -> Value {
json!({
"decision": {
"input_version": self.input_version.get(),
"revision": self.revision.as_str()
},
"id": self.id.as_str(),
"kind": "decision"
})
}
}
#[derive(Clone, Debug, Eq, PartialEq)]
pub struct SplitBranch {
steps: Vec<StepNode>,
}
impl SplitBranch {
#[must_use]
pub fn new(steps: Vec<StepNode>) -> Self {
Self { steps }
}
#[must_use]
pub fn steps(&self) -> &[StepNode] {
&self.steps
}
#[must_use]
pub fn id(&self) -> Option<&NodeId> {
self.steps.first().map(StepNode::id)
}
fn manifest_value(&self) -> Value {
Value::Array(self.steps.iter().map(StepNode::manifest_value).collect())
}
}
#[derive(Clone, Debug, Eq, PartialEq)]
pub struct SplitNode {
id: NodeId,
branches: Vec<SplitBranch>,
join: NodeId,
budget: SplitBudget,
failure_policy: LocalFailurePolicy,
}
impl SplitNode {
#[must_use]
pub fn new(id: NodeId, branches: Vec<SplitBranch>, join: NodeId, budget: SplitBudget) -> Self {
Self {
id,
branches,
join,
budget,
failure_policy: LocalFailurePolicy::default(),
}
}
#[must_use]
pub const fn with_failure_policy(mut self, failure_policy: LocalFailurePolicy) -> Self {
self.failure_policy = failure_policy;
self
}
#[must_use]
pub const fn id(&self) -> &NodeId {
&self.id
}
#[must_use]
pub fn branches(&self) -> &[SplitBranch] {
&self.branches
}
#[must_use]
pub const fn join(&self) -> &NodeId {
&self.join
}
#[must_use]
pub const fn budget(&self) -> SplitBudget {
self.budget
}
#[must_use]
pub const fn failure_policy(&self) -> LocalFailurePolicy {
self.failure_policy
}
fn manifest_value(&self) -> Value {
json!({
"branches": self.branches.iter().map(SplitBranch::manifest_value).collect::<Vec<_>>(),
"failure_policy": self.failure_policy.as_str(),
"id": self.id.as_str(),
"join": self.join.as_str(),
"kind": "split"
})
}
}
#[derive(Clone, Debug, Eq, PartialEq)]
pub struct JoinNode {
id: NodeId,
}
impl JoinNode {
#[must_use]
pub const fn new(id: NodeId) -> Self {
Self { id }
}
#[must_use]
pub const fn id(&self) -> &NodeId {
&self.id
}
fn manifest_value(&self) -> Value {
json!({
"id": self.id.as_str(),
"kind": "join"
})
}
}
#[derive(Clone, Copy, Debug, Eq, Hash, Ord, PartialEq, PartialOrd)]
pub struct PartitionCount(u16);
impl PartitionCount {
pub fn new(value: u16) -> Result<Self, PlanError> {
if value == 0 || value > MAX_PARTITIONS {
return Err(PlanError::InvalidPartitionCount {
max: MAX_PARTITIONS,
});
}
Ok(Self(value))
}
#[must_use]
pub const fn get(self) -> u16 {
self.0
}
}
#[derive(Clone, Debug, Eq, PartialEq)]
pub struct PartitionedStepNode {
id: NodeId,
step_name: StepName,
worker: StepNode,
partitioner: ComponentRevision,
aggregation: ComponentRevision,
partitions: PartitionCount,
budget: PartitionBudget,
failure_policy: LocalFailurePolicy,
start: StartControls,
}
impl PartitionedStepNode {
#[allow(clippy::too_many_arguments)]
#[must_use]
pub fn new(
id: NodeId,
step_name: StepName,
worker: StepNode,
partitioner: ComponentRevision,
aggregation: ComponentRevision,
partitions: PartitionCount,
budget: PartitionBudget,
) -> Self {
Self {
id,
step_name,
worker,
partitioner,
aggregation,
partitions,
budget,
failure_policy: LocalFailurePolicy::default(),
start: StartControls::default(),
}
}
#[must_use]
pub const fn with_failure_policy(mut self, failure_policy: LocalFailurePolicy) -> Self {
self.failure_policy = failure_policy;
self
}
#[must_use]
pub const fn with_start_controls(mut self, start: StartControls) -> Self {
self.start = start;
self
}
#[must_use]
pub const fn id(&self) -> &NodeId {
&self.id
}
#[must_use]
pub const fn step_name(&self) -> &StepName {
&self.step_name
}
#[must_use]
pub const fn worker(&self) -> &StepNode {
&self.worker
}
#[must_use]
pub const fn partitioner(&self) -> &ComponentRevision {
&self.partitioner
}
#[must_use]
pub const fn aggregation(&self) -> &ComponentRevision {
&self.aggregation
}
#[must_use]
pub const fn partition_count(&self) -> PartitionCount {
self.partitions
}
#[must_use]
pub const fn budget(&self) -> PartitionBudget {
self.budget
}
#[must_use]
pub const fn failure_policy(&self) -> LocalFailurePolicy {
self.failure_policy
}
#[must_use]
pub const fn start_controls(&self) -> StartControls {
self.start
}
fn manifest_value(&self) -> Value {
json!({
"aggregation": self.aggregation.as_str(),
"failure_policy": self.failure_policy.as_str(),
"id": self.id.as_str(),
"kind": "partitioned_step",
"partition_count": self.partitions.get(),
"partitioner": self.partitioner.as_str(),
"start": start_controls_manifest(self.start),
"step_name": self.step_name.as_str(),
"worker": self.worker.manifest_value()
})
}
}
#[derive(Clone, Debug, Eq, PartialEq)]
#[non_exhaustive]
pub enum FlowNode {
Step(Box<StepNode>),
Decision(DecisionNode),
Split(Box<SplitNode>),
Join(JoinNode),
PartitionedStep(Box<PartitionedStepNode>),
}
impl FlowNode {
#[must_use]
pub fn step(node: StepNode) -> Self {
Self::Step(Box::new(node))
}
#[must_use]
pub const fn decision(node: DecisionNode) -> Self {
Self::Decision(node)
}
#[must_use]
pub fn split(node: SplitNode) -> Self {
Self::Split(Box::new(node))
}
#[must_use]
pub const fn join(node: JoinNode) -> Self {
Self::Join(node)
}
#[must_use]
pub fn partitioned_step(node: PartitionedStepNode) -> Self {
Self::PartitionedStep(Box::new(node))
}
#[must_use]
pub const fn id(&self) -> &NodeId {
match self {
Self::Step(node) => node.id(),
Self::Decision(node) => node.id(),
Self::Split(node) => node.id(),
Self::Join(node) => node.id(),
Self::PartitionedStep(node) => node.id(),
}
}
fn manifest_value(&self) -> Value {
match self {
Self::Step(node) => node.manifest_value(),
Self::Decision(node) => node.manifest_value(),
Self::Split(node) => node.manifest_value(),
Self::Join(node) => node.manifest_value(),
Self::PartitionedStep(node) => node.manifest_value(),
}
}
}
#[derive(Clone, Debug, Eq, PartialEq)]
pub struct FlowTransition {
source: NodeId,
pattern: ExitPattern,
target: FlowTarget,
}
impl FlowTransition {
#[must_use]
pub const fn new(source: NodeId, pattern: ExitPattern, target: FlowTarget) -> Self {
Self {
source,
pattern,
target,
}
}
#[must_use]
pub const fn source(&self) -> &NodeId {
&self.source
}
#[must_use]
pub const fn pattern(&self) -> &ExitPattern {
&self.pattern
}
#[must_use]
pub const fn target(&self) -> &FlowTarget {
&self.target
}
fn manifest_value(&self) -> Value {
json!({
"pattern": self.pattern.as_str(),
"source": self.source.as_str(),
"target": flow_target_manifest(&self.target)
})
}
}
#[derive(Clone, Debug, Default, Eq, PartialEq)]
pub struct FlowGraph {
entry: Option<NodeId>,
nodes: Vec<FlowNode>,
transitions: Vec<FlowTransition>,
}
impl FlowGraph {
#[must_use]
pub fn new(entry: NodeId) -> Self {
Self {
entry: Some(entry),
nodes: Vec::new(),
transitions: Vec::new(),
}
}
#[must_use]
pub fn with_node(mut self, node: FlowNode) -> Self {
self.nodes.push(node);
self
}
#[must_use]
pub fn with_transition(mut self, transition: FlowTransition) -> Self {
self.transitions.push(transition);
self
}
pub fn with_sequence(self, source: NodeId, next: FlowTarget) -> Result<Self, PlanError> {
Ok(self
.with_transition(FlowTransition::new(
source.clone(),
ExitPattern::new("FAILED")?,
FlowTarget::Terminal(TerminalKind::Fail),
))
.with_transition(FlowTransition::new(source, ExitPattern::new("*")?, next)))
}
pub fn compile(
self,
job_name: &JobName,
revision: DefinitionRevision,
) -> Result<CompiledExecutionPlan, PlanError> {
let entry = self.entry.ok_or(PlanError::MissingEntryNode)?;
if self.nodes.len() > MAX_NODES {
return Err(PlanError::TooManyNodes { max: MAX_NODES });
}
if self.transitions.len() > MAX_TRANSITIONS {
return Err(PlanError::TooManyTransitions {
max: MAX_TRANSITIONS,
});
}
let mut nodes = BTreeMap::new();
for node in self.nodes {
if nodes.insert(node.id().clone(), node.clone()).is_some() {
return Err(PlanError::DuplicateNodeId {
node: node.id().clone(),
});
}
}
if !nodes.contains_key(&entry) {
return Err(PlanError::UndefinedNode {
node: entry.clone(),
});
}
let local_scale = check_local_scale_subset(&entry, &nodes)?;
let mut outgoing: BTreeMap<NodeId, Vec<FlowTransition>> = BTreeMap::new();
for transition in self.transitions {
if !nodes.contains_key(transition.source()) {
return Err(PlanError::UndefinedNode {
node: transition.source().clone(),
});
}
if let FlowTarget::Node(target) = transition.target()
&& !nodes.contains_key(target)
{
return Err(PlanError::UndefinedNode {
node: target.clone(),
});
}
if let FlowTarget::Node(target) = transition.target()
&& matches!(nodes.get(target), Some(FlowNode::Join(_)))
{
return Err(PlanError::JoinHasExternalEntry {
join: target.clone(),
});
}
if matches!(nodes.get(transition.source()), Some(FlowNode::Split(_))) {
return Err(PlanError::SplitHasExplicitTransition {
split: transition.source().clone(),
});
}
let edges = outgoing.entry(transition.source().clone()).or_default();
if edges.len() == MAX_OUTGOING_TRANSITIONS {
return Err(PlanError::TooManyOutgoingTransitions {
node: transition.source().clone(),
max: MAX_OUTGOING_TRANSITIONS,
});
}
edges.push(transition);
}
for (id, node) in &nodes {
if matches!(node, FlowNode::Split(_)) {
continue;
}
let edges = outgoing
.get(id)
.filter(|edges| !edges.is_empty())
.ok_or_else(|| PlanError::MissingTransition { node: id.clone() })?;
check_unambiguous(id, edges)?;
}
let mut compiled: BTreeMap<NodeId, Vec<FlowTransition>> = outgoing;
for edges in compiled.values_mut() {
edges.sort_by(|left, right| {
right
.pattern()
.specificity()
.cmp(&left.pattern().specificity())
.then_with(|| left.pattern().cmp(right.pattern()))
.then_with(|| left.target().sort_key().cmp(&right.target().sort_key()))
});
}
check_reachable_and_acyclic(&entry, &nodes, &compiled)?;
let manifest = flow_manifest(job_name, &entry, &nodes, &compiled, local_scale);
let canonical = serde_json::to_vec(&manifest)
.map_err(|_| PlanError::Manifest(DefinitionError::ManifestEncoding))?;
let definition = DefinitionIdentity::from_flow_manifest(job_name, revision, &canonical)
.map_err(PlanError::Manifest)?;
Ok(CompiledExecutionPlan {
definition,
entry,
nodes,
transitions: compiled,
})
}
}
fn check_local_scale_subset(
entry: &NodeId,
nodes: &BTreeMap<NodeId, FlowNode>,
) -> Result<bool, PlanError> {
let mut embedded_ids = BTreeSet::new();
let mut join_owners: BTreeMap<NodeId, NodeId> = BTreeMap::new();
let mut local_scale = false;
for (id, node) in nodes {
match node {
FlowNode::Split(split) => {
local_scale = true;
if id == entry {
return Err(PlanError::SplitIsEntry { split: id.clone() });
}
if !(2..=MAX_SPLIT_BRANCHES).contains(&split.branches().len()) {
return Err(PlanError::InvalidSplitBranchCount {
split: id.clone(),
min: 2,
max: MAX_SPLIT_BRANCHES,
});
}
if usize::from(split.budget().max_parallel_branches()) > split.branches().len() {
return Err(PlanError::ParallelBudgetExceedsBranches {
split: id.clone(),
branches: split.branches().len(),
});
}
if !matches!(nodes.get(split.join()), Some(FlowNode::Join(_))) {
return Err(PlanError::InvalidSplitJoin {
split: id.clone(),
join: split.join().clone(),
});
}
if let Some(first) = join_owners.insert(split.join().clone(), id.clone()) {
return Err(PlanError::JoinHasMultipleOwners {
join: split.join().clone(),
first,
second: id.clone(),
});
}
for branch in split.branches() {
if !(1..=MAX_BRANCH_STEPS).contains(&branch.steps().len()) {
return Err(PlanError::InvalidBranchLength {
split: id.clone(),
max: MAX_BRANCH_STEPS,
});
}
for step in branch.steps() {
if nodes.contains_key(step.id()) || !embedded_ids.insert(step.id().clone())
{
return Err(PlanError::DuplicateNodeId {
node: step.id().clone(),
});
}
}
}
}
FlowNode::Join(_) => {
local_scale = true;
}
FlowNode::PartitionedStep(partitioned) => {
local_scale = true;
let worker = partitioned.worker().id();
if nodes.contains_key(worker) || !embedded_ids.insert(worker.clone()) {
return Err(PlanError::DuplicateNodeId {
node: worker.clone(),
});
}
}
FlowNode::Step(_) | FlowNode::Decision(_) => {}
}
}
if nodes.len().saturating_add(embedded_ids.len()) > MAX_NODES {
return Err(PlanError::TooManyNodes { max: MAX_NODES });
}
for (id, node) in nodes {
if matches!(node, FlowNode::Join(_)) && !join_owners.contains_key(id) {
return Err(PlanError::OrphanJoin { join: id.clone() });
}
}
Ok(local_scale)
}
fn check_unambiguous(node: &NodeId, edges: &[FlowTransition]) -> Result<(), PlanError> {
for (index, left) in edges.iter().enumerate() {
for right in &edges[index + 1..] {
if left.pattern().specificity() == right.pattern().specificity()
&& left.pattern().intersects(right.pattern())
{
return Err(PlanError::AmbiguousTransition {
node: node.clone(),
first: left.pattern().clone(),
second: right.pattern().clone(),
});
}
}
}
Ok(())
}
fn check_reachable_and_acyclic(
entry: &NodeId,
nodes: &BTreeMap<NodeId, FlowNode>,
transitions: &BTreeMap<NodeId, Vec<FlowTransition>>,
) -> Result<(), PlanError> {
let mut visited = BTreeSet::new();
let mut on_path = BTreeSet::new();
visit(entry, nodes, transitions, &mut visited, &mut on_path)?;
for id in nodes.keys() {
if !visited.contains(id) {
return Err(PlanError::UnreachableNode { node: id.clone() });
}
}
Ok(())
}
fn visit(
node: &NodeId,
nodes: &BTreeMap<NodeId, FlowNode>,
transitions: &BTreeMap<NodeId, Vec<FlowTransition>>,
visited: &mut BTreeSet<NodeId>,
on_path: &mut BTreeSet<NodeId>,
) -> Result<(), PlanError> {
if on_path.contains(node) {
return Err(PlanError::CyclicGraph { node: node.clone() });
}
if !visited.insert(node.clone()) {
return Ok(());
}
on_path.insert(node.clone());
if let Some(FlowNode::Split(split)) = nodes.get(node) {
visit(split.join(), nodes, transitions, visited, on_path)?;
}
if let Some(edges) = transitions.get(node) {
for edge in edges {
if let FlowTarget::Node(target) = edge.target() {
visit(target, nodes, transitions, visited, on_path)?;
}
}
}
on_path.remove(node);
Ok(())
}
fn start_controls_manifest(controls: StartControls) -> Value {
json!({
"allow_start_if_complete": controls.allow_start_if_complete(),
"start_limit": controls.start_limit().get()
})
}
fn flow_target_manifest(target: &FlowTarget) -> Value {
match target {
FlowTarget::Node(id) => json!({ "node": id.as_str() }),
FlowTarget::Terminal(kind) => json!({ "terminal": kind.as_str() }),
}
}
fn chunk_declaration_manifest(revisions: &ChunkComponentRevisions) -> Value {
let mut value = json!({
"checkpoint": {
"schema": revisions.checkpoint_schema().as_str(),
"version": revisions.checkpoint_schema_version().get()
},
"components": {
"checkpoint": revisions.checkpoint().as_str(),
"processor": revisions.processor().as_str(),
"reader": revisions.reader().as_str(),
"writer": revisions.writer().as_str()
},
"context": {
"schema": revisions.context_schema().as_str(),
"version": revisions.context_schema_version().get()
},
"delivery_mode": revisions.delivery_mode().manifest_name()
});
if revisions.in_flight_policy() == InFlightPolicy::RollbackChunk
&& let Some(object) = value.as_object_mut()
{
object.insert(
"in_flight_policy".to_owned(),
Value::String("rollback_chunk".to_owned()),
);
}
value
}
fn fault_manifest_value(policy: &FaultPolicy) -> Value {
let backoff = policy.backoff();
let rules: Vec<Value> = policy
.classifier()
.rules()
.iter()
.map(|rule| {
json!({
"category": rule.category().as_str(),
"phase": rule.phase().as_str(),
"retryable": rule.action().is_retryable(),
"skip": rule
.action()
.skip_disposition()
.map_or(Value::Null, |skip| Value::String(skip.as_str().to_owned()))
})
})
.collect();
json!({
"backoff": {
"initial_ms": u64::try_from(backoff.initial().as_millis()).unwrap_or(u64::MAX),
"kind": backoff.kind().as_str(),
"maximum_ms": u64::try_from(backoff.maximum().as_millis()).unwrap_or(u64::MAX),
"multiplier": backoff.multiplier()
},
"classifier": {
"revision": policy.classifier().revision().as_str(),
"rules": rules
},
"retry_limit": policy.retry_limit().get(),
"retry_state_limit": policy.retry_state_limit().get(),
"skip_limit": policy.skip_limit().get()
})
}
fn flow_manifest(
job_name: &JobName,
entry: &NodeId,
nodes: &BTreeMap<NodeId, FlowNode>,
transitions: &BTreeMap<NodeId, Vec<FlowTransition>>,
local_scale: bool,
) -> Value {
let node_values: Vec<Value> = nodes.values().map(FlowNode::manifest_value).collect();
let transition_values: Vec<Value> = transitions
.values()
.flat_map(|edges| edges.iter().map(FlowTransition::manifest_value))
.collect();
json!({
"entry": entry.as_str(),
"format": if local_scale {
oxide_batch_core::MANIFEST_FORMAT_LOCAL_SCALE
} else {
oxide_batch_core::MANIFEST_FORMAT_FLOW
},
"job": job_name.as_str(),
"nodes": node_values,
"transitions": transition_values
})
}
#[derive(Clone, Debug, Eq, PartialEq)]
pub struct CompiledExecutionPlan {
definition: DefinitionIdentity,
entry: NodeId,
nodes: BTreeMap<NodeId, FlowNode>,
transitions: BTreeMap<NodeId, Vec<FlowTransition>>,
}
impl CompiledExecutionPlan {
#[doc(hidden)]
pub fn compatibility_one_step(
definition: DefinitionIdentity,
step: StepNode,
) -> Result<Self, PlanError> {
let entry = step.id().clone();
let mut nodes = BTreeMap::new();
nodes.insert(entry.clone(), FlowNode::step(step));
let mut edges = Vec::with_capacity(3);
for (code, terminal) in [
("COMPLETED", TerminalKind::Complete),
("FAILED", TerminalKind::Fail),
("STOPPED", TerminalKind::Stop),
] {
edges.push(FlowTransition::new(
entry.clone(),
ExitPattern::new(code)?,
FlowTarget::Terminal(terminal),
));
}
check_unambiguous(&entry, &edges)?;
let mut transitions = BTreeMap::new();
transitions.insert(entry.clone(), edges);
Ok(Self {
definition,
entry,
nodes,
transitions,
})
}
#[must_use]
pub const fn definition_identity(&self) -> &DefinitionIdentity {
&self.definition
}
#[must_use]
pub const fn manifest_format(&self) -> u16 {
self.definition.manifest_format()
}
#[must_use]
pub const fn fingerprint(&self) -> &[u8; 32] {
self.definition.manifest_digest()
}
#[must_use]
pub const fn entry(&self) -> &NodeId {
&self.entry
}
#[must_use]
pub fn node_count(&self) -> usize {
self.nodes.len()
}
#[must_use]
pub fn transition_count(&self) -> usize {
self.transitions.values().map(Vec::len).sum()
}
#[must_use]
pub fn node(&self, id: &NodeId) -> Option<&FlowNode> {
self.nodes.get(id)
}
#[must_use]
pub fn nodes(&self) -> impl ExactSizeIterator<Item = (&NodeId, &FlowNode)> {
self.nodes.iter()
}
#[must_use]
pub fn transitions(&self, id: &NodeId) -> &[FlowTransition] {
self.transitions.get(id).map_or(&[], Vec::as_slice)
}
pub fn select_target(
&self,
id: &NodeId,
code: &ExitCode,
) -> Result<&FlowTarget, FlowSelectionError> {
let edges = self
.transitions
.get(id)
.ok_or_else(|| FlowSelectionError::UnknownNode { node: id.clone() })?;
edges
.iter()
.find(|edge| edge.pattern().matches(code))
.map(FlowTransition::target)
.ok_or_else(|| FlowSelectionError::UnmappedExitOutcome {
node: id.clone(),
code: code.clone(),
})
}
}
#[derive(Clone, Debug, Eq, PartialEq)]
#[non_exhaustive]
pub enum PlanError {
MissingEntryNode,
DuplicateNodeId {
node: NodeId,
},
UndefinedNode {
node: NodeId,
},
MissingTransition {
node: NodeId,
},
AmbiguousTransition {
node: NodeId,
first: ExitPattern,
second: ExitPattern,
},
UnreachableNode {
node: NodeId,
},
CyclicGraph {
node: NodeId,
},
TooManyNodes {
max: usize,
},
TooManyTransitions {
max: usize,
},
TooManyOutgoingTransitions {
node: NodeId,
max: usize,
},
InvalidPattern {
max_bytes: usize,
},
ZeroDecisionInputVersion,
InvalidSplitBranchCount {
split: NodeId,
min: usize,
max: usize,
},
InvalidBranchLength {
split: NodeId,
max: usize,
},
SplitIsEntry {
split: NodeId,
},
InvalidSplitJoin {
split: NodeId,
join: NodeId,
},
JoinHasMultipleOwners {
join: NodeId,
first: NodeId,
second: NodeId,
},
OrphanJoin {
join: NodeId,
},
JoinHasExternalEntry {
join: NodeId,
},
SplitHasExplicitTransition {
split: NodeId,
},
ParallelBudgetExceedsBranches {
split: NodeId,
branches: usize,
},
InvalidParallelBranchBudget {
max: usize,
},
InvalidPartitionWorkerBudget {
max: u8,
},
InsufficientPoolCapacity {
required: u32,
configured: u32,
},
InvalidPartitionCount {
max: u16,
},
Token(DefinitionError),
Manifest(DefinitionError),
}
impl fmt::Display for PlanError {
#[allow(
clippy::too_many_lines,
reason = "each typed plan rejection retains one stable redacted diagnostic"
)]
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
match self {
Self::MissingEntryNode => formatter.write_str("flow graph has no entry node"),
Self::DuplicateNodeId { node } => {
write!(
formatter,
"node {} is declared more than once",
node.as_str()
)
}
Self::UndefinedNode { node } => {
write!(formatter, "node {} is not declared", node.as_str())
}
Self::MissingTransition { node } => {
write!(
formatter,
"node {} has no outgoing transition",
node.as_str()
)
}
Self::AmbiguousTransition {
node,
first,
second,
} => write!(
formatter,
"node {} patterns {first} and {second} are equally specific and overlap",
node.as_str()
),
Self::UnreachableNode { node } => {
write!(
formatter,
"node {} is unreachable from the entry node",
node.as_str()
)
}
Self::CyclicGraph { node } => {
write!(formatter, "node {} closes a cycle", node.as_str())
}
Self::TooManyNodes { max } => write!(formatter, "flow graph exceeds {max} nodes"),
Self::TooManyTransitions { max } => {
write!(formatter, "flow graph exceeds {max} transitions")
}
Self::TooManyOutgoingTransitions { node, max } => write!(
formatter,
"node {} exceeds {max} outgoing transitions",
node.as_str()
),
Self::InvalidPattern { max_bytes } => write!(
formatter,
"exit pattern must be 1 to {max_bytes} bytes without control characters"
),
Self::ZeroDecisionInputVersion => {
formatter.write_str("decision input version must be nonzero")
}
Self::InvalidSplitBranchCount { split, min, max } => write!(
formatter,
"split {} must declare {min} to {max} branches",
split.as_str()
),
Self::InvalidBranchLength { split, max } => write!(
formatter,
"split {} branches must declare 1 to {max} steps",
split.as_str()
),
Self::SplitIsEntry { split } => {
write!(
formatter,
"split {} cannot be the entry node",
split.as_str()
)
}
Self::InvalidSplitJoin { split, join } => write!(
formatter,
"split {} does not own declared join {}",
split.as_str(),
join.as_str()
),
Self::JoinHasMultipleOwners {
join,
first,
second,
} => write!(
formatter,
"join {} is owned by both splits {} and {}",
join.as_str(),
first.as_str(),
second.as_str()
),
Self::OrphanJoin { join } => {
write!(formatter, "join {} has no owning split", join.as_str())
}
Self::JoinHasExternalEntry { join } => write!(
formatter,
"join {} can be entered only by its owning split",
join.as_str()
),
Self::SplitHasExplicitTransition { split } => write!(
formatter,
"split {} reaches only its declared join",
split.as_str()
),
Self::ParallelBudgetExceedsBranches { split, branches } => write!(
formatter,
"split {} parallel budget exceeds its {branches} branches",
split.as_str()
),
Self::InvalidParallelBranchBudget { max } => {
write!(formatter, "parallel branch budget must be 1 to {max}")
}
Self::InvalidPartitionWorkerBudget { max } => {
write!(formatter, "partition worker budget must be 1 to {max}")
}
Self::InsufficientPoolCapacity {
required,
configured,
} => write!(
formatter,
"repository pool size {configured} cannot supply required capacity {required}"
),
Self::InvalidPartitionCount { max } => {
write!(formatter, "partition count must be 1 to {max}")
}
Self::Token(error) => write!(formatter, "flow graph token is invalid: {error}"),
Self::Manifest(error) => {
write!(formatter, "flow manifest could not be encoded: {error}")
}
}
}
}
impl Error for PlanError {
fn source(&self) -> Option<&(dyn Error + 'static)> {
match self {
Self::Token(error) | Self::Manifest(error) => Some(error),
_ => None,
}
}
}
impl From<DefinitionError> for PlanError {
fn from(error: DefinitionError) -> Self {
Self::Token(error)
}
}
#[derive(Clone, Debug, Eq, PartialEq)]
#[non_exhaustive]
pub enum FlowSelectionError {
UnknownNode {
node: NodeId,
},
UnmappedExitOutcome {
node: NodeId,
code: ExitCode,
},
}
impl fmt::Display for FlowSelectionError {
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
match self {
Self::UnknownNode { node } => {
write!(
formatter,
"node {} is not part of the compiled plan",
node.as_str()
)
}
Self::UnmappedExitOutcome { node, code } => write!(
formatter,
"node {} declares no transition for exit outcome {code}",
node.as_str()
),
}
}
}
impl Error for FlowSelectionError {}