use std::collections::{BTreeMap, BTreeSet};
use std::error::Error;
use std::fmt;
use serde_json::json;
use sha2::{Digest, Sha256};
use crate::{ChunkSize, JobName, NodeId, StateSchemaId, StateSchemaVersion, StepName};
const MAX_TOKEN_BYTES: usize = 128;
pub const MAX_NODES: usize = 1_024;
pub const MAX_TRANSITIONS: usize = 4_096;
pub const MAX_REPEAT_INTERCEPTORS: usize = 32;
pub const MAX_REPEAT_NESTING_DEPTH: usize = 8;
pub const MAX_DEFINITION_UPGRADE_NODES: usize = MAX_NODES;
pub const MAX_DEFINITION_UPGRADE_TRANSFORMS: usize = MAX_TRANSITIONS;
pub(crate) const MAX_MANIFEST_BYTES: usize = 64 * 1024;
pub const MANIFEST_FORMAT_ONE_STEP: u16 = 1;
pub const MANIFEST_FORMAT_FLOW: u16 = 2;
pub const MANIFEST_FORMAT_LOCAL_SCALE: u16 = 3;
pub const MANIFEST_FORMAT_ADVANCED_FLOW: u16 = 4;
pub(crate) const SUPPORTED_MANIFEST_FORMAT: u16 = MANIFEST_FORMAT_ADVANCED_FLOW;
const LEGACY_REVISION: &str = "__m1_repository_port_v1";
const LEGACY_MANIFEST: &[u8] =
br#"{"format":1,"repository_port":"m1","revision":"__m1_repository_port_v1"}"#;
pub fn validate_token(value: &str, kind: DefinitionTokenKind) -> Result<(), DefinitionError> {
if value.is_empty() {
return Err(DefinitionError::EmptyToken { kind });
}
if value.len() > MAX_TOKEN_BYTES {
return Err(DefinitionError::TokenTooLong {
kind,
max_bytes: MAX_TOKEN_BYTES,
});
}
if value.trim() != value {
return Err(DefinitionError::SurroundingWhitespace { kind });
}
if value.chars().any(char::is_control) {
return Err(DefinitionError::ControlCharacter { kind });
}
Ok(())
}
#[macro_export]
macro_rules! definition_token {
($name:ident, $kind:expr, $docs:literal) => {
#[doc = $docs]
#[derive(Clone, Debug, Eq, Hash, Ord, PartialEq, PartialOrd)]
pub struct $name(String);
impl $name {
pub fn new(value: impl Into<String>) -> Result<Self, DefinitionError> {
let value = value.into();
validate_token(&value, $kind)?;
Ok(Self(value))
}
#[must_use]
pub fn as_str(&self) -> &str {
&self.0
}
}
};
}
definition_token!(
DefinitionRevision,
DefinitionTokenKind::Revision,
"An application-owned audit label for one restart-relevant definition."
);
#[derive(Clone, Debug, Eq, PartialEq)]
pub struct ParameterIncrementerIdentity {
kind: DefinitionRevision,
revision: DefinitionRevision,
}
impl ParameterIncrementerIdentity {
#[must_use]
pub const fn new(kind: DefinitionRevision, revision: DefinitionRevision) -> Self {
Self { kind, revision }
}
#[must_use]
pub const fn kind(&self) -> &DefinitionRevision {
&self.kind
}
#[must_use]
pub const fn revision(&self) -> &DefinitionRevision {
&self.revision
}
}
definition_token!(
DefinitionArtifactId,
DefinitionTokenKind::Artifact,
"A bounded application-owned identity for the executable artifact that assembles one definition."
);
definition_token!(
DefinitionUpgradeKey,
DefinitionTokenKind::Upgrade,
"An application-owned key for one directed definition compatibility edge."
);
definition_token!(
DefinitionUpgradeRevision,
DefinitionTokenKind::Upgrade,
"A bounded application-owned revision for one deterministic compatible-upgrade implementation."
);
definition_token!(
DefinitionStateTransformId,
DefinitionTokenKind::Upgrade,
"A stable bounded identity for one durable state family member transformed by a compatible upgrade."
);
definition_token!(
RepeatId,
DefinitionTokenKind::Repeat,
"A stable logical identifier for one repeat definition."
);
#[derive(Clone, Debug, Eq, PartialEq)]
pub struct StepDefinitionUpgrade {
source: StepName,
target: StepName,
}
impl StepDefinitionUpgrade {
#[must_use]
pub const fn new(source: StepName, target: StepName) -> Self {
Self { source, target }
}
#[must_use]
pub const fn source(&self) -> &StepName {
&self.source
}
#[must_use]
pub const fn target(&self) -> &StepName {
&self.target
}
}
#[derive(Clone, Copy, Debug, Eq, Hash, Ord, PartialEq, PartialOrd)]
#[non_exhaustive]
pub enum DefinitionStateKind {
Checkpoint,
ExecutionContext,
ComponentState,
ScopeResolution,
Repeat,
FaultState,
}
impl DefinitionStateKind {
#[must_use]
pub const fn as_str(self) -> &'static str {
match self {
Self::Checkpoint => "checkpoint",
Self::ExecutionContext => "execution-context",
Self::ComponentState => "component-state",
Self::ScopeResolution => "scope-resolution",
Self::Repeat => "repeat",
Self::FaultState => "fault-state",
}
}
}
#[derive(Clone, Debug, Eq, PartialEq)]
pub struct DefinitionNodeUpgrade {
source: NodeId,
target: NodeId,
}
impl DefinitionNodeUpgrade {
#[must_use]
pub const fn new(source: NodeId, target: NodeId) -> Self {
Self { source, target }
}
#[must_use]
pub const fn source(&self) -> &NodeId {
&self.source
}
#[must_use]
pub const fn target(&self) -> &NodeId {
&self.target
}
}
#[derive(Clone, Debug, Eq, PartialEq)]
pub struct DefinitionStateTransform {
source_node: NodeId,
target_node: NodeId,
kind: DefinitionStateKind,
state_id: DefinitionStateTransformId,
source_schema: StateSchemaId,
source_version: StateSchemaVersion,
target_schema: StateSchemaId,
target_version: StateSchemaVersion,
revision: DefinitionUpgradeRevision,
}
impl DefinitionStateTransform {
#[allow(clippy::too_many_arguments)]
#[must_use]
pub const fn new(
source_node: NodeId,
target_node: NodeId,
kind: DefinitionStateKind,
state_id: DefinitionStateTransformId,
source_schema: StateSchemaId,
source_version: StateSchemaVersion,
target_schema: StateSchemaId,
target_version: StateSchemaVersion,
revision: DefinitionUpgradeRevision,
) -> Self {
Self {
source_node,
target_node,
kind,
state_id,
source_schema,
source_version,
target_schema,
target_version,
revision,
}
}
#[must_use]
pub const fn source_node(&self) -> &NodeId {
&self.source_node
}
#[must_use]
pub const fn target_node(&self) -> &NodeId {
&self.target_node
}
#[must_use]
pub const fn kind(&self) -> DefinitionStateKind {
self.kind
}
#[must_use]
pub const fn state_id(&self) -> &DefinitionStateTransformId {
&self.state_id
}
#[must_use]
pub const fn source_schema(&self) -> &StateSchemaId {
&self.source_schema
}
#[must_use]
pub const fn source_version(&self) -> StateSchemaVersion {
self.source_version
}
#[must_use]
pub const fn target_schema(&self) -> &StateSchemaId {
&self.target_schema
}
#[must_use]
pub const fn target_version(&self) -> StateSchemaVersion {
self.target_version
}
#[must_use]
pub const fn revision(&self) -> &DefinitionUpgradeRevision {
&self.revision
}
}
#[derive(Clone)]
pub struct DefinitionStateTransformer {
revision: DefinitionUpgradeRevision,
apply: fn(&[u8]) -> Result<Vec<u8>, DefinitionStateTransformError>,
}
impl DefinitionStateTransformer {
#[must_use]
pub const fn new(
revision: DefinitionUpgradeRevision,
apply: fn(&[u8]) -> Result<Vec<u8>, DefinitionStateTransformError>,
) -> Self {
Self { revision, apply }
}
#[must_use]
pub const fn revision(&self) -> &DefinitionUpgradeRevision {
&self.revision
}
pub fn apply(&self, payload: &[u8]) -> Result<Vec<u8>, DefinitionStateTransformError> {
(self.apply)(payload)
}
}
impl fmt::Debug for DefinitionStateTransformer {
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
formatter
.debug_struct("DefinitionStateTransformer")
.field("revision", &self.revision)
.finish_non_exhaustive()
}
}
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
#[non_exhaustive]
pub enum DefinitionStateTransformError {
InvalidSource,
TransformFailed,
}
impl fmt::Display for DefinitionStateTransformError {
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
formatter.write_str(match self {
Self::InvalidSource => "definition-upgrade source state is invalid",
Self::TransformFailed => "definition-upgrade state transform failed",
})
}
}
impl Error for DefinitionStateTransformError {}
#[derive(Clone, Debug)]
pub struct DefinitionUpgradeExecution {
upgrade: DefinitionUpgrade,
transformers: BTreeMap<DefinitionUpgradeRevision, DefinitionStateTransformer>,
}
impl DefinitionUpgradeExecution {
pub fn new(
upgrade: DefinitionUpgrade,
transformers: impl IntoIterator<Item = DefinitionStateTransformer>,
) -> Result<Self, DefinitionError> {
if !upgrade.is_transformed() {
return Err(DefinitionError::LegacyUpgradeExecution);
}
let mut implementations = BTreeMap::new();
for transformer in transformers {
if implementations
.insert(transformer.revision().clone(), transformer)
.is_some()
{
return Err(DefinitionError::DuplicateStateTransformer);
}
}
let required = upgrade
.transforms()
.iter()
.map(|transform| transform.revision().clone())
.collect::<BTreeSet<_>>();
let supplied = implementations.keys().cloned().collect::<BTreeSet<_>>();
if required != supplied {
return Err(DefinitionError::StateTransformerSetMismatch);
}
Ok(Self {
upgrade,
transformers: implementations,
})
}
#[must_use]
pub const fn upgrade(&self) -> &DefinitionUpgrade {
&self.upgrade
}
#[must_use]
pub fn transformer(
&self,
transform: &DefinitionStateTransform,
) -> Option<&DefinitionStateTransformer> {
self.transformers.get(transform.revision())
}
}
#[derive(Clone, Debug, Eq, PartialEq)]
pub struct DefinitionUpgrade {
key: DefinitionUpgradeKey,
revision: Option<DefinitionUpgradeRevision>,
from: DefinitionIdentity,
to: DefinitionIdentity,
step_mapping: BTreeMap<StepName, StepName>,
node_mapping: BTreeMap<NodeId, NodeId>,
transforms: Vec<DefinitionStateTransform>,
}
impl DefinitionUpgrade {
pub fn new(
key: DefinitionUpgradeKey,
from: DefinitionIdentity,
to: DefinitionIdentity,
steps: impl IntoIterator<Item = StepDefinitionUpgrade>,
) -> Result<Self, DefinitionError> {
if from.manifest_digest() == to.manifest_digest() {
return Err(DefinitionError::UpgradeSelfEdge);
}
let mut step_mapping = BTreeMap::new();
let mut targets = BTreeSet::new();
for step in steps {
if step_mapping
.insert(step.source().clone(), step.target().clone())
.is_some()
{
return Err(DefinitionError::DuplicateSourceStep);
}
if !targets.insert(step.target().clone()) {
return Err(DefinitionError::DuplicateTargetStep);
}
}
if step_mapping.is_empty() {
return Err(DefinitionError::EmptyStepMapping);
}
Ok(Self {
key,
revision: None,
from,
to,
step_mapping,
node_mapping: BTreeMap::new(),
transforms: Vec::new(),
})
}
pub fn transformed(
key: DefinitionUpgradeKey,
revision: DefinitionUpgradeRevision,
from: DefinitionIdentity,
to: DefinitionIdentity,
nodes: impl IntoIterator<Item = DefinitionNodeUpgrade>,
transforms: impl IntoIterator<Item = DefinitionStateTransform>,
) -> Result<Self, DefinitionError> {
if from.manifest_digest() == to.manifest_digest() {
return Err(DefinitionError::UpgradeSelfEdge);
}
let mut node_mapping = BTreeMap::new();
let mut targets = BTreeSet::new();
for node in nodes {
if node_mapping.len() >= MAX_DEFINITION_UPGRADE_NODES {
return Err(DefinitionError::TooManyUpgradeNodes {
max: MAX_DEFINITION_UPGRADE_NODES,
});
}
if node_mapping
.insert(node.source().clone(), node.target().clone())
.is_some()
{
return Err(DefinitionError::DuplicateSourceNode);
}
if !targets.insert(node.target().clone()) {
return Err(DefinitionError::DuplicateTargetNode);
}
}
if node_mapping.is_empty() {
return Err(DefinitionError::EmptyStepMapping);
}
let source_nodes = definition_upgrade_source_nodes(&from)?;
if node_mapping.keys().any(|node| !source_nodes.contains(node)) {
return Err(DefinitionError::UnknownUpgradeSourceNode);
}
if source_nodes
.iter()
.any(|node| !node_mapping.contains_key(node))
{
return Err(DefinitionError::MissingUpgradeSourceNode);
}
let target_nodes = definition_upgrade_source_nodes(&to)?;
if node_mapping
.values()
.any(|node| !target_nodes.contains(node))
{
return Err(DefinitionError::UnknownUpgradeTargetNode);
}
let transforms = transforms.into_iter().collect::<Vec<_>>();
if transforms.len() > MAX_DEFINITION_UPGRADE_TRANSFORMS {
return Err(DefinitionError::TooManyUpgradeTransforms {
max: MAX_DEFINITION_UPGRADE_TRANSFORMS,
});
}
let mut transform_keys = BTreeSet::new();
for transform in &transforms {
if node_mapping.get(transform.source_node()) != Some(transform.target_node()) {
return Err(DefinitionError::StateTransformOutsideNodeMapping);
}
if !transform_keys.insert((
transform.source_node().clone(),
transform.kind(),
transform.state_id().clone(),
)) {
return Err(DefinitionError::DuplicateStateTransform);
}
}
validate_required_definition_state_transforms(&from, &to, &node_mapping, &transforms)?;
Ok(Self {
key,
revision: Some(revision),
from,
to,
step_mapping: BTreeMap::new(),
node_mapping,
transforms,
})
}
#[must_use]
pub const fn key(&self) -> &DefinitionUpgradeKey {
&self.key
}
#[must_use]
pub const fn revision(&self) -> Option<&DefinitionUpgradeRevision> {
self.revision.as_ref()
}
#[must_use]
pub const fn from(&self) -> &DefinitionIdentity {
&self.from
}
#[must_use]
pub const fn to(&self) -> &DefinitionIdentity {
&self.to
}
#[must_use]
pub fn step_mapping(&self) -> &BTreeMap<StepName, StepName> {
&self.step_mapping
}
#[must_use]
pub fn node_mapping(&self) -> &BTreeMap<NodeId, NodeId> {
&self.node_mapping
}
#[must_use]
pub fn transforms(&self) -> &[DefinitionStateTransform] {
&self.transforms
}
#[must_use]
pub const fn is_transformed(&self) -> bool {
self.revision.is_some()
}
}
#[derive(Clone, Debug, Eq, PartialEq)]
struct DefinitionDurableStateContract {
kind: DefinitionStateKind,
state_id: Option<String>,
schema: StateSchemaId,
version: StateSchemaVersion,
}
fn validate_required_definition_state_transforms(
from: &DefinitionIdentity,
to: &DefinitionIdentity,
node_mapping: &BTreeMap<NodeId, NodeId>,
transforms: &[DefinitionStateTransform],
) -> Result<(), DefinitionError> {
for (source_node, target_node) in node_mapping {
let source_contracts = definition_durable_state_contracts(from, source_node)?;
let target_contracts = definition_durable_state_contracts(to, target_node)?;
for source in source_contracts {
let target = target_contracts
.iter()
.find(|candidate| {
candidate.kind == source.kind && candidate.state_id == source.state_id
})
.ok_or(DefinitionError::MissingTargetStateContract)?;
let matching = transforms
.iter()
.filter(|transform| {
transform.source_node() == source_node
&& transform.target_node() == target_node
&& transform.kind() == source.kind
&& source
.state_id
.as_deref()
.is_none_or(|state_id| transform.state_id().as_str() == state_id)
})
.collect::<Vec<_>>();
if matching.len() > 1 {
return Err(DefinitionError::DuplicateStateTransform);
}
let changed = source.schema != target.schema || source.version != target.version;
let Some(transform) = matching.first().copied() else {
if changed {
return Err(DefinitionError::MissingRequiredStateTransform);
}
continue;
};
if transform.source_schema() != &source.schema
|| transform.source_version() != source.version
|| transform.target_schema() != &target.schema
|| transform.target_version() != target.version
{
return Err(DefinitionError::StateTransformSchemaMismatch);
}
}
}
Ok(())
}
fn definition_durable_state_contracts(
definition: &DefinitionIdentity,
node_id: &NodeId,
) -> Result<Vec<DefinitionDurableStateContract>, DefinitionError> {
let document: serde_json::Value = serde_json::from_slice(definition.canonical_manifest())
.map_err(|_| DefinitionError::ManifestEncoding)?;
if definition.manifest_format() == MANIFEST_FORMAT_ONE_STEP {
let object = document
.as_object()
.ok_or(DefinitionError::ManifestEncoding)?;
let step = object
.get("step")
.and_then(serde_json::Value::as_str)
.ok_or(DefinitionError::ManifestEncoding)?;
if step != node_id.as_str() {
return Err(DefinitionError::UnknownUpgradeSourceNode);
}
return one_step_durable_state_contracts(object);
}
let node = find_definition_node(&document, node_id).ok_or(DefinitionError::ManifestEncoding)?;
graph_node_durable_state_contracts(node)
}
fn one_step_durable_state_contracts(
object: &serde_json::Map<String, serde_json::Value>,
) -> Result<Vec<DefinitionDurableStateContract>, DefinitionError> {
if object.get("kind").and_then(serde_json::Value::as_str) != Some("chunk") {
return Ok(Vec::new());
}
Ok(vec![
schema_contract(
DefinitionStateKind::Checkpoint,
None,
object
.get("checkpoint")
.ok_or(DefinitionError::ManifestEncoding)?,
"schema",
)?,
schema_contract(
DefinitionStateKind::ExecutionContext,
None,
object
.get("context")
.ok_or(DefinitionError::ManifestEncoding)?,
"schema",
)?,
])
}
fn graph_node_durable_state_contracts(
node: &serde_json::Map<String, serde_json::Value>,
) -> Result<Vec<DefinitionDurableStateContract>, DefinitionError> {
let mut contracts = Vec::new();
match node.get("kind").and_then(serde_json::Value::as_str) {
Some("step") => {
let step = node
.get("step")
.and_then(serde_json::Value::as_object)
.ok_or(DefinitionError::ManifestEncoding)?;
if step.get("kind").and_then(serde_json::Value::as_str) == Some("chunk") {
let declaration = step
.get("declaration")
.and_then(serde_json::Value::as_object)
.ok_or(DefinitionError::ManifestEncoding)?;
contracts.push(schema_contract(
DefinitionStateKind::Checkpoint,
None,
declaration
.get("checkpoint")
.ok_or(DefinitionError::ManifestEncoding)?,
"schema",
)?);
contracts.push(schema_contract(
DefinitionStateKind::ExecutionContext,
None,
declaration
.get("context")
.ok_or(DefinitionError::ManifestEncoding)?,
"schema",
)?);
}
if let Some(repeat) = node.get("repeat") {
collect_repeat_state_contracts(repeat, &mut contracts)?;
}
}
Some("custom_leaf") => {
contracts.push(schema_contract(
DefinitionStateKind::ExecutionContext,
None,
node.get("state_schema")
.ok_or(DefinitionError::ManifestEncoding)?,
"id",
)?);
}
_ => {}
}
Ok(contracts)
}
fn collect_repeat_state_contracts(
repeat: &serde_json::Value,
contracts: &mut Vec<DefinitionDurableStateContract>,
) -> Result<(), DefinitionError> {
let object = repeat
.as_object()
.ok_or(DefinitionError::ManifestEncoding)?;
let id = object
.get("id")
.and_then(serde_json::Value::as_str)
.ok_or(DefinitionError::ManifestEncoding)?;
contracts.push(schema_contract(
DefinitionStateKind::Repeat,
Some(id.to_owned()),
object
.get("state")
.ok_or(DefinitionError::ManifestEncoding)?,
"schema",
)?);
if let Some(nested) = object.get("nested") {
collect_repeat_state_contracts(nested, contracts)?;
}
Ok(())
}
fn schema_contract(
kind: DefinitionStateKind,
state_id: Option<String>,
value: &serde_json::Value,
schema_key: &str,
) -> Result<DefinitionDurableStateContract, DefinitionError> {
let object = value.as_object().ok_or(DefinitionError::ManifestEncoding)?;
let schema = object
.get(schema_key)
.and_then(serde_json::Value::as_str)
.ok_or(DefinitionError::ManifestEncoding)?;
let version = object
.get("version")
.and_then(serde_json::Value::as_u64)
.and_then(|value| u32::try_from(value).ok())
.and_then(|value| StateSchemaVersion::new(value).ok())
.ok_or(DefinitionError::ManifestEncoding)?;
Ok(DefinitionDurableStateContract {
kind,
state_id,
schema: StateSchemaId::new(schema).map_err(|_| DefinitionError::ManifestEncoding)?,
version,
})
}
fn find_definition_node<'a>(
value: &'a serde_json::Value,
node_id: &NodeId,
) -> Option<&'a serde_json::Map<String, serde_json::Value>> {
match value {
serde_json::Value::Object(object) => {
let is_graph_node = matches!(
object.get("kind").and_then(serde_json::Value::as_str),
Some(
"step"
| "custom_leaf"
| "decision"
| "join"
| "split"
| "nested_flow"
| "nested_job"
| "partitioned_step"
)
);
if is_graph_node
&& object.get("id").and_then(serde_json::Value::as_str) == Some(node_id.as_str())
{
return Some(object);
}
object
.values()
.find_map(|child| find_definition_node(child, node_id))
}
serde_json::Value::Array(values) => values
.iter()
.find_map(|child| find_definition_node(child, node_id)),
_ => None,
}
}
fn definition_upgrade_source_nodes(
definition: &DefinitionIdentity,
) -> Result<BTreeSet<NodeId>, DefinitionError> {
let document: serde_json::Value = serde_json::from_slice(definition.canonical_manifest())
.map_err(|_| DefinitionError::ManifestEncoding)?;
let object = document
.as_object()
.ok_or(DefinitionError::ManifestEncoding)?;
if definition.manifest_format() == MANIFEST_FORMAT_ONE_STEP {
let step = object
.get("step")
.and_then(serde_json::Value::as_str)
.ok_or(DefinitionError::ManifestEncoding)?;
return Ok(BTreeSet::from([NodeId::new(step)?]));
}
let nodes = object
.get("nodes")
.and_then(serde_json::Value::as_array)
.ok_or(DefinitionError::ManifestEncoding)?;
let mut result = BTreeSet::new();
collect_upgrade_node_ids(nodes, &mut result)?;
Ok(result)
}
fn collect_upgrade_node_ids(
nodes: &[serde_json::Value],
result: &mut BTreeSet<NodeId>,
) -> Result<(), DefinitionError> {
for node in nodes {
let object = node.as_object().ok_or(DefinitionError::ManifestEncoding)?;
if let Some(id) = object.get("id").and_then(serde_json::Value::as_str) {
result.insert(NodeId::new(id)?);
}
for key in ["members", "steps", "nodes"] {
if let Some(children) = object.get(key).and_then(serde_json::Value::as_array) {
collect_upgrade_node_ids(children, result)?;
}
}
if let Some(branches) = object.get("branches").and_then(serde_json::Value::as_array) {
for branch in branches {
let branch = branch
.as_object()
.ok_or(DefinitionError::ManifestEncoding)?;
for key in ["members", "steps", "nodes"] {
if let Some(children) = branch.get(key).and_then(serde_json::Value::as_array) {
collect_upgrade_node_ids(children, result)?;
}
}
}
}
}
Ok(())
}
definition_token!(
ComponentRevision,
DefinitionTokenKind::Component,
"An application-owned revision token for one opaque executable component."
);
definition_token!(
ClassifierRevision,
DefinitionTokenKind::Classifier,
"An application-owned revision token for one bounded fault classifier."
);
definition_token!(
ComponentStreamIdentity,
DefinitionTokenKind::Component,
"A stable, owner-scoped `ItemStream` namespace token.\n\nThis is the durable component-state namespace: it is never derived from a\ndisplay name, a memory address, or runtime object identity, and it\nparticipates in the restart-relevant chunk definition fingerprint alongside\nthe other component revisions."
);
#[derive(Clone, Debug, Eq, PartialEq)]
pub struct ChunkComponentRevisions {
reader: ComponentRevision,
processor: ComponentRevision,
writer: ComponentRevision,
checkpoint: ComponentRevision,
restart: ChunkRestartContract,
streams: BTreeMap<ComponentStreamIdentity, ComponentRevision>,
completion_policy: Option<ComponentRevision>,
}
impl ChunkComponentRevisions {
#[must_use]
pub const fn new(
reader: ComponentRevision,
processor: ComponentRevision,
writer: ComponentRevision,
checkpoint: ComponentRevision,
restart: ChunkRestartContract,
) -> Self {
Self {
reader,
processor,
writer,
checkpoint,
restart,
streams: BTreeMap::new(),
completion_policy: None,
}
}
#[must_use]
pub fn with_stream_revision(
mut self,
identity: ComponentStreamIdentity,
revision: ComponentRevision,
) -> Self {
self.streams.insert(identity, revision);
self
}
pub fn stream_revisions(
&self,
) -> impl Iterator<Item = (&ComponentStreamIdentity, &ComponentRevision)> {
self.streams.iter()
}
#[must_use]
pub fn with_completion_policy_revision(mut self, revision: ComponentRevision) -> Self {
self.completion_policy = Some(revision);
self
}
#[must_use]
pub const fn completion_policy_revision(&self) -> Option<&ComponentRevision> {
self.completion_policy.as_ref()
}
#[must_use]
pub const fn delivery_mode(&self) -> ChunkDeliveryMode {
self.restart.delivery_mode
}
#[must_use]
pub const fn in_flight_policy(&self) -> InFlightPolicy {
self.restart.in_flight_policy
}
#[must_use]
pub const fn reader(&self) -> &ComponentRevision {
&self.reader
}
#[must_use]
pub const fn processor(&self) -> &ComponentRevision {
&self.processor
}
#[must_use]
pub const fn writer(&self) -> &ComponentRevision {
&self.writer
}
#[must_use]
pub const fn checkpoint(&self) -> &ComponentRevision {
&self.checkpoint
}
#[must_use]
pub const fn checkpoint_schema(&self) -> &StateSchemaId {
&self.restart.checkpoint_schema
}
#[must_use]
pub const fn checkpoint_schema_version(&self) -> StateSchemaVersion {
self.restart.checkpoint_schema_version
}
#[must_use]
pub const fn context_schema(&self) -> &StateSchemaId {
&self.restart.context_schema
}
#[must_use]
pub const fn context_schema_version(&self) -> StateSchemaVersion {
self.restart.context_schema_version
}
}
#[derive(Clone, Copy, Debug, Default, Eq, Hash, Ord, PartialEq, PartialOrd)]
#[non_exhaustive]
pub enum InFlightPolicy {
#[default]
FinishChunk,
RollbackChunk,
}
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
#[non_exhaustive]
pub enum ChunkDeliveryMode {
AtomicSameResource,
AtLeastOnce,
}
impl ChunkDeliveryMode {
#[must_use]
pub const fn manifest_name(self) -> &'static str {
match self {
Self::AtomicSameResource => "atomic_same_resource",
Self::AtLeastOnce => "at_least_once",
}
}
}
#[derive(Clone, Debug, Eq, PartialEq)]
pub struct ChunkRestartContract {
checkpoint_schema: StateSchemaId,
checkpoint_schema_version: StateSchemaVersion,
context_schema: StateSchemaId,
context_schema_version: StateSchemaVersion,
delivery_mode: ChunkDeliveryMode,
in_flight_policy: InFlightPolicy,
}
impl ChunkRestartContract {
#[must_use]
pub const fn new(
checkpoint_schema: StateSchemaId,
checkpoint_schema_version: StateSchemaVersion,
context_schema: StateSchemaId,
context_schema_version: StateSchemaVersion,
delivery_mode: ChunkDeliveryMode,
) -> Self {
Self {
checkpoint_schema,
checkpoint_schema_version,
context_schema,
context_schema_version,
delivery_mode,
in_flight_policy: InFlightPolicy::FinishChunk,
}
}
#[must_use]
pub const fn with_in_flight_policy(mut self, policy: InFlightPolicy) -> Self {
self.in_flight_policy = policy;
self
}
}
#[derive(Clone, Eq, PartialEq)]
pub struct DefinitionIdentity {
job_name: Option<JobName>,
revision: DefinitionRevision,
manifest_format: u16,
manifest_digest: [u8; 32],
canonical_manifest: Box<[u8]>,
}
impl DefinitionIdentity {
#[must_use]
pub fn legacy() -> Self {
Self::from_canonical(
None,
DefinitionRevision(LEGACY_REVISION.to_owned()),
LEGACY_MANIFEST.to_vec(),
MANIFEST_FORMAT_ONE_STEP,
)
}
pub fn tasklet(
job_name: &JobName,
step_name: &StepName,
revision: DefinitionRevision,
component_revision: &ComponentRevision,
) -> Result<Self, DefinitionError> {
let manifest = json!({
"component": {
"tasklet": component_revision.as_str()
},
"delivery_mode": "best_effort",
"format": MANIFEST_FORMAT_ONE_STEP,
"job": job_name.as_str(),
"kind": "tasklet",
"restart_state": "none",
"step": step_name.as_str(),
"transaction_boundary": "tasklet_completion"
});
Self::encode(job_name.clone(), revision, &manifest)
}
pub fn chunk(
job_name: &JobName,
step_name: &StepName,
chunk_size: ChunkSize,
revision: DefinitionRevision,
components: &ChunkComponentRevisions,
) -> Result<Self, DefinitionError> {
let mut manifest = json!({
"chunk_size": chunk_size.get(),
"components": {
"checkpoint": components.checkpoint.as_str(),
"processor": components.processor.as_str(),
"reader": components.reader.as_str(),
"writer": components.writer.as_str()
},
"context": {
"schema": components.restart.context_schema.as_str(),
"version": components.restart.context_schema_version.get()
},
"checkpoint": {
"schema": components.restart.checkpoint_schema.as_str(),
"version": components.restart.checkpoint_schema_version.get()
},
"delivery_mode": components.restart.delivery_mode.manifest_name(),
"format": MANIFEST_FORMAT_ONE_STEP,
"job": job_name.as_str(),
"kind": "chunk",
"step": step_name.as_str(),
"transaction_boundary": "chunk"
});
if components.restart.in_flight_policy == InFlightPolicy::RollbackChunk
&& let Some(object) = manifest.as_object_mut()
{
object.insert(
"in_flight_policy".to_owned(),
serde_json::Value::String("rollback_chunk".to_owned()),
);
}
if !components.streams.is_empty()
&& let Some(object) = manifest.as_object_mut()
{
let streams: Vec<serde_json::Value> = components
.streams
.iter()
.map(|(identity, revision)| {
json!({
"identity": identity.as_str(),
"revision": revision.as_str(),
})
})
.collect();
object.insert("streams".to_owned(), serde_json::Value::Array(streams));
}
if let Some(policy_revision) = &components.completion_policy
&& let Some(object) = manifest.as_object_mut()
{
object.insert(
"completion_policy".to_owned(),
serde_json::Value::String(policy_revision.as_str().to_owned()),
);
}
Self::encode(job_name.clone(), revision, &manifest)
}
pub fn from_flow_manifest(
job_name: &JobName,
revision: DefinitionRevision,
canonical: &[u8],
) -> Result<Self, DefinitionError> {
if canonical.len() > MAX_MANIFEST_BYTES {
return Err(DefinitionError::ManifestTooLarge {
max_bytes: MAX_MANIFEST_BYTES,
});
}
let document: serde_json::Value =
serde_json::from_slice(canonical).map_err(|_| DefinitionError::ManifestEncoding)?;
let reencoded =
serde_json::to_vec(&document).map_err(|_| DefinitionError::ManifestEncoding)?;
if !document.is_object() || reencoded != canonical {
return Err(DefinitionError::ManifestEncoding);
}
let format = document
.get("format")
.and_then(serde_json::Value::as_u64)
.and_then(|value| u16::try_from(value).ok())
.filter(|value| {
matches!(
*value,
MANIFEST_FORMAT_FLOW
| MANIFEST_FORMAT_LOCAL_SCALE
| MANIFEST_FORMAT_ADVANCED_FLOW
)
})
.ok_or(DefinitionError::ManifestEncoding)?;
Ok(Self::from_canonical(
Some(job_name.clone()),
revision,
canonical.to_vec(),
format,
))
}
fn encode(
job_name: JobName,
revision: DefinitionRevision,
manifest: &serde_json::Value,
) -> Result<Self, DefinitionError> {
let canonical =
serde_json::to_vec(manifest).map_err(|_| DefinitionError::ManifestEncoding)?;
if canonical.len() > MAX_MANIFEST_BYTES {
return Err(DefinitionError::ManifestTooLarge {
max_bytes: MAX_MANIFEST_BYTES,
});
}
Ok(Self::from_canonical(
Some(job_name),
revision,
canonical,
MANIFEST_FORMAT_ONE_STEP,
))
}
fn from_canonical(
job_name: Option<JobName>,
revision: DefinitionRevision,
canonical: Vec<u8>,
format: u16,
) -> Self {
let digest: [u8; 32] = Sha256::digest(&canonical).into();
Self {
job_name,
revision,
manifest_format: format,
manifest_digest: digest,
canonical_manifest: canonical.into_boxed_slice(),
}
}
#[doc(hidden)]
pub fn from_durable(
job_name: JobName,
revision: DefinitionRevision,
manifest_format: u16,
canonical: &[u8],
expected_digest: [u8; 32],
) -> Result<Self, DefinitionError> {
if canonical.len() > MAX_MANIFEST_BYTES {
return Err(DefinitionError::ManifestTooLarge {
max_bytes: MAX_MANIFEST_BYTES,
});
}
check_manifest_format(manifest_format).map_err(|_| DefinitionError::ManifestEncoding)?;
let document: serde_json::Value =
serde_json::from_slice(canonical).map_err(|_| DefinitionError::ManifestEncoding)?;
let reencoded =
serde_json::to_vec(&document).map_err(|_| DefinitionError::ManifestEncoding)?;
if !document.is_object()
|| reencoded != canonical
|| document.get("format").and_then(serde_json::Value::as_u64)
!= Some(u64::from(manifest_format))
{
return Err(DefinitionError::ManifestEncoding);
}
let identity = Self::from_canonical(
Some(job_name),
revision,
canonical.to_vec(),
manifest_format,
);
if identity.manifest_digest != expected_digest {
return Err(DefinitionError::ManifestEncoding);
}
Ok(identity)
}
#[must_use]
pub const fn revision(&self) -> &DefinitionRevision {
&self.revision
}
#[must_use]
pub const fn job_name(&self) -> Option<&JobName> {
self.job_name.as_ref()
}
#[must_use]
pub const fn manifest_format(&self) -> u16 {
self.manifest_format
}
#[must_use]
pub const fn manifest_digest(&self) -> &[u8; 32] {
&self.manifest_digest
}
#[must_use]
pub fn canonical_manifest(&self) -> &[u8] {
&self.canonical_manifest
}
}
pub const fn check_manifest_format(format: u16) -> Result<(), ManifestError> {
if format == 0 {
return Err(ManifestError::MissingFormat);
}
if format > SUPPORTED_MANIFEST_FORMAT {
return Err(ManifestError::UnsupportedFormat {
format,
supported: SUPPORTED_MANIFEST_FORMAT,
});
}
Ok(())
}
impl fmt::Debug for DefinitionIdentity {
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
formatter
.debug_struct("DefinitionIdentity")
.field("job_name", &self.job_name)
.field("revision", &self.revision)
.field("manifest_format", &self.manifest_format)
.field(
"digest_prefix",
&DigestPrefix([
self.manifest_digest[0],
self.manifest_digest[1],
self.manifest_digest[2],
self.manifest_digest[3],
]),
)
.field("canonical_manifest", &"<redacted>")
.finish()
}
}
struct DigestPrefix([u8; 4]);
impl fmt::Debug for DigestPrefix {
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
for byte in self.0 {
write!(formatter, "{byte:02x}")?;
}
Ok(())
}
}
#[derive(Clone, Debug, Eq, PartialEq)]
pub struct DefinitionManifest {
format: u16,
digest: [u8; 32],
job_name: Option<JobName>,
node_count: Option<usize>,
transition_count: Option<usize>,
}
impl DefinitionManifest {
pub fn read(bytes: &[u8]) -> Result<Self, ManifestError> {
if bytes.len() > MAX_MANIFEST_BYTES {
return Err(ManifestError::TooLarge {
max_bytes: MAX_MANIFEST_BYTES,
});
}
let document: serde_json::Value =
serde_json::from_slice(bytes).map_err(|_| ManifestError::MalformedJson)?;
let members = document.as_object().ok_or(ManifestError::NotAnObject)?;
if contains_float(&document) {
return Err(ManifestError::FloatValue);
}
let reencoded = serde_json::to_vec(&document).map_err(|_| ManifestError::MalformedJson)?;
if reencoded != bytes {
return Err(ManifestError::NonCanonicalEncoding);
}
let format = members
.get("format")
.and_then(serde_json::Value::as_u64)
.and_then(|format| u16::try_from(format).ok())
.ok_or(ManifestError::MissingFormat)?;
check_manifest_format(format)?;
if format != MANIFEST_FORMAT_ADVANCED_FLOW
&& (members.contains_key("parameter_incrementer")
|| contains_object_key(&document, "restart_policy"))
{
return Err(ManifestError::MalformedGraph);
}
let job_name = members
.get("job")
.and_then(serde_json::Value::as_str)
.map(JobName::new)
.transpose()
.map_err(|_| ManifestError::InvalidJobName)?;
let (node_count, transition_count) = match format {
MANIFEST_FORMAT_FLOW | MANIFEST_FORMAT_LOCAL_SCALE => {
let nodes_value = members.get("nodes").ok_or(ManifestError::MalformedGraph)?;
if contains_object_key(nodes_value, "repeat") {
return Err(ManifestError::MalformedGraph);
}
let nodes = array_len(Some(nodes_value))?;
let transitions = array_len(members.get("transitions"))?;
ensure_graph_bounds(nodes, transitions)?;
(Some(nodes), Some(transitions))
}
MANIFEST_FORMAT_ADVANCED_FLOW => {
validate_parameter_incrementer_manifest(members)?;
validate_restart_policy_manifest(&document)?;
let (nodes, transitions) = advanced_graph_counts(members)?;
ensure_graph_bounds(nodes, transitions)?;
if let Some(scoped) = members.get("scoped_components") {
let scoped = scoped.as_array().ok_or(ManifestError::MalformedGraph)?;
if scoped.is_empty() {
return Err(ManifestError::MalformedGraph);
}
let mut job_scoped = 0_usize;
let mut step_scoped = 0_usize;
for component in scoped {
let scope = component
.as_object()
.and_then(|object| object.get("scope"))
.and_then(serde_json::Value::as_str)
.ok_or(ManifestError::MalformedGraph)?;
let count = match scope {
"job" => &mut job_scoped,
"step" => &mut step_scoped,
_ => return Err(ManifestError::MalformedGraph),
};
*count += 1;
if *count > crate::MAX_SCOPED_COMPONENTS {
return Err(ManifestError::MalformedGraph);
}
}
}
validate_repeat_manifests(members)?;
(Some(nodes), Some(transitions))
}
_ => (None, None),
};
Ok(Self {
format,
digest: Sha256::digest(bytes).into(),
job_name,
node_count,
transition_count,
})
}
pub fn read_verified(bytes: &[u8], expected: &[u8; 32]) -> Result<Self, ManifestError> {
let manifest = Self::read(bytes)?;
if &manifest.digest != expected {
return Err(ManifestError::DigestMismatch);
}
Ok(manifest)
}
#[must_use]
pub const fn format(&self) -> u16 {
self.format
}
#[must_use]
pub const fn digest(&self) -> &[u8; 32] {
&self.digest
}
#[must_use]
pub const fn job_name(&self) -> Option<&JobName> {
self.job_name.as_ref()
}
#[must_use]
pub const fn node_count(&self) -> Option<usize> {
self.node_count
}
#[must_use]
pub const fn transition_count(&self) -> Option<usize> {
self.transition_count
}
}
fn contains_object_key(value: &serde_json::Value, key: &str) -> bool {
match value {
serde_json::Value::Object(object) => object
.iter()
.any(|(member, value)| member == key || contains_object_key(value, key)),
serde_json::Value::Array(values) => {
values.iter().any(|value| contains_object_key(value, key))
}
_ => false,
}
}
fn validate_parameter_incrementer_manifest(
members: &serde_json::Map<String, serde_json::Value>,
) -> Result<(), ManifestError> {
let Some(value) = members.get("parameter_incrementer") else {
return Ok(());
};
let object = value.as_object().ok_or(ManifestError::MalformedGraph)?;
let required = ["kind", "revision"];
if object.len() != required.len() || object.keys().any(|key| !required.contains(&key.as_str()))
{
return Err(ManifestError::MalformedGraph);
}
let kind = object
.get("kind")
.and_then(serde_json::Value::as_str)
.ok_or(ManifestError::MalformedGraph)?;
let revision = object
.get("revision")
.and_then(serde_json::Value::as_str)
.ok_or(ManifestError::MalformedGraph)?;
DefinitionRevision::new(kind).map_err(|_| ManifestError::MalformedGraph)?;
DefinitionRevision::new(revision).map_err(|_| ManifestError::MalformedGraph)?;
Ok(())
}
fn validate_restart_policy_manifest(document: &serde_json::Value) -> Result<(), ManifestError> {
fn visit(value: &serde_json::Value, root: bool) -> Result<(), ManifestError> {
match value {
serde_json::Value::Object(object) => {
if let Some(policy) = object.get("restart_policy") {
let is_start_controls = object.contains_key("start_limit")
&& object.contains_key("allow_start_if_complete");
if (!root && !is_start_controls) || policy.as_str() != Some("not_restartable") {
return Err(ManifestError::MalformedGraph);
}
}
for child in object.values() {
visit(child, false)?;
}
}
serde_json::Value::Array(values) => {
for child in values {
visit(child, false)?;
}
}
_ => {}
}
Ok(())
}
visit(document, true)
}
fn ensure_graph_bounds(nodes: usize, transitions: usize) -> Result<(), ManifestError> {
if nodes > MAX_NODES || transitions > MAX_TRANSITIONS {
return Err(ManifestError::GraphOutOfBounds {
max_nodes: MAX_NODES,
max_transitions: MAX_TRANSITIONS,
});
}
Ok(())
}
fn advanced_graph_counts(
members: &serde_json::Map<String, serde_json::Value>,
) -> Result<(usize, usize), ManifestError> {
let nodes = members
.get("nodes")
.and_then(serde_json::Value::as_array)
.ok_or(ManifestError::MalformedGraph)?;
let transitions = array_len(members.get("transitions"))?;
let mut materialized_nodes = nodes.len();
let mut materialized_transitions = transitions;
for node in nodes {
let object = node.as_object().ok_or(ManifestError::MalformedGraph)?;
match object.get("kind").and_then(serde_json::Value::as_str) {
Some("step" | "decision" | "join") => {}
Some("nested_job") => validate_nested_job_manifest(object)?,
Some("partitioned_step") => {
if !object
.get("worker")
.is_some_and(serde_json::Value::is_object)
{
return Err(ManifestError::MalformedGraph);
}
materialized_nodes =
materialized_nodes
.checked_add(1)
.ok_or(ManifestError::GraphOutOfBounds {
max_nodes: MAX_NODES,
max_transitions: MAX_TRANSITIONS,
})?;
}
Some("split") => {
let branches = object
.get("branches")
.and_then(serde_json::Value::as_array)
.ok_or(ManifestError::MalformedGraph)?;
materialized_nodes = materialized_nodes.checked_add(branches.len()).ok_or(
ManifestError::GraphOutOfBounds {
max_nodes: MAX_NODES,
max_transitions: MAX_TRANSITIONS,
},
)?;
for branch in branches {
let branch = branch.as_object().ok_or(ManifestError::MalformedGraph)?;
match branch.get("kind").and_then(serde_json::Value::as_str) {
Some("linear") => {
materialized_nodes = materialized_nodes
.checked_add(array_len(branch.get("steps"))?)
.ok_or(ManifestError::GraphOutOfBounds {
max_nodes: MAX_NODES,
max_transitions: MAX_TRANSITIONS,
})?;
}
Some("flow") => {
if !branch
.get("members")
.is_some_and(serde_json::Value::is_array)
|| branch
.get("entry")
.and_then(serde_json::Value::as_str)
.is_none()
{
return Err(ManifestError::MalformedGraph);
}
}
_ => return Err(ManifestError::MalformedGraph),
}
}
}
Some("nested_flow") => {
let exits = array_len(object.get("exits"))?;
if !object
.get("members")
.is_some_and(serde_json::Value::is_array)
|| object
.get("entry")
.and_then(serde_json::Value::as_str)
.is_none()
{
return Err(ManifestError::MalformedGraph);
}
materialized_transitions = materialized_transitions.checked_add(exits).ok_or(
ManifestError::GraphOutOfBounds {
max_nodes: MAX_NODES,
max_transitions: MAX_TRANSITIONS,
},
)?;
}
_ => return Err(ManifestError::MalformedGraph),
}
ensure_graph_bounds(materialized_nodes, materialized_transitions)?;
}
Ok((materialized_nodes, materialized_transitions))
}
fn validate_repeat_manifests(
members: &serde_json::Map<String, serde_json::Value>,
) -> Result<(), ManifestError> {
let nodes = members
.get("nodes")
.and_then(serde_json::Value::as_array)
.ok_or(ManifestError::MalformedGraph)?;
let mut repeat_ids = BTreeSet::new();
for node in nodes {
validate_node_repeat(node, &mut repeat_ids)?;
}
Ok(())
}
fn validate_node_repeat(
node: &serde_json::Value,
repeat_ids: &mut BTreeSet<String>,
) -> Result<(), ManifestError> {
let object = node.as_object().ok_or(ManifestError::MalformedGraph)?;
let kind = object.get("kind").and_then(serde_json::Value::as_str);
if kind != Some("step") && object.contains_key("repeat") {
return Err(ManifestError::MalformedGraph);
}
match kind {
Some("step") => {
if let Some(repeat) = object.get("repeat") {
validate_repeat_definition(repeat, repeat_ids, 1)?;
}
}
Some("partitioned_step") => {
validate_node_repeat(
object.get("worker").ok_or(ManifestError::MalformedGraph)?,
repeat_ids,
)?;
}
Some("split") => {
let branches = object
.get("branches")
.and_then(serde_json::Value::as_array)
.ok_or(ManifestError::MalformedGraph)?;
for branch in branches {
let branch = branch.as_object().ok_or(ManifestError::MalformedGraph)?;
if branch.get("kind").and_then(serde_json::Value::as_str) == Some("linear") {
let steps = branch
.get("steps")
.and_then(serde_json::Value::as_array)
.ok_or(ManifestError::MalformedGraph)?;
for step in steps {
validate_node_repeat(step, repeat_ids)?;
}
}
}
}
_ => {}
}
Ok(())
}
fn validate_repeat_definition(
value: &serde_json::Value,
repeat_ids: &mut BTreeSet<String>,
depth: usize,
) -> Result<(), ManifestError> {
if depth > MAX_REPEAT_NESTING_DEPTH {
return Err(ManifestError::MalformedGraph);
}
let object = value.as_object().ok_or(ManifestError::MalformedGraph)?;
let allowed = ["id", "interceptors", "nested", "policy", "state"];
if object.keys().any(|key| !allowed.contains(&key.as_str())) {
return Err(ManifestError::MalformedGraph);
}
let id = object
.get("id")
.and_then(serde_json::Value::as_str)
.ok_or(ManifestError::MalformedGraph)?;
validate_token(id, DefinitionTokenKind::Repeat).map_err(|_| ManifestError::MalformedGraph)?;
if !repeat_ids.insert(id.to_owned()) {
return Err(ManifestError::MalformedGraph);
}
validate_repeat_policy(object.get("policy").ok_or(ManifestError::MalformedGraph)?)?;
validate_repeat_state(object.get("state").ok_or(ManifestError::MalformedGraph)?)?;
let interceptors = object
.get("interceptors")
.and_then(serde_json::Value::as_array)
.ok_or(ManifestError::MalformedGraph)?;
if interceptors.len() > MAX_REPEAT_INTERCEPTORS {
return Err(ManifestError::MalformedGraph);
}
let mut interceptor_ids = BTreeSet::new();
for interceptor in interceptors {
validate_repeat_interceptor(interceptor, &mut interceptor_ids)?;
}
if let Some(nested) = object.get("nested") {
validate_repeat_definition(nested, repeat_ids, depth.saturating_add(1))?;
}
Ok(())
}
fn validate_repeat_policy(value: &serde_json::Value) -> Result<(), ManifestError> {
let object = value.as_object().ok_or(ManifestError::MalformedGraph)?;
let required = ["configuration", "kind", "revision"];
if object.len() != required.len() || object.keys().any(|key| !required.contains(&key.as_str()))
{
return Err(ManifestError::MalformedGraph);
}
let configuration = object
.get("configuration")
.and_then(serde_json::Value::as_str)
.ok_or(ManifestError::MalformedGraph)?;
validate_token(configuration, DefinitionTokenKind::RepeatPolicy)
.map_err(|_| ManifestError::MalformedGraph)?;
let kind = object
.get("kind")
.and_then(serde_json::Value::as_str)
.ok_or(ManifestError::MalformedGraph)?;
validate_token(kind, DefinitionTokenKind::RepeatPolicy)
.map_err(|_| ManifestError::MalformedGraph)?;
let revision = object
.get("revision")
.and_then(serde_json::Value::as_str)
.ok_or(ManifestError::MalformedGraph)?;
ComponentRevision::new(revision.to_owned()).map_err(|_| ManifestError::MalformedGraph)?;
Ok(())
}
fn validate_repeat_interceptor(
value: &serde_json::Value,
ids: &mut BTreeSet<String>,
) -> Result<(), ManifestError> {
let object = value.as_object().ok_or(ManifestError::MalformedGraph)?;
let required = ["id", "kind", "revision"];
if object.len() != required.len() || object.keys().any(|key| !required.contains(&key.as_str()))
{
return Err(ManifestError::MalformedGraph);
}
let id = object
.get("id")
.and_then(serde_json::Value::as_str)
.ok_or(ManifestError::MalformedGraph)?;
validate_token(id, DefinitionTokenKind::RepeatInterceptor)
.map_err(|_| ManifestError::MalformedGraph)?;
if !ids.insert(id.to_owned()) {
return Err(ManifestError::MalformedGraph);
}
let kind = object
.get("kind")
.and_then(serde_json::Value::as_str)
.ok_or(ManifestError::MalformedGraph)?;
validate_token(kind, DefinitionTokenKind::RepeatInterceptor)
.map_err(|_| ManifestError::MalformedGraph)?;
let revision = object
.get("revision")
.and_then(serde_json::Value::as_str)
.ok_or(ManifestError::MalformedGraph)?;
ComponentRevision::new(revision.to_owned()).map_err(|_| ManifestError::MalformedGraph)?;
Ok(())
}
fn validate_repeat_state(value: &serde_json::Value) -> Result<(), ManifestError> {
let object = value.as_object().ok_or(ManifestError::MalformedGraph)?;
let required = ["schema", "version"];
if object.len() != required.len() || object.keys().any(|key| !required.contains(&key.as_str()))
{
return Err(ManifestError::MalformedGraph);
}
let schema = object
.get("schema")
.and_then(serde_json::Value::as_str)
.ok_or(ManifestError::MalformedGraph)?;
StateSchemaId::new(schema.to_owned()).map_err(|_| ManifestError::MalformedGraph)?;
let version = object
.get("version")
.and_then(serde_json::Value::as_u64)
.and_then(|value| u32::try_from(value).ok())
.ok_or(ManifestError::MalformedGraph)?;
StateSchemaVersion::new(version).map_err(|_| ManifestError::MalformedGraph)?;
Ok(())
}
fn validate_nested_job_manifest(
object: &serde_json::Map<String, serde_json::Value>,
) -> Result<(), ManifestError> {
if !object
.get("child")
.is_some_and(serde_json::Value::is_object)
|| !object
.get("parameters")
.is_some_and(serde_json::Value::is_array)
|| object
.get("mapping_revision")
.and_then(serde_json::Value::as_str)
.is_none()
{
return Err(ManifestError::MalformedGraph);
}
Ok(())
}
fn array_len(value: Option<&serde_json::Value>) -> Result<usize, ManifestError> {
value
.and_then(serde_json::Value::as_array)
.map(Vec::len)
.ok_or(ManifestError::MalformedGraph)
}
fn contains_float(value: &serde_json::Value) -> bool {
match value {
serde_json::Value::Number(number) => number.as_i64().is_none() && number.as_u64().is_none(),
serde_json::Value::Array(values) => values.iter().any(contains_float),
serde_json::Value::Object(members) => members.values().any(contains_float),
_ => false,
}
}
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
#[non_exhaustive]
pub enum ManifestError {
TooLarge {
max_bytes: usize,
},
MalformedJson,
NotAnObject,
NonCanonicalEncoding,
FloatValue,
MissingFormat,
UnsupportedFormat {
format: u16,
supported: u16,
},
MalformedGraph,
GraphOutOfBounds {
max_nodes: usize,
max_transitions: usize,
},
InvalidJobName,
DigestMismatch,
}
impl fmt::Display for ManifestError {
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
match self {
Self::TooLarge { max_bytes } => {
write!(formatter, "definition manifest exceeds {max_bytes} bytes")
}
Self::MalformedJson => formatter.write_str("definition manifest is not valid JSON"),
Self::NotAnObject => formatter.write_str("definition manifest is not a JSON object"),
Self::NonCanonicalEncoding => {
formatter.write_str("definition manifest is not canonically encoded")
}
Self::FloatValue => {
formatter.write_str("definition manifest contains a floating-point value")
}
Self::MissingFormat => {
formatter.write_str("definition manifest has no usable format member")
}
Self::UnsupportedFormat { format, supported } => write!(
formatter,
"definition manifest format {format} is newer than the supported format {supported}"
),
Self::MalformedGraph => {
formatter.write_str("flow manifest has no readable node and transition members")
}
Self::GraphOutOfBounds {
max_nodes,
max_transitions,
} => write!(
formatter,
"flow manifest exceeds {max_nodes} nodes or {max_transitions} transitions"
),
Self::InvalidJobName => {
formatter.write_str("definition manifest binds an invalid job name")
}
Self::DigestMismatch => {
formatter.write_str("definition manifest does not match its fingerprint")
}
}
}
}
impl Error for ManifestError {}
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
#[non_exhaustive]
pub enum DefinitionTokenKind {
Revision,
Component,
Artifact,
Upgrade,
Classifier,
Node,
Decider,
Repeat,
RepeatPolicy,
RepeatInterceptor,
}
#[derive(Clone, Debug, Eq, PartialEq)]
#[non_exhaustive]
pub enum DefinitionError {
ZeroStartLimit,
EmptyToken {
kind: DefinitionTokenKind,
},
TokenTooLong {
kind: DefinitionTokenKind,
max_bytes: usize,
},
SurroundingWhitespace {
kind: DefinitionTokenKind,
},
ControlCharacter {
kind: DefinitionTokenKind,
},
ManifestEncoding,
ManifestTooLarge {
max_bytes: usize,
},
UpgradeSelfEdge,
EmptyStepMapping,
DuplicateSourceStep,
DuplicateTargetStep,
TooManyUpgradeNodes {
max: usize,
},
TooManyUpgradeTransforms {
max: usize,
},
DuplicateSourceNode,
DuplicateTargetNode,
DuplicateStateTransform,
StateTransformOutsideNodeMapping,
UnknownUpgradeSourceNode,
MissingUpgradeSourceNode,
UnknownUpgradeTargetNode,
MissingTargetStateContract,
MissingRequiredStateTransform,
StateTransformSchemaMismatch,
LegacyUpgradeExecution,
DuplicateStateTransformer,
StateTransformerSetMismatch,
DeliveryModeMismatch,
CompatibilityLowering,
RuntimeStreamNotDeclared {
namespace: ComponentStreamIdentity,
},
DeclaredStreamMissingRuntime {
namespace: ComponentStreamIdentity,
},
DuplicateRuntimeStream {
namespace: ComponentStreamIdentity,
},
NonRestartableStream {
namespace: ComponentStreamIdentity,
},
CompletionPolicyFingerprintPanic,
CompletionPolicyFingerprintMissing,
CompletionPolicyRevisionMismatch,
}
impl fmt::Display for DefinitionError {
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
match self {
Self::ZeroStartLimit => formatter.write_str("start limit must be nonzero"),
Self::EmptyToken { kind } => write!(formatter, "{kind:?} token must not be empty"),
Self::TokenTooLong { kind, max_bytes } => {
write!(formatter, "{kind:?} token exceeds {max_bytes} bytes")
}
Self::SurroundingWhitespace { kind } => {
write!(formatter, "{kind:?} token has surrounding whitespace")
}
Self::ControlCharacter { kind } => {
write!(formatter, "{kind:?} token contains a control character")
}
Self::ManifestEncoding => formatter.write_str("definition manifest encoding failed"),
Self::ManifestTooLarge { max_bytes } => {
write!(formatter, "definition manifest exceeds {max_bytes} bytes")
}
Self::UpgradeSelfEdge => formatter.write_str("definition upgrade is a self-edge"),
Self::EmptyStepMapping => formatter.write_str("definition upgrade has no step mapping"),
Self::DuplicateSourceStep => {
formatter.write_str("definition upgrade repeats a source step")
}
Self::DuplicateTargetStep => {
formatter.write_str("definition upgrade reuses a target step")
}
Self::TooManyUpgradeNodes { max } => {
write!(
formatter,
"definition upgrade exceeds {max} mapped source nodes"
)
}
Self::TooManyUpgradeTransforms { max } => {
write!(
formatter,
"definition upgrade exceeds {max} state transforms"
)
}
Self::DuplicateSourceNode => {
formatter.write_str("definition upgrade repeats a durable source node")
}
Self::DuplicateTargetNode => {
formatter.write_str("definition upgrade reuses a durable target node")
}
Self::DuplicateStateTransform => {
formatter.write_str("definition upgrade repeats a state transform")
}
Self::StateTransformOutsideNodeMapping => {
formatter.write_str("definition state transform is outside the node mapping")
}
Self::UnknownUpgradeSourceNode => {
formatter.write_str("definition upgrade maps an unknown source node")
}
Self::MissingUpgradeSourceNode => {
formatter.write_str("definition upgrade omits a durable source node")
}
Self::UnknownUpgradeTargetNode => {
formatter.write_str("definition upgrade maps to an unknown target node")
}
Self::MissingTargetStateContract => {
formatter.write_str("definition upgrade drops a durable source state contract")
}
Self::MissingRequiredStateTransform => {
formatter.write_str("definition upgrade omits a required state schema transform")
}
Self::StateTransformSchemaMismatch => {
formatter.write_str("definition state transform disagrees with manifest schemas")
}
Self::LegacyUpgradeExecution => formatter
.write_str("legacy definition upgrade has no transformed execution contract"),
Self::DuplicateStateTransformer => {
formatter.write_str("definition upgrade repeats a live state transformer revision")
}
Self::StateTransformerSetMismatch => formatter
.write_str("definition upgrade live transformer set does not match metadata"),
Self::DeliveryModeMismatch => formatter
.write_str("fault runtime and restart contract declare different delivery modes"),
Self::CompatibilityLowering => {
formatter.write_str("one-step compatibility lowering produced an invalid plan")
}
Self::RuntimeStreamNotDeclared { namespace } => write!(
formatter,
"runtime ItemStream namespace {:?} is not declared in the chunk definition",
namespace.as_str()
),
Self::DeclaredStreamMissingRuntime { namespace } => write!(
formatter,
"declared stream namespace {:?} has no registered runtime ItemStream",
namespace.as_str()
),
Self::DuplicateRuntimeStream { namespace } => write!(
formatter,
"runtime ItemStream namespace {:?} was registered more than once",
namespace.as_str()
),
Self::NonRestartableStream { namespace } => write!(
formatter,
"ItemStream namespace {:?} declares NotRestartable, so this step cannot claim restartability",
namespace.as_str()
),
Self::CompletionPolicyFingerprintPanic => {
formatter.write_str("completion policy fingerprint() panicked")
}
Self::CompletionPolicyFingerprintMissing => formatter
.write_str("custom completion policy must provide a restart-relevant fingerprint"),
Self::CompletionPolicyRevisionMismatch => formatter.write_str(
"the step's live completion-policy fingerprint does not match the \
completion-policy revision declared in the bound component revisions",
),
}
}
}
impl Error for DefinitionError {}