use chrono::{DateTime, Utc};
use serde::{Deserialize, Serialize};
use serde_json::{Map, Value};
use crate::record::{
DecisionAction, DecisionEvent, DecisionFamily, DecisionOutcome, Effect, PayloadRef,
PolicyVersion,
};
pub const TASK_ENVELOPE_FORMAT_VERSION: u32 = 1;
pub const BASE_RETRY_DELAY_MS: u64 = 1_000;
pub const MAX_RETRY_DELAY_MS: u64 = 300_000;
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum ErrorClass {
Transient,
RateLimited,
Timeout,
InvalidInput,
DependencyFailure,
ResourceExhausted,
Cancelled,
Unknown,
}
impl ErrorClass {
pub fn is_retryable(self) -> bool {
!matches!(self, ErrorClass::InvalidInput | ErrorClass::Cancelled)
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
#[serde(tag = "decision", rename_all = "snake_case")]
pub enum RetryDecision {
Retry {
after_ms: u64,
},
Dead,
Fail,
}
pub fn backoff_delay_ms(attempt: u32, uniform: f64) -> u64 {
let exponent = attempt.saturating_sub(1).min(20);
let exponential = BASE_RETRY_DELAY_MS
.saturating_mul(1u64 << exponent)
.min(MAX_RETRY_DELAY_MS);
(uniform.clamp(0.0, 1.0) * exponential as f64) as u64
}
pub fn classify_retry(
effect: Effect,
class: ErrorClass,
attempt: u32,
max_attempts: u32,
uniform: f64,
) -> RetryDecision {
if !effect.is_freely_repeatable() || !class.is_retryable() {
return RetryDecision::Fail;
}
if attempt >= max_attempts {
return RetryDecision::Dead;
}
RetryDecision::Retry {
after_ms: backoff_delay_ms(attempt, uniform),
}
}
pub fn retry_legal_actions(
effect: Effect,
class: ErrorClass,
attempt: u32,
max_attempts: u32,
) -> Vec<DecisionAction> {
if effect.is_freely_repeatable() && class.is_retryable() && attempt < max_attempts {
vec![
DecisionAction::Retry {
attempt: attempt + 1,
},
DecisionAction::Abort,
]
} else {
vec![DecisionAction::Abort]
}
}
pub fn retry_selected_action(decision: &RetryDecision, attempt: u32) -> DecisionAction {
match decision {
RetryDecision::Retry { .. } => DecisionAction::Retry {
attempt: attempt + 1,
},
RetryDecision::Dead | RetryDecision::Fail => DecisionAction::Abort,
}
}
#[allow(clippy::too_many_arguments)]
pub fn retry_decision_event(
run_id: impl Into<String>,
thread_id: impl Into<String>,
seq: u64,
effect: Effect,
class: ErrorClass,
attempt: u32,
max_attempts: u32,
dependency_latency_ms: Option<u64>,
decision: &RetryDecision,
policy_version: &PolicyVersion,
decided_at: DateTime<Utc>,
) -> DecisionEvent {
let run_id = run_id.into();
let mut features = Map::new();
features.insert(
"failure_class".to_owned(),
serde_json::to_value(class).unwrap_or(Value::Null),
);
features.insert("attempt".to_owned(), Value::from(attempt));
features.insert("max_attempts".to_owned(), Value::from(max_attempts));
features.insert(
"effect".to_owned(),
serde_json::to_value(effect).unwrap_or(Value::Null),
);
if let Some(latency) = dependency_latency_ms {
features.insert("dependency_latency_ms".to_owned(), Value::from(latency));
}
let outcome = match decision {
RetryDecision::Retry { .. } => None,
RetryDecision::Dead | RetryDecision::Fail => Some(DecisionOutcome::Failure),
};
DecisionEvent {
id: format!("{run_id}:d{seq}"),
run_id,
thread_id: thread_id.into(),
seq,
family: DecisionFamily::Retry,
features,
legal_actions: retry_legal_actions(effect, class, attempt, max_attempts),
selected: retry_selected_action(decision, attempt),
propensity: 1.0,
policy_version: policy_version.clone(),
outcome,
decided_at,
}
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct ArtifactContract {
pub kind: String,
#[serde(default)]
pub max_bytes: Option<u64>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub schema: Option<serde_json::Value>,
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct TaskBudget {
pub max_attempts: u32,
#[serde(default)]
pub timeout_ms: Option<u64>,
}
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
pub struct TaskEnvelope {
#[serde(default = "current_envelope_version")]
pub format_version: u32,
pub task_id: String,
#[serde(default)]
pub parent: Option<String>,
pub sender: String,
pub recipient: String,
#[serde(default)]
pub worker_version: Option<String>,
pub input: PayloadRef,
#[serde(default)]
pub output_contract: Option<ArtifactContract>,
#[serde(default)]
pub deadline: Option<DateTime<Utc>>,
#[serde(default)]
pub budget: Option<TaskBudget>,
#[serde(default)]
pub idempotency_key: Option<String>,
#[serde(default = "default_effect")]
pub effect: Effect,
}
fn current_envelope_version() -> u32 {
TASK_ENVELOPE_FORMAT_VERSION
}
fn default_effect() -> Effect {
Effect::NonIdempotent
}
impl TaskEnvelope {
pub fn new(
task_id: impl Into<String>,
sender: impl Into<String>,
recipient: impl Into<String>,
input: PayloadRef,
) -> Self {
Self {
format_version: TASK_ENVELOPE_FORMAT_VERSION,
task_id: task_id.into(),
parent: None,
sender: sender.into(),
recipient: recipient.into(),
worker_version: None,
input,
output_contract: None,
deadline: None,
budget: None,
idempotency_key: None,
effect: Effect::NonIdempotent,
}
}
}
#[cfg(test)]
mod tests {
use super::*;
use serde_json::json;
#[test]
fn error_class_retryability() {
for class in [
ErrorClass::Transient,
ErrorClass::RateLimited,
ErrorClass::Timeout,
ErrorClass::DependencyFailure,
ErrorClass::ResourceExhausted,
ErrorClass::Unknown,
] {
assert!(class.is_retryable(), "{class:?} must be retryable");
}
assert!(!ErrorClass::InvalidInput.is_retryable());
assert!(!ErrorClass::Cancelled.is_retryable());
}
#[test]
fn error_class_serde_names_are_snake_case() {
assert_eq!(
serde_json::to_value(ErrorClass::DependencyFailure).unwrap(),
json!("dependency_failure")
);
assert_eq!(
serde_json::to_value(ErrorClass::ResourceExhausted).unwrap(),
json!("resource_exhausted")
);
}
#[test]
fn backoff_bounds_and_cap() {
assert_eq!(backoff_delay_ms(1, 0.0), 0);
assert_eq!(backoff_delay_ms(8, 0.0), 0);
assert!(backoff_delay_ms(1, 0.999) < BASE_RETRY_DELAY_MS);
assert!(backoff_delay_ms(2, 0.999) < 2 * BASE_RETRY_DELAY_MS);
assert_eq!(backoff_delay_ms(1, 1.0), BASE_RETRY_DELAY_MS);
assert_eq!(backoff_delay_ms(20, 1.0), MAX_RETRY_DELAY_MS);
assert_eq!(backoff_delay_ms(u32::MAX, 1.0), MAX_RETRY_DELAY_MS);
}
#[test]
fn classify_retry_gates_in_order() {
let repeatable = Effect::Idempotent;
assert_eq!(
classify_retry(Effect::NonIdempotent, ErrorClass::Transient, 1, 5, 0.5),
RetryDecision::Fail
);
assert_eq!(
classify_retry(Effect::Compensatable, ErrorClass::Timeout, 1, 5, 0.5),
RetryDecision::Fail
);
assert_eq!(
classify_retry(repeatable, ErrorClass::InvalidInput, 1, 5, 0.5),
RetryDecision::Fail
);
assert_eq!(
classify_retry(repeatable, ErrorClass::Cancelled, 1, 5, 0.5),
RetryDecision::Fail
);
assert_eq!(
classify_retry(repeatable, ErrorClass::Unknown, 5, 5, 0.5),
RetryDecision::Dead
);
assert_eq!(
classify_retry(repeatable, ErrorClass::Transient, 1, 0, 0.5),
RetryDecision::Dead
);
match classify_retry(repeatable, ErrorClass::RateLimited, 1, 5, 0.5) {
RetryDecision::Retry { after_ms } => {
assert!(after_ms <= BASE_RETRY_DELAY_MS);
}
other => panic!("expected Retry, got {other:?}"),
}
}
#[test]
fn retry_decision_serde_shape() {
assert_eq!(
serde_json::to_value(RetryDecision::Retry { after_ms: 500 }).unwrap(),
json!({"decision": "retry", "after_ms": 500})
);
assert_eq!(
serde_json::to_value(RetryDecision::Dead).unwrap(),
json!({"decision": "dead"})
);
assert_eq!(
serde_json::to_value(RetryDecision::Fail).unwrap(),
json!({"decision": "fail"})
);
}
#[test]
fn retry_legal_actions_mirror_the_classifier_gates() {
let repeatable = Effect::Idempotent;
assert_eq!(
retry_legal_actions(repeatable, ErrorClass::Timeout, 1, 3),
vec![DecisionAction::Retry { attempt: 2 }, DecisionAction::Abort]
);
assert_eq!(
retry_legal_actions(repeatable, ErrorClass::Timeout, 3, 3),
vec![DecisionAction::Abort]
);
assert_eq!(
retry_legal_actions(repeatable, ErrorClass::Transient, 1, 0),
vec![DecisionAction::Abort]
);
assert_eq!(
retry_legal_actions(repeatable, ErrorClass::InvalidInput, 1, 3),
vec![DecisionAction::Abort]
);
assert_eq!(
retry_legal_actions(Effect::NonIdempotent, ErrorClass::Transient, 1, 3),
vec![DecisionAction::Abort]
);
}
#[test]
fn retry_decision_event_selected_action_stays_inside_the_legal_set() {
let effects = [
Effect::Pure,
Effect::ReadOnly,
Effect::Idempotent,
Effect::NonIdempotent,
Effect::Compensatable,
];
let classes = [
ErrorClass::Transient,
ErrorClass::RateLimited,
ErrorClass::Timeout,
ErrorClass::InvalidInput,
ErrorClass::DependencyFailure,
ErrorClass::ResourceExhausted,
ErrorClass::Cancelled,
ErrorClass::Unknown,
];
let decided_at = DateTime::<Utc>::from_timestamp_millis(1_760_000_000_000).unwrap();
for effect in effects {
for class in classes {
for (attempt, max_attempts) in [(1, 3), (3, 3), (1, 0)] {
let decision = classify_retry(effect, class, attempt, max_attempts, 0.5);
let event = retry_decision_event(
"run-1",
"thread-1",
1,
effect,
class,
attempt,
max_attempts,
None,
&decision,
&PolicyVersion::default(),
decided_at,
);
assert!(
event.legal_actions.contains(&event.selected),
"selected {:?} must be inside legal set {:?} for {effect:?}/{class:?} \
attempt {attempt}/{max_attempts}",
event.selected,
event.legal_actions,
);
}
}
}
}
#[test]
fn retry_decision_event_shape_and_outcome_mapping() {
let decided_at = DateTime::<Utc>::from_timestamp_millis(1_760_000_000_000).unwrap();
let decision = classify_retry(Effect::Idempotent, ErrorClass::Timeout, 1, 3, 0.5);
let event = retry_decision_event(
"run-9",
"thread-2",
3,
Effect::Idempotent,
ErrorClass::Timeout,
1,
3,
Some(840),
&decision,
&PolicyVersion::new("policy-0123456789ab"),
decided_at,
);
assert_eq!(event.id, "run-9:d3");
assert_eq!(event.seq, 3);
assert_eq!(event.family, DecisionFamily::Retry);
assert_eq!(
event.selected,
DecisionAction::Retry { attempt: 2 },
"the failure was attempt 1; the decision schedules attempt 2"
);
assert_eq!(event.propensity, 1.0);
assert!(event.outcome.is_none());
assert_eq!(event.features.get("failure_class"), Some(&json!("timeout")));
assert_eq!(event.features.get("attempt"), Some(&json!(1)));
assert_eq!(
event.features.get("dependency_latency_ms"),
Some(&json!(840))
);
let dead = retry_decision_event(
"run-9",
"thread-2",
4,
Effect::Idempotent,
ErrorClass::Timeout,
3,
3,
None,
&RetryDecision::Dead,
&PolicyVersion::default(),
decided_at,
);
assert_eq!(dead.selected, DecisionAction::Abort);
assert_eq!(dead.outcome, Some(DecisionOutcome::Failure));
assert!(
!dead.features.contains_key("dependency_latency_ms"),
"an unmeasured latency is absent from the wire, not null"
);
}
#[test]
fn envelope_minimal_json_deserializes_with_defaults() {
let minimal = json!({
"task_id": "t-1",
"sender": "run-9",
"recipient": "pool-default",
"input": {"kind": "inline", "value": {"n": 1}},
});
let envelope: TaskEnvelope = serde_json::from_value(minimal).unwrap();
assert_eq!(envelope.format_version, TASK_ENVELOPE_FORMAT_VERSION);
assert_eq!(envelope.effect, Effect::NonIdempotent);
assert!(envelope.parent.is_none());
assert!(envelope.idempotency_key.is_none());
assert!(envelope.worker_version.is_none());
assert!(envelope.deadline.is_none());
assert!(envelope.budget.is_none());
}
#[test]
fn envelope_serde_roundtrip() {
let mut envelope = TaskEnvelope::new(
"t-7",
"run-9:node-a",
"pool-default",
PayloadRef::inline(json!({"n": 1})),
);
envelope.parent = Some("run-9:3".into());
envelope.output_contract = Some(ArtifactContract {
kind: "application/json".into(),
max_bytes: Some(65_536),
schema: None,
});
envelope.deadline = DateTime::<Utc>::from_timestamp_millis(1_760_000_000_000);
envelope.budget = Some(TaskBudget {
max_attempts: 5,
timeout_ms: Some(30_000),
});
envelope.idempotency_key = Some("run-9:charge:7".into());
envelope.worker_version = Some("activity-worker/1.4.0".into());
envelope.effect = Effect::Idempotent;
let back: TaskEnvelope =
serde_json::from_str(&serde_json::to_string(&envelope).unwrap()).unwrap();
assert_eq!(envelope, back);
}
}