rusty_agent_runtime/durable.rs
1//! Durable Work contracts (R0.6): the shared retry taxonomy and task envelope.
2//!
3//! This module freezes the wire shapes that the R0.6 durable-activity system
4//! builds on — the Postgres task queue in `rusty-agent-server`, the worker SDK in
5//! `rusty-worker`, and the scheduler between them. Nothing here performs I/O
6//! or scheduling; these are pure data contracts plus the one policy function
7//! both sides must agree on ([`classify_retry`]).
8//!
9//! The two pillars:
10//!
11//! - [`ErrorClass`] + [`RetryDecision`] — the retry taxonomy. Every failed
12//! task attempt is classified into a closed set of error classes; the
13//! class, the declared [`Effect`] of the work, and the attempt ordinal map
14//! to exactly one decision: retry after a delay, dead-letter, or fail.
15//! - [`TaskEnvelope`] — the unit of durable work: who sent it, who executes
16//! it, the input reference, the artifact contract, the deadline, the
17//! budget, and the idempotency key that makes retry *effective-once*
18//! rather than duplicated.
19//!
20//! The composition rule that makes retry safe comes from the Flight
21//! Recorder: [`classify_retry`] refuses to silently retry any effect that is
22//! not [`Effect::is_freely_repeatable`], so an `Idempotent` declaration —
23//! with a stable idempotency key — is what unlocks automatic retry.
24//!
25//! Golden-file tests under `tests/golden/` pin the serialized shapes; any
26//! accidental contract drift fails CI. To bless an intentional change,
27//! re-run with `UPDATE_GOLDEN=1` and review the diff.
28
29use chrono::{DateTime, Utc};
30use serde::{Deserialize, Serialize};
31use serde_json::{Map, Value};
32
33use crate::record::{
34 DecisionAction, DecisionEvent, DecisionFamily, DecisionOutcome, Effect, PayloadRef,
35 PolicyVersion,
36};
37
38/// The current wire-format version of [`TaskEnvelope`].
39///
40/// Bump only on a breaking change to the envelope; additive evolution uses
41/// serde defaults instead so previously queued tasks keep deserializing.
42pub const TASK_ENVELOPE_FORMAT_VERSION: u32 = 1;
43
44/// The base delay of the retry backoff schedule, in milliseconds.
45///
46/// Retry `n` (1-based) draws a delay uniformly from
47/// `[0, BASE_RETRY_DELAY_MS * 2^(n-1)]`, capped at [`MAX_RETRY_DELAY_MS`].
48pub const BASE_RETRY_DELAY_MS: u64 = 1_000;
49
50/// The cap of the retry backoff schedule, in milliseconds (5 minutes).
51///
52/// The cap bounds worst-case queue latency for a stuck dependency while
53/// keeping retry pressure negligible: at the cap, one task retries at most
54/// 12 times per hour.
55pub const MAX_RETRY_DELAY_MS: u64 = 300_000;
56
57/// Why a task attempt failed. Closed set; the scheduler matches exhaustively
58/// on it, and new classes are additive (old senders still deserialize).
59///
60/// The class is declared by the executor of the work — the worker that ran
61/// the handler or the transport that carried it — not inferred from logs.
62/// Each variant documents its retry semantics; the mechanical mapping lives
63/// in [`classify_retry`].
64#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)]
65#[serde(rename_all = "snake_case")]
66pub enum ErrorClass {
67 /// A transient fault with no lasting cause: connection reset, broken
68 /// pipe, a dependency's internal hiccup. Retry with backoff; the same
69 /// request is expected to succeed later.
70 Transient,
71
72 /// The callee asked to be slowed down (HTTP 429, `Retry-After`).
73 /// Retry with backoff; a callee-supplied `Retry-After` value floors the
74 /// delay (applied by the scheduler, outside this contract).
75 RateLimited,
76
77 /// The attempt did not finish within its deadline. Retry with backoff,
78 /// but note the ambiguity: the work may have partially executed, which
79 /// is exactly why the [`Effect`] gate in [`classify_retry`] exists — a
80 /// timed-out non-idempotent effect must not be re-attempted silently.
81 Timeout,
82
83 /// The input is malformed or violates the callee's contract (HTTP 400,
84 /// schema validation). Never retried: the same bytes will fail the same
85 /// way on every attempt. Fails the task immediately.
86 InvalidInput,
87
88 /// An upstream dependency the task needs is down or degraded (database
89 /// unreachable, model endpoint 5xx). Retry with backoff; distinct from
90 /// [`ErrorClass::Transient`] so operators can distinguish "our wiring"
91 /// from "their outage" in telemetry and alerting.
92 DependencyFailure,
93
94 /// The worker or the callee is out of capacity: memory pressure,
95 /// connection-pool exhaustion, quota exhaustion. Retry with backoff —
96 /// ideally on a different worker (placement is the scheduler's concern;
97 /// this contract only carries the classification).
98 ResourceExhausted,
99
100 /// The attempt ended because the task was cancelled. Control flow, not
101 /// a failure: never retried, never dead-lettered. Cancellation
102 /// propagation (wave 2) classifies interrupted attempts this way so the
103 /// retry machinery stays out of the cancellation path.
104 Cancelled,
105
106 /// The failure could not be classified — the handler returned an
107 /// unclassified error, or the worker died mid-attempt (lease expiry
108 /// classifies here). Retried with backoff up to the attempt limit, then
109 /// dead-lettered: unknown failures are the DLQ's primary input, since
110 /// they are the ones that need human eyes.
111 Unknown,
112}
113
114impl ErrorClass {
115 /// Whether this class is retryable at all. `InvalidInput` and
116 /// `Cancelled` are the only non-retryable classes; everything else —
117 /// including `Unknown` — may be re-attempted up to the attempt limit.
118 ///
119 /// Retryability of the class is necessary but not sufficient for a
120 /// retry: [`classify_retry`] additionally gates on the work's declared
121 /// [`Effect`].
122 pub fn is_retryable(self) -> bool {
123 !matches!(self, ErrorClass::InvalidInput | ErrorClass::Cancelled)
124 }
125}
126
127/// What the scheduler does with a failed task attempt.
128///
129/// Serialized with internal tagging (`{"decision": "retry", "after_ms": …}`)
130/// so a decision recorded in the journal or the queue's attempt log is
131/// self-describing.
132#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
133#[serde(tag = "decision", rename_all = "snake_case")]
134pub enum RetryDecision {
135 /// Re-queue the task to become visible again after `after_ms`
136 /// milliseconds. The idempotency key and input are unchanged; the
137 /// attempt counter increments.
138 Retry {
139 /// Delay before the task becomes visible to workers again, in
140 /// milliseconds. Computed by [`backoff_delay_ms`].
141 after_ms: u64,
142 },
143
144 /// Move the task to the dead-letter queue: a retryable failure class
145 /// whose attempts are exhausted, or an `Unknown` failure that kept
146 /// recurring. DLQ entries are operator-visible evidence — they carry the
147 /// full attempt history and can be re-driven by hand after the cause is
148 /// fixed.
149 Dead,
150
151 /// Fail the task immediately: a non-retryable class, or any failure of
152 /// work whose declared [`Effect`] is not freely repeatable. The error
153 /// surfaces to the sender; nothing is retried and nothing is
154 /// dead-lettered (there is nothing a human can fix by re-driving the
155 /// same input).
156 Fail,
157}
158
159/// Full-jitter exponential backoff, in milliseconds.
160///
161/// Retry `attempt` (1-based — `1` is the first retry after the initial
162/// failure) draws uniformly from `[0, base * 2^(attempt-1)]` where `base` is
163/// [`BASE_RETRY_DELAY_MS`], capped at [`MAX_RETRY_DELAY_MS`]. Full jitter —
164/// rather than a fixed fraction of the exponential — is what decorrelates a
165/// fleet of tasks that failed together when a shared dependency recovers
166/// (the thundering-herd problem); see the design doc for the citation.
167///
168/// `uniform` is a sample from `[0, 1)`; values outside are clamped. Source
169/// it from the run's `RngSource` so seeded runs reproduce their retry
170/// schedules exactly.
171pub fn backoff_delay_ms(attempt: u32, uniform: f64) -> u64 {
172 let exponent = attempt.saturating_sub(1).min(20);
173 let exponential = BASE_RETRY_DELAY_MS
174 .saturating_mul(1u64 << exponent)
175 .min(MAX_RETRY_DELAY_MS);
176 (uniform.clamp(0.0, 1.0) * exponential as f64) as u64
177}
178
179/// Map a failed attempt to exactly one [`RetryDecision`].
180///
181/// This is the single place the retry policy lives, shared verbatim by the
182/// server scheduler and the worker SDK so both sides of the queue always
183/// agree. The order of the gates is the policy:
184///
185/// 1. **Effect gate.** Work whose declared [`Effect`] is not
186/// [`Effect::is_freely_repeatable`] is never silently retried — a timed
187/// out `NonIdempotent` charge may already have happened. This is the
188/// composition with the Flight Recorder: the `Idempotent`
189/// classification is what makes retry safe, and it is checked here,
190/// not assumed.
191/// 2. **Class gate.** `InvalidInput` and `Cancelled` fail immediately; all
192/// other classes are retryable ([`ErrorClass::is_retryable`]).
193/// 3. **Attempt gate.** A retryable failure with `attempt >= max_attempts`
194/// goes to the dead-letter queue (`attempt` counts attempts made so far,
195/// starting at 1 for the first failure; `max_attempts == 0` means no
196/// retries at all).
197/// 4. Otherwise: retry, with the delay from [`backoff_delay_ms`].
198pub fn classify_retry(
199 effect: Effect,
200 class: ErrorClass,
201 attempt: u32,
202 max_attempts: u32,
203 uniform: f64,
204) -> RetryDecision {
205 if !effect.is_freely_repeatable() || !class.is_retryable() {
206 return RetryDecision::Fail;
207 }
208 if attempt >= max_attempts {
209 return RetryDecision::Dead;
210 }
211 RetryDecision::Retry {
212 after_ms: backoff_delay_ms(attempt, uniform),
213 }
214}
215
216/// The closed legal-action set of a retry decision (R0.8 wave 4).
217///
218/// Mirrors [`classify_retry`]'s gates exactly: when a retry is legal
219/// (freely-repeatable effect, retryable class, attempt budget remaining) the
220/// legal set is `[Retry { attempt: attempt + 1 }, Abort]`; otherwise retrying
221/// is not a legal action and the set collapses to `[Abort]`. The set is
222/// computed from the same inputs as the decision itself, so a
223/// [`DecisionEvent`] built by [`retry_decision_event`] can never record a
224/// selected action outside its legal set.
225pub fn retry_legal_actions(
226 effect: Effect,
227 class: ErrorClass,
228 attempt: u32,
229 max_attempts: u32,
230) -> Vec<DecisionAction> {
231 if effect.is_freely_repeatable() && class.is_retryable() && attempt < max_attempts {
232 vec![
233 DecisionAction::Retry {
234 attempt: attempt + 1,
235 },
236 DecisionAction::Abort,
237 ]
238 } else {
239 vec![DecisionAction::Abort]
240 }
241}
242
243/// The [`DecisionAction`] a [`RetryDecision`] corresponds to, given the
244/// 1-based attempt ordinal that failed.
245///
246/// A `Retry` decision takes retry ordinal `attempt + 1` (the failure was
247/// attempt `attempt`; the decision schedules the next one). `Dead` and
248/// `Fail` both map to `Abort`: dead-lettering is giving up on the automatic
249/// path, and the dead-letter queue entry is the queue's own evidence, not a
250/// decision action.
251pub fn retry_selected_action(decision: &RetryDecision, attempt: u32) -> DecisionAction {
252 match decision {
253 RetryDecision::Retry { .. } => DecisionAction::Retry {
254 attempt: attempt + 1,
255 },
256 RetryDecision::Dead | RetryDecision::Fail => DecisionAction::Abort,
257 }
258}
259
260/// Build the [`DecisionEvent`] for one retry decision (R0.8 wave 4).
261///
262/// This is the emission contract between the scheduler (which owns the
263/// decision inputs) and the journal: given exactly the values
264/// [`classify_retry`] was called with plus its output, produce the evidence
265/// record. Features pin the observation vocabulary — `failure_class`,
266/// `attempt`, `max_attempts`, `effect`, and `dependency_latency_ms` when the
267/// caller measured one — so offline evaluation reads stable keys.
268///
269/// **Propensity honesty.** Every v1 policy — including the static floor —
270/// is deterministic: it assigns probability 1 to the selected action and 0
271/// to every other legal action. The event therefore records `propensity:
272/// 1.0`. That is the truthful value, but it also means v1's evidence cannot
273/// support inverse-propensity scoring (division by a zero propensity is
274/// undefined): off-policy evaluation over v1 decisions is restricted to
275/// policies that would have taken the same action. Learned stochastic
276/// policies must record their true propensity at decision time; this
277/// function's `1.0` is the deterministic-policy degenerate case, not a
278/// placeholder.
279///
280/// `outcome` is `None` for a `Retry` decision (the re-attempt has not
281/// happened yet) and `Some(DecisionOutcome::Failure)` for `Dead`/`Fail`
282/// (the operation is over and did not complete).
283#[allow(clippy::too_many_arguments)]
284pub fn retry_decision_event(
285 run_id: impl Into<String>,
286 thread_id: impl Into<String>,
287 seq: u64,
288 effect: Effect,
289 class: ErrorClass,
290 attempt: u32,
291 max_attempts: u32,
292 dependency_latency_ms: Option<u64>,
293 decision: &RetryDecision,
294 policy_version: &PolicyVersion,
295 decided_at: DateTime<Utc>,
296) -> DecisionEvent {
297 let run_id = run_id.into();
298 let mut features = Map::new();
299 features.insert(
300 "failure_class".to_owned(),
301 serde_json::to_value(class).unwrap_or(Value::Null),
302 );
303 features.insert("attempt".to_owned(), Value::from(attempt));
304 features.insert("max_attempts".to_owned(), Value::from(max_attempts));
305 features.insert(
306 "effect".to_owned(),
307 serde_json::to_value(effect).unwrap_or(Value::Null),
308 );
309 if let Some(latency) = dependency_latency_ms {
310 features.insert("dependency_latency_ms".to_owned(), Value::from(latency));
311 }
312 let outcome = match decision {
313 RetryDecision::Retry { .. } => None,
314 RetryDecision::Dead | RetryDecision::Fail => Some(DecisionOutcome::Failure),
315 };
316 DecisionEvent {
317 id: format!("{run_id}:d{seq}"),
318 run_id,
319 thread_id: thread_id.into(),
320 seq,
321 family: DecisionFamily::Retry,
322 features,
323 legal_actions: retry_legal_actions(effect, class, attempt, max_attempts),
324 selected: retry_selected_action(decision, attempt),
325 propensity: 1.0,
326 policy_version: policy_version.clone(),
327 outcome,
328 decided_at,
329 }
330}
331
332/// The artifact contract of a [`TaskEnvelope`]: what the recipient is
333/// expected to produce and the sender to consume.
334///
335/// v1 carries a kind identifier and an optional size bound only — enough
336/// for the recipient to know it is producing the right *shape* of artifact
337/// and for the queue to refuse to store an out-of-contract result. R0.7
338/// (Agent Fabric wave 1) adds the optional `schema` field — additive with a
339/// serde default, omitted from the wire when unset, so pre-R0.7 contracts
340/// and readers see no shape change.
341#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
342pub struct ArtifactContract {
343 /// Artifact kind identifier — a media type (`application/json`), a
344 /// schema URI, or an application-defined kind name. Interpreted by the
345 /// sender and recipient; the queue treats it as opaque.
346 pub kind: String,
347
348 /// Upper bound on the serialized artifact size in bytes. `None` means
349 /// unbounded within the queue's own storage limits.
350 #[serde(default)]
351 pub max_bytes: Option<u64>,
352
353 /// Optional JSON Schema (draft 2020-12, per the design's open-questions
354 /// default) the artifact payload must validate against (R0.7). Declared
355 /// on the contract so an unacceptable message can fail fast as
356 /// [`ErrorClass::InvalidInput`] at submission — never retried — instead
357 /// of dead-lettering after the attempt budget is spent. `None` means
358 /// kind-and-size checking only, exactly the pre-R0.7 semantics.
359 ///
360 /// Wave 1 pins the field's *shape* (stored and golden-tested); payload
361 /// validation against it is wired at the mailbox submission path in a
362 /// later wave — the schema dialect constraint (design open question 5)
363 /// settles first.
364 #[serde(default, skip_serializing_if = "Option::is_none")]
365 pub schema: Option<serde_json::Value>,
366}
367
368/// The budget a task may consume across all of its attempts.
369#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
370pub struct TaskBudget {
371 /// Maximum number of attempts, counting the initial one. Once reached,
372 /// a retryable failure dead-letters instead of retrying (see
373 /// [`classify_retry`]).
374 pub max_attempts: u32,
375
376 /// Per-attempt wall-clock timeout in milliseconds. An attempt that
377 /// exceeds it is classified [`ErrorClass::Timeout`] by the worker.
378 /// `None` defers to the queue's default.
379 #[serde(default)]
380 pub timeout_ms: Option<u64>,
381}
382
383/// The unit of durable work: one task handed from a sender to a recipient
384/// through the queue.
385///
386/// The envelope is everything the scheduler and the recipient need and
387/// nothing they don't — the input travels as a [`PayloadRef`] (inline for
388/// small values, content-addressed above the inline threshold), so the
389/// queue row stays cheap to scan and the envelope composes with the Flight
390/// Recorder's artifact addressing.
391///
392/// Every field after `format_version` that was added after v1 carries a
393/// serde default, so envelopes queued by an older client keep deserializing;
394/// the default [`Effect`] is the conservative `NonIdempotent`, meaning an
395/// undeclared task is never silently retried.
396#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
397pub struct TaskEnvelope {
398 /// Envelope format version; [`TASK_ENVELOPE_FORMAT_VERSION`] for
399 /// anything written now.
400 #[serde(default = "current_envelope_version")]
401 pub format_version: u32,
402
403 /// Unique task id, assigned by the sender. Server-side ids are
404 /// tenant-namespaced like every other id (`{tenant}/…`); the envelope
405 /// itself is namespace-agnostic.
406 pub task_id: String,
407
408 /// The causal parent: the run event or task that created this one
409 /// (`{run_id}:{seq}` or a task id). Task lineage composes with the
410 /// journal's causal chain, so a fan-out of durable tasks is one
411 /// connected evidence tree. `None` for root tasks.
412 #[serde(default)]
413 pub parent: Option<String>,
414
415 /// Who submitted the task — a run id, a node name, or an agent
416 /// identity. Used for attribution, quotas, and cancellation scoping.
417 pub sender: String,
418
419 /// Who executes the task: a worker pool name (the queue routes by pool)
420 /// or a specific worker identity for pinned placement.
421 pub recipient: String,
422
423 /// Version pin (R0.6 wave 3): the exact worker version string this task
424 /// may be dispatched to. A run started against worker version `w1`
425 /// stamps its tasks with `w1`, so a mid-run deploy never changes
426 /// semantics under an in-flight execution — the scheduler keeps handing
427 /// the task to `w1`-advertising workers until it finishes.
428 ///
429 /// The pin is an *exact string match*, deliberately: semver range
430 /// matching (`^1.4`) makes a claim's outcome depend on the version
431 /// grammar's rules rather than on what the run actually recorded, and
432 /// range resolution belongs to the scheduler's policy, not to a frozen
433 /// wire contract. Ranges are documented future work; `None` (the
434 /// default) means unpinned — any worker may claim the task.
435 #[serde(default)]
436 pub worker_version: Option<String>,
437
438 /// The task input, inline or content-addressed. Small inputs travel
439 /// inside the queue row; large ones are artifacts the recipient resolves
440 /// through the journal's artifact map.
441 pub input: PayloadRef,
442
443 /// What the task is expected to produce. `None` means the result is an
444 /// unconstrained JSON value (the common case for node executions).
445 #[serde(default)]
446 pub output_contract: Option<ArtifactContract>,
447
448 /// Wall-clock deadline for the whole task, across all attempts. The
449 /// scheduler does not re-queue a task whose deadline has passed; the
450 /// worker treats an expired deadline as [`ErrorClass::Cancelled`].
451 #[serde(default)]
452 pub deadline: Option<DateTime<Utc>>,
453
454 /// Attempt and timeout budget. `None` means the queue's defaults.
455 #[serde(default)]
456 pub budget: Option<TaskBudget>,
457
458 /// The deduplication key for effectively-once execution: the queue
459 /// refuses a duplicate submission with an existing key, and the
460 /// recipient passes it to the effect it performs. Required for any task
461 /// whose `effect` is [`Effect::Idempotent`] — the key is what the
462 /// idempotency declaration *means* at the wire. `None` is honest only
463 /// for `Pure` / `ReadOnly` work.
464 #[serde(default)]
465 pub idempotency_key: Option<String>,
466
467 /// The declared effect classification of the work this task performs —
468 /// the same taxonomy the Flight Recorder journals. Defaults to the
469 /// conservative `NonIdempotent`, so a task that does not declare its
470 /// effect is never silently retried; declaring `Idempotent` (with an
471 /// idempotency key) is what unlocks automatic retry.
472 #[serde(default = "default_effect")]
473 pub effect: Effect,
474}
475
476fn current_envelope_version() -> u32 {
477 TASK_ENVELOPE_FORMAT_VERSION
478}
479
480fn default_effect() -> Effect {
481 Effect::NonIdempotent
482}
483
484impl TaskEnvelope {
485 /// A minimal envelope: sender, recipient, and input, with every optional
486 /// field unset and the effect conservatively [`Effect::NonIdempotent`].
487 /// Set the optional fields directly; they are public.
488 pub fn new(
489 task_id: impl Into<String>,
490 sender: impl Into<String>,
491 recipient: impl Into<String>,
492 input: PayloadRef,
493 ) -> Self {
494 Self {
495 format_version: TASK_ENVELOPE_FORMAT_VERSION,
496 task_id: task_id.into(),
497 parent: None,
498 sender: sender.into(),
499 recipient: recipient.into(),
500 worker_version: None,
501 input,
502 output_contract: None,
503 deadline: None,
504 budget: None,
505 idempotency_key: None,
506 effect: Effect::NonIdempotent,
507 }
508 }
509}
510
511#[cfg(test)]
512mod tests {
513 use super::*;
514 use serde_json::json;
515
516 #[test]
517 fn error_class_retryability() {
518 for class in [
519 ErrorClass::Transient,
520 ErrorClass::RateLimited,
521 ErrorClass::Timeout,
522 ErrorClass::DependencyFailure,
523 ErrorClass::ResourceExhausted,
524 ErrorClass::Unknown,
525 ] {
526 assert!(class.is_retryable(), "{class:?} must be retryable");
527 }
528 assert!(!ErrorClass::InvalidInput.is_retryable());
529 assert!(!ErrorClass::Cancelled.is_retryable());
530 }
531
532 #[test]
533 fn error_class_serde_names_are_snake_case() {
534 assert_eq!(
535 serde_json::to_value(ErrorClass::DependencyFailure).unwrap(),
536 json!("dependency_failure")
537 );
538 assert_eq!(
539 serde_json::to_value(ErrorClass::ResourceExhausted).unwrap(),
540 json!("resource_exhausted")
541 );
542 }
543
544 #[test]
545 fn backoff_bounds_and_cap() {
546 // Zero jitter always yields zero delay.
547 assert_eq!(backoff_delay_ms(1, 0.0), 0);
548 assert_eq!(backoff_delay_ms(8, 0.0), 0);
549 // First retry tops out at the base delay; later retries grow
550 // exponentially and clamp at the 5-minute cap.
551 assert!(backoff_delay_ms(1, 0.999) < BASE_RETRY_DELAY_MS);
552 assert!(backoff_delay_ms(2, 0.999) < 2 * BASE_RETRY_DELAY_MS);
553 assert_eq!(backoff_delay_ms(1, 1.0), BASE_RETRY_DELAY_MS);
554 assert_eq!(backoff_delay_ms(20, 1.0), MAX_RETRY_DELAY_MS);
555 // A saturating attempt ordinal cannot overflow.
556 assert_eq!(backoff_delay_ms(u32::MAX, 1.0), MAX_RETRY_DELAY_MS);
557 }
558
559 #[test]
560 fn classify_retry_gates_in_order() {
561 let repeatable = Effect::Idempotent;
562
563 // Gate 1: a non-freely-repeatable effect is never silently retried,
564 // even for an eminently retryable class.
565 assert_eq!(
566 classify_retry(Effect::NonIdempotent, ErrorClass::Transient, 1, 5, 0.5),
567 RetryDecision::Fail
568 );
569 assert_eq!(
570 classify_retry(Effect::Compensatable, ErrorClass::Timeout, 1, 5, 0.5),
571 RetryDecision::Fail
572 );
573
574 // Gate 2: non-retryable classes fail immediately.
575 assert_eq!(
576 classify_retry(repeatable, ErrorClass::InvalidInput, 1, 5, 0.5),
577 RetryDecision::Fail
578 );
579 assert_eq!(
580 classify_retry(repeatable, ErrorClass::Cancelled, 1, 5, 0.5),
581 RetryDecision::Fail
582 );
583
584 // Gate 3: exhausting the attempt budget dead-letters.
585 assert_eq!(
586 classify_retry(repeatable, ErrorClass::Unknown, 5, 5, 0.5),
587 RetryDecision::Dead
588 );
589 assert_eq!(
590 classify_retry(repeatable, ErrorClass::Transient, 1, 0, 0.5),
591 RetryDecision::Dead
592 );
593
594 // Otherwise: retry with a jittered delay within the bound.
595 match classify_retry(repeatable, ErrorClass::RateLimited, 1, 5, 0.5) {
596 RetryDecision::Retry { after_ms } => {
597 assert!(after_ms <= BASE_RETRY_DELAY_MS);
598 }
599 other => panic!("expected Retry, got {other:?}"),
600 }
601 }
602
603 #[test]
604 fn retry_decision_serde_shape() {
605 assert_eq!(
606 serde_json::to_value(RetryDecision::Retry { after_ms: 500 }).unwrap(),
607 json!({"decision": "retry", "after_ms": 500})
608 );
609 assert_eq!(
610 serde_json::to_value(RetryDecision::Dead).unwrap(),
611 json!({"decision": "dead"})
612 );
613 assert_eq!(
614 serde_json::to_value(RetryDecision::Fail).unwrap(),
615 json!({"decision": "fail"})
616 );
617 }
618
619 #[test]
620 fn retry_legal_actions_mirror_the_classifier_gates() {
621 let repeatable = Effect::Idempotent;
622
623 // A retryable failure with budget remaining: retry or abort.
624 assert_eq!(
625 retry_legal_actions(repeatable, ErrorClass::Timeout, 1, 3),
626 vec![DecisionAction::Retry { attempt: 2 }, DecisionAction::Abort]
627 );
628 // Budget exhausted: retrying is not a legal action.
629 assert_eq!(
630 retry_legal_actions(repeatable, ErrorClass::Timeout, 3, 3),
631 vec![DecisionAction::Abort]
632 );
633 assert_eq!(
634 retry_legal_actions(repeatable, ErrorClass::Transient, 1, 0),
635 vec![DecisionAction::Abort]
636 );
637 // Non-retryable class or non-repeatable effect: abort only.
638 assert_eq!(
639 retry_legal_actions(repeatable, ErrorClass::InvalidInput, 1, 3),
640 vec![DecisionAction::Abort]
641 );
642 assert_eq!(
643 retry_legal_actions(Effect::NonIdempotent, ErrorClass::Transient, 1, 3),
644 vec![DecisionAction::Abort]
645 );
646 }
647
648 #[test]
649 fn retry_decision_event_selected_action_stays_inside_the_legal_set() {
650 // Property: for every combination the classifier can see, the event's
651 // selected action is a member of its legal set.
652 let effects = [
653 Effect::Pure,
654 Effect::ReadOnly,
655 Effect::Idempotent,
656 Effect::NonIdempotent,
657 Effect::Compensatable,
658 ];
659 let classes = [
660 ErrorClass::Transient,
661 ErrorClass::RateLimited,
662 ErrorClass::Timeout,
663 ErrorClass::InvalidInput,
664 ErrorClass::DependencyFailure,
665 ErrorClass::ResourceExhausted,
666 ErrorClass::Cancelled,
667 ErrorClass::Unknown,
668 ];
669 let decided_at = DateTime::<Utc>::from_timestamp_millis(1_760_000_000_000).unwrap();
670 for effect in effects {
671 for class in classes {
672 for (attempt, max_attempts) in [(1, 3), (3, 3), (1, 0)] {
673 let decision = classify_retry(effect, class, attempt, max_attempts, 0.5);
674 let event = retry_decision_event(
675 "run-1",
676 "thread-1",
677 1,
678 effect,
679 class,
680 attempt,
681 max_attempts,
682 None,
683 &decision,
684 &PolicyVersion::default(),
685 decided_at,
686 );
687 assert!(
688 event.legal_actions.contains(&event.selected),
689 "selected {:?} must be inside legal set {:?} for {effect:?}/{class:?} \
690 attempt {attempt}/{max_attempts}",
691 event.selected,
692 event.legal_actions,
693 );
694 }
695 }
696 }
697 }
698
699 #[test]
700 fn retry_decision_event_shape_and_outcome_mapping() {
701 let decided_at = DateTime::<Utc>::from_timestamp_millis(1_760_000_000_000).unwrap();
702 let decision = classify_retry(Effect::Idempotent, ErrorClass::Timeout, 1, 3, 0.5);
703 let event = retry_decision_event(
704 "run-9",
705 "thread-2",
706 3,
707 Effect::Idempotent,
708 ErrorClass::Timeout,
709 1,
710 3,
711 Some(840),
712 &decision,
713 &PolicyVersion::new("policy-0123456789ab"),
714 decided_at,
715 );
716 assert_eq!(event.id, "run-9:d3");
717 assert_eq!(event.seq, 3);
718 assert_eq!(event.family, DecisionFamily::Retry);
719 assert_eq!(
720 event.selected,
721 DecisionAction::Retry { attempt: 2 },
722 "the failure was attempt 1; the decision schedules attempt 2"
723 );
724 // Deterministic v1 policies record the degenerate propensity, 1.0.
725 assert_eq!(event.propensity, 1.0);
726 // A retry decision has no outcome until the re-attempt completes.
727 assert!(event.outcome.is_none());
728 assert_eq!(event.features.get("failure_class"), Some(&json!("timeout")));
729 assert_eq!(event.features.get("attempt"), Some(&json!(1)));
730 assert_eq!(
731 event.features.get("dependency_latency_ms"),
732 Some(&json!(840))
733 );
734
735 // Terminal decisions record the failure outcome immediately.
736 let dead = retry_decision_event(
737 "run-9",
738 "thread-2",
739 4,
740 Effect::Idempotent,
741 ErrorClass::Timeout,
742 3,
743 3,
744 None,
745 &RetryDecision::Dead,
746 &PolicyVersion::default(),
747 decided_at,
748 );
749 assert_eq!(dead.selected, DecisionAction::Abort);
750 assert_eq!(dead.outcome, Some(DecisionOutcome::Failure));
751 assert!(
752 !dead.features.contains_key("dependency_latency_ms"),
753 "an unmeasured latency is absent from the wire, not null"
754 );
755 }
756
757 #[test]
758 fn envelope_minimal_json_deserializes_with_defaults() {
759 // A v1 envelope carrying only the required fields — the shape an
760 // older, smaller client would write — loads with honest defaults:
761 // current format version, no options set, conservative effect.
762 let minimal = json!({
763 "task_id": "t-1",
764 "sender": "run-9",
765 "recipient": "pool-default",
766 "input": {"kind": "inline", "value": {"n": 1}},
767 });
768 let envelope: TaskEnvelope = serde_json::from_value(minimal).unwrap();
769 assert_eq!(envelope.format_version, TASK_ENVELOPE_FORMAT_VERSION);
770 assert_eq!(envelope.effect, Effect::NonIdempotent);
771 assert!(envelope.parent.is_none());
772 assert!(envelope.idempotency_key.is_none());
773 assert!(envelope.worker_version.is_none());
774 assert!(envelope.deadline.is_none());
775 assert!(envelope.budget.is_none());
776 }
777
778 #[test]
779 fn envelope_serde_roundtrip() {
780 let mut envelope = TaskEnvelope::new(
781 "t-7",
782 "run-9:node-a",
783 "pool-default",
784 PayloadRef::inline(json!({"n": 1})),
785 );
786 envelope.parent = Some("run-9:3".into());
787 envelope.output_contract = Some(ArtifactContract {
788 kind: "application/json".into(),
789 max_bytes: Some(65_536),
790 schema: None,
791 });
792 envelope.deadline = DateTime::<Utc>::from_timestamp_millis(1_760_000_000_000);
793 envelope.budget = Some(TaskBudget {
794 max_attempts: 5,
795 timeout_ms: Some(30_000),
796 });
797 envelope.idempotency_key = Some("run-9:charge:7".into());
798 envelope.worker_version = Some("activity-worker/1.4.0".into());
799 envelope.effect = Effect::Idempotent;
800
801 let back: TaskEnvelope =
802 serde_json::from_str(&serde_json::to_string(&envelope).unwrap()).unwrap();
803 assert_eq!(envelope, back);
804 }
805}