Skip to main content

rusty_agent_runtime/
durable.rs

1//! Durable Work contracts (R0.6): the shared retry taxonomy and task envelope.
2//!
3//! This module freezes the wire shapes that the R0.6 durable-activity system
4//! builds on — the Postgres task queue in `rusty-agent-server`, the worker SDK in
5//! `rusty-worker`, and the scheduler between them. Nothing here performs I/O
6//! or scheduling; these are pure data contracts plus the one policy function
7//! both sides must agree on ([`classify_retry`]).
8//!
9//! The two pillars:
10//!
11//! - [`ErrorClass`] + [`RetryDecision`] — the retry taxonomy. Every failed
12//!   task attempt is classified into a closed set of error classes; the
13//!   class, the declared [`Effect`] of the work, and the attempt ordinal map
14//!   to exactly one decision: retry after a delay, dead-letter, or fail.
15//! - [`TaskEnvelope`] — the unit of durable work: who sent it, who executes
16//!   it, the input reference, the artifact contract, the deadline, the
17//!   budget, and the idempotency key that makes retry *effective-once*
18//!   rather than duplicated.
19//!
20//! The composition rule that makes retry safe comes from the Flight
21//! Recorder: [`classify_retry`] refuses to silently retry any effect that is
22//! not [`Effect::is_freely_repeatable`], so an `Idempotent` declaration —
23//! with a stable idempotency key — is what unlocks automatic retry.
24//!
25//! Golden-file tests under `tests/golden/` pin the serialized shapes; any
26//! accidental contract drift fails CI. To bless an intentional change,
27//! re-run with `UPDATE_GOLDEN=1` and review the diff.
28
29use chrono::{DateTime, Utc};
30use serde::{Deserialize, Serialize};
31use serde_json::{Map, Value};
32
33use crate::record::{
34    DecisionAction, DecisionEvent, DecisionFamily, DecisionOutcome, Effect, PayloadRef,
35    PolicyVersion,
36};
37
38/// The current wire-format version of [`TaskEnvelope`].
39///
40/// Bump only on a breaking change to the envelope; additive evolution uses
41/// serde defaults instead so previously queued tasks keep deserializing.
42pub const TASK_ENVELOPE_FORMAT_VERSION: u32 = 1;
43
44/// The base delay of the retry backoff schedule, in milliseconds.
45///
46/// Retry `n` (1-based) draws a delay uniformly from
47/// `[0, BASE_RETRY_DELAY_MS * 2^(n-1)]`, capped at [`MAX_RETRY_DELAY_MS`].
48pub const BASE_RETRY_DELAY_MS: u64 = 1_000;
49
50/// The cap of the retry backoff schedule, in milliseconds (5 minutes).
51///
52/// The cap bounds worst-case queue latency for a stuck dependency while
53/// keeping retry pressure negligible: at the cap, one task retries at most
54/// 12 times per hour.
55pub const MAX_RETRY_DELAY_MS: u64 = 300_000;
56
57/// Why a task attempt failed. Closed set; the scheduler matches exhaustively
58/// on it, and new classes are additive (old senders still deserialize).
59///
60/// The class is declared by the executor of the work — the worker that ran
61/// the handler or the transport that carried it — not inferred from logs.
62/// Each variant documents its retry semantics; the mechanical mapping lives
63/// in [`classify_retry`].
64#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)]
65#[serde(rename_all = "snake_case")]
66pub enum ErrorClass {
67    /// A transient fault with no lasting cause: connection reset, broken
68    /// pipe, a dependency's internal hiccup. Retry with backoff; the same
69    /// request is expected to succeed later.
70    Transient,
71
72    /// The callee asked to be slowed down (HTTP 429, `Retry-After`).
73    /// Retry with backoff; a callee-supplied `Retry-After` value floors the
74    /// delay (applied by the scheduler, outside this contract).
75    RateLimited,
76
77    /// The attempt did not finish within its deadline. Retry with backoff,
78    /// but note the ambiguity: the work may have partially executed, which
79    /// is exactly why the [`Effect`] gate in [`classify_retry`] exists — a
80    /// timed-out non-idempotent effect must not be re-attempted silently.
81    Timeout,
82
83    /// The input is malformed or violates the callee's contract (HTTP 400,
84    /// schema validation). Never retried: the same bytes will fail the same
85    /// way on every attempt. Fails the task immediately.
86    InvalidInput,
87
88    /// An upstream dependency the task needs is down or degraded (database
89    /// unreachable, model endpoint 5xx). Retry with backoff; distinct from
90    /// [`ErrorClass::Transient`] so operators can distinguish "our wiring"
91    /// from "their outage" in telemetry and alerting.
92    DependencyFailure,
93
94    /// The worker or the callee is out of capacity: memory pressure,
95    /// connection-pool exhaustion, quota exhaustion. Retry with backoff —
96    /// ideally on a different worker (placement is the scheduler's concern;
97    /// this contract only carries the classification).
98    ResourceExhausted,
99
100    /// The attempt ended because the task was cancelled. Control flow, not
101    /// a failure: never retried, never dead-lettered. Cancellation
102    /// propagation (wave 2) classifies interrupted attempts this way so the
103    /// retry machinery stays out of the cancellation path.
104    Cancelled,
105
106    /// The failure could not be classified — the handler returned an
107    /// unclassified error, or the worker died mid-attempt (lease expiry
108    /// classifies here). Retried with backoff up to the attempt limit, then
109    /// dead-lettered: unknown failures are the DLQ's primary input, since
110    /// they are the ones that need human eyes.
111    Unknown,
112}
113
114impl ErrorClass {
115    /// Whether this class is retryable at all. `InvalidInput` and
116    /// `Cancelled` are the only non-retryable classes; everything else —
117    /// including `Unknown` — may be re-attempted up to the attempt limit.
118    ///
119    /// Retryability of the class is necessary but not sufficient for a
120    /// retry: [`classify_retry`] additionally gates on the work's declared
121    /// [`Effect`].
122    pub fn is_retryable(self) -> bool {
123        !matches!(self, ErrorClass::InvalidInput | ErrorClass::Cancelled)
124    }
125}
126
127/// What the scheduler does with a failed task attempt.
128///
129/// Serialized with internal tagging (`{"decision": "retry", "after_ms": …}`)
130/// so a decision recorded in the journal or the queue's attempt log is
131/// self-describing.
132#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
133#[serde(tag = "decision", rename_all = "snake_case")]
134pub enum RetryDecision {
135    /// Re-queue the task to become visible again after `after_ms`
136    /// milliseconds. The idempotency key and input are unchanged; the
137    /// attempt counter increments.
138    Retry {
139        /// Delay before the task becomes visible to workers again, in
140        /// milliseconds. Computed by [`backoff_delay_ms`].
141        after_ms: u64,
142    },
143
144    /// Move the task to the dead-letter queue: a retryable failure class
145    /// whose attempts are exhausted, or an `Unknown` failure that kept
146    /// recurring. DLQ entries are operator-visible evidence — they carry the
147    /// full attempt history and can be re-driven by hand after the cause is
148    /// fixed.
149    Dead,
150
151    /// Fail the task immediately: a non-retryable class, or any failure of
152    /// work whose declared [`Effect`] is not freely repeatable. The error
153    /// surfaces to the sender; nothing is retried and nothing is
154    /// dead-lettered (there is nothing a human can fix by re-driving the
155    /// same input).
156    Fail,
157}
158
159/// Full-jitter exponential backoff, in milliseconds.
160///
161/// Retry `attempt` (1-based — `1` is the first retry after the initial
162/// failure) draws uniformly from `[0, base * 2^(attempt-1)]` where `base` is
163/// [`BASE_RETRY_DELAY_MS`], capped at [`MAX_RETRY_DELAY_MS`]. Full jitter —
164/// rather than a fixed fraction of the exponential — is what decorrelates a
165/// fleet of tasks that failed together when a shared dependency recovers
166/// (the thundering-herd problem); see the design doc for the citation.
167///
168/// `uniform` is a sample from `[0, 1)`; values outside are clamped. Source
169/// it from the run's `RngSource` so seeded runs reproduce their retry
170/// schedules exactly.
171pub fn backoff_delay_ms(attempt: u32, uniform: f64) -> u64 {
172    let exponent = attempt.saturating_sub(1).min(20);
173    let exponential = BASE_RETRY_DELAY_MS
174        .saturating_mul(1u64 << exponent)
175        .min(MAX_RETRY_DELAY_MS);
176    (uniform.clamp(0.0, 1.0) * exponential as f64) as u64
177}
178
179/// Map a failed attempt to exactly one [`RetryDecision`].
180///
181/// This is the single place the retry policy lives, shared verbatim by the
182/// server scheduler and the worker SDK so both sides of the queue always
183/// agree. The order of the gates is the policy:
184///
185/// 1. **Effect gate.** Work whose declared [`Effect`] is not
186///    [`Effect::is_freely_repeatable`] is never silently retried — a timed
187///    out `NonIdempotent` charge may already have happened. This is the
188///    composition with the Flight Recorder: the `Idempotent`
189///    classification is what makes retry safe, and it is checked here,
190///    not assumed.
191/// 2. **Class gate.** `InvalidInput` and `Cancelled` fail immediately; all
192///    other classes are retryable ([`ErrorClass::is_retryable`]).
193/// 3. **Attempt gate.** A retryable failure with `attempt >= max_attempts`
194///    goes to the dead-letter queue (`attempt` counts attempts made so far,
195///    starting at 1 for the first failure; `max_attempts == 0` means no
196///    retries at all).
197/// 4. Otherwise: retry, with the delay from [`backoff_delay_ms`].
198pub fn classify_retry(
199    effect: Effect,
200    class: ErrorClass,
201    attempt: u32,
202    max_attempts: u32,
203    uniform: f64,
204) -> RetryDecision {
205    if !effect.is_freely_repeatable() || !class.is_retryable() {
206        return RetryDecision::Fail;
207    }
208    if attempt >= max_attempts {
209        return RetryDecision::Dead;
210    }
211    RetryDecision::Retry {
212        after_ms: backoff_delay_ms(attempt, uniform),
213    }
214}
215
216/// The closed legal-action set of a retry decision (R0.8 wave 4).
217///
218/// Mirrors [`classify_retry`]'s gates exactly: when a retry is legal
219/// (freely-repeatable effect, retryable class, attempt budget remaining) the
220/// legal set is `[Retry { attempt: attempt + 1 }, Abort]`; otherwise retrying
221/// is not a legal action and the set collapses to `[Abort]`. The set is
222/// computed from the same inputs as the decision itself, so a
223/// [`DecisionEvent`] built by [`retry_decision_event`] can never record a
224/// selected action outside its legal set.
225pub fn retry_legal_actions(
226    effect: Effect,
227    class: ErrorClass,
228    attempt: u32,
229    max_attempts: u32,
230) -> Vec<DecisionAction> {
231    if effect.is_freely_repeatable() && class.is_retryable() && attempt < max_attempts {
232        vec![
233            DecisionAction::Retry {
234                attempt: attempt + 1,
235            },
236            DecisionAction::Abort,
237        ]
238    } else {
239        vec![DecisionAction::Abort]
240    }
241}
242
243/// The [`DecisionAction`] a [`RetryDecision`] corresponds to, given the
244/// 1-based attempt ordinal that failed.
245///
246/// A `Retry` decision takes retry ordinal `attempt + 1` (the failure was
247/// attempt `attempt`; the decision schedules the next one). `Dead` and
248/// `Fail` both map to `Abort`: dead-lettering is giving up on the automatic
249/// path, and the dead-letter queue entry is the queue's own evidence, not a
250/// decision action.
251pub fn retry_selected_action(decision: &RetryDecision, attempt: u32) -> DecisionAction {
252    match decision {
253        RetryDecision::Retry { .. } => DecisionAction::Retry {
254            attempt: attempt + 1,
255        },
256        RetryDecision::Dead | RetryDecision::Fail => DecisionAction::Abort,
257    }
258}
259
260/// Build the [`DecisionEvent`] for one retry decision (R0.8 wave 4).
261///
262/// This is the emission contract between the scheduler (which owns the
263/// decision inputs) and the journal: given exactly the values
264/// [`classify_retry`] was called with plus its output, produce the evidence
265/// record. Features pin the observation vocabulary — `failure_class`,
266/// `attempt`, `max_attempts`, `effect`, and `dependency_latency_ms` when the
267/// caller measured one — so offline evaluation reads stable keys.
268///
269/// **Propensity honesty.** Every v1 policy — including the static floor —
270/// is deterministic: it assigns probability 1 to the selected action and 0
271/// to every other legal action. The event therefore records `propensity:
272/// 1.0`. That is the truthful value, but it also means v1's evidence cannot
273/// support inverse-propensity scoring (division by a zero propensity is
274/// undefined): off-policy evaluation over v1 decisions is restricted to
275/// policies that would have taken the same action. Learned stochastic
276/// policies must record their true propensity at decision time; this
277/// function's `1.0` is the deterministic-policy degenerate case, not a
278/// placeholder.
279///
280/// `outcome` is `None` for a `Retry` decision (the re-attempt has not
281/// happened yet) and `Some(DecisionOutcome::Failure)` for `Dead`/`Fail`
282/// (the operation is over and did not complete).
283#[allow(clippy::too_many_arguments)]
284pub fn retry_decision_event(
285    run_id: impl Into<String>,
286    thread_id: impl Into<String>,
287    seq: u64,
288    effect: Effect,
289    class: ErrorClass,
290    attempt: u32,
291    max_attempts: u32,
292    dependency_latency_ms: Option<u64>,
293    decision: &RetryDecision,
294    policy_version: &PolicyVersion,
295    decided_at: DateTime<Utc>,
296) -> DecisionEvent {
297    let run_id = run_id.into();
298    let mut features = Map::new();
299    features.insert(
300        "failure_class".to_owned(),
301        serde_json::to_value(class).unwrap_or(Value::Null),
302    );
303    features.insert("attempt".to_owned(), Value::from(attempt));
304    features.insert("max_attempts".to_owned(), Value::from(max_attempts));
305    features.insert(
306        "effect".to_owned(),
307        serde_json::to_value(effect).unwrap_or(Value::Null),
308    );
309    if let Some(latency) = dependency_latency_ms {
310        features.insert("dependency_latency_ms".to_owned(), Value::from(latency));
311    }
312    let outcome = match decision {
313        RetryDecision::Retry { .. } => None,
314        RetryDecision::Dead | RetryDecision::Fail => Some(DecisionOutcome::Failure),
315    };
316    DecisionEvent {
317        id: format!("{run_id}:d{seq}"),
318        run_id,
319        thread_id: thread_id.into(),
320        seq,
321        family: DecisionFamily::Retry,
322        features,
323        legal_actions: retry_legal_actions(effect, class, attempt, max_attempts),
324        selected: retry_selected_action(decision, attempt),
325        propensity: 1.0,
326        policy_version: policy_version.clone(),
327        outcome,
328        decided_at,
329    }
330}
331
332/// The artifact contract of a [`TaskEnvelope`]: what the recipient is
333/// expected to produce and the sender to consume.
334///
335/// v1 carries a kind identifier and an optional size bound only — enough
336/// for the recipient to know it is producing the right *shape* of artifact
337/// and for the queue to refuse to store an out-of-contract result. R0.7
338/// (Agent Fabric wave 1) adds the optional `schema` field — additive with a
339/// serde default, omitted from the wire when unset, so pre-R0.7 contracts
340/// and readers see no shape change.
341#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
342pub struct ArtifactContract {
343    /// Artifact kind identifier — a media type (`application/json`), a
344    /// schema URI, or an application-defined kind name. Interpreted by the
345    /// sender and recipient; the queue treats it as opaque.
346    pub kind: String,
347
348    /// Upper bound on the serialized artifact size in bytes. `None` means
349    /// unbounded within the queue's own storage limits.
350    #[serde(default)]
351    pub max_bytes: Option<u64>,
352
353    /// Optional JSON Schema (draft 2020-12, per the design's open-questions
354    /// default) the artifact payload must validate against (R0.7). Declared
355    /// on the contract so an unacceptable message can fail fast as
356    /// [`ErrorClass::InvalidInput`] at submission — never retried — instead
357    /// of dead-lettering after the attempt budget is spent. `None` means
358    /// kind-and-size checking only, exactly the pre-R0.7 semantics.
359    ///
360    /// Wave 1 pins the field's *shape* (stored and golden-tested); payload
361    /// validation against it is wired at the mailbox submission path in a
362    /// later wave — the schema dialect constraint (design open question 5)
363    /// settles first.
364    #[serde(default, skip_serializing_if = "Option::is_none")]
365    pub schema: Option<serde_json::Value>,
366}
367
368/// The budget a task may consume across all of its attempts.
369#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
370pub struct TaskBudget {
371    /// Maximum number of attempts, counting the initial one. Once reached,
372    /// a retryable failure dead-letters instead of retrying (see
373    /// [`classify_retry`]).
374    pub max_attempts: u32,
375
376    /// Per-attempt wall-clock timeout in milliseconds. An attempt that
377    /// exceeds it is classified [`ErrorClass::Timeout`] by the worker.
378    /// `None` defers to the queue's default.
379    #[serde(default)]
380    pub timeout_ms: Option<u64>,
381}
382
383/// The unit of durable work: one task handed from a sender to a recipient
384/// through the queue.
385///
386/// The envelope is everything the scheduler and the recipient need and
387/// nothing they don't — the input travels as a [`PayloadRef`] (inline for
388/// small values, content-addressed above the inline threshold), so the
389/// queue row stays cheap to scan and the envelope composes with the Flight
390/// Recorder's artifact addressing.
391///
392/// Every field after `format_version` that was added after v1 carries a
393/// serde default, so envelopes queued by an older client keep deserializing;
394/// the default [`Effect`] is the conservative `NonIdempotent`, meaning an
395/// undeclared task is never silently retried.
396#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
397pub struct TaskEnvelope {
398    /// Envelope format version; [`TASK_ENVELOPE_FORMAT_VERSION`] for
399    /// anything written now.
400    #[serde(default = "current_envelope_version")]
401    pub format_version: u32,
402
403    /// Unique task id, assigned by the sender. Server-side ids are
404    /// tenant-namespaced like every other id (`{tenant}/…`); the envelope
405    /// itself is namespace-agnostic.
406    pub task_id: String,
407
408    /// The causal parent: the run event or task that created this one
409    /// (`{run_id}:{seq}` or a task id). Task lineage composes with the
410    /// journal's causal chain, so a fan-out of durable tasks is one
411    /// connected evidence tree. `None` for root tasks.
412    #[serde(default)]
413    pub parent: Option<String>,
414
415    /// Who submitted the task — a run id, a node name, or an agent
416    /// identity. Used for attribution, quotas, and cancellation scoping.
417    pub sender: String,
418
419    /// Who executes the task: a worker pool name (the queue routes by pool)
420    /// or a specific worker identity for pinned placement.
421    pub recipient: String,
422
423    /// Version pin (R0.6 wave 3): the exact worker version string this task
424    /// may be dispatched to. A run started against worker version `w1`
425    /// stamps its tasks with `w1`, so a mid-run deploy never changes
426    /// semantics under an in-flight execution — the scheduler keeps handing
427    /// the task to `w1`-advertising workers until it finishes.
428    ///
429    /// The pin is an *exact string match*, deliberately: semver range
430    /// matching (`^1.4`) makes a claim's outcome depend on the version
431    /// grammar's rules rather than on what the run actually recorded, and
432    /// range resolution belongs to the scheduler's policy, not to a frozen
433    /// wire contract. Ranges are documented future work; `None` (the
434    /// default) means unpinned — any worker may claim the task.
435    #[serde(default)]
436    pub worker_version: Option<String>,
437
438    /// The task input, inline or content-addressed. Small inputs travel
439    /// inside the queue row; large ones are artifacts the recipient resolves
440    /// through the journal's artifact map.
441    pub input: PayloadRef,
442
443    /// What the task is expected to produce. `None` means the result is an
444    /// unconstrained JSON value (the common case for node executions).
445    #[serde(default)]
446    pub output_contract: Option<ArtifactContract>,
447
448    /// Wall-clock deadline for the whole task, across all attempts. The
449    /// scheduler does not re-queue a task whose deadline has passed; the
450    /// worker treats an expired deadline as [`ErrorClass::Cancelled`].
451    #[serde(default)]
452    pub deadline: Option<DateTime<Utc>>,
453
454    /// Attempt and timeout budget. `None` means the queue's defaults.
455    #[serde(default)]
456    pub budget: Option<TaskBudget>,
457
458    /// The deduplication key for effectively-once execution: the queue
459    /// refuses a duplicate submission with an existing key, and the
460    /// recipient passes it to the effect it performs. Required for any task
461    /// whose `effect` is [`Effect::Idempotent`] — the key is what the
462    /// idempotency declaration *means* at the wire. `None` is honest only
463    /// for `Pure` / `ReadOnly` work.
464    #[serde(default)]
465    pub idempotency_key: Option<String>,
466
467    /// The declared effect classification of the work this task performs —
468    /// the same taxonomy the Flight Recorder journals. Defaults to the
469    /// conservative `NonIdempotent`, so a task that does not declare its
470    /// effect is never silently retried; declaring `Idempotent` (with an
471    /// idempotency key) is what unlocks automatic retry.
472    #[serde(default = "default_effect")]
473    pub effect: Effect,
474}
475
476fn current_envelope_version() -> u32 {
477    TASK_ENVELOPE_FORMAT_VERSION
478}
479
480fn default_effect() -> Effect {
481    Effect::NonIdempotent
482}
483
484impl TaskEnvelope {
485    /// A minimal envelope: sender, recipient, and input, with every optional
486    /// field unset and the effect conservatively [`Effect::NonIdempotent`].
487    /// Set the optional fields directly; they are public.
488    pub fn new(
489        task_id: impl Into<String>,
490        sender: impl Into<String>,
491        recipient: impl Into<String>,
492        input: PayloadRef,
493    ) -> Self {
494        Self {
495            format_version: TASK_ENVELOPE_FORMAT_VERSION,
496            task_id: task_id.into(),
497            parent: None,
498            sender: sender.into(),
499            recipient: recipient.into(),
500            worker_version: None,
501            input,
502            output_contract: None,
503            deadline: None,
504            budget: None,
505            idempotency_key: None,
506            effect: Effect::NonIdempotent,
507        }
508    }
509}
510
511#[cfg(test)]
512mod tests {
513    use super::*;
514    use serde_json::json;
515
516    #[test]
517    fn error_class_retryability() {
518        for class in [
519            ErrorClass::Transient,
520            ErrorClass::RateLimited,
521            ErrorClass::Timeout,
522            ErrorClass::DependencyFailure,
523            ErrorClass::ResourceExhausted,
524            ErrorClass::Unknown,
525        ] {
526            assert!(class.is_retryable(), "{class:?} must be retryable");
527        }
528        assert!(!ErrorClass::InvalidInput.is_retryable());
529        assert!(!ErrorClass::Cancelled.is_retryable());
530    }
531
532    #[test]
533    fn error_class_serde_names_are_snake_case() {
534        assert_eq!(
535            serde_json::to_value(ErrorClass::DependencyFailure).unwrap(),
536            json!("dependency_failure")
537        );
538        assert_eq!(
539            serde_json::to_value(ErrorClass::ResourceExhausted).unwrap(),
540            json!("resource_exhausted")
541        );
542    }
543
544    #[test]
545    fn backoff_bounds_and_cap() {
546        // Zero jitter always yields zero delay.
547        assert_eq!(backoff_delay_ms(1, 0.0), 0);
548        assert_eq!(backoff_delay_ms(8, 0.0), 0);
549        // First retry tops out at the base delay; later retries grow
550        // exponentially and clamp at the 5-minute cap.
551        assert!(backoff_delay_ms(1, 0.999) < BASE_RETRY_DELAY_MS);
552        assert!(backoff_delay_ms(2, 0.999) < 2 * BASE_RETRY_DELAY_MS);
553        assert_eq!(backoff_delay_ms(1, 1.0), BASE_RETRY_DELAY_MS);
554        assert_eq!(backoff_delay_ms(20, 1.0), MAX_RETRY_DELAY_MS);
555        // A saturating attempt ordinal cannot overflow.
556        assert_eq!(backoff_delay_ms(u32::MAX, 1.0), MAX_RETRY_DELAY_MS);
557    }
558
559    #[test]
560    fn classify_retry_gates_in_order() {
561        let repeatable = Effect::Idempotent;
562
563        // Gate 1: a non-freely-repeatable effect is never silently retried,
564        // even for an eminently retryable class.
565        assert_eq!(
566            classify_retry(Effect::NonIdempotent, ErrorClass::Transient, 1, 5, 0.5),
567            RetryDecision::Fail
568        );
569        assert_eq!(
570            classify_retry(Effect::Compensatable, ErrorClass::Timeout, 1, 5, 0.5),
571            RetryDecision::Fail
572        );
573
574        // Gate 2: non-retryable classes fail immediately.
575        assert_eq!(
576            classify_retry(repeatable, ErrorClass::InvalidInput, 1, 5, 0.5),
577            RetryDecision::Fail
578        );
579        assert_eq!(
580            classify_retry(repeatable, ErrorClass::Cancelled, 1, 5, 0.5),
581            RetryDecision::Fail
582        );
583
584        // Gate 3: exhausting the attempt budget dead-letters.
585        assert_eq!(
586            classify_retry(repeatable, ErrorClass::Unknown, 5, 5, 0.5),
587            RetryDecision::Dead
588        );
589        assert_eq!(
590            classify_retry(repeatable, ErrorClass::Transient, 1, 0, 0.5),
591            RetryDecision::Dead
592        );
593
594        // Otherwise: retry with a jittered delay within the bound.
595        match classify_retry(repeatable, ErrorClass::RateLimited, 1, 5, 0.5) {
596            RetryDecision::Retry { after_ms } => {
597                assert!(after_ms <= BASE_RETRY_DELAY_MS);
598            }
599            other => panic!("expected Retry, got {other:?}"),
600        }
601    }
602
603    #[test]
604    fn retry_decision_serde_shape() {
605        assert_eq!(
606            serde_json::to_value(RetryDecision::Retry { after_ms: 500 }).unwrap(),
607            json!({"decision": "retry", "after_ms": 500})
608        );
609        assert_eq!(
610            serde_json::to_value(RetryDecision::Dead).unwrap(),
611            json!({"decision": "dead"})
612        );
613        assert_eq!(
614            serde_json::to_value(RetryDecision::Fail).unwrap(),
615            json!({"decision": "fail"})
616        );
617    }
618
619    #[test]
620    fn retry_legal_actions_mirror_the_classifier_gates() {
621        let repeatable = Effect::Idempotent;
622
623        // A retryable failure with budget remaining: retry or abort.
624        assert_eq!(
625            retry_legal_actions(repeatable, ErrorClass::Timeout, 1, 3),
626            vec![DecisionAction::Retry { attempt: 2 }, DecisionAction::Abort]
627        );
628        // Budget exhausted: retrying is not a legal action.
629        assert_eq!(
630            retry_legal_actions(repeatable, ErrorClass::Timeout, 3, 3),
631            vec![DecisionAction::Abort]
632        );
633        assert_eq!(
634            retry_legal_actions(repeatable, ErrorClass::Transient, 1, 0),
635            vec![DecisionAction::Abort]
636        );
637        // Non-retryable class or non-repeatable effect: abort only.
638        assert_eq!(
639            retry_legal_actions(repeatable, ErrorClass::InvalidInput, 1, 3),
640            vec![DecisionAction::Abort]
641        );
642        assert_eq!(
643            retry_legal_actions(Effect::NonIdempotent, ErrorClass::Transient, 1, 3),
644            vec![DecisionAction::Abort]
645        );
646    }
647
648    #[test]
649    fn retry_decision_event_selected_action_stays_inside_the_legal_set() {
650        // Property: for every combination the classifier can see, the event's
651        // selected action is a member of its legal set.
652        let effects = [
653            Effect::Pure,
654            Effect::ReadOnly,
655            Effect::Idempotent,
656            Effect::NonIdempotent,
657            Effect::Compensatable,
658        ];
659        let classes = [
660            ErrorClass::Transient,
661            ErrorClass::RateLimited,
662            ErrorClass::Timeout,
663            ErrorClass::InvalidInput,
664            ErrorClass::DependencyFailure,
665            ErrorClass::ResourceExhausted,
666            ErrorClass::Cancelled,
667            ErrorClass::Unknown,
668        ];
669        let decided_at = DateTime::<Utc>::from_timestamp_millis(1_760_000_000_000).unwrap();
670        for effect in effects {
671            for class in classes {
672                for (attempt, max_attempts) in [(1, 3), (3, 3), (1, 0)] {
673                    let decision = classify_retry(effect, class, attempt, max_attempts, 0.5);
674                    let event = retry_decision_event(
675                        "run-1",
676                        "thread-1",
677                        1,
678                        effect,
679                        class,
680                        attempt,
681                        max_attempts,
682                        None,
683                        &decision,
684                        &PolicyVersion::default(),
685                        decided_at,
686                    );
687                    assert!(
688                        event.legal_actions.contains(&event.selected),
689                        "selected {:?} must be inside legal set {:?} for {effect:?}/{class:?} \
690                         attempt {attempt}/{max_attempts}",
691                        event.selected,
692                        event.legal_actions,
693                    );
694                }
695            }
696        }
697    }
698
699    #[test]
700    fn retry_decision_event_shape_and_outcome_mapping() {
701        let decided_at = DateTime::<Utc>::from_timestamp_millis(1_760_000_000_000).unwrap();
702        let decision = classify_retry(Effect::Idempotent, ErrorClass::Timeout, 1, 3, 0.5);
703        let event = retry_decision_event(
704            "run-9",
705            "thread-2",
706            3,
707            Effect::Idempotent,
708            ErrorClass::Timeout,
709            1,
710            3,
711            Some(840),
712            &decision,
713            &PolicyVersion::new("policy-0123456789ab"),
714            decided_at,
715        );
716        assert_eq!(event.id, "run-9:d3");
717        assert_eq!(event.seq, 3);
718        assert_eq!(event.family, DecisionFamily::Retry);
719        assert_eq!(
720            event.selected,
721            DecisionAction::Retry { attempt: 2 },
722            "the failure was attempt 1; the decision schedules attempt 2"
723        );
724        // Deterministic v1 policies record the degenerate propensity, 1.0.
725        assert_eq!(event.propensity, 1.0);
726        // A retry decision has no outcome until the re-attempt completes.
727        assert!(event.outcome.is_none());
728        assert_eq!(event.features.get("failure_class"), Some(&json!("timeout")));
729        assert_eq!(event.features.get("attempt"), Some(&json!(1)));
730        assert_eq!(
731            event.features.get("dependency_latency_ms"),
732            Some(&json!(840))
733        );
734
735        // Terminal decisions record the failure outcome immediately.
736        let dead = retry_decision_event(
737            "run-9",
738            "thread-2",
739            4,
740            Effect::Idempotent,
741            ErrorClass::Timeout,
742            3,
743            3,
744            None,
745            &RetryDecision::Dead,
746            &PolicyVersion::default(),
747            decided_at,
748        );
749        assert_eq!(dead.selected, DecisionAction::Abort);
750        assert_eq!(dead.outcome, Some(DecisionOutcome::Failure));
751        assert!(
752            !dead.features.contains_key("dependency_latency_ms"),
753            "an unmeasured latency is absent from the wire, not null"
754        );
755    }
756
757    #[test]
758    fn envelope_minimal_json_deserializes_with_defaults() {
759        // A v1 envelope carrying only the required fields — the shape an
760        // older, smaller client would write — loads with honest defaults:
761        // current format version, no options set, conservative effect.
762        let minimal = json!({
763            "task_id": "t-1",
764            "sender": "run-9",
765            "recipient": "pool-default",
766            "input": {"kind": "inline", "value": {"n": 1}},
767        });
768        let envelope: TaskEnvelope = serde_json::from_value(minimal).unwrap();
769        assert_eq!(envelope.format_version, TASK_ENVELOPE_FORMAT_VERSION);
770        assert_eq!(envelope.effect, Effect::NonIdempotent);
771        assert!(envelope.parent.is_none());
772        assert!(envelope.idempotency_key.is_none());
773        assert!(envelope.worker_version.is_none());
774        assert!(envelope.deadline.is_none());
775        assert!(envelope.budget.is_none());
776    }
777
778    #[test]
779    fn envelope_serde_roundtrip() {
780        let mut envelope = TaskEnvelope::new(
781            "t-7",
782            "run-9:node-a",
783            "pool-default",
784            PayloadRef::inline(json!({"n": 1})),
785        );
786        envelope.parent = Some("run-9:3".into());
787        envelope.output_contract = Some(ArtifactContract {
788            kind: "application/json".into(),
789            max_bytes: Some(65_536),
790            schema: None,
791        });
792        envelope.deadline = DateTime::<Utc>::from_timestamp_millis(1_760_000_000_000);
793        envelope.budget = Some(TaskBudget {
794            max_attempts: 5,
795            timeout_ms: Some(30_000),
796        });
797        envelope.idempotency_key = Some("run-9:charge:7".into());
798        envelope.worker_version = Some("activity-worker/1.4.0".into());
799        envelope.effect = Effect::Idempotent;
800
801        let back: TaskEnvelope =
802            serde_json::from_str(&serde_json::to_string(&envelope).unwrap()).unwrap();
803        assert_eq!(envelope, back);
804    }
805}