Skip to main content

rusty_agent_runtime/
durable.rs

1//! Durable Work contracts (R0.6): the shared retry taxonomy and task envelope.
2//!
3//! This module freezes the wire shapes that the R0.6 durable-activity system
4//! builds on — the Postgres task queue in `rusty-agent-server`, the worker SDK in
5//! `rusty-worker`, and the scheduler between them. Nothing here performs I/O
6//! or scheduling; these are pure data contracts plus the one policy function
7//! both sides must agree on ([`classify_retry`]).
8//!
9//! The two pillars:
10//!
11//! - [`ErrorClass`] + [`RetryDecision`] — the retry taxonomy. Every failed
12//!   task attempt is classified into a closed set of error classes; the
13//!   class, the declared [`Effect`] of the work, and the attempt ordinal map
14//!   to exactly one decision: retry after a delay, dead-letter, or fail.
15//! - [`TaskEnvelope`] — the unit of durable work: who sent it, who executes
16//!   it, the input reference, the artifact contract, the deadline, the
17//!   budget, and the idempotency key that makes retry *effective-once*
18//!   rather than duplicated.
19//!
20//! The composition rule that makes retry safe comes from the Flight
21//! Recorder: [`classify_retry`] refuses to silently retry any effect that is
22//! not [`Effect::is_freely_repeatable`], so an `Idempotent` declaration —
23//! with a stable idempotency key — is what unlocks automatic retry.
24//!
25//! Golden-file tests under `tests/golden/` pin the serialized shapes; any
26//! accidental contract drift fails CI. To bless an intentional change,
27//! re-run with `UPDATE_GOLDEN=1` and review the diff.
28
29use chrono::{DateTime, Utc};
30use serde::{Deserialize, Serialize};
31use serde_json::{Map, Value};
32
33use crate::record::{
34    DecisionAction, DecisionEvent, DecisionFamily, DecisionOutcome, Effect, PayloadRef,
35    PolicyVersion,
36};
37
38/// The current wire-format version of [`TaskEnvelope`].
39///
40/// Bump only on a breaking change to the envelope; additive evolution uses
41/// serde defaults instead so previously queued tasks keep deserializing.
42pub const TASK_ENVELOPE_FORMAT_VERSION: u32 = 1;
43
44/// The base delay of the retry backoff schedule, in milliseconds.
45///
46/// Retry `n` (1-based) draws a delay uniformly from
47/// `[0, BASE_RETRY_DELAY_MS * 2^(n-1)]`, capped at [`MAX_RETRY_DELAY_MS`].
48pub const BASE_RETRY_DELAY_MS: u64 = 1_000;
49
50/// The cap of the retry backoff schedule, in milliseconds (5 minutes).
51///
52/// The cap bounds worst-case queue latency for a stuck dependency while
53/// keeping retry pressure negligible: at the cap, one task retries at most
54/// 12 times per hour.
55pub const MAX_RETRY_DELAY_MS: u64 = 300_000;
56
57/// Why a task attempt failed. Closed set; the scheduler matches exhaustively
58/// on it, and new classes are additive (old senders still deserialize).
59///
60/// The class is declared by the executor of the work — the worker that ran
61/// the handler or the transport that carried it — not inferred from logs.
62/// Each variant documents its retry semantics; the mechanical mapping lives
63/// in [`classify_retry`].
64#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)]
65#[serde(rename_all = "snake_case")]
66pub enum ErrorClass {
67    /// A transient fault with no lasting cause: connection reset, broken
68    /// pipe, a dependency's internal hiccup. Retry with backoff; the same
69    /// request is expected to succeed later.
70    Transient,
71
72    /// The callee asked to be slowed down (HTTP 429, `Retry-After`).
73    /// Retry with backoff; a callee-supplied `Retry-After` value floors the
74    /// delay (applied by the scheduler, outside this contract).
75    RateLimited,
76
77    /// The attempt did not finish within its deadline. Retry with backoff,
78    /// but note the ambiguity: the work may have partially executed, which
79    /// is exactly why the [`Effect`] gate in [`classify_retry`] exists — a
80    /// timed-out non-idempotent effect must not be re-attempted silently.
81    Timeout,
82
83    /// The input is malformed or violates the callee's contract (HTTP 400,
84    /// schema validation). Never retried: the same bytes will fail the same
85    /// way on every attempt. Fails the task immediately.
86    InvalidInput,
87
88    /// An upstream dependency the task needs is down or degraded (database
89    /// unreachable, model endpoint 5xx). Retry with backoff; distinct from
90    /// [`ErrorClass::Transient`] so operators can distinguish "our wiring"
91    /// from "their outage" in telemetry and alerting.
92    DependencyFailure,
93
94    /// The worker or the callee is out of capacity: memory pressure,
95    /// connection-pool exhaustion, quota exhaustion. Retry with backoff —
96    /// ideally on a different worker (placement is the scheduler's concern;
97    /// this contract only carries the classification).
98    ResourceExhausted,
99
100    /// The attempt ended because the task was cancelled. Control flow, not
101    /// a failure: never retried, never dead-lettered. Cancellation
102    /// propagation (wave 2) classifies interrupted attempts this way so the
103    /// retry machinery stays out of the cancellation path.
104    Cancelled,
105
106    /// The failure could not be classified — the handler returned an
107    /// unclassified error, or the worker died mid-attempt (lease expiry
108    /// classifies here). Retried with backoff up to the attempt limit, then
109    /// dead-lettered: unknown failures are the DLQ's primary input, since
110    /// they are the ones that need human eyes.
111    Unknown,
112}
113
114impl ErrorClass {
115    /// Whether this class is retryable at all. `InvalidInput` and
116    /// `Cancelled` are the only non-retryable classes; everything else —
117    /// including `Unknown` — may be re-attempted up to the attempt limit.
118    ///
119    /// Retryability of the class is necessary but not sufficient for a
120    /// retry: [`classify_retry`] additionally gates on the work's declared
121    /// [`Effect`].
122    pub fn is_retryable(self) -> bool {
123        !matches!(self, ErrorClass::InvalidInput | ErrorClass::Cancelled)
124    }
125}
126
127/// What the scheduler does with a failed task attempt.
128///
129/// Serialized with internal tagging (`{"decision": "retry", "after_ms": …}`)
130/// so a decision recorded in the journal or the queue's attempt log is
131/// self-describing.
132#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
133#[serde(tag = "decision", rename_all = "snake_case")]
134pub enum RetryDecision {
135    /// Re-queue the task to become visible again after `after_ms`
136    /// milliseconds. The idempotency key and input are unchanged; the
137    /// attempt counter increments.
138    Retry {
139        /// Delay before the task becomes visible to workers again, in
140        /// milliseconds. Computed by [`backoff_delay_ms`].
141        after_ms: u64,
142    },
143
144    /// Move the task to the dead-letter queue: a retryable failure class
145    /// whose attempts are exhausted, or an `Unknown` failure that kept
146    /// recurring. DLQ entries are operator-visible evidence — they carry the
147    /// full attempt history and can be re-driven by hand after the cause is
148    /// fixed.
149    Dead,
150
151    /// Fail the task immediately: a non-retryable class, or any failure of
152    /// work whose declared [`Effect`] is not freely repeatable. The error
153    /// surfaces to the sender; nothing is retried and nothing is
154    /// dead-lettered (there is nothing a human can fix by re-driving the
155    /// same input).
156    Fail,
157}
158
159/// Full-jitter exponential backoff, in milliseconds.
160///
161/// Retry `attempt` (1-based — `1` is the first retry after the initial
162/// failure) draws uniformly from `[0, base * 2^(attempt-1)]` where `base` is
163/// [`BASE_RETRY_DELAY_MS`], capped at [`MAX_RETRY_DELAY_MS`]. Full jitter —
164/// rather than a fixed fraction of the exponential — is what decorrelates a
165/// fleet of tasks that failed together when a shared dependency recovers
166/// (the thundering-herd problem); see the design doc for the citation.
167///
168/// `uniform` is a sample from `[0, 1)`; values outside are clamped. Source
169/// it from the run's `RngSource` so seeded runs reproduce their retry
170/// schedules exactly.
171pub fn backoff_delay_ms(attempt: u32, uniform: f64) -> u64 {
172    backoff_delay_ms_with(attempt, uniform, BASE_RETRY_DELAY_MS, MAX_RETRY_DELAY_MS)
173}
174
175/// The backoff schedule [`backoff_delay_ms`] computes, drawn from
176/// policy-supplied `base_ms`/`cap_ms` instead of the floor constants
177/// (R0.10 wave 3). The schedule's *shape* — full jitter over a doubling
178/// window — is not learnable: the Wave 1 headroom experiment's published
179/// scar is that a fixed per-class delay table underperforms the jittered
180/// exponential on latency, so learned policies tune the base, the cap, and
181/// the budget, never the shape.
182pub fn backoff_delay_ms_with(attempt: u32, uniform: f64, base_ms: u64, cap_ms: u64) -> u64 {
183    let exponent = attempt.saturating_sub(1).min(20);
184    let exponential = base_ms.saturating_mul(1u64 << exponent).min(cap_ms);
185    (uniform.clamp(0.0, 1.0) * exponential as f64) as u64
186}
187
188/// Map a failed attempt to exactly one [`RetryDecision`].
189///
190/// This is the single place the retry policy lives, shared verbatim by the
191/// server scheduler and the worker SDK so both sides of the queue always
192/// agree. It decides with the static floor's constants — the read path
193/// every decision falls back to when no learned policy is in force
194/// (R0.10 wave 3's application loop is [`classify_retry_with_policy`],
195/// which this delegates to with the floor's parameters). The order of the
196/// gates is the policy:
197///
198/// 1. **Effect gate.** Work whose declared [`Effect`] is not
199///    [`Effect::is_freely_repeatable`] is never silently retried — a timed
200///    out `NonIdempotent` charge may already have happened. This is the
201///    composition with the Flight Recorder: the `Idempotent`
202///    classification is what makes retry safe, and it is checked here,
203///    not assumed.
204/// 2. **Class gate.** `InvalidInput` and `Cancelled` fail immediately; all
205///    other classes are retryable ([`ErrorClass::is_retryable`]).
206/// 3. **Attempt gate.** A retryable failure with `attempt >= max_attempts`
207///    goes to the dead-letter queue (`attempt` counts attempts made so far,
208///    starting at 1 for the first failure; `max_attempts == 0` means no
209///    retries at all).
210/// 4. Otherwise: retry, with the delay from [`backoff_delay_ms`].
211pub fn classify_retry(
212    effect: Effect,
213    class: ErrorClass,
214    attempt: u32,
215    max_attempts: u32,
216    uniform: f64,
217) -> RetryDecision {
218    classify_retry_with_policy(
219        effect,
220        class,
221        attempt,
222        &ResolvedRetryParameters::floor(max_attempts),
223        uniform,
224    )
225}
226
227/// The parameters one retry decision is made with (R0.10 wave 3): the
228/// resolution of the active policy's retry parameters for one error class
229/// against the task's declared attempt budget.
230///
231/// Produced by [`resolve_retry_parameters`]; consumed by
232/// [`classify_retry_with_policy`]. Carrying the resolution as a value —
233/// rather than re-reading the policy at the decision point — is what makes
234/// the decision's inputs journaled evidence: the features record the
235/// numbers decided with, not a pointer to state that may have moved.
236#[derive(Debug, Clone, Copy, PartialEq, Eq)]
237pub struct ResolvedRetryParameters {
238    /// Base of the backoff schedule, in milliseconds.
239    pub base_delay_ms: u64,
240
241    /// Cap of the backoff schedule, in milliseconds.
242    pub max_delay_ms: u64,
243
244    /// Effective attempt budget (the task's declared budget, narrowed by a
245    /// learned budget when one is in force).
246    pub max_attempts: u32,
247}
248
249impl ResolvedRetryParameters {
250    /// The floor's resolution: the module constants and the task's own
251    /// budget, untouched. This is exactly what [`classify_retry`] decided
252    /// with before the policy plane existed.
253    pub fn floor(max_attempts: u32) -> Self {
254        Self {
255            base_delay_ms: BASE_RETRY_DELAY_MS,
256            max_delay_ms: MAX_RETRY_DELAY_MS,
257            max_attempts,
258        }
259    }
260}
261
262/// Resolve the active policy's retry parameters for one error class
263/// against the task's declared attempt budget (R0.10 wave 3 — the
264/// application loop's read path).
265///
266/// Fail-closed, in order:
267///
268/// - **An invalid policy steers nothing.** Parameters that fail
269///   [`crate::record::RetryPolicyParameters::validate`] resolve to the
270///   floor — validation is the promotion gate's job, but the decision
271///   point re-checks rather than trusting the write path, because a
272///   registry body read after a partial migration is the decision point's
273///   problem, not the gate's.
274/// - **A class with a learned entry** decides by that entry's schedule;
275///   its budget *narrows* the task's declared budget (`min`), never
276///   widens it — the task's budget is the sender's declaration, the same
277///   rule the concurrency family's quota ceiling follows.
278/// - **A flat learned schedule** (flat values differing from the floor's,
279///   no per-class entry) applies the same way.
280/// - **Otherwise the floor decides**: the module constants and the task's
281///   own budget, byte-identical to pre-learning behavior.
282pub fn resolve_retry_parameters(
283    policy: &crate::record::ExecutorPolicy,
284    class: ErrorClass,
285    task_max_attempts: u32,
286) -> ResolvedRetryParameters {
287    let retry = &policy.retry;
288    if retry.validate().is_err() {
289        return ResolvedRetryParameters::floor(task_max_attempts);
290    }
291    let floor_flat = crate::record::ExecutorPolicy::static_v0().retry.flat();
292    let flat_learned = retry.flat() != floor_flat;
293    let entry = retry.per_class.as_ref().and_then(|table| table.get(&class));
294    match (entry, flat_learned) {
295        (Some(schedule), _) => ResolvedRetryParameters {
296            base_delay_ms: schedule.base_delay_ms,
297            max_delay_ms: schedule.max_delay_ms,
298            max_attempts: schedule.max_attempts.min(task_max_attempts),
299        },
300        (None, true) => ResolvedRetryParameters {
301            base_delay_ms: retry.base_delay_ms,
302            max_delay_ms: retry.max_delay_ms,
303            max_attempts: retry.max_attempts.min(task_max_attempts),
304        },
305        (None, false) => ResolvedRetryParameters::floor(task_max_attempts),
306    }
307}
308
309/// [`classify_retry`] with policy-resolved parameters (R0.10 wave 3 — the
310/// application loop's decision path).
311///
312/// The gates never move: effect gate, class gate, attempt gate are
313/// policy-independent and evaluated in the same order as the floor's.
314/// What a learned policy tunes is the *numbers* — the backoff base and cap
315/// the delay draws from, and (narrowed) the attempt budget — never which
316/// classes or effects are retryable.
317pub fn classify_retry_with_policy(
318    effect: Effect,
319    class: ErrorClass,
320    attempt: u32,
321    resolved: &ResolvedRetryParameters,
322    uniform: f64,
323) -> RetryDecision {
324    if !effect.is_freely_repeatable() || !class.is_retryable() {
325        return RetryDecision::Fail;
326    }
327    if attempt >= resolved.max_attempts {
328        return RetryDecision::Dead;
329    }
330    RetryDecision::Retry {
331        after_ms: backoff_delay_ms_with(
332            attempt,
333            uniform,
334            resolved.base_delay_ms,
335            resolved.max_delay_ms,
336        ),
337    }
338}
339
340/// The closed legal-action set of a retry decision (R0.8 wave 4).
341///
342/// Mirrors [`classify_retry`]'s gates exactly: when a retry is legal
343/// (freely-repeatable effect, retryable class, attempt budget remaining) the
344/// legal set is `[Retry { attempt: attempt + 1 }, Abort]`; otherwise retrying
345/// is not a legal action and the set collapses to `[Abort]`. The set is
346/// computed from the same inputs as the decision itself, so a
347/// [`DecisionEvent`] built by [`retry_decision_event`] can never record a
348/// selected action outside its legal set.
349pub fn retry_legal_actions(
350    effect: Effect,
351    class: ErrorClass,
352    attempt: u32,
353    max_attempts: u32,
354) -> Vec<DecisionAction> {
355    if effect.is_freely_repeatable() && class.is_retryable() && attempt < max_attempts {
356        vec![
357            DecisionAction::Retry {
358                attempt: attempt + 1,
359            },
360            DecisionAction::Abort,
361        ]
362    } else {
363        vec![DecisionAction::Abort]
364    }
365}
366
367/// The [`DecisionAction`] a [`RetryDecision`] corresponds to, given the
368/// 1-based attempt ordinal that failed.
369///
370/// A `Retry` decision takes retry ordinal `attempt + 1` (the failure was
371/// attempt `attempt`; the decision schedules the next one). `Dead` and
372/// `Fail` both map to `Abort`: dead-lettering is giving up on the automatic
373/// path, and the dead-letter queue entry is the queue's own evidence, not a
374/// decision action.
375pub fn retry_selected_action(decision: &RetryDecision, attempt: u32) -> DecisionAction {
376    match decision {
377        RetryDecision::Retry { .. } => DecisionAction::Retry {
378            attempt: attempt + 1,
379        },
380        RetryDecision::Dead | RetryDecision::Fail => DecisionAction::Abort,
381    }
382}
383
384/// Build the [`DecisionEvent`] for one retry decision (R0.8 wave 4).
385///
386/// This is the emission contract between the scheduler (which owns the
387/// decision inputs) and the journal: given exactly the values
388/// [`classify_retry`] was called with plus its output, produce the evidence
389/// record. Features pin the observation vocabulary — `failure_class`,
390/// `attempt`, `max_attempts`, `effect`, and `dependency_latency_ms` when the
391/// caller measured one — so offline evaluation reads stable keys.
392///
393/// **Propensity honesty.** Every v1 policy — including the static floor —
394/// is deterministic: it assigns probability 1 to the selected action and 0
395/// to every other legal action. The event therefore records `propensity:
396/// 1.0`. That is the truthful value, but it also means v1's evidence cannot
397/// support inverse-propensity scoring (division by a zero propensity is
398/// undefined): off-policy evaluation over v1 decisions is restricted to
399/// policies that would have taken the same action. Learned stochastic
400/// policies must record their true propensity at decision time; this
401/// function's `1.0` is the deterministic-policy degenerate case, not a
402/// placeholder.
403///
404/// `outcome` is `None` for a `Retry` decision (the re-attempt has not
405/// happened yet) and `Some(DecisionOutcome::Failure)` for `Dead`/`Fail`
406/// (the operation is over and did not complete).
407#[allow(clippy::too_many_arguments)]
408pub fn retry_decision_event(
409    run_id: impl Into<String>,
410    thread_id: impl Into<String>,
411    seq: u64,
412    effect: Effect,
413    class: ErrorClass,
414    attempt: u32,
415    max_attempts: u32,
416    dependency_latency_ms: Option<u64>,
417    decision: &RetryDecision,
418    policy_version: &PolicyVersion,
419    decided_at: DateTime<Utc>,
420) -> DecisionEvent {
421    let run_id = run_id.into();
422    let mut features = Map::new();
423    features.insert(
424        "failure_class".to_owned(),
425        serde_json::to_value(class).unwrap_or(Value::Null),
426    );
427    features.insert("attempt".to_owned(), Value::from(attempt));
428    features.insert("max_attempts".to_owned(), Value::from(max_attempts));
429    features.insert(
430        "effect".to_owned(),
431        serde_json::to_value(effect).unwrap_or(Value::Null),
432    );
433    if let Some(latency) = dependency_latency_ms {
434        features.insert("dependency_latency_ms".to_owned(), Value::from(latency));
435    }
436    let outcome = match decision {
437        RetryDecision::Retry { .. } => None,
438        RetryDecision::Dead | RetryDecision::Fail => Some(DecisionOutcome::Failure),
439    };
440    DecisionEvent {
441        id: format!("{run_id}:d{seq}"),
442        run_id,
443        thread_id: thread_id.into(),
444        seq,
445        family: DecisionFamily::Retry,
446        features,
447        legal_actions: retry_legal_actions(effect, class, attempt, max_attempts),
448        selected: retry_selected_action(decision, attempt),
449        propensity: 1.0,
450        policy_version: policy_version.clone(),
451        // Acting by construction: the v1 emission point records the decision
452        // that executed. The twin's shadow pairs mark roles explicitly.
453        role: None,
454        outcome,
455        decided_at,
456    }
457}
458
459// --------------------------------------------------------------------- //
460// The timeout family (R0.10 wave 3)
461//
462// The floor's stance is "no bound in force" (`TimeoutPolicyParameters`
463// with every field `None`); a promoted policy resolves to a concrete
464// bound per callee or a default. The decision point's evidence contract
465// mirrors the retry family's: the closed legal set (the declared ladder),
466// the acting policy's version, the degenerate propensity.
467// --------------------------------------------------------------------- //
468
469/// The latency snapshot a timeout decision is made from: the callee's
470/// journaled completion percentiles over the feature window (design open
471/// question 4). Assembled server-side from rolling evidence and journaled
472/// as the feature map at decision time — the feature is then evidence the
473/// twin can reproduce, not a query re-run against mutated state.
474#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
475pub struct LatencyPercentiles {
476    /// Median completion latency, in milliseconds.
477    pub p50_ms: u64,
478
479    /// 95th-percentile completion latency, in milliseconds.
480    pub p95_ms: u64,
481
482    /// 99th-percentile completion latency, in milliseconds.
483    pub p99_ms: u64,
484
485    /// Completions the percentiles were computed over.
486    pub samples: u64,
487}
488
489/// Resolve the active policy's timeout bound for one callee (R0.10 wave 3
490/// — the application loop's read path for the timeout family).
491///
492/// Fail-closed, mirroring [`resolve_retry_parameters`]: an invalid policy
493/// steers nothing (`None` — no bound, the floor's stance); a callee with a
494/// learned entry resolves to it; otherwise the policy's default; otherwise
495/// no bound. A resolved bound is clamped to the declared ceiling and never
496/// below [`crate::record::MIN_TIMEOUT_RUNG_MS`] — validation is the
497/// promotion gate's job, but the clamp is the decision point's own
498/// defense, because a bound below the minimum rung aborts ordinary work
499/// early, a correctness hazard no read path may produce.
500pub fn resolve_timeout_bound_ms(
501    policy: &crate::record::ExecutorPolicy,
502    callee: Option<&str>,
503) -> Option<u64> {
504    let timeout = &policy.timeout;
505    if timeout.validate().is_err() {
506        return None;
507    }
508    let bound = callee
509        .and_then(|name| timeout.per_callee.as_ref()?.get(name).copied())
510        .or(timeout.default_millis)?;
511    Some(
512        bound
513            .max(crate::record::MIN_TIMEOUT_RUNG_MS)
514            .min(timeout.max_millis.unwrap_or(u64::MAX)),
515    )
516}
517
518/// The closed legal-action set of a timeout decision (R0.10 wave 3): one
519/// [`DecisionAction::SetTimeout`] per declared ladder rung, ascending. The
520/// top rung models "no bound in force" — the floor's stance — exactly as
521/// the twin and the Wave 1 experiment model it (the lease boundary, the
522/// queue's worst-case discovery latency for hung work).
523pub fn timeout_legal_actions(ladder: &[u64]) -> Vec<DecisionAction> {
524    ladder
525        .iter()
526        .map(|millis| DecisionAction::SetTimeout { millis: *millis })
527        .collect()
528}
529
530/// The [`DecisionAction`] a resolved bound corresponds to on `ladder`: the
531/// smallest rung at or above the bound (a policy's bound is honored by the
532/// first rung that covers it — rounding down would silently tighten the
533/// bound the policy declared), and the top rung when no bound is in force.
534pub fn timeout_selected_action(bound: Option<u64>, ladder: &[u64]) -> DecisionAction {
535    let millis = match bound {
536        Some(bound) => ladder
537            .iter()
538            .copied()
539            .find(|rung| *rung >= bound)
540            .or_else(|| ladder.last().copied())
541            .unwrap_or(bound),
542        None => ladder
543            .last()
544            .copied()
545            .unwrap_or(crate::record::MIN_TIMEOUT_RUNG_MS),
546    };
547    DecisionAction::SetTimeout { millis }
548}
549
550/// Build the [`DecisionEvent`] for one timeout decision (R0.10 wave 3) —
551/// the emission point the design's family-2 telemetry row names.
552///
553/// Features pin the observation vocabulary: `callee` (when the decision
554/// point knows the callee identity), `effect`, `attempt`, and the
555/// callee's journaled latency snapshot (`latency_p50_ms` / `p95` / `p99`
556/// / `latency_samples`) when one was assembled. `bound_ms` is the resolved
557/// bound — `None` when no timeout policy is in force, so the floor's
558/// decisions are distinguishable from a concrete bound on the wire.
559///
560/// Propensity honesty follows the retry family's contract: the acting
561/// policy is deterministic, so the event records `propensity: 1.0` — the
562/// truthful degenerate case, and the same off-policy restriction it
563/// implies. `outcome` is always `None` at decision time: whether the bound
564/// held is the operation's own journaled evidence.
565#[allow(clippy::too_many_arguments)]
566pub fn timeout_decision_event(
567    run_id: impl Into<String>,
568    thread_id: impl Into<String>,
569    seq: u64,
570    callee: Option<&str>,
571    effect: Effect,
572    attempt: u32,
573    percentiles: Option<&LatencyPercentiles>,
574    bound_ms: Option<u64>,
575    ladder: &[u64],
576    policy_version: &PolicyVersion,
577    decided_at: DateTime<Utc>,
578) -> DecisionEvent {
579    let run_id = run_id.into();
580    let mut features = Map::new();
581    if let Some(callee) = callee {
582        features.insert("callee".to_owned(), Value::from(callee));
583    }
584    features.insert(
585        "effect".to_owned(),
586        serde_json::to_value(effect).unwrap_or(Value::Null),
587    );
588    features.insert("attempt".to_owned(), Value::from(attempt));
589    if let Some(percentiles) = percentiles {
590        features.insert("latency_p50_ms".to_owned(), Value::from(percentiles.p50_ms));
591        features.insert("latency_p95_ms".to_owned(), Value::from(percentiles.p95_ms));
592        features.insert("latency_p99_ms".to_owned(), Value::from(percentiles.p99_ms));
593        features.insert(
594            "latency_samples".to_owned(),
595            Value::from(percentiles.samples),
596        );
597    }
598    match bound_ms {
599        Some(bound) => features.insert("bound_ms".to_owned(), Value::from(bound)),
600        None => features.insert("bound_ms".to_owned(), Value::Null),
601    };
602    DecisionEvent {
603        id: format!("{run_id}:d{seq}"),
604        run_id,
605        thread_id: thread_id.into(),
606        seq,
607        family: DecisionFamily::Timeout,
608        legal_actions: timeout_legal_actions(ladder),
609        selected: timeout_selected_action(bound_ms, ladder),
610        features,
611        propensity: 1.0,
612        policy_version: policy_version.clone(),
613        // Acting by construction: this emission point records the decision
614        // that executed. The twin's shadow pairs mark roles explicitly.
615        role: None,
616        outcome: None,
617        decided_at,
618    }
619}
620
621/// The artifact contract of a [`TaskEnvelope`]: what the recipient is
622/// expected to produce and the sender to consume.
623///
624/// v1 carries a kind identifier and an optional size bound only — enough
625/// for the recipient to know it is producing the right *shape* of artifact
626/// and for the queue to refuse to store an out-of-contract result. R0.7
627/// (Agent Fabric wave 1) adds the optional `schema` field — additive with a
628/// serde default, omitted from the wire when unset, so pre-R0.7 contracts
629/// and readers see no shape change.
630#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
631pub struct ArtifactContract {
632    /// Artifact kind identifier — a media type (`application/json`), a
633    /// schema URI, or an application-defined kind name. Interpreted by the
634    /// sender and recipient; the queue treats it as opaque.
635    pub kind: String,
636
637    /// Upper bound on the serialized artifact size in bytes. `None` means
638    /// unbounded within the queue's own storage limits.
639    #[serde(default)]
640    pub max_bytes: Option<u64>,
641
642    /// Optional JSON Schema (draft 2020-12, per the design's open-questions
643    /// default) the artifact payload must validate against (R0.7). Declared
644    /// on the contract so an unacceptable message can fail fast as
645    /// [`ErrorClass::InvalidInput`] at submission — never retried — instead
646    /// of dead-lettering after the attempt budget is spent. `None` means
647    /// kind-and-size checking only, exactly the pre-R0.7 semantics.
648    ///
649    /// Wave 1 pins the field's *shape* (stored and golden-tested); payload
650    /// validation against it is wired at the mailbox submission path in a
651    /// later wave — the schema dialect constraint (design open question 5)
652    /// settles first.
653    #[serde(default, skip_serializing_if = "Option::is_none")]
654    pub schema: Option<serde_json::Value>,
655}
656
657/// The budget a task may consume across all of its attempts.
658#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
659pub struct TaskBudget {
660    /// Maximum number of attempts, counting the initial one. Once reached,
661    /// a retryable failure dead-letters instead of retrying (see
662    /// [`classify_retry`]).
663    pub max_attempts: u32,
664
665    /// Per-attempt wall-clock timeout in milliseconds. An attempt that
666    /// exceeds it is classified [`ErrorClass::Timeout`] by the worker.
667    /// `None` defers to the queue's default.
668    #[serde(default)]
669    pub timeout_ms: Option<u64>,
670}
671
672/// The unit of durable work: one task handed from a sender to a recipient
673/// through the queue.
674///
675/// The envelope is everything the scheduler and the recipient need and
676/// nothing they don't — the input travels as a [`PayloadRef`] (inline for
677/// small values, content-addressed above the inline threshold), so the
678/// queue row stays cheap to scan and the envelope composes with the Flight
679/// Recorder's artifact addressing.
680///
681/// Every field after `format_version` that was added after v1 carries a
682/// serde default, so envelopes queued by an older client keep deserializing;
683/// the default [`Effect`] is the conservative `NonIdempotent`, meaning an
684/// undeclared task is never silently retried.
685#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
686pub struct TaskEnvelope {
687    /// Envelope format version; [`TASK_ENVELOPE_FORMAT_VERSION`] for
688    /// anything written now.
689    #[serde(default = "current_envelope_version")]
690    pub format_version: u32,
691
692    /// Unique task id, assigned by the sender. Server-side ids are
693    /// tenant-namespaced like every other id (`{tenant}/…`); the envelope
694    /// itself is namespace-agnostic.
695    pub task_id: String,
696
697    /// The causal parent: the run event or task that created this one
698    /// (`{run_id}:{seq}` or a task id). Task lineage composes with the
699    /// journal's causal chain, so a fan-out of durable tasks is one
700    /// connected evidence tree. `None` for root tasks.
701    #[serde(default)]
702    pub parent: Option<String>,
703
704    /// Who submitted the task — a run id, a node name, or an agent
705    /// identity. Used for attribution, quotas, and cancellation scoping.
706    pub sender: String,
707
708    /// Who executes the task: a worker pool name (the queue routes by pool)
709    /// or a specific worker identity for pinned placement.
710    pub recipient: String,
711
712    /// Version pin (R0.6 wave 3): the exact worker version string this task
713    /// may be dispatched to. A run started against worker version `w1`
714    /// stamps its tasks with `w1`, so a mid-run deploy never changes
715    /// semantics under an in-flight execution — the scheduler keeps handing
716    /// the task to `w1`-advertising workers until it finishes.
717    ///
718    /// The pin is an *exact string match*, deliberately: semver range
719    /// matching (`^1.4`) makes a claim's outcome depend on the version
720    /// grammar's rules rather than on what the run actually recorded, and
721    /// range resolution belongs to the scheduler's policy, not to a frozen
722    /// wire contract. Ranges are documented future work; `None` (the
723    /// default) means unpinned — any worker may claim the task.
724    #[serde(default)]
725    pub worker_version: Option<String>,
726
727    /// The task input, inline or content-addressed. Small inputs travel
728    /// inside the queue row; large ones are artifacts the recipient resolves
729    /// through the journal's artifact map.
730    pub input: PayloadRef,
731
732    /// What the task is expected to produce. `None` means the result is an
733    /// unconstrained JSON value (the common case for node executions).
734    #[serde(default)]
735    pub output_contract: Option<ArtifactContract>,
736
737    /// Wall-clock deadline for the whole task, across all attempts. The
738    /// scheduler does not re-queue a task whose deadline has passed; the
739    /// worker treats an expired deadline as [`ErrorClass::Cancelled`].
740    #[serde(default)]
741    pub deadline: Option<DateTime<Utc>>,
742
743    /// Attempt and timeout budget. `None` means the queue's defaults.
744    #[serde(default)]
745    pub budget: Option<TaskBudget>,
746
747    /// The deduplication key for effectively-once execution: the queue
748    /// refuses a duplicate submission with an existing key, and the
749    /// recipient passes it to the effect it performs. Required for any task
750    /// whose `effect` is [`Effect::Idempotent`] — the key is what the
751    /// idempotency declaration *means* at the wire. `None` is honest only
752    /// for `Pure` / `ReadOnly` work.
753    #[serde(default)]
754    pub idempotency_key: Option<String>,
755
756    /// The declared effect classification of the work this task performs —
757    /// the same taxonomy the Flight Recorder journals. Defaults to the
758    /// conservative `NonIdempotent`, so a task that does not declare its
759    /// effect is never silently retried; declaring `Idempotent` (with an
760    /// idempotency key) is what unlocks automatic retry.
761    #[serde(default = "default_effect")]
762    pub effect: Effect,
763}
764
765fn current_envelope_version() -> u32 {
766    TASK_ENVELOPE_FORMAT_VERSION
767}
768
769fn default_effect() -> Effect {
770    Effect::NonIdempotent
771}
772
773impl TaskEnvelope {
774    /// A minimal envelope: sender, recipient, and input, with every optional
775    /// field unset and the effect conservatively [`Effect::NonIdempotent`].
776    /// Set the optional fields directly; they are public.
777    pub fn new(
778        task_id: impl Into<String>,
779        sender: impl Into<String>,
780        recipient: impl Into<String>,
781        input: PayloadRef,
782    ) -> Self {
783        Self {
784            format_version: TASK_ENVELOPE_FORMAT_VERSION,
785            task_id: task_id.into(),
786            parent: None,
787            sender: sender.into(),
788            recipient: recipient.into(),
789            worker_version: None,
790            input,
791            output_contract: None,
792            deadline: None,
793            budget: None,
794            idempotency_key: None,
795            effect: Effect::NonIdempotent,
796        }
797    }
798}
799
800#[cfg(test)]
801mod tests {
802    use super::*;
803    use serde_json::json;
804
805    #[test]
806    fn error_class_retryability() {
807        for class in [
808            ErrorClass::Transient,
809            ErrorClass::RateLimited,
810            ErrorClass::Timeout,
811            ErrorClass::DependencyFailure,
812            ErrorClass::ResourceExhausted,
813            ErrorClass::Unknown,
814        ] {
815            assert!(class.is_retryable(), "{class:?} must be retryable");
816        }
817        assert!(!ErrorClass::InvalidInput.is_retryable());
818        assert!(!ErrorClass::Cancelled.is_retryable());
819    }
820
821    #[test]
822    fn error_class_serde_names_are_snake_case() {
823        assert_eq!(
824            serde_json::to_value(ErrorClass::DependencyFailure).unwrap(),
825            json!("dependency_failure")
826        );
827        assert_eq!(
828            serde_json::to_value(ErrorClass::ResourceExhausted).unwrap(),
829            json!("resource_exhausted")
830        );
831    }
832
833    #[test]
834    fn backoff_bounds_and_cap() {
835        // Zero jitter always yields zero delay.
836        assert_eq!(backoff_delay_ms(1, 0.0), 0);
837        assert_eq!(backoff_delay_ms(8, 0.0), 0);
838        // First retry tops out at the base delay; later retries grow
839        // exponentially and clamp at the 5-minute cap.
840        assert!(backoff_delay_ms(1, 0.999) < BASE_RETRY_DELAY_MS);
841        assert!(backoff_delay_ms(2, 0.999) < 2 * BASE_RETRY_DELAY_MS);
842        assert_eq!(backoff_delay_ms(1, 1.0), BASE_RETRY_DELAY_MS);
843        assert_eq!(backoff_delay_ms(20, 1.0), MAX_RETRY_DELAY_MS);
844        // A saturating attempt ordinal cannot overflow.
845        assert_eq!(backoff_delay_ms(u32::MAX, 1.0), MAX_RETRY_DELAY_MS);
846    }
847
848    #[test]
849    fn classify_retry_gates_in_order() {
850        let repeatable = Effect::Idempotent;
851
852        // Gate 1: a non-freely-repeatable effect is never silently retried,
853        // even for an eminently retryable class.
854        assert_eq!(
855            classify_retry(Effect::NonIdempotent, ErrorClass::Transient, 1, 5, 0.5),
856            RetryDecision::Fail
857        );
858        assert_eq!(
859            classify_retry(Effect::Compensatable, ErrorClass::Timeout, 1, 5, 0.5),
860            RetryDecision::Fail
861        );
862
863        // Gate 2: non-retryable classes fail immediately.
864        assert_eq!(
865            classify_retry(repeatable, ErrorClass::InvalidInput, 1, 5, 0.5),
866            RetryDecision::Fail
867        );
868        assert_eq!(
869            classify_retry(repeatable, ErrorClass::Cancelled, 1, 5, 0.5),
870            RetryDecision::Fail
871        );
872
873        // Gate 3: exhausting the attempt budget dead-letters.
874        assert_eq!(
875            classify_retry(repeatable, ErrorClass::Unknown, 5, 5, 0.5),
876            RetryDecision::Dead
877        );
878        assert_eq!(
879            classify_retry(repeatable, ErrorClass::Transient, 1, 0, 0.5),
880            RetryDecision::Dead
881        );
882
883        // Otherwise: retry with a jittered delay within the bound.
884        match classify_retry(repeatable, ErrorClass::RateLimited, 1, 5, 0.5) {
885            RetryDecision::Retry { after_ms } => {
886                assert!(after_ms <= BASE_RETRY_DELAY_MS);
887            }
888            other => panic!("expected Retry, got {other:?}"),
889        }
890    }
891
892    #[test]
893    fn retry_decision_serde_shape() {
894        assert_eq!(
895            serde_json::to_value(RetryDecision::Retry { after_ms: 500 }).unwrap(),
896            json!({"decision": "retry", "after_ms": 500})
897        );
898        assert_eq!(
899            serde_json::to_value(RetryDecision::Dead).unwrap(),
900            json!({"decision": "dead"})
901        );
902        assert_eq!(
903            serde_json::to_value(RetryDecision::Fail).unwrap(),
904            json!({"decision": "fail"})
905        );
906    }
907
908    #[test]
909    fn retry_legal_actions_mirror_the_classifier_gates() {
910        let repeatable = Effect::Idempotent;
911
912        // A retryable failure with budget remaining: retry or abort.
913        assert_eq!(
914            retry_legal_actions(repeatable, ErrorClass::Timeout, 1, 3),
915            vec![DecisionAction::Retry { attempt: 2 }, DecisionAction::Abort]
916        );
917        // Budget exhausted: retrying is not a legal action.
918        assert_eq!(
919            retry_legal_actions(repeatable, ErrorClass::Timeout, 3, 3),
920            vec![DecisionAction::Abort]
921        );
922        assert_eq!(
923            retry_legal_actions(repeatable, ErrorClass::Transient, 1, 0),
924            vec![DecisionAction::Abort]
925        );
926        // Non-retryable class or non-repeatable effect: abort only.
927        assert_eq!(
928            retry_legal_actions(repeatable, ErrorClass::InvalidInput, 1, 3),
929            vec![DecisionAction::Abort]
930        );
931        assert_eq!(
932            retry_legal_actions(Effect::NonIdempotent, ErrorClass::Transient, 1, 3),
933            vec![DecisionAction::Abort]
934        );
935    }
936
937    #[test]
938    fn retry_decision_event_selected_action_stays_inside_the_legal_set() {
939        // Property: for every combination the classifier can see, the event's
940        // selected action is a member of its legal set.
941        let effects = [
942            Effect::Pure,
943            Effect::ReadOnly,
944            Effect::Idempotent,
945            Effect::NonIdempotent,
946            Effect::Compensatable,
947        ];
948        let classes = [
949            ErrorClass::Transient,
950            ErrorClass::RateLimited,
951            ErrorClass::Timeout,
952            ErrorClass::InvalidInput,
953            ErrorClass::DependencyFailure,
954            ErrorClass::ResourceExhausted,
955            ErrorClass::Cancelled,
956            ErrorClass::Unknown,
957        ];
958        let decided_at = DateTime::<Utc>::from_timestamp_millis(1_760_000_000_000).unwrap();
959        for effect in effects {
960            for class in classes {
961                for (attempt, max_attempts) in [(1, 3), (3, 3), (1, 0)] {
962                    let decision = classify_retry(effect, class, attempt, max_attempts, 0.5);
963                    let event = retry_decision_event(
964                        "run-1",
965                        "thread-1",
966                        1,
967                        effect,
968                        class,
969                        attempt,
970                        max_attempts,
971                        None,
972                        &decision,
973                        &PolicyVersion::default(),
974                        decided_at,
975                    );
976                    assert!(
977                        event.legal_actions.contains(&event.selected),
978                        "selected {:?} must be inside legal set {:?} for {effect:?}/{class:?} \
979                         attempt {attempt}/{max_attempts}",
980                        event.selected,
981                        event.legal_actions,
982                    );
983                }
984            }
985        }
986    }
987
988    #[test]
989    fn retry_decision_event_shape_and_outcome_mapping() {
990        let decided_at = DateTime::<Utc>::from_timestamp_millis(1_760_000_000_000).unwrap();
991        let decision = classify_retry(Effect::Idempotent, ErrorClass::Timeout, 1, 3, 0.5);
992        let event = retry_decision_event(
993            "run-9",
994            "thread-2",
995            3,
996            Effect::Idempotent,
997            ErrorClass::Timeout,
998            1,
999            3,
1000            Some(840),
1001            &decision,
1002            &PolicyVersion::new("policy-0123456789ab"),
1003            decided_at,
1004        );
1005        assert_eq!(event.id, "run-9:d3");
1006        assert_eq!(event.seq, 3);
1007        assert_eq!(event.family, DecisionFamily::Retry);
1008        assert_eq!(
1009            event.selected,
1010            DecisionAction::Retry { attempt: 2 },
1011            "the failure was attempt 1; the decision schedules attempt 2"
1012        );
1013        // Deterministic v1 policies record the degenerate propensity, 1.0.
1014        assert_eq!(event.propensity, 1.0);
1015        // A retry decision has no outcome until the re-attempt completes.
1016        assert!(event.outcome.is_none());
1017        assert_eq!(event.features.get("failure_class"), Some(&json!("timeout")));
1018        assert_eq!(event.features.get("attempt"), Some(&json!(1)));
1019        assert_eq!(
1020            event.features.get("dependency_latency_ms"),
1021            Some(&json!(840))
1022        );
1023
1024        // Terminal decisions record the failure outcome immediately.
1025        let dead = retry_decision_event(
1026            "run-9",
1027            "thread-2",
1028            4,
1029            Effect::Idempotent,
1030            ErrorClass::Timeout,
1031            3,
1032            3,
1033            None,
1034            &RetryDecision::Dead,
1035            &PolicyVersion::default(),
1036            decided_at,
1037        );
1038        assert_eq!(dead.selected, DecisionAction::Abort);
1039        assert_eq!(dead.outcome, Some(DecisionOutcome::Failure));
1040        assert!(
1041            !dead.features.contains_key("dependency_latency_ms"),
1042            "an unmeasured latency is absent from the wire, not null"
1043        );
1044    }
1045
1046    #[test]
1047    fn envelope_minimal_json_deserializes_with_defaults() {
1048        // A v1 envelope carrying only the required fields — the shape an
1049        // older, smaller client would write — loads with honest defaults:
1050        // current format version, no options set, conservative effect.
1051        let minimal = json!({
1052            "task_id": "t-1",
1053            "sender": "run-9",
1054            "recipient": "pool-default",
1055            "input": {"kind": "inline", "value": {"n": 1}},
1056        });
1057        let envelope: TaskEnvelope = serde_json::from_value(minimal).unwrap();
1058        assert_eq!(envelope.format_version, TASK_ENVELOPE_FORMAT_VERSION);
1059        assert_eq!(envelope.effect, Effect::NonIdempotent);
1060        assert!(envelope.parent.is_none());
1061        assert!(envelope.idempotency_key.is_none());
1062        assert!(envelope.worker_version.is_none());
1063        assert!(envelope.deadline.is_none());
1064        assert!(envelope.budget.is_none());
1065    }
1066
1067    #[test]
1068    fn envelope_serde_roundtrip() {
1069        let mut envelope = TaskEnvelope::new(
1070            "t-7",
1071            "run-9:node-a",
1072            "pool-default",
1073            PayloadRef::inline(json!({"n": 1})),
1074        );
1075        envelope.parent = Some("run-9:3".into());
1076        envelope.output_contract = Some(ArtifactContract {
1077            kind: "application/json".into(),
1078            max_bytes: Some(65_536),
1079            schema: None,
1080        });
1081        envelope.deadline = DateTime::<Utc>::from_timestamp_millis(1_760_000_000_000);
1082        envelope.budget = Some(TaskBudget {
1083            max_attempts: 5,
1084            timeout_ms: Some(30_000),
1085        });
1086        envelope.idempotency_key = Some("run-9:charge:7".into());
1087        envelope.worker_version = Some("activity-worker/1.4.0".into());
1088        envelope.effect = Effect::Idempotent;
1089
1090        let back: TaskEnvelope =
1091            serde_json::from_str(&serde_json::to_string(&envelope).unwrap()).unwrap();
1092        assert_eq!(envelope, back);
1093    }
1094}