rusty_agent_runtime/durable.rs
1//! Durable Work contracts (R0.6): the shared retry taxonomy and task envelope.
2//!
3//! This module freezes the wire shapes that the R0.6 durable-activity system
4//! builds on — the Postgres task queue in `rusty-agent-server`, the worker SDK in
5//! `rusty-worker`, and the scheduler between them. Nothing here performs I/O
6//! or scheduling; these are pure data contracts plus the one policy function
7//! both sides must agree on ([`classify_retry`]).
8//!
9//! The two pillars:
10//!
11//! - [`ErrorClass`] + [`RetryDecision`] — the retry taxonomy. Every failed
12//! task attempt is classified into a closed set of error classes; the
13//! class, the declared [`Effect`] of the work, and the attempt ordinal map
14//! to exactly one decision: retry after a delay, dead-letter, or fail.
15//! - [`TaskEnvelope`] — the unit of durable work: who sent it, who executes
16//! it, the input reference, the artifact contract, the deadline, the
17//! budget, and the idempotency key that makes retry *effective-once*
18//! rather than duplicated.
19//!
20//! The composition rule that makes retry safe comes from the Flight
21//! Recorder: [`classify_retry`] refuses to silently retry any effect that is
22//! not [`Effect::is_freely_repeatable`], so an `Idempotent` declaration —
23//! with a stable idempotency key — is what unlocks automatic retry.
24//!
25//! Golden-file tests under `tests/golden/` pin the serialized shapes; any
26//! accidental contract drift fails CI. To bless an intentional change,
27//! re-run with `UPDATE_GOLDEN=1` and review the diff.
28
29use chrono::{DateTime, Utc};
30use serde::{Deserialize, Serialize};
31use serde_json::{Map, Value};
32
33use crate::record::{
34 DecisionAction, DecisionEvent, DecisionFamily, DecisionOutcome, Effect, PayloadRef,
35 PolicyVersion,
36};
37
38/// The current wire-format version of [`TaskEnvelope`].
39///
40/// Bump only on a breaking change to the envelope; additive evolution uses
41/// serde defaults instead so previously queued tasks keep deserializing.
42pub const TASK_ENVELOPE_FORMAT_VERSION: u32 = 1;
43
44/// The base delay of the retry backoff schedule, in milliseconds.
45///
46/// Retry `n` (1-based) draws a delay uniformly from
47/// `[0, BASE_RETRY_DELAY_MS * 2^(n-1)]`, capped at [`MAX_RETRY_DELAY_MS`].
48pub const BASE_RETRY_DELAY_MS: u64 = 1_000;
49
50/// The cap of the retry backoff schedule, in milliseconds (5 minutes).
51///
52/// The cap bounds worst-case queue latency for a stuck dependency while
53/// keeping retry pressure negligible: at the cap, one task retries at most
54/// 12 times per hour.
55pub const MAX_RETRY_DELAY_MS: u64 = 300_000;
56
57/// Why a task attempt failed. Closed set; the scheduler matches exhaustively
58/// on it, and new classes are additive (old senders still deserialize).
59///
60/// The class is declared by the executor of the work — the worker that ran
61/// the handler or the transport that carried it — not inferred from logs.
62/// Each variant documents its retry semantics; the mechanical mapping lives
63/// in [`classify_retry`].
64#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)]
65#[serde(rename_all = "snake_case")]
66pub enum ErrorClass {
67 /// A transient fault with no lasting cause: connection reset, broken
68 /// pipe, a dependency's internal hiccup. Retry with backoff; the same
69 /// request is expected to succeed later.
70 Transient,
71
72 /// The callee asked to be slowed down (HTTP 429, `Retry-After`).
73 /// Retry with backoff; a callee-supplied `Retry-After` value floors the
74 /// delay (applied by the scheduler, outside this contract).
75 RateLimited,
76
77 /// The attempt did not finish within its deadline. Retry with backoff,
78 /// but note the ambiguity: the work may have partially executed, which
79 /// is exactly why the [`Effect`] gate in [`classify_retry`] exists — a
80 /// timed-out non-idempotent effect must not be re-attempted silently.
81 Timeout,
82
83 /// The input is malformed or violates the callee's contract (HTTP 400,
84 /// schema validation). Never retried: the same bytes will fail the same
85 /// way on every attempt. Fails the task immediately.
86 InvalidInput,
87
88 /// An upstream dependency the task needs is down or degraded (database
89 /// unreachable, model endpoint 5xx). Retry with backoff; distinct from
90 /// [`ErrorClass::Transient`] so operators can distinguish "our wiring"
91 /// from "their outage" in telemetry and alerting.
92 DependencyFailure,
93
94 /// The worker or the callee is out of capacity: memory pressure,
95 /// connection-pool exhaustion, quota exhaustion. Retry with backoff —
96 /// ideally on a different worker (placement is the scheduler's concern;
97 /// this contract only carries the classification).
98 ResourceExhausted,
99
100 /// The attempt ended because the task was cancelled. Control flow, not
101 /// a failure: never retried, never dead-lettered. Cancellation
102 /// propagation (wave 2) classifies interrupted attempts this way so the
103 /// retry machinery stays out of the cancellation path.
104 Cancelled,
105
106 /// The failure could not be classified — the handler returned an
107 /// unclassified error, or the worker died mid-attempt (lease expiry
108 /// classifies here). Retried with backoff up to the attempt limit, then
109 /// dead-lettered: unknown failures are the DLQ's primary input, since
110 /// they are the ones that need human eyes.
111 Unknown,
112}
113
114impl ErrorClass {
115 /// Whether this class is retryable at all. `InvalidInput` and
116 /// `Cancelled` are the only non-retryable classes; everything else —
117 /// including `Unknown` — may be re-attempted up to the attempt limit.
118 ///
119 /// Retryability of the class is necessary but not sufficient for a
120 /// retry: [`classify_retry`] additionally gates on the work's declared
121 /// [`Effect`].
122 pub fn is_retryable(self) -> bool {
123 !matches!(self, ErrorClass::InvalidInput | ErrorClass::Cancelled)
124 }
125}
126
127/// What the scheduler does with a failed task attempt.
128///
129/// Serialized with internal tagging (`{"decision": "retry", "after_ms": …}`)
130/// so a decision recorded in the journal or the queue's attempt log is
131/// self-describing.
132#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
133#[serde(tag = "decision", rename_all = "snake_case")]
134pub enum RetryDecision {
135 /// Re-queue the task to become visible again after `after_ms`
136 /// milliseconds. The idempotency key and input are unchanged; the
137 /// attempt counter increments.
138 Retry {
139 /// Delay before the task becomes visible to workers again, in
140 /// milliseconds. Computed by [`backoff_delay_ms`].
141 after_ms: u64,
142 },
143
144 /// Move the task to the dead-letter queue: a retryable failure class
145 /// whose attempts are exhausted, or an `Unknown` failure that kept
146 /// recurring. DLQ entries are operator-visible evidence — they carry the
147 /// full attempt history and can be re-driven by hand after the cause is
148 /// fixed.
149 Dead,
150
151 /// Fail the task immediately: a non-retryable class, or any failure of
152 /// work whose declared [`Effect`] is not freely repeatable. The error
153 /// surfaces to the sender; nothing is retried and nothing is
154 /// dead-lettered (there is nothing a human can fix by re-driving the
155 /// same input).
156 Fail,
157}
158
159/// Full-jitter exponential backoff, in milliseconds.
160///
161/// Retry `attempt` (1-based — `1` is the first retry after the initial
162/// failure) draws uniformly from `[0, base * 2^(attempt-1)]` where `base` is
163/// [`BASE_RETRY_DELAY_MS`], capped at [`MAX_RETRY_DELAY_MS`]. Full jitter —
164/// rather than a fixed fraction of the exponential — is what decorrelates a
165/// fleet of tasks that failed together when a shared dependency recovers
166/// (the thundering-herd problem); see the design doc for the citation.
167///
168/// `uniform` is a sample from `[0, 1)`; values outside are clamped. Source
169/// it from the run's `RngSource` so seeded runs reproduce their retry
170/// schedules exactly.
171pub fn backoff_delay_ms(attempt: u32, uniform: f64) -> u64 {
172 backoff_delay_ms_with(attempt, uniform, BASE_RETRY_DELAY_MS, MAX_RETRY_DELAY_MS)
173}
174
175/// The backoff schedule [`backoff_delay_ms`] computes, drawn from
176/// policy-supplied `base_ms`/`cap_ms` instead of the floor constants
177/// (R0.10 wave 3). The schedule's *shape* — full jitter over a doubling
178/// window — is not learnable: the Wave 1 headroom experiment's published
179/// scar is that a fixed per-class delay table underperforms the jittered
180/// exponential on latency, so learned policies tune the base, the cap, and
181/// the budget, never the shape.
182pub fn backoff_delay_ms_with(attempt: u32, uniform: f64, base_ms: u64, cap_ms: u64) -> u64 {
183 let exponent = attempt.saturating_sub(1).min(20);
184 let exponential = base_ms.saturating_mul(1u64 << exponent).min(cap_ms);
185 (uniform.clamp(0.0, 1.0) * exponential as f64) as u64
186}
187
188/// Map a failed attempt to exactly one [`RetryDecision`].
189///
190/// This is the single place the retry policy lives, shared verbatim by the
191/// server scheduler and the worker SDK so both sides of the queue always
192/// agree. It decides with the static floor's constants — the read path
193/// every decision falls back to when no learned policy is in force
194/// (R0.10 wave 3's application loop is [`classify_retry_with_policy`],
195/// which this delegates to with the floor's parameters). The order of the
196/// gates is the policy:
197///
198/// 1. **Effect gate.** Work whose declared [`Effect`] is not
199/// [`Effect::is_freely_repeatable`] is never silently retried — a timed
200/// out `NonIdempotent` charge may already have happened. This is the
201/// composition with the Flight Recorder: the `Idempotent`
202/// classification is what makes retry safe, and it is checked here,
203/// not assumed.
204/// 2. **Class gate.** `InvalidInput` and `Cancelled` fail immediately; all
205/// other classes are retryable ([`ErrorClass::is_retryable`]).
206/// 3. **Attempt gate.** A retryable failure with `attempt >= max_attempts`
207/// goes to the dead-letter queue (`attempt` counts attempts made so far,
208/// starting at 1 for the first failure; `max_attempts == 0` means no
209/// retries at all).
210/// 4. Otherwise: retry, with the delay from [`backoff_delay_ms`].
211pub fn classify_retry(
212 effect: Effect,
213 class: ErrorClass,
214 attempt: u32,
215 max_attempts: u32,
216 uniform: f64,
217) -> RetryDecision {
218 classify_retry_with_policy(
219 effect,
220 class,
221 attempt,
222 &ResolvedRetryParameters::floor(max_attempts),
223 uniform,
224 )
225}
226
227/// The parameters one retry decision is made with (R0.10 wave 3): the
228/// resolution of the active policy's retry parameters for one error class
229/// against the task's declared attempt budget.
230///
231/// Produced by [`resolve_retry_parameters`]; consumed by
232/// [`classify_retry_with_policy`]. Carrying the resolution as a value —
233/// rather than re-reading the policy at the decision point — is what makes
234/// the decision's inputs journaled evidence: the features record the
235/// numbers decided with, not a pointer to state that may have moved.
236#[derive(Debug, Clone, Copy, PartialEq, Eq)]
237pub struct ResolvedRetryParameters {
238 /// Base of the backoff schedule, in milliseconds.
239 pub base_delay_ms: u64,
240
241 /// Cap of the backoff schedule, in milliseconds.
242 pub max_delay_ms: u64,
243
244 /// Effective attempt budget (the task's declared budget, narrowed by a
245 /// learned budget when one is in force).
246 pub max_attempts: u32,
247}
248
249impl ResolvedRetryParameters {
250 /// The floor's resolution: the module constants and the task's own
251 /// budget, untouched. This is exactly what [`classify_retry`] decided
252 /// with before the policy plane existed.
253 pub fn floor(max_attempts: u32) -> Self {
254 Self {
255 base_delay_ms: BASE_RETRY_DELAY_MS,
256 max_delay_ms: MAX_RETRY_DELAY_MS,
257 max_attempts,
258 }
259 }
260}
261
262/// Resolve the active policy's retry parameters for one error class
263/// against the task's declared attempt budget (R0.10 wave 3 — the
264/// application loop's read path).
265///
266/// Fail-closed, in order:
267///
268/// - **An invalid policy steers nothing.** Parameters that fail
269/// [`crate::record::RetryPolicyParameters::validate`] resolve to the
270/// floor — validation is the promotion gate's job, but the decision
271/// point re-checks rather than trusting the write path, because a
272/// registry body read after a partial migration is the decision point's
273/// problem, not the gate's.
274/// - **A class with a learned entry** decides by that entry's schedule;
275/// its budget *narrows* the task's declared budget (`min`), never
276/// widens it — the task's budget is the sender's declaration, the same
277/// rule the concurrency family's quota ceiling follows.
278/// - **A flat learned schedule** (flat values differing from the floor's,
279/// no per-class entry) applies the same way.
280/// - **Otherwise the floor decides**: the module constants and the task's
281/// own budget, byte-identical to pre-learning behavior.
282pub fn resolve_retry_parameters(
283 policy: &crate::record::ExecutorPolicy,
284 class: ErrorClass,
285 task_max_attempts: u32,
286) -> ResolvedRetryParameters {
287 let retry = &policy.retry;
288 if retry.validate().is_err() {
289 return ResolvedRetryParameters::floor(task_max_attempts);
290 }
291 let floor_flat = crate::record::ExecutorPolicy::static_v0().retry.flat();
292 let flat_learned = retry.flat() != floor_flat;
293 let entry = retry.per_class.as_ref().and_then(|table| table.get(&class));
294 match (entry, flat_learned) {
295 (Some(schedule), _) => ResolvedRetryParameters {
296 base_delay_ms: schedule.base_delay_ms,
297 max_delay_ms: schedule.max_delay_ms,
298 max_attempts: schedule.max_attempts.min(task_max_attempts),
299 },
300 (None, true) => ResolvedRetryParameters {
301 base_delay_ms: retry.base_delay_ms,
302 max_delay_ms: retry.max_delay_ms,
303 max_attempts: retry.max_attempts.min(task_max_attempts),
304 },
305 (None, false) => ResolvedRetryParameters::floor(task_max_attempts),
306 }
307}
308
309/// [`classify_retry`] with policy-resolved parameters (R0.10 wave 3 — the
310/// application loop's decision path).
311///
312/// The gates never move: effect gate, class gate, attempt gate are
313/// policy-independent and evaluated in the same order as the floor's.
314/// What a learned policy tunes is the *numbers* — the backoff base and cap
315/// the delay draws from, and (narrowed) the attempt budget — never which
316/// classes or effects are retryable.
317pub fn classify_retry_with_policy(
318 effect: Effect,
319 class: ErrorClass,
320 attempt: u32,
321 resolved: &ResolvedRetryParameters,
322 uniform: f64,
323) -> RetryDecision {
324 if !effect.is_freely_repeatable() || !class.is_retryable() {
325 return RetryDecision::Fail;
326 }
327 if attempt >= resolved.max_attempts {
328 return RetryDecision::Dead;
329 }
330 RetryDecision::Retry {
331 after_ms: backoff_delay_ms_with(
332 attempt,
333 uniform,
334 resolved.base_delay_ms,
335 resolved.max_delay_ms,
336 ),
337 }
338}
339
340/// The closed legal-action set of a retry decision (R0.8 wave 4).
341///
342/// Mirrors [`classify_retry`]'s gates exactly: when a retry is legal
343/// (freely-repeatable effect, retryable class, attempt budget remaining) the
344/// legal set is `[Retry { attempt: attempt + 1 }, Abort]`; otherwise retrying
345/// is not a legal action and the set collapses to `[Abort]`. The set is
346/// computed from the same inputs as the decision itself, so a
347/// [`DecisionEvent`] built by [`retry_decision_event`] can never record a
348/// selected action outside its legal set.
349pub fn retry_legal_actions(
350 effect: Effect,
351 class: ErrorClass,
352 attempt: u32,
353 max_attempts: u32,
354) -> Vec<DecisionAction> {
355 if effect.is_freely_repeatable() && class.is_retryable() && attempt < max_attempts {
356 vec![
357 DecisionAction::Retry {
358 attempt: attempt + 1,
359 },
360 DecisionAction::Abort,
361 ]
362 } else {
363 vec![DecisionAction::Abort]
364 }
365}
366
367/// The [`DecisionAction`] a [`RetryDecision`] corresponds to, given the
368/// 1-based attempt ordinal that failed.
369///
370/// A `Retry` decision takes retry ordinal `attempt + 1` (the failure was
371/// attempt `attempt`; the decision schedules the next one). `Dead` and
372/// `Fail` both map to `Abort`: dead-lettering is giving up on the automatic
373/// path, and the dead-letter queue entry is the queue's own evidence, not a
374/// decision action.
375pub fn retry_selected_action(decision: &RetryDecision, attempt: u32) -> DecisionAction {
376 match decision {
377 RetryDecision::Retry { .. } => DecisionAction::Retry {
378 attempt: attempt + 1,
379 },
380 RetryDecision::Dead | RetryDecision::Fail => DecisionAction::Abort,
381 }
382}
383
384/// Build the [`DecisionEvent`] for one retry decision (R0.8 wave 4).
385///
386/// This is the emission contract between the scheduler (which owns the
387/// decision inputs) and the journal: given exactly the values
388/// [`classify_retry`] was called with plus its output, produce the evidence
389/// record. Features pin the observation vocabulary — `failure_class`,
390/// `attempt`, `max_attempts`, `effect`, and `dependency_latency_ms` when the
391/// caller measured one — so offline evaluation reads stable keys.
392///
393/// **Propensity honesty.** Every v1 policy — including the static floor —
394/// is deterministic: it assigns probability 1 to the selected action and 0
395/// to every other legal action. The event therefore records `propensity:
396/// 1.0`. That is the truthful value, but it also means v1's evidence cannot
397/// support inverse-propensity scoring (division by a zero propensity is
398/// undefined): off-policy evaluation over v1 decisions is restricted to
399/// policies that would have taken the same action. Learned stochastic
400/// policies must record their true propensity at decision time; this
401/// function's `1.0` is the deterministic-policy degenerate case, not a
402/// placeholder.
403///
404/// `outcome` is `None` for a `Retry` decision (the re-attempt has not
405/// happened yet) and `Some(DecisionOutcome::Failure)` for `Dead`/`Fail`
406/// (the operation is over and did not complete).
407#[allow(clippy::too_many_arguments)]
408pub fn retry_decision_event(
409 run_id: impl Into<String>,
410 thread_id: impl Into<String>,
411 seq: u64,
412 effect: Effect,
413 class: ErrorClass,
414 attempt: u32,
415 max_attempts: u32,
416 dependency_latency_ms: Option<u64>,
417 decision: &RetryDecision,
418 policy_version: &PolicyVersion,
419 decided_at: DateTime<Utc>,
420) -> DecisionEvent {
421 let run_id = run_id.into();
422 let mut features = Map::new();
423 features.insert(
424 "failure_class".to_owned(),
425 serde_json::to_value(class).unwrap_or(Value::Null),
426 );
427 features.insert("attempt".to_owned(), Value::from(attempt));
428 features.insert("max_attempts".to_owned(), Value::from(max_attempts));
429 features.insert(
430 "effect".to_owned(),
431 serde_json::to_value(effect).unwrap_or(Value::Null),
432 );
433 if let Some(latency) = dependency_latency_ms {
434 features.insert("dependency_latency_ms".to_owned(), Value::from(latency));
435 }
436 let outcome = match decision {
437 RetryDecision::Retry { .. } => None,
438 RetryDecision::Dead | RetryDecision::Fail => Some(DecisionOutcome::Failure),
439 };
440 DecisionEvent {
441 id: format!("{run_id}:d{seq}"),
442 run_id,
443 thread_id: thread_id.into(),
444 seq,
445 family: DecisionFamily::Retry,
446 features,
447 legal_actions: retry_legal_actions(effect, class, attempt, max_attempts),
448 selected: retry_selected_action(decision, attempt),
449 propensity: 1.0,
450 policy_version: policy_version.clone(),
451 // Acting by construction: the v1 emission point records the decision
452 // that executed. The twin's shadow pairs mark roles explicitly.
453 role: None,
454 outcome,
455 decided_at,
456 }
457}
458
459// --------------------------------------------------------------------- //
460// The timeout family (R0.10 wave 3)
461//
462// The floor's stance is "no bound in force" (`TimeoutPolicyParameters`
463// with every field `None`); a promoted policy resolves to a concrete
464// bound per callee or a default. The decision point's evidence contract
465// mirrors the retry family's: the closed legal set (the declared ladder),
466// the acting policy's version, the degenerate propensity.
467// --------------------------------------------------------------------- //
468
469/// The latency snapshot a timeout decision is made from: the callee's
470/// journaled completion percentiles over the feature window (design open
471/// question 4). Assembled server-side from rolling evidence and journaled
472/// as the feature map at decision time — the feature is then evidence the
473/// twin can reproduce, not a query re-run against mutated state.
474#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
475pub struct LatencyPercentiles {
476 /// Median completion latency, in milliseconds.
477 pub p50_ms: u64,
478
479 /// 95th-percentile completion latency, in milliseconds.
480 pub p95_ms: u64,
481
482 /// 99th-percentile completion latency, in milliseconds.
483 pub p99_ms: u64,
484
485 /// Completions the percentiles were computed over.
486 pub samples: u64,
487}
488
489/// Resolve the active policy's timeout bound for one callee (R0.10 wave 3
490/// — the application loop's read path for the timeout family).
491///
492/// Fail-closed, mirroring [`resolve_retry_parameters`]: an invalid policy
493/// steers nothing (`None` — no bound, the floor's stance); a callee with a
494/// learned entry resolves to it; otherwise the policy's default; otherwise
495/// no bound. A resolved bound is clamped to the declared ceiling and never
496/// below [`crate::record::MIN_TIMEOUT_RUNG_MS`] — validation is the
497/// promotion gate's job, but the clamp is the decision point's own
498/// defense, because a bound below the minimum rung aborts ordinary work
499/// early, a correctness hazard no read path may produce.
500pub fn resolve_timeout_bound_ms(
501 policy: &crate::record::ExecutorPolicy,
502 callee: Option<&str>,
503) -> Option<u64> {
504 let timeout = &policy.timeout;
505 if timeout.validate().is_err() {
506 return None;
507 }
508 let bound = callee
509 .and_then(|name| timeout.per_callee.as_ref()?.get(name).copied())
510 .or(timeout.default_millis)?;
511 Some(
512 bound
513 .max(crate::record::MIN_TIMEOUT_RUNG_MS)
514 .min(timeout.max_millis.unwrap_or(u64::MAX)),
515 )
516}
517
518/// The closed legal-action set of a timeout decision (R0.10 wave 3): one
519/// [`DecisionAction::SetTimeout`] per declared ladder rung, ascending. The
520/// top rung models "no bound in force" — the floor's stance — exactly as
521/// the twin and the Wave 1 experiment model it (the lease boundary, the
522/// queue's worst-case discovery latency for hung work).
523pub fn timeout_legal_actions(ladder: &[u64]) -> Vec<DecisionAction> {
524 ladder
525 .iter()
526 .map(|millis| DecisionAction::SetTimeout { millis: *millis })
527 .collect()
528}
529
530/// The [`DecisionAction`] a resolved bound corresponds to on `ladder`: the
531/// smallest rung at or above the bound (a policy's bound is honored by the
532/// first rung that covers it — rounding down would silently tighten the
533/// bound the policy declared), and the top rung when no bound is in force.
534pub fn timeout_selected_action(bound: Option<u64>, ladder: &[u64]) -> DecisionAction {
535 let millis = match bound {
536 Some(bound) => ladder
537 .iter()
538 .copied()
539 .find(|rung| *rung >= bound)
540 .or_else(|| ladder.last().copied())
541 .unwrap_or(bound),
542 None => ladder
543 .last()
544 .copied()
545 .unwrap_or(crate::record::MIN_TIMEOUT_RUNG_MS),
546 };
547 DecisionAction::SetTimeout { millis }
548}
549
550/// Build the [`DecisionEvent`] for one timeout decision (R0.10 wave 3) —
551/// the emission point the design's family-2 telemetry row names.
552///
553/// Features pin the observation vocabulary: `callee` (when the decision
554/// point knows the callee identity), `effect`, `attempt`, and the
555/// callee's journaled latency snapshot (`latency_p50_ms` / `p95` / `p99`
556/// / `latency_samples`) when one was assembled. `bound_ms` is the resolved
557/// bound — `None` when no timeout policy is in force, so the floor's
558/// decisions are distinguishable from a concrete bound on the wire.
559///
560/// Propensity honesty follows the retry family's contract: the acting
561/// policy is deterministic, so the event records `propensity: 1.0` — the
562/// truthful degenerate case, and the same off-policy restriction it
563/// implies. `outcome` is always `None` at decision time: whether the bound
564/// held is the operation's own journaled evidence.
565#[allow(clippy::too_many_arguments)]
566pub fn timeout_decision_event(
567 run_id: impl Into<String>,
568 thread_id: impl Into<String>,
569 seq: u64,
570 callee: Option<&str>,
571 effect: Effect,
572 attempt: u32,
573 percentiles: Option<&LatencyPercentiles>,
574 bound_ms: Option<u64>,
575 ladder: &[u64],
576 policy_version: &PolicyVersion,
577 decided_at: DateTime<Utc>,
578) -> DecisionEvent {
579 let run_id = run_id.into();
580 let mut features = Map::new();
581 if let Some(callee) = callee {
582 features.insert("callee".to_owned(), Value::from(callee));
583 }
584 features.insert(
585 "effect".to_owned(),
586 serde_json::to_value(effect).unwrap_or(Value::Null),
587 );
588 features.insert("attempt".to_owned(), Value::from(attempt));
589 if let Some(percentiles) = percentiles {
590 features.insert("latency_p50_ms".to_owned(), Value::from(percentiles.p50_ms));
591 features.insert("latency_p95_ms".to_owned(), Value::from(percentiles.p95_ms));
592 features.insert("latency_p99_ms".to_owned(), Value::from(percentiles.p99_ms));
593 features.insert(
594 "latency_samples".to_owned(),
595 Value::from(percentiles.samples),
596 );
597 }
598 match bound_ms {
599 Some(bound) => features.insert("bound_ms".to_owned(), Value::from(bound)),
600 None => features.insert("bound_ms".to_owned(), Value::Null),
601 };
602 DecisionEvent {
603 id: format!("{run_id}:d{seq}"),
604 run_id,
605 thread_id: thread_id.into(),
606 seq,
607 family: DecisionFamily::Timeout,
608 legal_actions: timeout_legal_actions(ladder),
609 selected: timeout_selected_action(bound_ms, ladder),
610 features,
611 propensity: 1.0,
612 policy_version: policy_version.clone(),
613 // Acting by construction: this emission point records the decision
614 // that executed. The twin's shadow pairs mark roles explicitly.
615 role: None,
616 outcome: None,
617 decided_at,
618 }
619}
620
621/// The artifact contract of a [`TaskEnvelope`]: what the recipient is
622/// expected to produce and the sender to consume.
623///
624/// v1 carries a kind identifier and an optional size bound only — enough
625/// for the recipient to know it is producing the right *shape* of artifact
626/// and for the queue to refuse to store an out-of-contract result. R0.7
627/// (Agent Fabric wave 1) adds the optional `schema` field — additive with a
628/// serde default, omitted from the wire when unset, so pre-R0.7 contracts
629/// and readers see no shape change.
630#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
631pub struct ArtifactContract {
632 /// Artifact kind identifier — a media type (`application/json`), a
633 /// schema URI, or an application-defined kind name. Interpreted by the
634 /// sender and recipient; the queue treats it as opaque.
635 pub kind: String,
636
637 /// Upper bound on the serialized artifact size in bytes. `None` means
638 /// unbounded within the queue's own storage limits.
639 #[serde(default)]
640 pub max_bytes: Option<u64>,
641
642 /// Optional JSON Schema (draft 2020-12, per the design's open-questions
643 /// default) the artifact payload must validate against (R0.7). Declared
644 /// on the contract so an unacceptable message can fail fast as
645 /// [`ErrorClass::InvalidInput`] at submission — never retried — instead
646 /// of dead-lettering after the attempt budget is spent. `None` means
647 /// kind-and-size checking only, exactly the pre-R0.7 semantics.
648 ///
649 /// Wave 1 pins the field's *shape* (stored and golden-tested); payload
650 /// validation against it is wired at the mailbox submission path in a
651 /// later wave — the schema dialect constraint (design open question 5)
652 /// settles first.
653 #[serde(default, skip_serializing_if = "Option::is_none")]
654 pub schema: Option<serde_json::Value>,
655}
656
657/// The budget a task may consume across all of its attempts.
658#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
659pub struct TaskBudget {
660 /// Maximum number of attempts, counting the initial one. Once reached,
661 /// a retryable failure dead-letters instead of retrying (see
662 /// [`classify_retry`]).
663 pub max_attempts: u32,
664
665 /// Per-attempt wall-clock timeout in milliseconds. An attempt that
666 /// exceeds it is classified [`ErrorClass::Timeout`] by the worker.
667 /// `None` defers to the queue's default.
668 #[serde(default)]
669 pub timeout_ms: Option<u64>,
670}
671
672/// The unit of durable work: one task handed from a sender to a recipient
673/// through the queue.
674///
675/// The envelope is everything the scheduler and the recipient need and
676/// nothing they don't — the input travels as a [`PayloadRef`] (inline for
677/// small values, content-addressed above the inline threshold), so the
678/// queue row stays cheap to scan and the envelope composes with the Flight
679/// Recorder's artifact addressing.
680///
681/// Every field after `format_version` that was added after v1 carries a
682/// serde default, so envelopes queued by an older client keep deserializing;
683/// the default [`Effect`] is the conservative `NonIdempotent`, meaning an
684/// undeclared task is never silently retried.
685#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
686pub struct TaskEnvelope {
687 /// Envelope format version; [`TASK_ENVELOPE_FORMAT_VERSION`] for
688 /// anything written now.
689 #[serde(default = "current_envelope_version")]
690 pub format_version: u32,
691
692 /// Unique task id, assigned by the sender. Server-side ids are
693 /// tenant-namespaced like every other id (`{tenant}/…`); the envelope
694 /// itself is namespace-agnostic.
695 pub task_id: String,
696
697 /// The causal parent: the run event or task that created this one
698 /// (`{run_id}:{seq}` or a task id). Task lineage composes with the
699 /// journal's causal chain, so a fan-out of durable tasks is one
700 /// connected evidence tree. `None` for root tasks.
701 #[serde(default)]
702 pub parent: Option<String>,
703
704 /// Who submitted the task — a run id, a node name, or an agent
705 /// identity. Used for attribution, quotas, and cancellation scoping.
706 pub sender: String,
707
708 /// Who executes the task: a worker pool name (the queue routes by pool)
709 /// or a specific worker identity for pinned placement.
710 pub recipient: String,
711
712 /// Version pin (R0.6 wave 3): the exact worker version string this task
713 /// may be dispatched to. A run started against worker version `w1`
714 /// stamps its tasks with `w1`, so a mid-run deploy never changes
715 /// semantics under an in-flight execution — the scheduler keeps handing
716 /// the task to `w1`-advertising workers until it finishes.
717 ///
718 /// The pin is an *exact string match*, deliberately: semver range
719 /// matching (`^1.4`) makes a claim's outcome depend on the version
720 /// grammar's rules rather than on what the run actually recorded, and
721 /// range resolution belongs to the scheduler's policy, not to a frozen
722 /// wire contract. Ranges are documented future work; `None` (the
723 /// default) means unpinned — any worker may claim the task.
724 #[serde(default)]
725 pub worker_version: Option<String>,
726
727 /// The task input, inline or content-addressed. Small inputs travel
728 /// inside the queue row; large ones are artifacts the recipient resolves
729 /// through the journal's artifact map.
730 pub input: PayloadRef,
731
732 /// What the task is expected to produce. `None` means the result is an
733 /// unconstrained JSON value (the common case for node executions).
734 #[serde(default)]
735 pub output_contract: Option<ArtifactContract>,
736
737 /// Wall-clock deadline for the whole task, across all attempts. The
738 /// scheduler does not re-queue a task whose deadline has passed; the
739 /// worker treats an expired deadline as [`ErrorClass::Cancelled`].
740 #[serde(default)]
741 pub deadline: Option<DateTime<Utc>>,
742
743 /// Attempt and timeout budget. `None` means the queue's defaults.
744 #[serde(default)]
745 pub budget: Option<TaskBudget>,
746
747 /// The deduplication key for effectively-once execution: the queue
748 /// refuses a duplicate submission with an existing key, and the
749 /// recipient passes it to the effect it performs. Required for any task
750 /// whose `effect` is [`Effect::Idempotent`] — the key is what the
751 /// idempotency declaration *means* at the wire. `None` is honest only
752 /// for `Pure` / `ReadOnly` work.
753 #[serde(default)]
754 pub idempotency_key: Option<String>,
755
756 /// The declared effect classification of the work this task performs —
757 /// the same taxonomy the Flight Recorder journals. Defaults to the
758 /// conservative `NonIdempotent`, so a task that does not declare its
759 /// effect is never silently retried; declaring `Idempotent` (with an
760 /// idempotency key) is what unlocks automatic retry.
761 #[serde(default = "default_effect")]
762 pub effect: Effect,
763}
764
765fn current_envelope_version() -> u32 {
766 TASK_ENVELOPE_FORMAT_VERSION
767}
768
769fn default_effect() -> Effect {
770 Effect::NonIdempotent
771}
772
773impl TaskEnvelope {
774 /// A minimal envelope: sender, recipient, and input, with every optional
775 /// field unset and the effect conservatively [`Effect::NonIdempotent`].
776 /// Set the optional fields directly; they are public.
777 pub fn new(
778 task_id: impl Into<String>,
779 sender: impl Into<String>,
780 recipient: impl Into<String>,
781 input: PayloadRef,
782 ) -> Self {
783 Self {
784 format_version: TASK_ENVELOPE_FORMAT_VERSION,
785 task_id: task_id.into(),
786 parent: None,
787 sender: sender.into(),
788 recipient: recipient.into(),
789 worker_version: None,
790 input,
791 output_contract: None,
792 deadline: None,
793 budget: None,
794 idempotency_key: None,
795 effect: Effect::NonIdempotent,
796 }
797 }
798}
799
800#[cfg(test)]
801mod tests {
802 use super::*;
803 use serde_json::json;
804
805 #[test]
806 fn error_class_retryability() {
807 for class in [
808 ErrorClass::Transient,
809 ErrorClass::RateLimited,
810 ErrorClass::Timeout,
811 ErrorClass::DependencyFailure,
812 ErrorClass::ResourceExhausted,
813 ErrorClass::Unknown,
814 ] {
815 assert!(class.is_retryable(), "{class:?} must be retryable");
816 }
817 assert!(!ErrorClass::InvalidInput.is_retryable());
818 assert!(!ErrorClass::Cancelled.is_retryable());
819 }
820
821 #[test]
822 fn error_class_serde_names_are_snake_case() {
823 assert_eq!(
824 serde_json::to_value(ErrorClass::DependencyFailure).unwrap(),
825 json!("dependency_failure")
826 );
827 assert_eq!(
828 serde_json::to_value(ErrorClass::ResourceExhausted).unwrap(),
829 json!("resource_exhausted")
830 );
831 }
832
833 #[test]
834 fn backoff_bounds_and_cap() {
835 // Zero jitter always yields zero delay.
836 assert_eq!(backoff_delay_ms(1, 0.0), 0);
837 assert_eq!(backoff_delay_ms(8, 0.0), 0);
838 // First retry tops out at the base delay; later retries grow
839 // exponentially and clamp at the 5-minute cap.
840 assert!(backoff_delay_ms(1, 0.999) < BASE_RETRY_DELAY_MS);
841 assert!(backoff_delay_ms(2, 0.999) < 2 * BASE_RETRY_DELAY_MS);
842 assert_eq!(backoff_delay_ms(1, 1.0), BASE_RETRY_DELAY_MS);
843 assert_eq!(backoff_delay_ms(20, 1.0), MAX_RETRY_DELAY_MS);
844 // A saturating attempt ordinal cannot overflow.
845 assert_eq!(backoff_delay_ms(u32::MAX, 1.0), MAX_RETRY_DELAY_MS);
846 }
847
848 #[test]
849 fn classify_retry_gates_in_order() {
850 let repeatable = Effect::Idempotent;
851
852 // Gate 1: a non-freely-repeatable effect is never silently retried,
853 // even for an eminently retryable class.
854 assert_eq!(
855 classify_retry(Effect::NonIdempotent, ErrorClass::Transient, 1, 5, 0.5),
856 RetryDecision::Fail
857 );
858 assert_eq!(
859 classify_retry(Effect::Compensatable, ErrorClass::Timeout, 1, 5, 0.5),
860 RetryDecision::Fail
861 );
862
863 // Gate 2: non-retryable classes fail immediately.
864 assert_eq!(
865 classify_retry(repeatable, ErrorClass::InvalidInput, 1, 5, 0.5),
866 RetryDecision::Fail
867 );
868 assert_eq!(
869 classify_retry(repeatable, ErrorClass::Cancelled, 1, 5, 0.5),
870 RetryDecision::Fail
871 );
872
873 // Gate 3: exhausting the attempt budget dead-letters.
874 assert_eq!(
875 classify_retry(repeatable, ErrorClass::Unknown, 5, 5, 0.5),
876 RetryDecision::Dead
877 );
878 assert_eq!(
879 classify_retry(repeatable, ErrorClass::Transient, 1, 0, 0.5),
880 RetryDecision::Dead
881 );
882
883 // Otherwise: retry with a jittered delay within the bound.
884 match classify_retry(repeatable, ErrorClass::RateLimited, 1, 5, 0.5) {
885 RetryDecision::Retry { after_ms } => {
886 assert!(after_ms <= BASE_RETRY_DELAY_MS);
887 }
888 other => panic!("expected Retry, got {other:?}"),
889 }
890 }
891
892 #[test]
893 fn retry_decision_serde_shape() {
894 assert_eq!(
895 serde_json::to_value(RetryDecision::Retry { after_ms: 500 }).unwrap(),
896 json!({"decision": "retry", "after_ms": 500})
897 );
898 assert_eq!(
899 serde_json::to_value(RetryDecision::Dead).unwrap(),
900 json!({"decision": "dead"})
901 );
902 assert_eq!(
903 serde_json::to_value(RetryDecision::Fail).unwrap(),
904 json!({"decision": "fail"})
905 );
906 }
907
908 #[test]
909 fn retry_legal_actions_mirror_the_classifier_gates() {
910 let repeatable = Effect::Idempotent;
911
912 // A retryable failure with budget remaining: retry or abort.
913 assert_eq!(
914 retry_legal_actions(repeatable, ErrorClass::Timeout, 1, 3),
915 vec![DecisionAction::Retry { attempt: 2 }, DecisionAction::Abort]
916 );
917 // Budget exhausted: retrying is not a legal action.
918 assert_eq!(
919 retry_legal_actions(repeatable, ErrorClass::Timeout, 3, 3),
920 vec![DecisionAction::Abort]
921 );
922 assert_eq!(
923 retry_legal_actions(repeatable, ErrorClass::Transient, 1, 0),
924 vec![DecisionAction::Abort]
925 );
926 // Non-retryable class or non-repeatable effect: abort only.
927 assert_eq!(
928 retry_legal_actions(repeatable, ErrorClass::InvalidInput, 1, 3),
929 vec![DecisionAction::Abort]
930 );
931 assert_eq!(
932 retry_legal_actions(Effect::NonIdempotent, ErrorClass::Transient, 1, 3),
933 vec![DecisionAction::Abort]
934 );
935 }
936
937 #[test]
938 fn retry_decision_event_selected_action_stays_inside_the_legal_set() {
939 // Property: for every combination the classifier can see, the event's
940 // selected action is a member of its legal set.
941 let effects = [
942 Effect::Pure,
943 Effect::ReadOnly,
944 Effect::Idempotent,
945 Effect::NonIdempotent,
946 Effect::Compensatable,
947 ];
948 let classes = [
949 ErrorClass::Transient,
950 ErrorClass::RateLimited,
951 ErrorClass::Timeout,
952 ErrorClass::InvalidInput,
953 ErrorClass::DependencyFailure,
954 ErrorClass::ResourceExhausted,
955 ErrorClass::Cancelled,
956 ErrorClass::Unknown,
957 ];
958 let decided_at = DateTime::<Utc>::from_timestamp_millis(1_760_000_000_000).unwrap();
959 for effect in effects {
960 for class in classes {
961 for (attempt, max_attempts) in [(1, 3), (3, 3), (1, 0)] {
962 let decision = classify_retry(effect, class, attempt, max_attempts, 0.5);
963 let event = retry_decision_event(
964 "run-1",
965 "thread-1",
966 1,
967 effect,
968 class,
969 attempt,
970 max_attempts,
971 None,
972 &decision,
973 &PolicyVersion::default(),
974 decided_at,
975 );
976 assert!(
977 event.legal_actions.contains(&event.selected),
978 "selected {:?} must be inside legal set {:?} for {effect:?}/{class:?} \
979 attempt {attempt}/{max_attempts}",
980 event.selected,
981 event.legal_actions,
982 );
983 }
984 }
985 }
986 }
987
988 #[test]
989 fn retry_decision_event_shape_and_outcome_mapping() {
990 let decided_at = DateTime::<Utc>::from_timestamp_millis(1_760_000_000_000).unwrap();
991 let decision = classify_retry(Effect::Idempotent, ErrorClass::Timeout, 1, 3, 0.5);
992 let event = retry_decision_event(
993 "run-9",
994 "thread-2",
995 3,
996 Effect::Idempotent,
997 ErrorClass::Timeout,
998 1,
999 3,
1000 Some(840),
1001 &decision,
1002 &PolicyVersion::new("policy-0123456789ab"),
1003 decided_at,
1004 );
1005 assert_eq!(event.id, "run-9:d3");
1006 assert_eq!(event.seq, 3);
1007 assert_eq!(event.family, DecisionFamily::Retry);
1008 assert_eq!(
1009 event.selected,
1010 DecisionAction::Retry { attempt: 2 },
1011 "the failure was attempt 1; the decision schedules attempt 2"
1012 );
1013 // Deterministic v1 policies record the degenerate propensity, 1.0.
1014 assert_eq!(event.propensity, 1.0);
1015 // A retry decision has no outcome until the re-attempt completes.
1016 assert!(event.outcome.is_none());
1017 assert_eq!(event.features.get("failure_class"), Some(&json!("timeout")));
1018 assert_eq!(event.features.get("attempt"), Some(&json!(1)));
1019 assert_eq!(
1020 event.features.get("dependency_latency_ms"),
1021 Some(&json!(840))
1022 );
1023
1024 // Terminal decisions record the failure outcome immediately.
1025 let dead = retry_decision_event(
1026 "run-9",
1027 "thread-2",
1028 4,
1029 Effect::Idempotent,
1030 ErrorClass::Timeout,
1031 3,
1032 3,
1033 None,
1034 &RetryDecision::Dead,
1035 &PolicyVersion::default(),
1036 decided_at,
1037 );
1038 assert_eq!(dead.selected, DecisionAction::Abort);
1039 assert_eq!(dead.outcome, Some(DecisionOutcome::Failure));
1040 assert!(
1041 !dead.features.contains_key("dependency_latency_ms"),
1042 "an unmeasured latency is absent from the wire, not null"
1043 );
1044 }
1045
1046 #[test]
1047 fn envelope_minimal_json_deserializes_with_defaults() {
1048 // A v1 envelope carrying only the required fields — the shape an
1049 // older, smaller client would write — loads with honest defaults:
1050 // current format version, no options set, conservative effect.
1051 let minimal = json!({
1052 "task_id": "t-1",
1053 "sender": "run-9",
1054 "recipient": "pool-default",
1055 "input": {"kind": "inline", "value": {"n": 1}},
1056 });
1057 let envelope: TaskEnvelope = serde_json::from_value(minimal).unwrap();
1058 assert_eq!(envelope.format_version, TASK_ENVELOPE_FORMAT_VERSION);
1059 assert_eq!(envelope.effect, Effect::NonIdempotent);
1060 assert!(envelope.parent.is_none());
1061 assert!(envelope.idempotency_key.is_none());
1062 assert!(envelope.worker_version.is_none());
1063 assert!(envelope.deadline.is_none());
1064 assert!(envelope.budget.is_none());
1065 }
1066
1067 #[test]
1068 fn envelope_serde_roundtrip() {
1069 let mut envelope = TaskEnvelope::new(
1070 "t-7",
1071 "run-9:node-a",
1072 "pool-default",
1073 PayloadRef::inline(json!({"n": 1})),
1074 );
1075 envelope.parent = Some("run-9:3".into());
1076 envelope.output_contract = Some(ArtifactContract {
1077 kind: "application/json".into(),
1078 max_bytes: Some(65_536),
1079 schema: None,
1080 });
1081 envelope.deadline = DateTime::<Utc>::from_timestamp_millis(1_760_000_000_000);
1082 envelope.budget = Some(TaskBudget {
1083 max_attempts: 5,
1084 timeout_ms: Some(30_000),
1085 });
1086 envelope.idempotency_key = Some("run-9:charge:7".into());
1087 envelope.worker_version = Some("activity-worker/1.4.0".into());
1088 envelope.effect = Effect::Idempotent;
1089
1090 let back: TaskEnvelope =
1091 serde_json::from_str(&serde_json::to_string(&envelope).unwrap()).unwrap();
1092 assert_eq!(envelope, back);
1093 }
1094}