rusty_agent_runtime/learn.rs
1//! The learning loop's candidate pipeline (R0.8 Rusty Learn, wave 3): the
2//! candidate contract, the evaluation composition seam, the promotion
3//! envelope and its gate, canary binding by seeded draw, and the
4//! active-version pointer with byte-exact rollback.
5//!
6//! The design doc is `docs/learn-design.md` ("The learning loop"). Its
7//! learning rule governs everything here: **no learning process may
8//! silently rewrite a production prompt, graph, policy, memory, or tool
9//! permission.** Learning produces an immutable candidate; the candidate is
10//! evaluated against recorded evidence; promotion is a journaled runtime
11//! transition bounded by a declared envelope; rollback re-points an
12//! immutable version pointer. What a framework does with a config file
13//! edit and a restart, this module does as an evidence-carrying state
14//! transition.
15//!
16//! - [`Candidate`] — the distiller's output: an immutable, versioned,
17//! content-addressed declaration of a proposed change. Identity is
18//! `sha256` over the canonical content ([`derive_candidate_id`]), so two
19//! distillations of the same change converge on one id and a tampered
20//! candidate fails its own address ([`Candidate::verify_address`]).
21//! Closed enum [`CandidateKind`]: `prompt` / `policy` / `memory_set` /
22//! `tool_permission`.
23//! - [`CandidateEvaluation`] — the journaled evaluation payload: the
24//! replay divergence summary, the experiment report pair, the dataset
25//! version, and the comparison verdict. The evaluator itself is a seam
26//! ([`CandidateEvaluator`]), not runtime code — the same boundary open
27//! question 2 draws for distillers, forced by the workspace's dependency
28//! direction: `rusty-eval` links the runtime, never the reverse, so the
29//! runtime owns the journaled *shape* of the evaluation and the
30//! application owns producing it through `rusty-eval`'s
31//! `ExperimentRunner` + `compare()`. [`EvaluationVerdict`] and
32//! [`EvaluationThresholds`] are the deliberate mirrors of `compare()`'s
33//! output and `CompareThresholds` — one serde-pinned record reference,
34//! the same way `rusty-eval` already meets the runtime at the `Journal`.
35//! - [`PromotionEnvelope`] — the declared, per-deployment standing
36//! approval: per candidate kind, what may auto-promote (the evidence
37//! thresholds), what requires a human approval, and what promotes into
38//! a canary. The gate is [`admit_promotion`], a pure function; refusal
39//! is a typed error ([`PromotionRefusal`]), never a silent no-op.
40//! Out-of-envelope promotion requires an
41//! [`crate::effects::ApprovalToken`] scoped to
42//! [`promotion_effect_id`] — derived over the candidate's content hash
43//! and target scope, so an approval for one candidate is
44//! non-transferable to another. This composes the effect kernel rather
45//! than inventing an approval parallel to it.
46//! - [`VersionPointer`] — the active version per surface (prompt name,
47//! policy scope, memory scope, tool grant) as an immutable pointer to a
48//! [`CandidateId`]. Every promotion is a pointer move; rollback
49//! re-points to the previous candidate and is byte-exact because
50//! candidates are content-addressed and immutable — the restored
51//! version is the one that previously served, not a reconstruction.
52//!
53//! Every lifecycle transition (created → evaluated → promoted → rolled
54//! back) journals through four additive [`RunEventKind`](crate::record::RunEventKind) variants —
55//! `CandidateCreated` / `CandidateEvaluated` / `CandidatePromoted` /
56//! `CandidateRolledBack` — the same evolution rule R0.6's `EffectReceipt`
57//! and R0.7's agent variants followed; old journals keep deserializing.
58//! Refused transitions are errors ([`LearnError`]), not silent no-ops.
59//!
60//! Golden-file tests under `tests/golden/` pin every wire shape in this
61//! module; any accidental contract drift fails CI.
62
63use std::collections::HashSet;
64use std::sync::Arc;
65
66use async_trait::async_trait;
67use chrono::{DateTime, Utc};
68use serde::{Deserialize, Serialize};
69use serde_json::Value;
70use sha2::{Digest, Sha256};
71use thiserror::Error;
72
73use crate::effects::{derive_effect_id, ApprovalToken, EffectId};
74use crate::error::{Result, RustyError};
75use crate::journal::JournalSnapshot;
76use crate::memory::{
77 apply_query, MemoryQuery, MemoryRecord, MemoryScope, MemoryStore, ProvenanceAuthor,
78 ScopeAddress,
79};
80use crate::record::{sha256_hex, DecisionFamily, RunManifest};
81
82fn invalid(message: impl Into<String>) -> RustyError {
83 // Candidate construction and application are state updates to the
84 // governed learning plane; contract validation failures reuse the
85 // invalid-update class rather than growing the error taxonomy for one
86 // module (the memory module's own convention).
87 RustyError::InvalidUpdate(message.into())
88}
89
90// --------------------------------------------------------------------- //
91// The candidate contract
92// --------------------------------------------------------------------- //
93
94/// A content-addressed candidate identity: lowercase hex SHA-256 over the
95/// candidate's canonical content ([`derive_candidate_id`]).
96///
97/// Transparent newtype so the type system — not convention — keeps
98/// candidate ids distinct from memory addresses, effect ids, and other
99/// digest strings (the [`PolicyVersion`] / [`EffectId`] precedent).
100#[derive(Debug, Clone, PartialEq, Eq, Hash, PartialOrd, Ord, Serialize, Deserialize)]
101#[serde(transparent)]
102pub struct CandidateId(String);
103
104impl CandidateId {
105 /// The hex digest.
106 pub fn as_str(&self) -> &str {
107 &self.0
108 }
109}
110
111impl std::fmt::Display for CandidateId {
112 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
113 f.write_str(&self.0)
114 }
115}
116
117impl From<String> for CandidateId {
118 /// Wrap a digest the caller already trusts — rehydration from storage,
119 /// a rollback receipt's `previous`. Minting ids from content is
120 /// [`derive_candidate_id`]'s job alone; nothing here re-verifies.
121 fn from(digest: String) -> Self {
122 Self(digest)
123 }
124}
125
126/// What a candidate would change. Closed enum — the gate, the pointer,
127/// and the evaluator match exhaustively on it.
128#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)]
129#[serde(rename_all = "snake_case")]
130pub enum CandidateKind {
131 /// New prompt text for one named prompt.
132 Prompt,
133 /// Executor policy parameters for one decision family.
134 Policy,
135 /// A set of memory records (adds and supersessions).
136 MemorySet,
137 /// A narrowed or widened tool grant.
138 ToolPermission,
139}
140
141/// The direction of a [`CandidateContent::ToolPermission`] change.
142/// Closed enum: the direction is the R0.8 contract — grant mechanics
143/// (which capabilities a narrowed grant drops, how a widened one is
144/// bounded) are R0.9's capsule-manifest work, and this release gates
145/// every tool-permission candidate behind a human approval regardless.
146#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)]
147#[serde(rename_all = "snake_case")]
148pub enum GrantDirection {
149 /// The grant shrinks (a capability is withdrawn).
150 Narrow,
151 /// The grant grows (a capability is added).
152 Widen,
153}
154
155/// The change a candidate declares, per [`CandidateKind`]. Serialized with
156/// internal tagging (`{"kind": "memory_set", …}`): the kind is part of
157/// the content address — a prompt text and a policy parameter set that
158/// happened to serialize alike must never converge.
159#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
160#[serde(tag = "kind", rename_all = "snake_case")]
161pub enum CandidateContent {
162 /// New prompt text. The text's digest — `sha256_hex` over its UTF-8
163 /// bytes — is exactly the pin [`RunManifest::pin_prompt`] records, so
164 /// the candidate and the run manifest speak one content address.
165 Prompt {
166 /// The prompt's name (the manifest pin's key).
167 name: String,
168 /// The proposed prompt text.
169 prompt: String,
170 },
171 /// Executor policy parameters for one
172 /// [`crate::record::DecisionFamily`] — backoff caps, timeout bounds,
173 /// concurrency limits. The closed legal-action sets stay closed: a
174 /// learned policy chooses among declared actions, never free-form
175 /// outputs.
176 Policy {
177 /// The decision family the parameters govern.
178 family: DecisionFamily,
179 /// The parameter set (schema is the policy registry's, wave 4).
180 parameters: Value,
181 },
182 /// A set of memory records at one scope: adds and supersessions.
183 /// Carrying full [`MemoryRecord`]s — not ids — is what lets the set
184 /// carry wave 2's attributed correction candidates: the records'
185 /// provenance (and therefore their own content addresses) is inside
186 /// this candidate's address.
187 MemorySet {
188 /// The scope the set applies at.
189 scope: ScopeAddress,
190 /// The records the set adds (already content-addressed, already
191 /// immutable — a record in two sets is the same record).
192 adds: Vec<MemoryRecord>,
193 /// Bare memory content addresses the set supersedes beyond the
194 /// adds' own `supersedes` fields.
195 #[serde(default, skip_serializing_if = "Vec::is_empty")]
196 supersedes: Vec<String>,
197 },
198 /// A narrowed or widened tool grant.
199 ToolPermission {
200 /// The tool whose grant changes.
201 tool: String,
202 /// The direction of the change (see [`GrantDirection`]).
203 direction: GrantDirection,
204 },
205}
206
207/// The content address of a candidate: `sha256` over the canonical
208/// serialization of its content — the one hashing primitive shared with
209/// artifact references and journal heads, over the canonical `serde_json`
210/// serialization [`crate::record::PayloadRef::content_hash`] also relies
211/// on (object keys sort deterministically).
212pub fn derive_candidate_id(content: &CandidateContent) -> Result<CandidateId> {
213 Ok(CandidateId(sha256_hex(&serde_json::to_vec(content)?)))
214}
215
216/// The evidence span a distiller read to produce a candidate — journaled
217/// with creation, because a candidate that cannot name its observations
218/// cannot be audited. Every field is optional and absent from the wire
219/// when empty: a hand-authored candidate may carry no span at all, and
220/// sparse evidence must not change the shape for dense readers.
221#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
222pub struct EvidenceSpan {
223 /// The recorded runs whose journals the distiller read. "Completed"
224 /// is load-bearing (the design's observe stage): learning reads
225 /// terminal evidence, never in-flight state.
226 #[serde(default, skip_serializing_if = "Vec::is_empty")]
227 pub run_ids: Vec<String>,
228
229 /// The corrections (R0.8 wave 2) the candidate folds in — the
230 /// reference distiller's input.
231 #[serde(default, skip_serializing_if = "Vec::is_empty")]
232 pub correction_ids: Vec<String>,
233
234 /// The memory records the distiller read (e.g. the attributed
235 /// candidates a `memory_set` carries).
236 #[serde(default, skip_serializing_if = "Vec::is_empty")]
237 pub memory_ids: Vec<String>,
238}
239
240impl EvidenceSpan {
241 /// `true` when no span is carried (the sparse wire shape).
242 pub fn is_empty(&self) -> bool {
243 self.run_ids.is_empty() && self.correction_ids.is_empty() && self.memory_ids.is_empty()
244 }
245}
246
247/// A proposed change: immutable, versioned, content-addressed.
248///
249/// The address covers the **content** — the change itself — and nothing
250/// else: two distillations of the same change converge on one id even
251/// when different distillers read different evidence to reach it (the
252/// design's identity-is-integrity rule). Attribution is not identity, so
253/// the distiller, the evidence span, and the learning instant travel
254/// beside the address rather than inside it — the inverse of
255/// [`MemoryRecord`]'s provenance-in-identity rule, deliberate: a memory
256/// record answers "who claims this," a candidate answers "what would
257/// change."
258///
259/// Immutable by construction: a changed candidate is a new id, and there
260/// is no in-place update anywhere in the pipeline.
261#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
262pub struct Candidate {
263 /// The content address ([`derive_candidate_id`]).
264 pub candidate_id: CandidateId,
265
266 /// The change, per kind.
267 pub content: CandidateContent,
268
269 /// The distiller's identity. Mandatory — journaled with creation
270 /// (`CandidateCreated`), because a candidate that cannot name its
271 /// distiller is indistinguishable from a config edit.
272 pub distilled_by: ProvenanceAuthor,
273
274 /// The evidence span the distiller read. Absent from the wire when
275 /// empty.
276 #[serde(default, skip_serializing_if = "EvidenceSpan::is_empty")]
277 pub evidence: EvidenceSpan,
278
279 /// When the distiller minted the candidate.
280 pub created_at: DateTime<Utc>,
281}
282
283impl Candidate {
284 /// Build a candidate, deriving its content address from `content`.
285 pub fn new(
286 content: CandidateContent,
287 distilled_by: ProvenanceAuthor,
288 evidence: EvidenceSpan,
289 created_at: DateTime<Utc>,
290 ) -> Result<Self> {
291 let candidate_id = derive_candidate_id(&content)?;
292 Ok(Self {
293 candidate_id,
294 content,
295 distilled_by,
296 evidence,
297 created_at,
298 })
299 }
300
301 /// The candidate's kind.
302 pub fn kind(&self) -> CandidateKind {
303 match &self.content {
304 CandidateContent::Prompt { .. } => CandidateKind::Prompt,
305 CandidateContent::Policy { .. } => CandidateKind::Policy,
306 CandidateContent::MemorySet { .. } => CandidateKind::MemorySet,
307 CandidateContent::ToolPermission { .. } => CandidateKind::ToolPermission,
308 }
309 }
310
311 /// The production surface this candidate would change — the pointer
312 /// key: `prompt:{name}` / `policy:{family}` / `memory:{scope}` /
313 /// `tool:{tool}`. One surface admits one active version, which is
314 /// what makes promotion a pointer move and rollback a re-pointing.
315 pub fn surface(&self) -> SurfaceKey {
316 match &self.content {
317 CandidateContent::Prompt { name, .. } => SurfaceKey::new(format!("prompt:{name}")),
318 CandidateContent::Policy { family, .. } => {
319 let family = serde_json::to_value(family)
320 .ok()
321 .and_then(|v| v.as_str().map(str::to_owned))
322 .unwrap_or_else(|| format!("{family:?}").to_lowercase());
323 SurfaceKey::new(format!("policy:{family}"))
324 }
325 CandidateContent::MemorySet { scope, .. } => {
326 SurfaceKey::new(format!("memory:{}", scope.as_address()))
327 }
328 CandidateContent::ToolPermission { tool, .. } => {
329 SurfaceKey::new(format!("tool:{tool}"))
330 }
331 }
332 }
333
334 /// Re-derive the content address and compare: `Ok` when the candidate
335 /// is the one its id names. Identity is integrity — the gate runs
336 /// this before any promotion, so a tampered candidate fails closed at
337 /// the boundary, not after a pointer moved.
338 pub fn verify_address(&self) -> std::result::Result<(), LearnError> {
339 let derived = derive_candidate_id(&self.content)
340 .map_err(|e| LearnError::UnaddressableContent(e.to_string()))?;
341 if derived != self.candidate_id {
342 return Err(LearnError::AddressMismatch {
343 claimed: self.candidate_id.clone(),
344 derived,
345 });
346 }
347 Ok(())
348 }
349
350 /// Apply a `prompt` candidate to a manifest: substitute the named
351 /// prompt's pin with the candidate's — the same digest
352 /// [`RunManifest::pin_prompt`] records, so the evaluation's manifest
353 /// speaks the production surface's content address. Every other kind
354 /// fails: a candidate applies only to the surface its kind declares.
355 pub fn apply_to_manifest(&self, manifest: &mut RunManifest) -> Result<()> {
356 let CandidateContent::Prompt { name, prompt } = &self.content else {
357 return Err(invalid(format!(
358 "only a `prompt` candidate applies to a run manifest; `{}` candidates apply \
359 to their own surfaces",
360 self.kind_str()
361 )));
362 };
363 manifest
364 .prompts
365 .insert(name.clone(), sha256_hex(prompt.as_bytes()));
366 Ok(())
367 }
368
369 fn kind_str(&self) -> &'static str {
370 match self.kind() {
371 CandidateKind::Prompt => "prompt",
372 CandidateKind::Policy => "policy",
373 CandidateKind::MemorySet => "memory_set",
374 CandidateKind::ToolPermission => "tool_permission",
375 }
376 }
377}
378
379/// A candidate's production surface: the pointer key. Transparent newtype
380/// over the canonical `{kind}:{scope}` string ([`Candidate::surface`]).
381#[derive(Debug, Clone, PartialEq, Eq, Hash, PartialOrd, Ord, Serialize, Deserialize)]
382#[serde(transparent)]
383pub struct SurfaceKey(String);
384
385impl SurfaceKey {
386 /// Wrap a surface string (built by [`Candidate::surface`]).
387 pub fn new(surface: impl Into<String>) -> Self {
388 Self(surface.into())
389 }
390
391 /// The surface string.
392 pub fn as_str(&self) -> &str {
393 &self.0
394 }
395}
396
397impl std::fmt::Display for SurfaceKey {
398 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
399 f.write_str(&self.0)
400 }
401}
402
403// --------------------------------------------------------------------- //
404// Effect keys and the approval scope
405// --------------------------------------------------------------------- //
406
407/// The derived idempotency key of a candidate creation:
408/// `candidate:{candidate_id}`. Content addressing makes retried
409/// submissions converge on one key.
410pub fn candidate_effect_key(candidate_id: &CandidateId) -> String {
411 format!("candidate:{candidate_id}")
412}
413
414/// The derived idempotency key of an evaluation:
415/// `evaluation:{candidate_id}:{dataset_version}`. Re-evaluation against
416/// the same dataset version converges; a new dataset version is a new
417/// evaluation by construction.
418pub fn evaluation_effect_key(candidate_id: &CandidateId, dataset_version: &str) -> String {
419 format!("evaluation:{candidate_id}:{dataset_version}")
420}
421
422/// The derived idempotency key of a promotion: `promotion:{candidate_id}`
423/// (the design's mechanism decision). Retried promotions converge, and
424/// recovery re-derives the same key.
425pub fn promotion_effect_key(candidate_id: &CandidateId) -> String {
426 format!("promotion:{candidate_id}")
427}
428
429/// The derived idempotency key of a rollback:
430/// `rollback:{surface}:{candidate_id}` — one rollback per candidate per
431/// surface.
432pub fn rollback_effect_key(surface: &SurfaceKey, candidate_id: &CandidateId) -> String {
433 format!("rollback:{surface}:{candidate_id}")
434}
435
436/// The effect kind promotions derive their deterministic id under.
437pub const PROMOTION_EFFECT_KIND: &str = "candidate_promotion";
438
439/// The deterministic effect id of a candidate's promotion —
440/// [`derive_effect_id`] over the target surface (scope), the promotion
441/// kind, the candidate's content hash (input hash), and the promotion
442/// effect key. This is the id an out-of-envelope [`ApprovalToken`] must
443/// be scoped to: the scope check makes an approval for one candidate
444/// non-transferable to another, and the token's `approved_by` gives the
445/// journaled promotion its attribution.
446pub fn promotion_effect_id(candidate: &Candidate) -> EffectId {
447 derive_effect_id(
448 candidate.surface().as_str(),
449 PROMOTION_EFFECT_KIND,
450 candidate.candidate_id.as_str(),
451 Some(&promotion_effect_key(&candidate.candidate_id)),
452 )
453}
454
455// --------------------------------------------------------------------- //
456// The evaluation composition seam
457// --------------------------------------------------------------------- //
458
459/// The comparison thresholds an evaluation ran under — the deliberate
460/// mirror of `rusty-eval`'s `CompareThresholds`, serde-identical by
461/// contract: the evaluator (which links `rusty-eval`) maps one onto the
462/// other field-for-field, and the journaled payload speaks this shape so
463/// the runtime never links the eval crate (the workspace's dependency
464/// direction is `rusty-eval` → runtime, never the reverse).
465#[derive(Debug, Clone, Copy, PartialEq, Serialize, Deserialize)]
466pub struct EvaluationThresholds {
467 /// Maximum tolerable absolute pass-rate drop, per assertion and per
468 /// case, before the comparison flags a regression.
469 pub max_pass_rate_drop: f64,
470
471 /// Maximum tolerable p95 latency ratio `candidate / baseline`.
472 pub max_latency_p95_ratio: f64,
473}
474
475impl Default for EvaluationThresholds {
476 /// 5-point pass-rate drop tolerance, 25% p95 latency tolerance — the
477 /// same defaults `CompareThresholds` declares.
478 fn default() -> Self {
479 Self {
480 max_pass_rate_drop: 0.05,
481 max_latency_p95_ratio: 1.25,
482 }
483 }
484}
485
486/// The comparison verdict the promotion gate reads — the deliberate
487/// mirror of the release-gate bits of `rusty-eval`'s `ComparisonReport`
488/// (`regressed` plus the target metric's movement), stated in terms the
489/// envelope's evidence thresholds consume: no regression **and**
490/// improvement on the target metric over the named dataset version.
491#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
492pub struct EvaluationVerdict {
493 /// `compare()`'s release-gate bit: any threshold breach.
494 pub regressed: bool,
495
496 /// The metric the envelope's improvement bar reads (e.g.
497 /// `run_pass_rate`, `case:{case_id}`, `assertion:{key}`).
498 pub target_metric: String,
499
500 /// The metric's baseline value, when the baseline names it.
501 #[serde(default, skip_serializing_if = "Option::is_none")]
502 pub baseline: Option<f64>,
503
504 /// The metric's candidate value, when the candidate names it.
505 #[serde(default, skip_serializing_if = "Option::is_none")]
506 pub candidate: Option<f64>,
507
508 /// `candidate - baseline` (negative is worse); absent when either
509 /// side lacks the metric — an improvement bar cannot clear on a
510 /// metric the evidence does not name.
511 #[serde(default, skip_serializing_if = "Option::is_none")]
512 pub delta: Option<f64>,
513}
514
515/// One recorded run that diverged under replay with the candidate
516/// applied: the fixture and the first disagreement, summarized (the full
517/// evidence is the replayed journal, reachable by fixture id).
518#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
519pub struct ReplayDivergence {
520 /// The replay fixture (a recorded run's identity) that diverged.
521 pub fixture_id: String,
522
523 /// The first disagreement, summarized (kind, sequence, request hash —
524 /// enough to locate it without dumping payloads).
525 pub detail: String,
526}
527
528/// The replay half of an evaluation: recorded runs re-driven with the
529/// candidate applied. Exact replay serves journaled effects, so the
530/// candidate's behavior is measured against identical evidence with zero
531/// outbound calls; divergence detection is the replay engine's own
532/// contract, consumed — not re-implemented — here.
533#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
534pub struct ReplaySummary {
535 /// Every fixture re-driven (recorded run identities), sorted for
536 /// determinism.
537 pub fixture_ids: Vec<String>,
538
539 /// How many fixtures replayed without divergence.
540 pub matched: usize,
541
542 /// Every fixture that diverged, sorted by fixture id.
543 #[serde(default, skip_serializing_if = "Vec::is_empty")]
544 pub divergences: Vec<ReplayDivergence>,
545}
546
547impl ReplaySummary {
548 /// `true` when every fixture replayed clean. The auto-promotion bar
549 /// requires it: a candidate that cannot reproduce its evidence has no
550 /// business in front of a comparison verdict.
551 pub fn is_clean(&self) -> bool {
552 self.divergences.is_empty() && self.matched == self.fixture_ids.len()
553 }
554}
555
556/// The journaled evaluation payload — the `CandidateEvaluated` event's
557/// output. The evaluation is evidence, not a log line: the verdict, the
558/// report pair, the dataset version, and the replay fixture ids, so the
559/// improvement is explainable afterward by walking ids (candidate →
560/// evaluation → reports → fixtures).
561#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
562pub struct CandidateEvaluation {
563 /// The evaluated candidate.
564 pub candidate_id: CandidateId,
565
566 /// The versioned dataset both reports ran against. Versioned, never
567 /// edited in place — the envelope's auto-promotion bar may name the
568 /// exact version a clean verdict must come from.
569 pub dataset_version: String,
570
571 /// The replay half: divergence summary plus the fixture ids.
572 pub replay: ReplaySummary,
573
574 /// The baseline experiment report, verbatim (`rusty-eval`'s
575 /// `ExperimentReport` serialized; carried as a value because the
576 /// runtime does not link the eval crate — see the module docs).
577 pub baseline_report: Value,
578
579 /// The candidate experiment report, same convention.
580 pub candidate_report: Value,
581
582 /// The comparison verdict the gate reads.
583 pub verdict: EvaluationVerdict,
584
585 /// The thresholds the comparison applied, echoed so an auditor reads
586 /// the bar the verdict was judged against, not a later default.
587 pub thresholds: EvaluationThresholds,
588
589 /// Who ran the evaluation (attribution is mandatory here too).
590 pub evaluated_by: ProvenanceAuthor,
591
592 /// When the evaluation ran.
593 pub evaluated_at: DateTime<Utc>,
594}
595
596/// What an evaluator is asked to do: run `candidate` over the declared
597/// evidence and produce the journaled [`CandidateEvaluation`].
598/// Wire-complete: the server's evaluate route accepts it as the request
599/// body.
600#[derive(Debug, Clone, Serialize, Deserialize)]
601pub struct EvaluationRequest {
602 /// The dataset version both reports must name.
603 pub dataset_version: String,
604
605 /// The metric the promotion gate reads.
606 pub target_metric: String,
607
608 /// The comparison thresholds.
609 pub thresholds: EvaluationThresholds,
610
611 /// The recorded run journals to re-drive with the candidate applied
612 /// (the replay half's evidence), already integrity-loadable.
613 pub replay_evidence: Vec<JournalSnapshot>,
614}
615
616/// The evaluation seam: composition, not duplication. The implementation
617/// builds `PreparedRun`s with the candidate applied (for `memory_set`,
618/// the candidate's records visible at their scope via
619/// [`CandidateOverlay`]; for `prompt`, the pinned prompt hash substituted
620/// via [`Candidate::apply_to_manifest`]) and drives them through the real
621/// executor — replay over [`EvaluationRequest::replay_evidence`] for the
622/// divergence half, `rusty-eval`'s `ExperimentRunner` over the versioned
623/// dataset for the report half, `compare()` for the verdict.
624///
625/// The runtime owns this trait rather than an implementation for the same
626/// reason it owns the distiller contract but not distillers (design open
627/// question 2), compounded by the workspace's dependency direction:
628/// `rusty-eval` links the runtime, never the reverse. Wave 4's release
629/// proof implements this trait over `rusty-eval`'s public API; wave 3
630/// proves the seam with scripted nodes.
631#[async_trait]
632pub trait CandidateEvaluator: Send + Sync + std::fmt::Debug {
633 /// Evaluate `candidate` against `request`'s evidence. The returned
634 /// evaluation must name the candidate and the request's dataset
635 /// version — the gate refuses mismatches.
636 async fn evaluate(
637 &self,
638 candidate: &Candidate,
639 request: &EvaluationRequest,
640 ) -> Result<CandidateEvaluation>;
641}
642
643// --------------------------------------------------------------------- //
644// Candidate application: the memory overlay
645// --------------------------------------------------------------------- //
646
647/// A `memory_set` candidate applied as a read lens over a live store:
648/// the candidate's records answer at their scope, its supersessions drop
649/// out of default retrieval, and everything else passes through. This is
650/// how evaluation (and, post-promotion, new traffic) sees candidate
651/// memory without a store migration — the candidate applied, as a store.
652///
653/// The lens is deliberately read-only: writes fail. An evaluation that
654/// writes through the candidate's lens would be mutating production
655/// memory mid-measurement — the silent-rewrite pattern the pipeline
656/// exists to forbid.
657#[derive(Debug)]
658pub struct CandidateOverlay {
659 base: Arc<dyn MemoryStore>,
660 adds: Vec<MemoryRecord>,
661 supersedes: HashSet<String>,
662}
663
664impl CandidateOverlay {
665 /// Apply `candidate` over `base`. Fails for non-`memory_set`
666 /// candidates: a candidate applies only to the surface its kind
667 /// declares.
668 pub fn new(base: Arc<dyn MemoryStore>, candidate: &Candidate) -> Result<Self> {
669 let CandidateContent::MemorySet {
670 adds, supersedes, ..
671 } = &candidate.content
672 else {
673 return Err(invalid(format!(
674 "only a `memory_set` candidate applies as a memory overlay; `{}` candidates \
675 apply to their own surfaces",
676 match candidate.kind() {
677 CandidateKind::Prompt => "prompt",
678 CandidateKind::Policy => "policy",
679 CandidateKind::MemorySet => unreachable!(),
680 CandidateKind::ToolPermission => "tool_permission",
681 }
682 )));
683 };
684 Ok(Self {
685 base,
686 adds: adds.clone(),
687 supersedes: supersedes.iter().cloned().collect(),
688 })
689 }
690}
691
692#[async_trait]
693impl MemoryStore for CandidateOverlay {
694 async fn put(&self, _record: &MemoryRecord) -> Result<bool> {
695 Err(invalid(
696 "the candidate overlay is a read lens: writes through it would mutate production \
697 memory mid-evaluation — the silent-rewrite pattern the candidate pipeline exists \
698 to forbid",
699 ))
700 }
701
702 async fn get(&self, memory_id: &str) -> Result<Option<MemoryRecord>> {
703 if let Some(record) = self.adds.iter().find(|r| r.memory_id == memory_id) {
704 return Ok(Some(record.clone()));
705 }
706 self.base.get(memory_id).await
707 }
708
709 async fn all(&self) -> Result<Vec<MemoryRecord>> {
710 // Merge by content address: a candidate record already in the base
711 // is the same record (identity is integrity), so the add wins by
712 // construction and nothing doubles.
713 let mut merged: Vec<MemoryRecord> = self
714 .base
715 .all()
716 .await?
717 .into_iter()
718 .filter(|record| {
719 !self
720 .adds
721 .iter()
722 .any(|add| add.memory_id == record.memory_id)
723 })
724 .collect();
725 merged.extend(self.adds.iter().cloned());
726 Ok(merged)
727 }
728
729 async fn remove(&self, _memory_id: &str) -> Result<bool> {
730 Err(invalid(
731 "the candidate overlay is a read lens: deletion through it would mutate production \
732 memory mid-evaluation",
733 ))
734 }
735
736 async fn query(&self, query: &MemoryQuery, now: DateTime<Utc>) -> Result<Vec<MemoryRecord>> {
737 // Core's matcher supplies every filter semantic, including the
738 // superseded set it computes from the merged universe (the adds'
739 // own `supersedes` fields included). The candidate's explicit
740 // supersession list is the overlay's own contribution: drop those
741 // ids unless the query asked for superseded records.
742 let matched = apply_query(&self.all().await?, query, now);
743 if query.include_superseded || self.supersedes.is_empty() {
744 return Ok(matched);
745 }
746 Ok(matched
747 .into_iter()
748 .filter(|record| !self.supersedes.contains(&record.memory_id))
749 .collect())
750 }
751}
752
753// --------------------------------------------------------------------- //
754// The promotion envelope and its gate
755// --------------------------------------------------------------------- //
756
757/// The evidence thresholds an auto-promotion must clear (the design's
758/// rule): a clean replay, `compare()` showing no regression, **and**
759/// improvement on the target metric — over the named dataset version,
760/// when the deployment names one.
761#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
762pub struct AutoPromotion {
763 /// The dataset version a clean verdict must come from. `None` accepts
764 /// the verdict's own version — honest for deployments that version
765 /// datasets per evaluation; a named version pins the promotion to the
766 /// exact evidence the deployment reviewed.
767 #[serde(default, skip_serializing_if = "Option::is_none")]
768 pub dataset_version: Option<String>,
769
770 /// The target-metric delta must exceed this (strictly). `0.0` is the
771 /// design's default: any improvement clears, parity does not.
772 pub min_improvement: f64,
773
774 /// The memory scopes a `memory_set` candidate may auto-promote at
775 /// (ignored for other kinds; empty = unrestricted). The R0.8 default
776 /// names `run` and `agent` (open question 6): wider scopes always
777 /// require a human approval this release.
778 #[serde(default, skip_serializing_if = "Vec::is_empty")]
779 pub scopes: Vec<MemoryScope>,
780}
781
782/// What the envelope declares for one candidate kind. Closed enum — the
783/// gate matches exhaustively.
784#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
785#[serde(tag = "mode", rename_all = "snake_case")]
786pub enum EnvelopeRule {
787 /// May auto-promote when the evaluation clears the declared evidence
788 /// thresholds. Inside the envelope, the envelope itself is the
789 /// standing approval — versioned and declared, not a silent default.
790 Auto(AutoPromotion),
791
792 /// Always requires a human [`ApprovalToken`] scoped to
793 /// [`promotion_effect_id`], whatever the evidence says. The token's
794 /// `approved_by` is the promotion's attribution.
795 Approval,
796
797 /// Auto-promotes into a canary: the candidate binds to the declared
798 /// fraction of new runs (admission picks by seeded draw —
799 /// [`canary_admits`] — so a recorded run reproduces its assignment),
800 /// the static version serving the rest. The evidence thresholds gate
801 /// entry into the canary, exactly as for [`EnvelopeRule::Auto`].
802 Canary {
803 /// The fraction of new runs the candidate serves, in `(0, 1]`.
804 fraction: f64,
805 /// The evidence thresholds gating canary entry.
806 auto: AutoPromotion,
807 },
808}
809
810/// The declared, per-deployment promotion envelope: per candidate kind,
811/// what may promote automatically, what requires review, and what
812/// canaries. Versioned (`envelope_version`) because the envelope is the
813/// standing approval — a journaled promotion names the envelope version
814/// it cleared, so the audit reads the declaration that was in force, not
815/// a later edit.
816#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
817pub struct PromotionEnvelope {
818 /// The deployment-declared envelope version.
819 pub envelope_version: String,
820
821 /// The rule for `prompt` candidates.
822 pub prompt: EnvelopeRule,
823
824 /// The rule for `policy` candidates.
825 pub policy: EnvelopeRule,
826
827 /// The rule for `memory_set` candidates.
828 pub memory_set: EnvelopeRule,
829
830 /// The rule for `tool_permission` candidates.
831 pub tool_permission: EnvelopeRule,
832}
833
834impl PromotionEnvelope {
835 /// The R0.8 default (design open question 6): `memory_set` candidates
836 /// at run and agent scope with a clean verdict may auto-promote;
837 /// `prompt`, `policy`, and `tool_permission` always require an
838 /// approval token this release.
839 pub fn r08_default() -> Self {
840 Self {
841 envelope_version: "r0.8-default".to_owned(),
842 prompt: EnvelopeRule::Approval,
843 policy: EnvelopeRule::Approval,
844 memory_set: EnvelopeRule::Auto(AutoPromotion {
845 dataset_version: None,
846 min_improvement: 0.0,
847 scopes: vec![MemoryScope::Run, MemoryScope::Agent],
848 }),
849 tool_permission: EnvelopeRule::Approval,
850 }
851 }
852
853 /// The rule for `kind`.
854 pub fn rule_for(&self, kind: CandidateKind) -> &EnvelopeRule {
855 match kind {
856 CandidateKind::Prompt => &self.prompt,
857 CandidateKind::Policy => &self.policy,
858 CandidateKind::MemorySet => &self.memory_set,
859 CandidateKind::ToolPermission => &self.tool_permission,
860 }
861 }
862
863 /// The declaration's own validity: canary fractions in `(0, 1]` and
864 /// improvement bars non-negative. Checked where envelopes are
865 /// declared (deployment configuration), so a malformed envelope fails
866 /// at startup, not at a promotion.
867 pub fn validate(&self) -> Result<()> {
868 for rule in [
869 &self.prompt,
870 &self.policy,
871 &self.memory_set,
872 &self.tool_permission,
873 ] {
874 match rule {
875 EnvelopeRule::Auto(auto) => check_auto(auto)?,
876 EnvelopeRule::Canary { fraction, auto } => {
877 if !(fraction.is_finite() && *fraction > 0.0 && *fraction <= 1.0) {
878 return Err(invalid(format!(
879 "canary fraction must be in (0, 1], got {fraction} — a fraction \
880 outside the interval binds nothing or everything, and both are \
881 silent full promotions or silent no-ops"
882 )));
883 }
884 check_auto(auto)?;
885 }
886 EnvelopeRule::Approval => {}
887 }
888 }
889 Ok(())
890 }
891}
892
893fn check_auto(auto: &AutoPromotion) -> Result<()> {
894 if !(auto.min_improvement.is_finite() && auto.min_improvement >= 0.0) {
895 return Err(invalid(format!(
896 "min_improvement must be finite and non-negative, got {} — a negative bar would \
897 auto-promote regressions, which is the one outcome the gate exists to refuse",
898 auto.min_improvement
899 )));
900 }
901 Ok(())
902}
903
904/// A canary binding: the candidate serves `fraction` of new runs on its
905/// surface, admission picking by seeded draw ([`canary_admits`]); the
906/// static version (or the pointer's full `active`) serves the rest.
907#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
908pub struct CanaryBinding {
909 /// The canaried candidate.
910 pub candidate_id: CandidateId,
911
912 /// The fraction of new runs the candidate serves, in `(0, 1]`.
913 pub fraction: f64,
914}
915
916/// Domain-separation prefix for [`canary_admits`] — canary draws must
917/// never collide with effect ids, journal heads, or any other digest in
918/// the system.
919pub const CANARY_DRAW_DOMAIN: &str = "rusty/canary-draw/v1";
920
921/// The canary admission draw: whether `run_id` binds `binding`'s
922/// candidate on `surface`.
923///
924/// **Seed derivation, documented (the design's reproducibility rule).**
925/// The draw is `SHA-256` over the newline-joined tuple
926/// `CANARY_DRAW_DOMAIN "\n" surface "\n" candidate_id "\n" run_id`,
927/// read as a `u64` from the digest's first eight bytes (big-endian) and
928/// compared against `fraction × u64::MAX`. Every input is journaled
929/// evidence — the surface and candidate id are on the pointer the run
930/// bound, the run id is the journal's own identity — so a recorded run
931/// re-derives its assignment exactly, and the draw needs no RNG state of
932/// its own. The candidate id's presence makes two canaries on one
933/// surface independent draws; the run id's presence makes one canary's
934/// assignments uniform across traffic.
935pub fn canary_admits(binding: &CanaryBinding, surface: &SurfaceKey, run_id: &str) -> bool {
936 if binding.fraction >= 1.0 {
937 return true;
938 }
939 let material = [
940 CANARY_DRAW_DOMAIN,
941 surface.as_str(),
942 binding.candidate_id.as_str(),
943 run_id,
944 ]
945 .join("\n");
946 // The first eight bytes of the digest as a big-endian u64: uniform
947 // over the digest space, so `draw < fraction × MAX` admits a
948 // `fraction` share of runs.
949 let mut hasher = Sha256::new();
950 hasher.update(material.as_bytes());
951 let digest = hasher.finalize();
952 let draw = u64::from_be_bytes(digest[..8].try_into().expect("eight bytes"));
953 (draw as f64) < binding.fraction * (u64::MAX as f64)
954}
955
956/// Why a promotion was refused. A dedicated error type (the
957/// [`crate::effects::EffectViolation`] precedent): refusals are contract
958/// outcomes surfaced to the caller, not runtime failures — and a refused
959/// promotion changes nothing.
960#[derive(Debug, Clone, PartialEq, Error)]
961pub enum PromotionRefusal {
962 /// The candidate was never evaluated. Promotion is gated on evidence;
963 /// there is no evidence-free path, not even with an approval — a
964 /// human approves *reviewed evidence*, never a blank check.
965 #[error(
966 "candidate `{candidate_id}` has no journaled evaluation: promotion is gated on \
967 evidence — a human approves reviewed evidence, never a blank check"
968 )]
969 NotEvaluated {
970 /// The candidate refused.
971 candidate_id: CandidateId,
972 },
973
974 /// The evaluation on record is not this candidate's.
975 #[error(
976 "evaluation names candidate `{evaluation}` but promotion was requested for \
977 `{candidate_id}` — the evidence must be the candidate's own"
978 )]
979 EvaluationMismatch {
980 /// The candidate refused.
981 candidate_id: CandidateId,
982 /// The candidate the journaled evaluation names.
983 evaluation: CandidateId,
984 },
985
986 /// The replay half diverged: the candidate could not reproduce its
987 /// recorded evidence.
988 #[error(
989 "candidate `{candidate_id}` diverged under replay ({divergences} fixture(s)): a \
990 candidate that cannot reproduce its evidence never auto-promotes"
991 )]
992 ReplayDiverged {
993 /// The candidate refused.
994 candidate_id: CandidateId,
995 /// How many fixtures diverged.
996 divergences: usize,
997 },
998
999 /// `compare()` flagged a regression beyond the declared thresholds.
1000 #[error(
1001 "candidate `{candidate_id}` regressed against its baseline: the auto-promotion bar \
1002 is no regression AND improvement — the evidence failed the first half"
1003 )]
1004 EvaluationRegressed {
1005 /// The candidate refused.
1006 candidate_id: CandidateId,
1007 },
1008
1009 /// The envelope names a dataset version the evaluation did not run.
1010 #[error(
1011 "envelope requires a clean verdict over dataset `{required}` but the evaluation \
1012 ran `{evaluated}` — the promotion bar names its evidence exactly"
1013 )]
1014 DatasetVersionMismatch {
1015 /// The version the envelope names.
1016 required: String,
1017 /// The version the evaluation ran.
1018 evaluated: String,
1019 },
1020
1021 /// A `memory_set` candidate at a scope the auto rule does not cover.
1022 #[error(
1023 "memory_set candidates at `{scope:?}` scope do not auto-promote under this envelope — \
1024 the R0.8 default covers run and agent scope; wider scopes require an approval"
1025 )]
1026 ScopeNotAutoPromotable {
1027 /// The scope that fell outside the rule.
1028 scope: MemoryScope,
1029 },
1030
1031 /// The target metric did not improve past the declared bar.
1032 #[error(
1033 "target metric moved by {delta:?} but the envelope requires improvement past \
1034 {required} — parity is not improvement, and the auto bar does not clear on it"
1035 )]
1036 InsufficientImprovement {
1037 /// The observed delta (`None`: the evidence did not name the
1038 /// metric at all).
1039 delta: Option<f64>,
1040 /// The bar the envelope declared.
1041 required: f64,
1042 },
1043
1044 /// The candidate falls outside the envelope's auto paths: a human
1045 /// approval scoped to the named effect id is required.
1046 #[error(
1047 "this promotion is outside the envelope: present an approval token scoped to \
1048 effect id {effect_id} — the scope check makes an approval for one candidate \
1049 non-transferable to another"
1050 )]
1051 RequiresApproval {
1052 /// The effect id the approval must be scoped to
1053 /// ([`promotion_effect_id`]).
1054 effect_id: EffectId,
1055 },
1056
1057 /// A token was presented, but it approves a different effect id.
1058 #[error(
1059 "approval token for {presented} does not admit this promotion ({required}) — an \
1060 approval is scoped to exactly one candidate's promotion and is not transferable"
1061 )]
1062 ApprovalMismatch {
1063 /// The effect id the admission requires.
1064 required: EffectId,
1065 /// The effect id the presented token approves.
1066 presented: EffectId,
1067 },
1068}
1069
1070/// The learn plane's typed errors: address integrity, lifecycle
1071/// transitions, and promotion refusals. Refused operations change
1072/// nothing — there is no silent no-op anywhere in the pipeline.
1073#[derive(Debug, Clone, PartialEq, Error)]
1074pub enum LearnError {
1075 /// A candidate's id does not re-derive from its content — the
1076 /// tampered-candidate case, failing closed at the gate.
1077 #[error(
1078 "candidate address mismatch: id {claimed} does not re-derive from the content \
1079 (derived {derived}) — identity is integrity, and a tampered candidate fails its \
1080 own address"
1081 )]
1082 AddressMismatch {
1083 /// The id the candidate claims.
1084 claimed: CandidateId,
1085 /// The id its content derives.
1086 derived: CandidateId,
1087 },
1088
1089 /// The candidate's content could not be serialized for addressing —
1090 /// unreachable for well-formed content, surfaced rather than
1091 /// panicked.
1092 #[error("candidate content could not be addressed: {0}")]
1093 UnaddressableContent(String),
1094
1095 /// A lifecycle transition the state machine does not admit (e.g.
1096 /// promoting a candidate that was never evaluated, rolling back one
1097 /// that was never promoted).
1098 #[error(
1099 "invalid candidate transition: cannot {action} from status `{from:?}` — refused \
1100 transitions are errors, not silent no-ops"
1101 )]
1102 InvalidTransition {
1103 /// The status the transition was attempted from.
1104 from: CandidateStatus,
1105 /// The attempted transition.
1106 action: &'static str,
1107 },
1108
1109 /// A receipt presented for one candidate names another.
1110 #[error(
1111 "receipt names candidate `{named}` but the record is `{candidate_id}` — a receipt \
1112 applies to the candidate it was minted for, never to another"
1113 )]
1114 ReceiptMismatch {
1115 /// The candidate the record holds.
1116 candidate_id: CandidateId,
1117 /// The candidate the receipt names.
1118 named: CandidateId,
1119 },
1120
1121 /// The promotion gate refused.
1122 #[error("{0}")]
1123 Refused(#[from] PromotionRefusal),
1124}
1125
1126/// The promotion gate's positive decision: the authority the promotion
1127/// clears under, journaled on the receipt — the envelope version (the
1128/// standing approval) or the token's `approved_by` — plus the canary
1129/// binding, when the promotion is a canary.
1130#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
1131pub struct PromotionDecision {
1132 /// Who authorized the promotion.
1133 pub authority: PromotionAuthority,
1134
1135 /// The canary binding, when the envelope declared a canary.
1136 #[serde(default, skip_serializing_if = "Option::is_none")]
1137 pub canary: Option<CanaryBinding>,
1138}
1139
1140/// The authority a promotion cleared under. Closed enum — the journaled
1141/// attribution.
1142#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
1143#[serde(tag = "authority", rename_all = "snake_case")]
1144pub enum PromotionAuthority {
1145 /// The envelope's auto path cleared: the envelope itself is the
1146 /// standing approval, and the journaled version is the declaration
1147 /// that was in force.
1148 Envelope {
1149 /// The envelope version that admitted the promotion.
1150 envelope_version: String,
1151 },
1152 /// A human approved out-of-envelope: the token's `approved_by` is the
1153 /// attribution.
1154 Approval {
1155 /// Who approved (the token's `approved_by`).
1156 approved_by: String,
1157 },
1158}
1159
1160/// The promotion gate (the design's promote stage), a pure function over
1161/// the envelope, the candidate, its journaled evaluation, and an optional
1162/// approval token.
1163///
1164/// Order of decisions: integrity first ([`Candidate::verify_address`] —
1165/// a tampered candidate fails closed before any policy reads it), then
1166/// evidence (a journaled evaluation of *this* candidate must exist —
1167/// approvals review evidence, they never replace it), then the envelope
1168/// rule for the candidate's kind: auto paths check the evidence
1169/// thresholds (clean replay, no regression, improvement past the
1170/// declared bar, over the named dataset version, at an admitted scope),
1171/// the approval path requires a token scoped to [`promotion_effect_id`].
1172/// Refusal is a typed [`PromotionRefusal`]; nothing here mutates
1173/// anything.
1174pub fn admit_promotion(
1175 envelope: &PromotionEnvelope,
1176 candidate: &Candidate,
1177 evaluation: Option<&CandidateEvaluation>,
1178 approval: Option<&ApprovalToken>,
1179) -> std::result::Result<PromotionDecision, LearnError> {
1180 candidate.verify_address()?;
1181 let evaluation = evaluation.ok_or(PromotionRefusal::NotEvaluated {
1182 candidate_id: candidate.candidate_id.clone(),
1183 })?;
1184 if evaluation.candidate_id != candidate.candidate_id {
1185 return Err(PromotionRefusal::EvaluationMismatch {
1186 candidate_id: candidate.candidate_id.clone(),
1187 evaluation: evaluation.candidate_id.clone(),
1188 }
1189 .into());
1190 }
1191 match envelope.rule_for(candidate.kind()) {
1192 EnvelopeRule::Auto(auto) => {
1193 check_evidence_bar(auto, candidate, evaluation)?;
1194 Ok(PromotionDecision {
1195 authority: PromotionAuthority::Envelope {
1196 envelope_version: envelope.envelope_version.clone(),
1197 },
1198 canary: None,
1199 })
1200 }
1201 EnvelopeRule::Canary { fraction, auto } => {
1202 check_evidence_bar(auto, candidate, evaluation)?;
1203 Ok(PromotionDecision {
1204 authority: PromotionAuthority::Envelope {
1205 envelope_version: envelope.envelope_version.clone(),
1206 },
1207 canary: Some(CanaryBinding {
1208 candidate_id: candidate.candidate_id.clone(),
1209 fraction: *fraction,
1210 }),
1211 })
1212 }
1213 EnvelopeRule::Approval => {
1214 let required = promotion_effect_id(candidate);
1215 let token = approval.ok_or(PromotionRefusal::RequiresApproval {
1216 effect_id: required.clone(),
1217 })?;
1218 if !token.admits(&required) {
1219 return Err(PromotionRefusal::ApprovalMismatch {
1220 required,
1221 presented: token.effect_id().clone(),
1222 }
1223 .into());
1224 }
1225 Ok(PromotionDecision {
1226 authority: PromotionAuthority::Approval {
1227 approved_by: token.approved_by().to_owned(),
1228 },
1229 canary: None,
1230 })
1231 }
1232 }
1233}
1234
1235/// The auto paths' evidence bar (the design's thresholds): clean replay,
1236/// no regression, improvement past the declared bar on the target
1237/// metric, over the named dataset version, at an admitted scope.
1238fn check_evidence_bar(
1239 auto: &AutoPromotion,
1240 candidate: &Candidate,
1241 evaluation: &CandidateEvaluation,
1242) -> std::result::Result<(), LearnError> {
1243 if !evaluation.replay.is_clean() {
1244 return Err(PromotionRefusal::ReplayDiverged {
1245 candidate_id: candidate.candidate_id.clone(),
1246 divergences: evaluation.replay.divergences.len(),
1247 }
1248 .into());
1249 }
1250 if evaluation.verdict.regressed {
1251 return Err(PromotionRefusal::EvaluationRegressed {
1252 candidate_id: candidate.candidate_id.clone(),
1253 }
1254 .into());
1255 }
1256 if let Some(required) = &auto.dataset_version {
1257 if required != &evaluation.dataset_version {
1258 return Err(PromotionRefusal::DatasetVersionMismatch {
1259 required: required.clone(),
1260 evaluated: evaluation.dataset_version.clone(),
1261 }
1262 .into());
1263 }
1264 }
1265 if let CandidateContent::MemorySet { scope, .. } = &candidate.content {
1266 if !auto.scopes.is_empty() && !auto.scopes.contains(&scope.scope) {
1267 return Err(PromotionRefusal::ScopeNotAutoPromotable { scope: scope.scope }.into());
1268 }
1269 }
1270 let clears = evaluation
1271 .verdict
1272 .delta
1273 .is_some_and(|delta| delta > auto.min_improvement);
1274 if !clears {
1275 return Err(PromotionRefusal::InsufficientImprovement {
1276 delta: evaluation.verdict.delta,
1277 required: auto.min_improvement,
1278 }
1279 .into());
1280 }
1281 Ok(())
1282}
1283
1284// --------------------------------------------------------------------- //
1285// The lifecycle record and the active-version pointer
1286// --------------------------------------------------------------------- //
1287
1288/// Where a candidate is in its lifecycle. Closed enum; transitions are
1289/// journaled events, and refused transitions are errors.
1290#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)]
1291#[serde(rename_all = "snake_case")]
1292pub enum CandidateStatus {
1293 /// Distilled, not yet evaluated.
1294 Created,
1295 /// Evaluated against recorded evidence; the evaluation is on record.
1296 Evaluated,
1297 /// Promoted: the surface's pointer names this candidate (fully, or as
1298 /// the canary).
1299 Promoted,
1300 /// Rolled back: the pointer re-pointed away from this candidate.
1301 RolledBack,
1302}
1303
1304/// The journaled promotion receipt — the `CandidatePromoted` event's
1305/// output payload. Plays the effect receipt's role for the learning
1306/// plane: the durable, attributable confirmation that the pointer moved.
1307#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
1308pub struct PromotionReceipt {
1309 /// The promoted candidate.
1310 pub candidate_id: CandidateId,
1311
1312 /// The surface whose pointer moved.
1313 pub surface: SurfaceKey,
1314
1315 /// The pointer's full-traffic value before the move (`None`: the
1316 /// static version served). Rollback re-points to exactly this — the
1317 /// byte-exactness comes from candidates being content-addressed and
1318 /// immutable, so `previous` *is* the version that served, not a
1319 /// reconstruction of it.
1320 #[serde(default, skip_serializing_if = "Option::is_none")]
1321 pub previous: Option<CandidateId>,
1322
1323 /// The gate's decision: the authority and the canary binding.
1324 pub decision: PromotionDecision,
1325
1326 /// When the promotion executed.
1327 pub promoted_at: DateTime<Utc>,
1328}
1329
1330/// The journaled rollback receipt — the `CandidateRolledBack` event's
1331/// output payload.
1332#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
1333pub struct RollbackReceipt {
1334 /// The surface whose pointer re-pointed.
1335 pub surface: SurfaceKey,
1336
1337 /// The candidate rolled back (the version that was serving).
1338 pub from: CandidateId,
1339
1340 /// The candidate re-pointed to (`None`: the static version serves
1341 /// again — revert-to-default is always a legal rollback).
1342 #[serde(default, skip_serializing_if = "Option::is_none")]
1343 pub to: Option<CandidateId>,
1344
1345 /// The causing evidence: what prompted the rollback (a drift
1346 /// observation's journal ids, an operator's statement naming the
1347 /// evidence). Free text by design — wave 4's drift monitor will
1348 /// write it with the same shape operators use now.
1349 pub cause: String,
1350
1351 /// When the rollback executed.
1352 pub rolled_back_at: DateTime<Utc>,
1353}
1354
1355/// A candidate and its lifecycle state: the store row both server
1356/// backends persist. Transitions go through the `apply_*` methods, which
1357/// enforce the state machine (created → evaluated → promoted → rolled
1358/// back) and refuse everything else as a typed error — the store trusts
1359/// what the gate admitted, exactly as the memory store trusts the write
1360/// gate.
1361#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
1362pub struct CandidateRecord {
1363 /// The candidate itself (immutable).
1364 pub candidate: Candidate,
1365
1366 /// Where it is in its lifecycle.
1367 pub status: CandidateStatus,
1368
1369 /// The journaled evaluation (set on evaluate; replaced on
1370 /// re-evaluation — a new dataset version is a new evaluation).
1371 #[serde(default, skip_serializing_if = "Option::is_none")]
1372 pub evaluation: Option<CandidateEvaluation>,
1373
1374 /// The journaled promotion receipt (set on promote).
1375 #[serde(default, skip_serializing_if = "Option::is_none")]
1376 pub promotion: Option<PromotionReceipt>,
1377
1378 /// The journaled rollback receipt (set on rollback).
1379 #[serde(default, skip_serializing_if = "Option::is_none")]
1380 pub rollback: Option<RollbackReceipt>,
1381}
1382
1383impl CandidateRecord {
1384 /// A freshly created candidate.
1385 pub fn new(candidate: Candidate) -> Self {
1386 Self {
1387 candidate,
1388 status: CandidateStatus::Created,
1389 evaluation: None,
1390 promotion: None,
1391 rollback: None,
1392 }
1393 }
1394
1395 /// The candidate's id.
1396 pub fn candidate_id(&self) -> &CandidateId {
1397 &self.candidate.candidate_id
1398 }
1399
1400 /// Record an evaluation: `created → evaluated`, or `evaluated →
1401 /// evaluated` for re-evaluation (a new dataset version is a new
1402 /// evaluation; the effect key names both). Refused from `promoted`
1403 /// and `rolled_back`: re-evaluating a settled candidate reads as a
1404 /// new distillation, and the model keeps that honest by requiring
1405 /// one.
1406 pub fn apply_evaluation(
1407 &mut self,
1408 evaluation: CandidateEvaluation,
1409 ) -> std::result::Result<(), LearnError> {
1410 match self.status {
1411 CandidateStatus::Created | CandidateStatus::Evaluated => {}
1412 other => {
1413 return Err(LearnError::InvalidTransition {
1414 from: other,
1415 action: "evaluate",
1416 })
1417 }
1418 }
1419 if evaluation.candidate_id != self.candidate.candidate_id {
1420 return Err(PromotionRefusal::EvaluationMismatch {
1421 candidate_id: self.candidate.candidate_id.clone(),
1422 evaluation: evaluation.candidate_id,
1423 }
1424 .into());
1425 }
1426 self.evaluation = Some(evaluation);
1427 self.status = CandidateStatus::Evaluated;
1428 Ok(())
1429 }
1430
1431 /// Record a promotion: `evaluated → promoted`. The gate
1432 /// ([`admit_promotion`]) must already have cleared — this method
1433 /// enforces the state machine, not the evidence bar.
1434 pub fn apply_promotion(
1435 &mut self,
1436 receipt: PromotionReceipt,
1437 ) -> std::result::Result<(), LearnError> {
1438 if self.status != CandidateStatus::Evaluated {
1439 return Err(LearnError::InvalidTransition {
1440 from: self.status,
1441 action: "promote",
1442 });
1443 }
1444 if receipt.candidate_id != self.candidate.candidate_id {
1445 return Err(LearnError::ReceiptMismatch {
1446 candidate_id: self.candidate.candidate_id.clone(),
1447 named: receipt.candidate_id,
1448 });
1449 }
1450 self.promotion = Some(receipt);
1451 self.status = CandidateStatus::Promoted;
1452 Ok(())
1453 }
1454
1455 /// Record a rollback: `promoted → rolled_back`.
1456 pub fn apply_rollback(
1457 &mut self,
1458 receipt: RollbackReceipt,
1459 ) -> std::result::Result<(), LearnError> {
1460 if self.status != CandidateStatus::Promoted {
1461 return Err(LearnError::InvalidTransition {
1462 from: self.status,
1463 action: "roll back",
1464 });
1465 }
1466 if receipt.from != self.candidate.candidate_id {
1467 return Err(LearnError::ReceiptMismatch {
1468 candidate_id: self.candidate.candidate_id.clone(),
1469 named: receipt.from,
1470 });
1471 }
1472 self.rollback = Some(receipt);
1473 self.status = CandidateStatus::RolledBack;
1474 Ok(())
1475 }
1476}
1477
1478/// The active version of one production surface: an immutable-pointer
1479/// move away from any version that ever served.
1480///
1481/// Two slots, exactly the design's canary shape: `active` is the
1482/// full-traffic version (`None` — the static version serves, the
1483/// documented floor); `canary` binds one candidate to a declared fraction
1484/// of new runs while `active` serves the rest. New runs bind the pointer
1485/// at admission (the canary by seeded draw — [`canary_admits`]);
1486/// in-flight runs keep the version their checkpoint header pins, the same
1487/// conservatism as worker-version pinning and manifest pinning.
1488#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
1489pub struct VersionPointer {
1490 /// The surface this pointer governs.
1491 pub surface: SurfaceKey,
1492
1493 /// The full-traffic candidate (`None`: the static version serves).
1494 #[serde(default, skip_serializing_if = "Option::is_none")]
1495 pub active: Option<CandidateId>,
1496
1497 /// The canary binding, when one is declared.
1498 #[serde(default, skip_serializing_if = "Option::is_none")]
1499 pub canary: Option<CanaryBinding>,
1500}
1501
1502impl VersionPointer {
1503 /// A pointer with nothing promoted: the static version serves.
1504 pub fn new(surface: SurfaceKey) -> Self {
1505 Self {
1506 surface,
1507 active: None,
1508 canary: None,
1509 }
1510 }
1511
1512 /// The pointer after a promotion: a canary promotion binds the
1513 /// candidate to its fraction (the full-traffic version keeps serving
1514 /// the rest); a full promotion moves `active` and clears any canary —
1515 /// a full promotion supersedes the experiment it graduated from.
1516 pub fn promoted(&self, receipt: &PromotionReceipt) -> VersionPointer {
1517 let mut next = self.clone();
1518 if let Some(binding) = &receipt.decision.canary {
1519 next.canary = Some(binding.clone());
1520 } else {
1521 next.active = Some(receipt.candidate_id.clone());
1522 next.canary = None;
1523 }
1524 next
1525 }
1526
1527 /// The pointer after a rollback: rolling back the full-traffic
1528 /// version re-points `active` to the receipt's `to`; rolling back the
1529 /// canary clears the binding. The caller validates that `from` is the
1530 /// serving version before building the receipt — this is the move,
1531 /// not the check.
1532 pub fn rolled_back(&self, receipt: &RollbackReceipt) -> VersionPointer {
1533 let mut next = self.clone();
1534 if self.active.as_ref() == Some(&receipt.from) {
1535 next.active = receipt.to.clone();
1536 }
1537 if self
1538 .canary
1539 .as_ref()
1540 .is_some_and(|binding| binding.candidate_id == receipt.from)
1541 {
1542 next.canary = None;
1543 }
1544 next
1545 }
1546}
1547
1548// --------------------------------------------------------------------- //
1549// R0.10 wave 3: the learned retry/timeout families
1550//
1551// Three pieces, all consuming the same journaled evidence:
1552//
1553// - **Distillation** ([`distill_retry_parameters`],
1554// [`distill_timeout_parameters`]) — closed-form learners that read
1555// `DecisionEvent`s and effect events and emit family parameter sets.
1556// The fitting is a grid search over declared envelopes, per design open
1557// question 1: the runtime owns small closed-form scorers serialized as
1558// the policy's parameters; the decision path stays a lookup.
1559// - **The twin-backed evaluator** ([`TwinCandidateEvaluator`]) — the
1560// `CandidateEvaluator` seam's policy-candidate implementation: replay
1561// the evidence span's fixtures with the candidate shadowing nothing and
1562// acting head-to-head against the floor on identical seeds and fault
1563// schedules, then verdict. A candidate that loses to the floor produces
1564// a regressed verdict, and the gate refuses it — promotion through the
1565// pipeline, never around it.
1566// - **Drift detection** ([`detect_policy_drift`]) — the promoted
1567// version's journaled outcomes against the baseline its promotion
1568// evaluation recorded; the revert itself is one activation of
1569// `static-v0` away (server side).
1570// --------------------------------------------------------------------- //
1571
1572use crate::durable::ErrorClass;
1573use crate::record::{
1574 derive_policy_version, BackoffParameters, DecisionAction, DecisionEvent, DecisionOutcome,
1575 Effect, EventStatus, ExecutorPolicy, PolicyVersion, RetryPolicyParameters, RunEvent,
1576 TimeoutPolicyParameters,
1577};
1578use crate::twin::{
1579 percentile, FaultSchedule, ParameterizedPolicy, Twin, TwinMetrics, TwinPolicy, TwinRunConfig,
1580 DEFAULT_TIMEOUT_LADDER,
1581};
1582
1583/// The charged cost of a dead-lettered item in the retry learner's model,
1584/// in milliseconds: the lease boundary — what a task that never
1585/// recovers costs the run before the queue's worst-case discovery. The
1586/// same accounting the Wave 1 experiment and the twin apply.
1587const DEAD_LETTER_PENALTY_MS: u64 = crate::durable::MAX_RETRY_DELAY_MS;
1588
1589/// How the retry distiller fits per-class schedules (R0.10 wave 3).
1590///
1591/// The grid is the learner's whole search space — closed-form scores over
1592/// declared values, so the fitted policy is inspectable as data and the
1593/// fitting itself is deterministic.
1594#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
1595pub struct RetryLearningConfig {
1596 /// Minimum evidence (retries observed plus terminal outcomes) before a
1597 /// class earns its own entry. Below it the class keeps the floor —
1598 /// sparse evidence must not produce confident policy.
1599 pub min_samples: usize,
1600
1601 /// The estimated wall-time margin over the floor a candidate schedule
1602 /// must clear to be emitted, in milliseconds. The Wave 1 experiment's
1603 /// published scar is that a cheap per-class table *underperformed* the
1604 /// floor's jittered exponential on latency; the margin is what keeps
1605 /// a marginal fit from demoting the floor.
1606 pub min_improvement_ms: u64,
1607
1608 /// Candidate backoff bases, in milliseconds.
1609 pub base_grid_ms: Vec<u64>,
1610
1611 /// Candidate backoff caps, in milliseconds.
1612 pub cap_grid_ms: Vec<u64>,
1613
1614 /// Candidate attempt budgets.
1615 pub max_attempts_grid: Vec<u32>,
1616}
1617
1618impl Default for RetryLearningConfig {
1619 fn default() -> Self {
1620 Self {
1621 min_samples: 4,
1622 min_improvement_ms: 2_000,
1623 base_grid_ms: vec![100, 250, 500, 1_000, 2_000],
1624 cap_grid_ms: vec![30_000, 120_000, 300_000],
1625 max_attempts_grid: vec![1, 2, 3, 5],
1626 }
1627 }
1628}
1629
1630/// One class's distilled retry evidence.
1631#[derive(Debug, Default)]
1632struct ClassEvidence {
1633 /// Decisions that selected `Retry`.
1634 retries: u64,
1635 /// Decisions journaled with a success outcome.
1636 successes: u64,
1637 /// Decisions journaled with a failure outcome (terminal: dead-letter
1638 /// or outright fail).
1639 failures: u64,
1640 /// Callee-supplied `Retry-After` floors observed at decision time.
1641 retry_after_ms: Vec<u64>,
1642}
1643
1644/// The estimated expected cost of one schedule on one class's evidence, in
1645/// milliseconds: expected backoff wall before recovery, plus the
1646/// dead-letter penalty weighted by the probability the budget runs out.
1647/// `p` is the estimated per-retry success probability; `retry_after` is
1648/// the world's own floor on any delay (a delay shorter than the callee's
1649/// `Retry-After` is not a shorter delay — it is the callee's delay).
1650fn retry_schedule_score(
1651 base_ms: u64,
1652 cap_ms: u64,
1653 max_attempts: u32,
1654 p: f64,
1655 retry_after: u64,
1656) -> f64 {
1657 let mut wall = 0.0;
1658 let mut survival = 1.0;
1659 for k in 0..max_attempts {
1660 let mean_delay =
1661 ((base_ms.saturating_mul(1u64 << k.min(20))).min(cap_ms) / 2).max(retry_after) as f64;
1662 wall += survival * mean_delay;
1663 survival *= 1.0 - p;
1664 }
1665 wall + survival * DEAD_LETTER_PENALTY_MS as f64
1666}
1667
1668/// Distill the retry family's parameters from journaled retry
1669/// [`DecisionEvent`]s (R0.10 wave 3).
1670///
1671/// What is learned, per error class with sufficient evidence: the backoff
1672/// base and cap and the attempt budget that minimize expected wall-to-
1673/// recovery charged with dead-letter risk — **never the schedule's
1674/// shape** (full jitter over a doubling window is the floor's; Wave 1's
1675/// scar is that fixed per-class delays lose to it), and never the gates.
1676/// A class whose failures never turned into completions earns the
1677/// permanent-failure stance (`max_attempts: 1` — abort after the first
1678/// failure, the design's "earlier abort under permanent ones"); a class
1679/// whose best grid score does not beat the floor's by
1680/// [`RetryLearningConfig::min_improvement_ms`] earns nothing — the floor
1681/// decides for it. The returned set carries the floor's flat schedule;
1682/// `per_class` is `None` when no class cleared the margin, in which case
1683/// the honest next step is no candidate at all.
1684pub fn distill_retry_parameters(
1685 events: &[DecisionEvent],
1686 config: &RetryLearningConfig,
1687) -> RetryPolicyParameters {
1688 let floor = ExecutorPolicy::static_v0().retry;
1689 let mut classes: std::collections::BTreeMap<ErrorClass, ClassEvidence> =
1690 std::collections::BTreeMap::new();
1691 for event in events {
1692 if event.family != crate::record::DecisionFamily::Retry {
1693 continue;
1694 }
1695 let class = event
1696 .features
1697 .get("failure_class")
1698 .and_then(|value| serde_json::from_value(value.clone()).ok())
1699 .unwrap_or(ErrorClass::Unknown);
1700 let evidence = classes.entry(class).or_default();
1701 if matches!(event.selected, DecisionAction::Retry { .. }) {
1702 evidence.retries += 1;
1703 }
1704 match event.outcome {
1705 Some(DecisionOutcome::Success) => evidence.successes += 1,
1706 Some(DecisionOutcome::Failure) => evidence.failures += 1,
1707 _ => {}
1708 }
1709 if let Some(retry_after) = event
1710 .features
1711 .get("retry_after_ms")
1712 .and_then(serde_json::Value::as_u64)
1713 {
1714 evidence.retry_after_ms.push(retry_after);
1715 }
1716 }
1717
1718 let mut table = std::collections::BTreeMap::new();
1719 for (class, evidence) in &classes {
1720 let total = evidence.retries + evidence.failures;
1721 if (total as usize) < config.min_samples {
1722 continue;
1723 }
1724 // A class with retries observed but no recorded success estimates
1725 // at the boundary — the permanent-failure stance — only when the
1726 // failures are themselves evidence (terminal outcomes), never on
1727 // absence alone.
1728 if evidence.successes == 0 && evidence.retries > 0 {
1729 if evidence.failures >= config.min_samples as u64 {
1730 table.insert(
1731 *class,
1732 BackoffParameters {
1733 base_delay_ms: floor.base_delay_ms,
1734 max_delay_ms: floor.max_delay_ms,
1735 max_attempts: 1,
1736 },
1737 );
1738 }
1739 continue;
1740 }
1741 if evidence.retries == 0 || evidence.successes == 0 {
1742 continue;
1743 }
1744 let p = evidence.successes as f64 / evidence.retries as f64;
1745 let retry_after = percentile(&evidence.retry_after_ms, 50.0);
1746 let floor_score = retry_schedule_score(
1747 floor.base_delay_ms,
1748 floor.max_delay_ms,
1749 floor.max_attempts,
1750 p,
1751 retry_after,
1752 );
1753 let mut best: Option<(f64, BackoffParameters)> = None;
1754 for &base in &config.base_grid_ms {
1755 for &cap in &config.cap_grid_ms {
1756 if base == 0 || base > cap {
1757 continue;
1758 }
1759 for &max_attempts in &config.max_attempts_grid {
1760 let score = retry_schedule_score(base, cap, max_attempts, p, retry_after);
1761 let candidate = BackoffParameters {
1762 base_delay_ms: base,
1763 max_delay_ms: cap,
1764 max_attempts,
1765 };
1766 if best
1767 .as_ref()
1768 .is_none_or(|(best_score, _)| score < *best_score)
1769 {
1770 best = Some((score, candidate));
1771 }
1772 }
1773 }
1774 }
1775 if let Some((score, schedule)) = best {
1776 if floor_score - score >= config.min_improvement_ms as f64 {
1777 table.insert(*class, schedule);
1778 }
1779 }
1780 }
1781
1782 RetryPolicyParameters {
1783 per_class: if table.is_empty() { None } else { Some(table) },
1784 ..floor
1785 }
1786}
1787
1788/// How the timeout distiller fits per-callee bounds (R0.10 wave 3).
1789#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
1790pub struct TimeoutLearningConfig {
1791 /// Minimum completions a callee must have journaled before a bound is
1792 /// fit to it. Below it the callee keeps no bound — sparse evidence
1793 /// must not produce confident policy.
1794 pub min_samples: usize,
1795
1796 /// The largest premature-abort fraction a bound may carry: the share
1797 /// of journaled completions that ran longer than the bound. The Wave
1798 /// 1 scar — a p99-plus-margin bound prematurely aborting heavy-tailed
1799 /// work — is a percentile *estimate* multiplied by a constant missing
1800 /// the tail mass the estimate cannot see; reading the abort fraction
1801 /// off the empirical distribution directly is the fix, and when no
1802 /// rung satisfies the tolerance the learner abstains (the floor keeps
1803 /// the callee) rather than shipping a bound that fails
1804 /// non-inferiority.
1805 pub abort_tolerance: f64,
1806
1807 /// The ladder the fitted bounds are chosen from, ascending. The top
1808 /// rung models "no bound in force" and is never emitted — a bound
1809 /// equal to it reclaims nothing the floor does not.
1810 pub ladder: Vec<u64>,
1811}
1812
1813impl Default for TimeoutLearningConfig {
1814 fn default() -> Self {
1815 Self {
1816 min_samples: 16,
1817 abort_tolerance: 0.01,
1818 ladder: DEFAULT_TIMEOUT_LADDER.to_vec(),
1819 }
1820 }
1821}
1822
1823/// Distill the timeout family's parameters from journaled effect events
1824/// (R0.10 wave 3). The callee identity is the event's `node_id`; only
1825/// completed calls (status `Ok` with a measured latency) are evidence —
1826/// a failure's latency says nothing about how long success takes.
1827///
1828/// Per callee: the smallest ladder rung whose empirical premature-abort
1829/// fraction (the share of completions that ran longer than the rung) is
1830/// within [`TimeoutLearningConfig::abort_tolerance`]. Callees with
1831/// insufficient evidence, with no qualifying rung (a tail heavier than
1832/// the ladder — the Wave 1 failure mode, abstained from rather than
1833/// shipped), or whose qualifying rung is the ladder's top get no entry.
1834/// `default_millis` stays `None` — the floor's stance for every callee
1835/// the evidence did not speak for; `max_millis` pins the ladder's top as
1836/// the declared ceiling when any entry exists.
1837pub fn distill_timeout_parameters(
1838 events: &[RunEvent],
1839 config: &TimeoutLearningConfig,
1840) -> TimeoutPolicyParameters {
1841 let mut latencies: std::collections::BTreeMap<String, Vec<u64>> =
1842 std::collections::BTreeMap::new();
1843 for event in events {
1844 if event.status != EventStatus::Ok {
1845 continue;
1846 }
1847 let (Some(callee), Some(latency)) = (event.node_id.clone(), event.latency_ms) else {
1848 continue;
1849 };
1850 latencies.entry(callee).or_default().push(latency);
1851 }
1852
1853 let mut table = std::collections::BTreeMap::new();
1854 let top = config.ladder.last().copied().unwrap_or(u64::MAX);
1855 for (callee, samples) in &latencies {
1856 if samples.len() < config.min_samples {
1857 continue;
1858 }
1859 let bound = config.ladder.iter().copied().find(|rung| {
1860 let aborted = samples.iter().filter(|latency| *latency > rung).count();
1861 (aborted as f64 / samples.len() as f64) <= config.abort_tolerance
1862 });
1863 match bound {
1864 Some(rung) if rung < top => {
1865 table.insert(callee.clone(), rung);
1866 }
1867 _ => {}
1868 }
1869 }
1870
1871 TimeoutPolicyParameters {
1872 default_millis: None,
1873 max_millis: if table.is_empty() { None } else { Some(top) },
1874 per_callee: if table.is_empty() { None } else { Some(table) },
1875 }
1876}
1877
1878/// The twin-backed [`CandidateEvaluator`] for `policy` candidates (R0.10
1879/// wave 3): the governance wiring's "evaluation happens in the twin",
1880/// landed. Every fixture in the request's evidence span is re-executed
1881/// twice on the same seed and fault schedule — the floor acting, then the
1882/// candidate's parameters acting through [`ParameterizedPolicy`] — and the
1883/// verdict is computed over the paired [`TwinMetrics`]: **non-inferior
1884/// completion**
1885/// (the release proof's constraint, applied at the gate) and improvement
1886/// on the request's target metric.
1887///
1888/// Verdict conventions, stated precisely: `regressed` is set when the
1889/// candidate's completion rate falls below the floor's by more than the
1890/// declared completion tolerance (see
1891/// [`TwinCandidateEvaluator::with_completion_tolerance`]) — on any single
1892/// fixture or in aggregate; `delta` is signed so that positive is better
1893/// on the named metric (for cost and latency metrics, lower is better,
1894/// so `delta = baseline − candidate`), keeping [`EvaluationVerdict`]'s
1895/// "negative is worse" invariant true for the gate. Both reports carry
1896/// every fixture's [`TwinReport`](crate::twin::TwinReport), so the
1897/// twin's validity bound travels with the evaluation (design open
1898/// question 5).
1899///
1900/// Determinism is the twin's own: same fixtures, same seed, same fault
1901/// schedule ⇒ the same metrics and the same verdict.
1902#[derive(Debug)]
1903pub struct TwinCandidateEvaluator {
1904 seed: u64,
1905 faults: FaultSchedule,
1906 workers: Vec<String>,
1907 max_attempts: u32,
1908 completion_tolerance: f64,
1909 evaluated_by: ProvenanceAuthor,
1910}
1911
1912impl TwinCandidateEvaluator {
1913 /// An evaluator drawing from `seed`, attributing every evaluation to
1914 /// `evaluated_by`, over an unfaulted recorded world, one worker, the
1915 /// floor's attempt budget, and strict completion parity (tolerance 0).
1916 pub fn new(seed: u64, evaluated_by: ProvenanceAuthor) -> Self {
1917 Self {
1918 seed,
1919 faults: FaultSchedule::new(seed),
1920 workers: vec!["worker-0".to_owned()],
1921 max_attempts: 3,
1922 completion_tolerance: 0.0,
1923 evaluated_by,
1924 }
1925 }
1926
1927 /// Builder-style: evaluate against this fault schedule — the faults
1928 /// rarer than any recorded window contains, which is the twin's
1929 /// reason to exist.
1930 pub fn with_faults(mut self, faults: FaultSchedule) -> Self {
1931 self.faults = faults;
1932 self
1933 }
1934
1935 /// Builder-style: declare the worker pool placement ranks.
1936 pub fn with_workers(mut self, workers: impl IntoIterator<Item = impl Into<String>>) -> Self {
1937 self.workers = workers.into_iter().map(Into::into).collect();
1938 self
1939 }
1940
1941 /// Builder-style: the attempt budget the world allows (a learned
1942 /// policy may narrow it, never widen it).
1943 pub fn with_max_attempts(mut self, max_attempts: u32) -> Self {
1944 self.max_attempts = max_attempts;
1945 self
1946 }
1947
1948 /// Builder-style: the completion-rate band within which the candidate
1949 /// is non-inferior. Zero is the honest default for a mechanical
1950 /// family: identical completion is achievable, so parity is the bar.
1951 pub fn with_completion_tolerance(mut self, tolerance: f64) -> Self {
1952 self.completion_tolerance = tolerance;
1953 self
1954 }
1955}
1956
1957/// Per-arm aggregates over the fixture set, computed once and shared by
1958/// the verdict and the reports.
1959#[derive(Debug, Clone, Copy, Default)]
1960struct ArmAggregate {
1961 items: usize,
1962 completed: usize,
1963 dead_lettered: usize,
1964 attempts: u64,
1965 cost_usd: f64,
1966 wall_time_ms: u64,
1967 latency_p95_ms: u64,
1968 runs: usize,
1969}
1970
1971impl ArmAggregate {
1972 fn fold(&mut self, metrics: &TwinMetrics) {
1973 self.items += metrics.items;
1974 self.completed += metrics.completed;
1975 self.dead_lettered += metrics.dead_lettered;
1976 self.attempts += metrics.attempts;
1977 self.cost_usd += metrics.cost_usd;
1978 self.wall_time_ms += metrics.wall_time_ms;
1979 self.latency_p95_ms = self.latency_p95_ms.max(metrics.item_latency_p95_ms);
1980 self.runs += 1;
1981 }
1982
1983 fn completion_rate(&self) -> f64 {
1984 if self.items == 0 {
1985 0.0
1986 } else {
1987 self.completed as f64 / self.items as f64
1988 }
1989 }
1990
1991 fn dead_letter_rate(&self) -> f64 {
1992 if self.items == 0 {
1993 0.0
1994 } else {
1995 self.dead_lettered as f64 / self.items as f64
1996 }
1997 }
1998
1999 fn mean_wall_ms(&self) -> f64 {
2000 if self.runs == 0 {
2001 0.0
2002 } else {
2003 self.wall_time_ms as f64 / self.runs as f64
2004 }
2005 }
2006
2007 /// The serialized aggregate both reports carry — also the shape
2008 /// [`DriftBaseline::from_twin_report`] reads back.
2009 fn as_value(&self) -> Value {
2010 serde_json::json!({
2011 "items": self.items,
2012 "completed": self.completed,
2013 "dead_lettered": self.dead_lettered,
2014 "attempts": self.attempts,
2015 "cost_usd": self.cost_usd,
2016 "mean_wall_time_ms": self.mean_wall_ms(),
2017 "item_latency_p95_ms": self.latency_p95_ms,
2018 "completion_rate": self.completion_rate(),
2019 "dead_letter_rate": self.dead_letter_rate(),
2020 })
2021 }
2022}
2023
2024#[async_trait]
2025impl CandidateEvaluator for TwinCandidateEvaluator {
2026 async fn evaluate(
2027 &self,
2028 candidate: &Candidate,
2029 request: &EvaluationRequest,
2030 ) -> Result<CandidateEvaluation> {
2031 let CandidateContent::Policy { family, parameters } = &candidate.content else {
2032 return Err(invalid(format!(
2033 "the twin-backed evaluator prices `policy` candidates; `{}` candidates \
2034 evaluate through the application's own evaluator",
2035 match candidate.kind() {
2036 CandidateKind::Prompt => "prompt",
2037 CandidateKind::Policy => unreachable!(),
2038 CandidateKind::MemorySet => "memory_set",
2039 CandidateKind::ToolPermission => "tool_permission",
2040 }
2041 )));
2042 };
2043 // The candidate as a concrete policy: parsed against the family's
2044 // contract and checked against its envelope — an out-of-envelope
2045 // candidate fails here, before a single twin run.
2046 let candidate_policy = ExecutorPolicy::static_v0()
2047 .with_family_parameters(*family, parameters.clone())
2048 .map_err(|e| {
2049 invalid(format!(
2050 "policy candidate cannot be evaluated: its parameters do not apply to \
2051 the floor ({e}) — an out-of-envelope candidate never reaches evidence"
2052 ))
2053 })?;
2054 let version = derive_policy_version(&candidate_policy)?;
2055 let acting: Arc<dyn TwinPolicy> =
2056 Arc::new(ParameterizedPolicy::new(candidate_policy, version.clone())?);
2057
2058 let mut floor_aggregate = ArmAggregate::default();
2059 let mut candidate_aggregate = ArmAggregate::default();
2060 let mut fixture_ids = Vec::with_capacity(request.replay_evidence.len());
2061 let mut matched = 0usize;
2062 let mut divergences = Vec::new();
2063 let mut floor_runs = Vec::new();
2064 let mut candidate_runs = Vec::new();
2065 let mut completion_breached = false;
2066
2067 for fixture in &request.replay_evidence {
2068 fixture_ids.push(fixture.run_id.clone());
2069 let outcome = (|| -> Result<_> {
2070 let twin = Twin::from_snapshot(fixture.clone())?;
2071 let config = TwinRunConfig::new(self.seed)
2072 .with_faults(self.faults.clone())
2073 .with_workers(self.workers.clone())
2074 .with_max_attempts(self.max_attempts);
2075 let floor_run = twin.run(&config)?;
2076 let candidate_run = twin.run(&config.with_acting(acting.clone()))?;
2077 Ok((floor_run, candidate_run))
2078 })();
2079 match outcome {
2080 Ok((floor_run, candidate_run)) => {
2081 matched += 1;
2082 // Per-fixture non-inferiority: a candidate that
2083 // completes less of one fixture's work than the floor
2084 // is a regression the aggregate must not average away.
2085 let floor_rate =
2086 floor_run.metrics.completed as f64 / floor_run.metrics.items.max(1) as f64;
2087 let candidate_rate = candidate_run.metrics.completed as f64
2088 / candidate_run.metrics.items.max(1) as f64;
2089 if candidate_rate < floor_rate - self.completion_tolerance {
2090 completion_breached = true;
2091 }
2092 floor_aggregate.fold(&floor_run.metrics);
2093 candidate_aggregate.fold(&candidate_run.metrics);
2094 floor_runs.push(serde_json::json!({
2095 "fixture": fixture.run_id,
2096 "metrics": floor_run.metrics,
2097 "report": floor_run.report,
2098 }));
2099 candidate_runs.push(serde_json::json!({
2100 "fixture": fixture.run_id,
2101 "metrics": candidate_run.metrics,
2102 "report": candidate_run.report,
2103 }));
2104 }
2105 Err(error) => {
2106 // A fixture the twin cannot re-execute is evidence the
2107 // evaluation could not verify — a divergence, fail
2108 // closed, the same rule the wave-4 evaluator applies.
2109 divergences.push(ReplayDivergence {
2110 fixture_id: fixture.run_id.clone(),
2111 detail: error.to_string(),
2112 });
2113 }
2114 }
2115 }
2116 fixture_ids.sort();
2117 divergences.sort_by(|a, b| a.fixture_id.cmp(&b.fixture_id));
2118
2119 let regressed = completion_breached
2120 || candidate_aggregate.completion_rate()
2121 < floor_aggregate.completion_rate() - self.completion_tolerance;
2122 let (baseline, candidate_value, delta) = match request.target_metric.as_str() {
2123 "completion_rate" => (
2124 Some(floor_aggregate.completion_rate()),
2125 Some(candidate_aggregate.completion_rate()),
2126 Some(candidate_aggregate.completion_rate() - floor_aggregate.completion_rate()),
2127 ),
2128 "cost_usd" => (
2129 Some(floor_aggregate.cost_usd),
2130 Some(candidate_aggregate.cost_usd),
2131 Some(floor_aggregate.cost_usd - candidate_aggregate.cost_usd),
2132 ),
2133 "wall_time_ms" => (
2134 Some(floor_aggregate.mean_wall_ms()),
2135 Some(candidate_aggregate.mean_wall_ms()),
2136 Some(floor_aggregate.mean_wall_ms() - candidate_aggregate.mean_wall_ms()),
2137 ),
2138 "item_latency_p95_ms" => (
2139 Some(floor_aggregate.latency_p95_ms as f64),
2140 Some(candidate_aggregate.latency_p95_ms as f64),
2141 Some(
2142 floor_aggregate.latency_p95_ms as f64
2143 - candidate_aggregate.latency_p95_ms as f64,
2144 ),
2145 ),
2146 // A metric the twin does not price: no values, no delta — an
2147 // improvement bar cannot clear on a metric the evidence does
2148 // not name (the gate's own rule).
2149 _ => (None, None, None),
2150 };
2151
2152 Ok(CandidateEvaluation {
2153 candidate_id: candidate.candidate_id.clone(),
2154 dataset_version: request.dataset_version.clone(),
2155 replay: ReplaySummary {
2156 fixture_ids,
2157 matched,
2158 divergences,
2159 },
2160 baseline_report: serde_json::json!({
2161 "arm": PolicyVersion::STATIC_V0,
2162 "aggregate": floor_aggregate.as_value(),
2163 "runs": floor_runs,
2164 }),
2165 candidate_report: serde_json::json!({
2166 "arm": version,
2167 "aggregate": candidate_aggregate.as_value(),
2168 "runs": candidate_runs,
2169 }),
2170 verdict: EvaluationVerdict {
2171 regressed,
2172 target_metric: request.target_metric.clone(),
2173 baseline,
2174 candidate: candidate_value,
2175 delta,
2176 },
2177 thresholds: request.thresholds,
2178 evaluated_by: self.evaluated_by.clone(),
2179 evaluated_at: Utc::now(),
2180 })
2181 }
2182}
2183
2184/// The promotion-time baseline drift is measured against: the metrics the
2185/// promoted version's own evaluation recorded for the arm it displaced
2186/// (the floor's recorded baseline — "is the promoted version regressing
2187/// against the evidence that promoted it", nothing deeper).
2188#[derive(Debug, Clone, Copy, PartialEq, Serialize, Deserialize)]
2189pub struct DriftBaseline {
2190 /// Baseline completion rate, in `[0, 1]`.
2191 pub completion_rate: f64,
2192
2193 /// Baseline dead-letter rate, in `[0, 1]`.
2194 pub dead_letter_rate: f64,
2195
2196 /// Baseline p95 decision latency (the journaled
2197 /// `dependency_latency_ms` feature), when the evidence carried one.
2198 #[serde(default, skip_serializing_if = "Option::is_none")]
2199 pub latency_p95_ms: Option<u64>,
2200}
2201
2202impl DriftBaseline {
2203 /// Read the baseline out of a serialized twin evaluation report — the
2204 /// `aggregate` section [`TwinCandidateEvaluator`] writes. `None` when
2205 /// the report names no such section (a non-twin evaluation, a
2206 /// hand-authored report): drift detection requires the promotion-time
2207 /// evidence, and it fails absent rather than guessing a baseline.
2208 pub fn from_twin_report(report: &Value) -> Option<Self> {
2209 let aggregate = report.get("aggregate")?;
2210 Some(Self {
2211 completion_rate: aggregate.get("completion_rate")?.as_f64()?,
2212 dead_letter_rate: aggregate.get("dead_letter_rate")?.as_f64()?,
2213 latency_p95_ms: aggregate.get("item_latency_p95_ms").and_then(Value::as_u64),
2214 })
2215 }
2216}
2217
2218/// The declared thresholds a drift check applies. All comparisons are
2219/// against the promotion-time baseline, on the acting version's journaled
2220/// outcomes only — shadow decisions are evidence for the *next* candidate,
2221/// never for the acting version's health.
2222#[derive(Debug, Clone, Copy, PartialEq, Serialize, Deserialize)]
2223pub struct DriftThresholds {
2224 /// Completion-rate drop from baseline that declares drift.
2225 pub max_completion_drop: f64,
2226
2227 /// Dead-letter-rate growth from baseline that declares drift.
2228 pub max_dead_letter_growth: f64,
2229
2230 /// p95 latency ratio `current / baseline` that declares drift (applied
2231 /// only when both sides carry the latency feature).
2232 pub max_latency_p95_ratio: f64,
2233
2234 /// Minimum terminal decisions before any verdict. Sparse evidence
2235 /// declares nothing — a drift verdict on three outcomes is noise
2236 /// wearing a label.
2237 pub min_decisions: usize,
2238}
2239
2240impl Default for DriftThresholds {
2241 fn default() -> Self {
2242 Self {
2243 max_completion_drop: 0.05,
2244 max_dead_letter_growth: 0.02,
2245 max_latency_p95_ratio: 1.25,
2246 min_decisions: 8,
2247 }
2248 }
2249}
2250
2251/// The drift check's verdict: the acting version's journaled outcomes
2252/// against its promotion-time baseline, with the reasons drift was (or
2253/// was not) declared. Carried as data — the report is the revert's
2254/// attributable cause.
2255#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
2256pub struct PolicyDriftReport {
2257 /// The version checked.
2258 pub version: PolicyVersion,
2259
2260 /// Acting decisions examined (shadow decisions excluded).
2261 pub decisions: usize,
2262
2263 /// Terminal decisions (outcomes recorded) the rates were computed
2264 /// over.
2265 pub terminal: usize,
2266
2267 /// The acting version's journaled completion rate.
2268 pub completion_rate: f64,
2269
2270 /// The acting version's journaled dead-letter rate.
2271 pub dead_letter_rate: f64,
2272
2273 /// The acting version's p95 decision latency, when carried.
2274 #[serde(default, skip_serializing_if = "Option::is_none")]
2275 pub latency_p95_ms: Option<u64>,
2276
2277 /// The baseline the check ran against.
2278 pub baseline: DriftBaseline,
2279
2280 /// Whether drift was declared.
2281 pub drifted: bool,
2282
2283 /// Why — one entry per breached threshold, or the reason nothing was
2284 /// declared (including insufficient evidence).
2285 pub reasons: Vec<String>,
2286}
2287
2288/// Detect drift on the acting policy's journaled outcomes (R0.10 wave 3).
2289///
2290/// Reads `decisions` for those made by `version` in an acting role
2291/// (`role` unset — every pre-twin decision — or
2292/// [`crate::record::DecisionRole::Acting`]); shadow decisions are
2293/// off-policy evidence, not the acting version's health. The signals are
2294/// the design's own: completion-rate drop, dead-letter growth, p95
2295/// latency ratio — declared thresholds on journaled metrics, honest about
2296/// answering "is the promoted version regressing against the evidence
2297/// that promoted it", nothing deeper. Pure: the revert that a drifted
2298/// verdict motivates is the caller's one activation of `static-v0`.
2299pub fn detect_policy_drift(
2300 decisions: &[DecisionEvent],
2301 version: &PolicyVersion,
2302 baseline: &DriftBaseline,
2303 thresholds: &DriftThresholds,
2304) -> PolicyDriftReport {
2305 let acting: Vec<&DecisionEvent> = decisions
2306 .iter()
2307 .filter(|event| {
2308 &event.policy_version == version
2309 && event.role != Some(crate::record::DecisionRole::Shadow)
2310 })
2311 .collect();
2312 let terminal: Vec<&DecisionEvent> = acting
2313 .iter()
2314 .filter(|event| {
2315 matches!(
2316 event.outcome,
2317 Some(DecisionOutcome::Success) | Some(DecisionOutcome::Failure)
2318 )
2319 })
2320 .copied()
2321 .collect();
2322 let successes = terminal
2323 .iter()
2324 .filter(|event| event.outcome == Some(DecisionOutcome::Success))
2325 .count();
2326 // Dead-lettered, as the retry family journals it: the Abort decision
2327 // with the attempt budget spent while the gates were open. The legal
2328 // set has already collapsed to `[Abort]` at that point, so the budget
2329 // and the gates are read back from the journaled features — the same
2330 // vocabulary `retry_decision_event` and the twin both pin.
2331 let dead_letters = terminal
2332 .iter()
2333 .filter(|event| {
2334 if event.outcome != Some(DecisionOutcome::Failure)
2335 || event.selected != DecisionAction::Abort
2336 {
2337 return false;
2338 }
2339 let budget_spent = match (
2340 event.features.get("attempt").and_then(Value::as_u64),
2341 event.features.get("max_attempts").and_then(Value::as_u64),
2342 ) {
2343 (Some(attempt), Some(max_attempts)) => attempt >= max_attempts,
2344 _ => false,
2345 };
2346 let effect_open = event
2347 .features
2348 .get("effect")
2349 .and_then(|value| serde_json::from_value::<Effect>(value.clone()).ok())
2350 .is_some_and(|effect| effect.is_freely_repeatable());
2351 let class_open = event
2352 .features
2353 .get("failure_class")
2354 .and_then(|value| serde_json::from_value::<ErrorClass>(value.clone()).ok())
2355 .is_some_and(|class| class.is_retryable());
2356 budget_spent && effect_open && class_open
2357 })
2358 .count();
2359 let latencies: Vec<u64> = acting
2360 .iter()
2361 .filter_map(|event| {
2362 event
2363 .features
2364 .get("dependency_latency_ms")
2365 .and_then(Value::as_u64)
2366 })
2367 .collect();
2368 let completion_rate = if terminal.is_empty() {
2369 0.0
2370 } else {
2371 successes as f64 / terminal.len() as f64
2372 };
2373 let dead_letter_rate = if terminal.is_empty() {
2374 0.0
2375 } else {
2376 dead_letters as f64 / terminal.len() as f64
2377 };
2378 let latency_p95_ms = (!latencies.is_empty()).then(|| percentile(&latencies, 95.0));
2379
2380 let mut reasons = Vec::new();
2381 let sufficient = terminal.len() >= thresholds.min_decisions;
2382 let mut drifted = false;
2383 if !sufficient {
2384 reasons.push(format!(
2385 "insufficient evidence: {} terminal decisions, fewer than the declared minimum {}",
2386 terminal.len(),
2387 thresholds.min_decisions
2388 ));
2389 } else {
2390 let completion_drop = baseline.completion_rate - completion_rate;
2391 if completion_drop > thresholds.max_completion_drop {
2392 drifted = true;
2393 reasons.push(format!(
2394 "completion rate {completion_rate:.3} is {completion_drop:.3} below the \
2395 promotion-time baseline {:.3} (threshold {:.3})",
2396 baseline.completion_rate, thresholds.max_completion_drop
2397 ));
2398 }
2399 let dead_letter_growth = dead_letter_rate - baseline.dead_letter_rate;
2400 if dead_letter_growth > thresholds.max_dead_letter_growth {
2401 drifted = true;
2402 reasons.push(format!(
2403 "dead-letter rate {dead_letter_rate:.3} grew {dead_letter_growth:.3} over the \
2404 promotion-time baseline {:.3} (threshold {:.3})",
2405 baseline.dead_letter_rate, thresholds.max_dead_letter_growth
2406 ));
2407 }
2408 if let (Some(current), Some(base)) = (latency_p95_ms, baseline.latency_p95_ms) {
2409 if base > 0 && current as f64 / base as f64 > thresholds.max_latency_p95_ratio {
2410 drifted = true;
2411 reasons.push(format!(
2412 "p95 decision latency {current} ms is {:.2}x the promotion-time baseline \
2413 {base} ms (threshold {:.2}x)",
2414 current as f64 / base as f64,
2415 thresholds.max_latency_p95_ratio
2416 ));
2417 }
2418 }
2419 }
2420
2421 PolicyDriftReport {
2422 version: version.clone(),
2423 decisions: acting.len(),
2424 terminal: terminal.len(),
2425 completion_rate,
2426 dead_letter_rate,
2427 latency_p95_ms,
2428 baseline: *baseline,
2429 drifted,
2430 reasons,
2431 }
2432}
2433
2434#[cfg(test)]
2435mod tests {
2436 use super::*;
2437 use crate::memory::{InMemoryMemoryStore, MemoryKind, MemoryProvenance, ValidityWindow};
2438 use serde_json::json;
2439
2440 fn ts(millis: i64) -> DateTime<Utc> {
2441 DateTime::<Utc>::from_timestamp_millis(millis).unwrap()
2442 }
2443
2444 fn distiller() -> ProvenanceAuthor {
2445 ProvenanceAuthor::Distiller {
2446 name: "correction-loop".into(),
2447 }
2448 }
2449
2450 fn memory_record(content: Value) -> MemoryRecord {
2451 MemoryRecord::new(
2452 MemoryKind::Fact,
2453 ScopeAddress::new(MemoryScope::Agent, "support-1"),
2454 MemoryProvenance {
2455 author: ProvenanceAuthor::Human {
2456 human_id: "amjad".into(),
2457 },
2458 evidence: Default::default(),
2459 written_at: ts(1_000),
2460 },
2461 1.0,
2462 ValidityWindow::starting(ts(500)),
2463 ts(1_000),
2464 content,
2465 )
2466 .unwrap()
2467 }
2468
2469 fn memory_set_candidate(content: Value) -> Candidate {
2470 Candidate::new(
2471 CandidateContent::MemorySet {
2472 scope: ScopeAddress::new(MemoryScope::Agent, "support-1"),
2473 adds: vec![memory_record(content)],
2474 supersedes: Vec::new(),
2475 },
2476 distiller(),
2477 EvidenceSpan {
2478 run_ids: vec!["run-1".into()],
2479 ..EvidenceSpan::default()
2480 },
2481 ts(2_000),
2482 )
2483 .unwrap()
2484 }
2485
2486 fn clean_evaluation(candidate: &Candidate, delta: f64) -> CandidateEvaluation {
2487 CandidateEvaluation {
2488 candidate_id: candidate.candidate_id.clone(),
2489 dataset_version: "v3".into(),
2490 replay: ReplaySummary {
2491 fixture_ids: vec!["run-1".into()],
2492 matched: 1,
2493 divergences: Vec::new(),
2494 },
2495 baseline_report: json!({"name": "support@v3", "summary": {"run_pass_rate": 0.5}}),
2496 candidate_report: json!({"name": "support@v3", "summary": {"run_pass_rate": 0.9}}),
2497 verdict: EvaluationVerdict {
2498 regressed: false,
2499 target_metric: "run_pass_rate".into(),
2500 baseline: Some(0.5),
2501 candidate: Some(0.5 + delta),
2502 delta: Some(delta),
2503 },
2504 thresholds: EvaluationThresholds::default(),
2505 evaluated_by: distiller(),
2506 evaluated_at: ts(3_000),
2507 }
2508 }
2509
2510 #[test]
2511 fn candidate_id_is_a_content_address() {
2512 let a = memory_set_candidate(json!({"tone": "warm"}));
2513 let b = memory_set_candidate(json!({"tone": "warm"}));
2514 let c = memory_set_candidate(json!({"tone": "cold"}));
2515 // Same change converges — even across distillers and instants…
2516 assert_eq!(a.candidate_id, b.candidate_id);
2517 let d = Candidate::new(
2518 a.content.clone(),
2519 ProvenanceAuthor::System,
2520 EvidenceSpan::default(),
2521 ts(9_999),
2522 )
2523 .unwrap();
2524 assert_eq!(a.candidate_id, d.candidate_id);
2525 // …changed content is a new id.
2526 assert_ne!(a.candidate_id, c.candidate_id);
2527 assert_eq!(a.candidate_id.as_str().len(), 64);
2528 }
2529
2530 #[test]
2531 fn tampered_candidate_fails_its_own_address() {
2532 let mut candidate = memory_set_candidate(json!({"tone": "warm"}));
2533 candidate.verify_address().unwrap();
2534 candidate.candidate_id = CandidateId("f".repeat(64));
2535 assert!(matches!(
2536 candidate.verify_address(),
2537 Err(LearnError::AddressMismatch { .. })
2538 ));
2539 }
2540
2541 #[test]
2542 fn surfaces_are_the_documented_keys() {
2543 let memory = memory_set_candidate(json!({}));
2544 assert_eq!(memory.surface().as_str(), "memory:agent:support-1");
2545 let prompt = Candidate::new(
2546 CandidateContent::Prompt {
2547 name: "system".into(),
2548 prompt: "You are careful.".into(),
2549 },
2550 distiller(),
2551 EvidenceSpan::default(),
2552 ts(2_000),
2553 )
2554 .unwrap();
2555 assert_eq!(prompt.surface().as_str(), "prompt:system");
2556 let policy = Candidate::new(
2557 CandidateContent::Policy {
2558 family: DecisionFamily::Retry,
2559 parameters: json!({"max_attempts": 4}),
2560 },
2561 distiller(),
2562 EvidenceSpan::default(),
2563 ts(2_000),
2564 )
2565 .unwrap();
2566 assert_eq!(policy.surface().as_str(), "policy:retry");
2567 let tool = Candidate::new(
2568 CandidateContent::ToolPermission {
2569 tool: "shell".into(),
2570 direction: GrantDirection::Narrow,
2571 },
2572 distiller(),
2573 EvidenceSpan::default(),
2574 ts(2_000),
2575 )
2576 .unwrap();
2577 assert_eq!(tool.surface().as_str(), "tool:shell");
2578 }
2579
2580 #[test]
2581 fn lifecycle_refuses_illegal_transitions() {
2582 let candidate = memory_set_candidate(json!({}));
2583 let mut record = CandidateRecord::new(candidate.clone());
2584 // Promote before evaluate: refused, typed.
2585 let receipt = PromotionReceipt {
2586 candidate_id: candidate.candidate_id.clone(),
2587 surface: candidate.surface(),
2588 previous: None,
2589 decision: PromotionDecision {
2590 authority: PromotionAuthority::Envelope {
2591 envelope_version: "r0.8-default".into(),
2592 },
2593 canary: None,
2594 },
2595 promoted_at: ts(4_000),
2596 };
2597 assert!(matches!(
2598 record.apply_promotion(receipt.clone()),
2599 Err(LearnError::InvalidTransition {
2600 from: CandidateStatus::Created,
2601 action: "promote",
2602 })
2603 ));
2604 // Evaluate, then promote, then refuse re-evaluation and allow rollback.
2605 record
2606 .apply_evaluation(clean_evaluation(&candidate, 0.4))
2607 .unwrap();
2608 record.apply_promotion(receipt).unwrap();
2609 assert!(matches!(
2610 record.apply_evaluation(clean_evaluation(&candidate, 0.5)),
2611 Err(LearnError::InvalidTransition {
2612 from: CandidateStatus::Promoted,
2613 action: "evaluate",
2614 })
2615 ));
2616 record
2617 .apply_rollback(RollbackReceipt {
2618 surface: candidate.surface(),
2619 from: candidate.candidate_id.clone(),
2620 to: None,
2621 cause: "operator: drift on run-9".into(),
2622 rolled_back_at: ts(5_000),
2623 })
2624 .unwrap();
2625 assert_eq!(record.status, CandidateStatus::RolledBack);
2626 }
2627
2628 #[test]
2629 fn gate_admits_in_envelope_and_refuses_out_of_it() {
2630 let envelope = PromotionEnvelope::r08_default();
2631 let candidate = memory_set_candidate(json!({"tone": "warm"}));
2632 let evaluation = clean_evaluation(&candidate, 0.4);
2633 // In-envelope: memory_set at agent scope, clean verdict → the
2634 // envelope version is the standing approval.
2635 let decision = admit_promotion(&envelope, &candidate, Some(&evaluation), None).unwrap();
2636 assert_eq!(
2637 decision.authority,
2638 PromotionAuthority::Envelope {
2639 envelope_version: "r0.8-default".into()
2640 }
2641 );
2642 // Out-of-envelope: prompt candidates always need a token in R0.8.
2643 let prompt = Candidate::new(
2644 CandidateContent::Prompt {
2645 name: "system".into(),
2646 prompt: "You are careful.".into(),
2647 },
2648 distiller(),
2649 EvidenceSpan::default(),
2650 ts(2_000),
2651 )
2652 .unwrap();
2653 let prompt_eval = clean_evaluation(&prompt, 0.4);
2654 let refused = admit_promotion(&envelope, &prompt, Some(&prompt_eval), None).unwrap_err();
2655 let effect_id = promotion_effect_id(&prompt);
2656 assert_eq!(
2657 refused,
2658 LearnError::Refused(PromotionRefusal::RequiresApproval {
2659 effect_id: effect_id.clone()
2660 })
2661 );
2662 // A token for another candidate's promotion does not transfer…
2663 let wrong = ApprovalToken::approve(promotion_effect_id(&candidate), "ops:amjad");
2664 assert!(matches!(
2665 admit_promotion(&envelope, &prompt, Some(&prompt_eval), Some(&wrong)),
2666 Err(LearnError::Refused(
2667 PromotionRefusal::ApprovalMismatch { .. }
2668 ))
2669 ));
2670 // …the correctly scoped token admits, with attribution.
2671 let token = ApprovalToken::approve(effect_id, "ops:amjad");
2672 let decision =
2673 admit_promotion(&envelope, &prompt, Some(&prompt_eval), Some(&token)).unwrap();
2674 assert_eq!(
2675 decision.authority,
2676 PromotionAuthority::Approval {
2677 approved_by: "ops:amjad".into()
2678 }
2679 );
2680 }
2681
2682 #[test]
2683 fn gate_enforces_the_evidence_bar() {
2684 let envelope = PromotionEnvelope::r08_default();
2685 let candidate = memory_set_candidate(json!({}));
2686 // Parity is not improvement.
2687 let parity = clean_evaluation(&candidate, 0.0);
2688 assert_eq!(
2689 admit_promotion(&envelope, &candidate, Some(&parity), None).unwrap_err(),
2690 LearnError::Refused(PromotionRefusal::InsufficientImprovement {
2691 delta: Some(0.0),
2692 required: 0.0,
2693 })
2694 );
2695 // Regression refuses even with improvement on the target metric.
2696 let mut regressed = clean_evaluation(&candidate, 0.4);
2697 regressed.verdict.regressed = true;
2698 assert!(matches!(
2699 admit_promotion(&envelope, &candidate, Some(®ressed), None),
2700 Err(LearnError::Refused(
2701 PromotionRefusal::EvaluationRegressed { .. }
2702 ))
2703 ));
2704 // Divergent replay refuses.
2705 let mut diverged = clean_evaluation(&candidate, 0.4);
2706 diverged.replay.divergences = vec![ReplayDivergence {
2707 fixture_id: "run-1".into(),
2708 detail: "seq 4: request hash mismatch".into(),
2709 }];
2710 assert!(matches!(
2711 admit_promotion(&envelope, &candidate, Some(&diverged), None),
2712 Err(LearnError::Refused(PromotionRefusal::ReplayDiverged { .. }))
2713 ));
2714 // A wider scope than the R0.8 default refuses.
2715 let tenant_scoped = Candidate::new(
2716 CandidateContent::MemorySet {
2717 scope: ScopeAddress::new(MemoryScope::Tenant, "default"),
2718 adds: vec![],
2719 supersedes: vec![],
2720 },
2721 distiller(),
2722 EvidenceSpan::default(),
2723 ts(2_000),
2724 )
2725 .unwrap();
2726 let evaluation = clean_evaluation(&tenant_scoped, 0.4);
2727 assert!(matches!(
2728 admit_promotion(&envelope, &tenant_scoped, Some(&evaluation), None),
2729 Err(LearnError::Refused(
2730 PromotionRefusal::ScopeNotAutoPromotable {
2731 scope: MemoryScope::Tenant
2732 }
2733 ))
2734 ));
2735 // No evaluation at all refuses, even for an in-envelope kind.
2736 assert!(matches!(
2737 admit_promotion(&envelope, &candidate, None, None),
2738 Err(LearnError::Refused(PromotionRefusal::NotEvaluated { .. }))
2739 ));
2740 }
2741
2742 #[test]
2743 fn canary_draw_is_deterministic_uniform_and_scoped() {
2744 let candidate = memory_set_candidate(json!({}));
2745 let surface = candidate.surface();
2746 let binding = CanaryBinding {
2747 candidate_id: candidate.candidate_id.clone(),
2748 fraction: 0.25,
2749 };
2750 // Deterministic per (surface, candidate, run).
2751 assert_eq!(
2752 canary_admits(&binding, &surface, "run-7"),
2753 canary_admits(&binding, &surface, "run-7")
2754 );
2755 // Full fraction admits everything.
2756 let full = CanaryBinding {
2757 candidate_id: candidate.candidate_id.clone(),
2758 fraction: 1.0,
2759 };
2760 for run in 0..32 {
2761 assert!(canary_admits(&full, &surface, &format!("run-{run}")));
2762 }
2763 // A quarter fraction admits roughly a quarter of traffic (loose
2764 // bounds: the draw is uniform over the digest space).
2765 let admitted = (0..1_000)
2766 .filter(|run| canary_admits(&binding, &surface, &format!("run-{run}")))
2767 .count();
2768 assert!(
2769 (150..=350).contains(&admitted),
2770 "fraction 0.25 admitted {admitted}/1000 runs"
2771 );
2772 }
2773
2774 #[test]
2775 fn pointer_moves_on_promotion_and_reponts_on_rollback() {
2776 let a = memory_set_candidate(json!({"v": "a"}));
2777 let b = memory_set_candidate(json!({"v": "b"}));
2778 let pointer = VersionPointer::new(a.surface());
2779 assert_eq!(pointer.active, None);
2780
2781 let promote_a = PromotionReceipt {
2782 candidate_id: a.candidate_id.clone(),
2783 surface: a.surface(),
2784 previous: None,
2785 decision: PromotionDecision {
2786 authority: PromotionAuthority::Envelope {
2787 envelope_version: "r0.8-default".into(),
2788 },
2789 canary: None,
2790 },
2791 promoted_at: ts(4_000),
2792 };
2793 let pointer = pointer.promoted(&promote_a);
2794 assert_eq!(pointer.active, Some(a.candidate_id.clone()));
2795
2796 let promote_b = PromotionReceipt {
2797 candidate_id: b.candidate_id.clone(),
2798 surface: a.surface(),
2799 previous: Some(a.candidate_id.clone()),
2800 decision: promote_a.decision.clone(),
2801 promoted_at: ts(5_000),
2802 };
2803 let pointer = pointer.promoted(&promote_b);
2804 assert_eq!(pointer.active, Some(b.candidate_id.clone()));
2805
2806 // Rollback re-points to the previous candidate — byte-exact,
2807 // because the candidate id IS the content.
2808 let pointer = pointer.rolled_back(&RollbackReceipt {
2809 surface: a.surface(),
2810 from: b.candidate_id.clone(),
2811 to: promote_b.previous.clone(),
2812 cause: "drift".into(),
2813 rolled_back_at: ts(6_000),
2814 });
2815 assert_eq!(pointer.active, Some(a.candidate_id.clone()));
2816 }
2817
2818 #[tokio::test]
2819 async fn overlay_applies_the_candidate_as_a_read_lens() {
2820 let base = Arc::new(InMemoryMemoryStore::new());
2821 let existing = memory_record(json!({"tone": "flat"}));
2822 base.put(&existing).await.unwrap();
2823
2824 let mut added = memory_record(json!({"tone": "warm"}));
2825 added.supersedes = Some(existing.memory_id.clone());
2826 let candidate = Candidate::new(
2827 CandidateContent::MemorySet {
2828 scope: ScopeAddress::new(MemoryScope::Agent, "support-1"),
2829 adds: vec![added.clone()],
2830 supersedes: Vec::new(),
2831 },
2832 distiller(),
2833 EvidenceSpan::default(),
2834 ts(2_000),
2835 )
2836 .unwrap();
2837 let overlay = CandidateOverlay::new(base.clone(), &candidate).unwrap();
2838 // The add answers…
2839 assert_eq!(overlay.get(&added.memory_id).await.unwrap(), Some(added));
2840 // …the base record still fetches by id…
2841 assert_eq!(
2842 overlay.get(&existing.memory_id).await.unwrap(),
2843 Some(existing.clone())
2844 );
2845 // …but drops out of default retrieval via the add's supersedes
2846 // field (core's matcher, over the merged universe)…
2847 let listed = overlay
2848 .query(&MemoryQuery::default(), Utc::now())
2849 .await
2850 .unwrap();
2851 assert_eq!(listed.len(), 1);
2852 // …and writes through the lens fail.
2853 assert!(overlay.put(&existing).await.is_err());
2854 }
2855}