use std::collections::HashSet;
use std::sync::Arc;
use async_trait::async_trait;
use chrono::{DateTime, Utc};
use serde::{Deserialize, Serialize};
use serde_json::Value;
use sha2::{Digest, Sha256};
use thiserror::Error;
use crate::effects::{derive_effect_id, ApprovalToken, EffectId};
use crate::error::{Result, RustyError};
use crate::journal::JournalSnapshot;
use crate::memory::{
apply_query, ContextBudget, MemoryQuery, MemoryRecord, MemoryScope, MemoryStore,
ProvenanceAuthor, ScopeAddress,
};
use crate::record::{sha256_hex, DecisionFamily, RunManifest};
fn invalid(message: impl Into<String>) -> RustyError {
RustyError::InvalidUpdate(message.into())
}
#[derive(Debug, Clone, PartialEq, Eq, Hash, PartialOrd, Ord, Serialize, Deserialize)]
#[serde(transparent)]
pub struct CandidateId(String);
impl CandidateId {
pub fn as_str(&self) -> &str {
&self.0
}
}
impl std::fmt::Display for CandidateId {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.write_str(&self.0)
}
}
impl From<String> for CandidateId {
fn from(digest: String) -> Self {
Self(digest)
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum CandidateKind {
Prompt,
Policy,
MemorySet,
ToolPermission,
ToolContract,
ModelSettings,
MemoryConfiguration,
MiddlewareComposition,
}
impl CandidateKind {
pub fn as_str(&self) -> &'static str {
match self {
CandidateKind::Prompt => "prompt",
CandidateKind::Policy => "policy",
CandidateKind::MemorySet => "memory_set",
CandidateKind::ToolPermission => "tool_permission",
CandidateKind::ToolContract => "tool_contract",
CandidateKind::ModelSettings => "model_settings",
CandidateKind::MemoryConfiguration => "memory_configuration",
CandidateKind::MiddlewareComposition => "middleware_composition",
}
}
}
impl std::fmt::Display for CandidateKind {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.write_str(self.as_str())
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum GrantDirection {
Narrow,
Widen,
}
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
#[serde(tag = "kind", rename_all = "snake_case")]
pub enum CandidateContent {
Prompt {
name: String,
prompt: String,
},
Policy {
family: DecisionFamily,
parameters: Value,
},
MemorySet {
scope: ScopeAddress,
adds: Vec<MemoryRecord>,
#[serde(default, skip_serializing_if = "Vec::is_empty")]
supersedes: Vec<String>,
},
ToolPermission {
tool: String,
direction: GrantDirection,
},
ToolContract {
tool: String,
schema: Value,
},
ModelSettings {
name: String,
model: String,
parameters: Value,
},
MemoryConfiguration {
name: String,
budget: ContextBudget,
default_filters: MemoryQuery,
schema_version: String,
},
MiddlewareComposition {
name: String,
layers: Vec<MiddlewareLayerConfig>,
},
}
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
pub struct MiddlewareLayerConfig {
pub layer: String,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub config: Option<Value>,
}
pub fn derive_candidate_id(content: &CandidateContent) -> Result<CandidateId> {
Ok(CandidateId(sha256_hex(&serde_json::to_vec(content)?)))
}
#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
pub struct EvidenceSpan {
#[serde(default, skip_serializing_if = "Vec::is_empty")]
pub run_ids: Vec<String>,
#[serde(default, skip_serializing_if = "Vec::is_empty")]
pub correction_ids: Vec<String>,
#[serde(default, skip_serializing_if = "Vec::is_empty")]
pub memory_ids: Vec<String>,
}
impl EvidenceSpan {
pub fn is_empty(&self) -> bool {
self.run_ids.is_empty() && self.correction_ids.is_empty() && self.memory_ids.is_empty()
}
}
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
pub struct Candidate {
pub candidate_id: CandidateId,
pub content: CandidateContent,
pub distilled_by: ProvenanceAuthor,
#[serde(default, skip_serializing_if = "EvidenceSpan::is_empty")]
pub evidence: EvidenceSpan,
pub created_at: DateTime<Utc>,
}
impl Candidate {
pub fn new(
content: CandidateContent,
distilled_by: ProvenanceAuthor,
evidence: EvidenceSpan,
created_at: DateTime<Utc>,
) -> Result<Self> {
let candidate_id = derive_candidate_id(&content)?;
Ok(Self {
candidate_id,
content,
distilled_by,
evidence,
created_at,
})
}
pub fn kind(&self) -> CandidateKind {
match &self.content {
CandidateContent::Prompt { .. } => CandidateKind::Prompt,
CandidateContent::Policy { .. } => CandidateKind::Policy,
CandidateContent::MemorySet { .. } => CandidateKind::MemorySet,
CandidateContent::ToolPermission { .. } => CandidateKind::ToolPermission,
CandidateContent::ToolContract { .. } => CandidateKind::ToolContract,
CandidateContent::ModelSettings { .. } => CandidateKind::ModelSettings,
CandidateContent::MemoryConfiguration { .. } => CandidateKind::MemoryConfiguration,
CandidateContent::MiddlewareComposition { .. } => CandidateKind::MiddlewareComposition,
}
}
pub fn surface(&self) -> SurfaceKey {
match &self.content {
CandidateContent::Prompt { name, .. } => surface_for_kind(CandidateKind::Prompt, name),
CandidateContent::Policy { family, .. } => {
let family = serde_json::to_value(family)
.ok()
.and_then(|v| v.as_str().map(str::to_owned))
.unwrap_or_else(|| format!("{family:?}").to_lowercase());
surface_for_kind(CandidateKind::Policy, &family)
}
CandidateContent::MemorySet { scope, .. } => {
surface_for_kind(CandidateKind::MemorySet, &scope.as_address())
}
CandidateContent::ToolPermission { tool, .. } => {
surface_for_kind(CandidateKind::ToolPermission, tool)
}
CandidateContent::ToolContract { tool, .. } => {
surface_for_kind(CandidateKind::ToolContract, tool)
}
CandidateContent::ModelSettings { name, .. } => {
surface_for_kind(CandidateKind::ModelSettings, name)
}
CandidateContent::MemoryConfiguration { name, .. } => {
surface_for_kind(CandidateKind::MemoryConfiguration, name)
}
CandidateContent::MiddlewareComposition { name, .. } => {
surface_for_kind(CandidateKind::MiddlewareComposition, name)
}
}
}
pub fn verify_address(&self) -> std::result::Result<(), LearnError> {
let derived = derive_candidate_id(&self.content)
.map_err(|e| LearnError::UnaddressableContent(e.to_string()))?;
if derived != self.candidate_id {
return Err(LearnError::AddressMismatch {
claimed: self.candidate_id.clone(),
derived,
});
}
Ok(())
}
pub fn apply_to_manifest(&self, manifest: &mut RunManifest) -> Result<()> {
let CandidateContent::Prompt { name, prompt } = &self.content else {
return Err(invalid(format!(
"only a `prompt` candidate applies to a run manifest; `{}` candidates apply \
to their own surfaces",
self.kind_str()
)));
};
manifest
.prompts
.insert(name.clone(), sha256_hex(prompt.as_bytes()));
Ok(())
}
fn kind_str(&self) -> &'static str {
self.kind().as_str()
}
}
pub fn surface_for_kind(kind: CandidateKind, name: &str) -> SurfaceKey {
let prefix = match kind {
CandidateKind::Prompt => "prompt",
CandidateKind::Policy => "policy",
CandidateKind::MemorySet => "memory",
CandidateKind::ToolPermission => "tool",
CandidateKind::ToolContract => "tool_contract",
CandidateKind::ModelSettings => "model_settings",
CandidateKind::MemoryConfiguration => "memory_config",
CandidateKind::MiddlewareComposition => "middleware",
};
SurfaceKey::new(format!("{prefix}:{name}"))
}
#[derive(Debug, Clone, PartialEq, Eq, Hash, PartialOrd, Ord, Serialize, Deserialize)]
#[serde(transparent)]
pub struct SurfaceKey(String);
impl SurfaceKey {
pub fn new(surface: impl Into<String>) -> Self {
Self(surface.into())
}
pub fn as_str(&self) -> &str {
&self.0
}
}
impl std::fmt::Display for SurfaceKey {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.write_str(&self.0)
}
}
pub const SURFACE_TAG_SEPARATOR: char = '@';
#[derive(Debug, Clone, PartialEq, Eq, Hash, PartialOrd, Ord, Serialize)]
#[serde(transparent)]
pub struct EnvironmentTag(String);
impl EnvironmentTag {
pub const MAX_LEN: usize = 64;
pub fn new(tag: impl Into<String>) -> Result<Self> {
let tag = tag.into();
let refuse = |reason: &str| invalid(format!("invalid environment tag {tag:?}: {reason}"));
if tag.is_empty() {
return Err(refuse(
"empty — an absent tag is the untagged surface, spelled `None`",
));
}
if tag.len() > Self::MAX_LEN {
return Err(refuse("over 64 bytes"));
}
if tag
.chars()
.any(|c| c.is_whitespace() || c.is_control() || c == SURFACE_TAG_SEPARATOR || c == '/')
{
return Err(refuse(
"carries whitespace, a control character, `@`, or `/` — the first two have no \
business in a key, the last two are the tag and tenant separators",
));
}
Ok(Self(tag))
}
pub fn as_str(&self) -> &str {
&self.0
}
}
impl std::fmt::Display for EnvironmentTag {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.write_str(&self.0)
}
}
impl<'de> Deserialize<'de> for EnvironmentTag {
fn deserialize<D>(deserializer: D) -> std::result::Result<Self, D::Error>
where
D: serde::Deserializer<'de>,
{
let raw = String::deserialize(deserializer)?;
EnvironmentTag::new(raw).map_err(serde::de::Error::custom)
}
}
impl SurfaceKey {
pub fn tagged(&self, tag: &EnvironmentTag) -> SurfaceKey {
SurfaceKey::new(format!(
"{}{}{}",
self.0,
SURFACE_TAG_SEPARATOR,
tag.as_str()
))
}
pub fn split_tag(&self) -> (SurfaceKey, Option<EnvironmentTag>) {
match self.0.rfind(SURFACE_TAG_SEPARATOR) {
Some(at) => {
let base = SurfaceKey::new(&self.0[..at]);
let tag = EnvironmentTag(self.0[at + 1..].to_owned());
(base, Some(tag))
}
None => (self.clone(), None),
}
}
}
pub fn candidate_effect_key(candidate_id: &CandidateId) -> String {
format!("candidate:{candidate_id}")
}
pub fn evaluation_effect_key(candidate_id: &CandidateId, dataset_version: &str) -> String {
format!("evaluation:{candidate_id}:{dataset_version}")
}
pub fn promotion_effect_key(candidate_id: &CandidateId) -> String {
format!("promotion:{candidate_id}")
}
pub fn rollback_effect_key(surface: &SurfaceKey, candidate_id: &CandidateId) -> String {
format!("rollback:{surface}:{candidate_id}")
}
pub const PROMOTION_EFFECT_KIND: &str = "candidate_promotion";
pub fn promotion_effect_id(candidate: &Candidate) -> EffectId {
derive_effect_id(
candidate.surface().as_str(),
PROMOTION_EFFECT_KIND,
candidate.candidate_id.as_str(),
Some(&promotion_effect_key(&candidate.candidate_id)),
)
}
#[derive(Debug, Clone, Copy, PartialEq, Serialize, Deserialize)]
pub struct EvaluationThresholds {
pub max_pass_rate_drop: f64,
pub max_latency_p95_ratio: f64,
}
impl Default for EvaluationThresholds {
fn default() -> Self {
Self {
max_pass_rate_drop: 0.05,
max_latency_p95_ratio: 1.25,
}
}
}
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
pub struct EvaluationVerdict {
pub regressed: bool,
pub target_metric: String,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub baseline: Option<f64>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub candidate: Option<f64>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub delta: Option<f64>,
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct ReplayDivergence {
pub fixture_id: String,
pub detail: String,
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct ReplaySummary {
pub fixture_ids: Vec<String>,
pub matched: usize,
#[serde(default, skip_serializing_if = "Vec::is_empty")]
pub divergences: Vec<ReplayDivergence>,
}
impl ReplaySummary {
pub fn is_clean(&self) -> bool {
self.divergences.is_empty() && self.matched == self.fixture_ids.len()
}
}
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
pub struct CandidateEvaluation {
pub candidate_id: CandidateId,
pub dataset_version: String,
pub replay: ReplaySummary,
pub baseline_report: Value,
pub candidate_report: Value,
pub verdict: EvaluationVerdict,
pub thresholds: EvaluationThresholds,
pub evaluated_by: ProvenanceAuthor,
pub evaluated_at: DateTime<Utc>,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct EvaluationRequest {
pub dataset_version: String,
pub target_metric: String,
pub thresholds: EvaluationThresholds,
pub replay_evidence: Vec<JournalSnapshot>,
}
#[async_trait]
pub trait CandidateEvaluator: Send + Sync + std::fmt::Debug {
async fn evaluate(
&self,
candidate: &Candidate,
request: &EvaluationRequest,
) -> Result<CandidateEvaluation>;
}
#[derive(Debug)]
pub struct CandidateOverlay {
base: Arc<dyn MemoryStore>,
adds: Vec<MemoryRecord>,
supersedes: HashSet<String>,
}
impl CandidateOverlay {
pub fn new(base: Arc<dyn MemoryStore>, candidate: &Candidate) -> Result<Self> {
let CandidateContent::MemorySet {
adds, supersedes, ..
} = &candidate.content
else {
return Err(invalid(format!(
"only a `memory_set` candidate applies as a memory overlay; `{}` candidates \
apply to their own surfaces",
candidate.kind().as_str()
)));
};
Ok(Self {
base,
adds: adds.clone(),
supersedes: supersedes.iter().cloned().collect(),
})
}
}
#[async_trait]
impl MemoryStore for CandidateOverlay {
async fn put(&self, _record: &MemoryRecord) -> Result<bool> {
Err(invalid(
"the candidate overlay is a read lens: writes through it would mutate production \
memory mid-evaluation — the silent-rewrite pattern the candidate pipeline exists \
to forbid",
))
}
async fn get(&self, memory_id: &str) -> Result<Option<MemoryRecord>> {
if let Some(record) = self.adds.iter().find(|r| r.memory_id == memory_id) {
return Ok(Some(record.clone()));
}
self.base.get(memory_id).await
}
async fn all(&self) -> Result<Vec<MemoryRecord>> {
let mut merged: Vec<MemoryRecord> = self
.base
.all()
.await?
.into_iter()
.filter(|record| {
!self
.adds
.iter()
.any(|add| add.memory_id == record.memory_id)
})
.collect();
merged.extend(self.adds.iter().cloned());
Ok(merged)
}
async fn remove(&self, _memory_id: &str) -> Result<bool> {
Err(invalid(
"the candidate overlay is a read lens: deletion through it would mutate production \
memory mid-evaluation",
))
}
async fn query(&self, query: &MemoryQuery, now: DateTime<Utc>) -> Result<Vec<MemoryRecord>> {
let matched = apply_query(&self.all().await?, query, now);
if query.include_superseded || self.supersedes.is_empty() {
return Ok(matched);
}
Ok(matched
.into_iter()
.filter(|record| !self.supersedes.contains(&record.memory_id))
.collect())
}
}
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
pub struct AutoPromotion {
#[serde(default, skip_serializing_if = "Option::is_none")]
pub dataset_version: Option<String>,
pub min_improvement: f64,
#[serde(default, skip_serializing_if = "Vec::is_empty")]
pub scopes: Vec<MemoryScope>,
}
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
#[serde(tag = "mode", rename_all = "snake_case")]
pub enum EnvelopeRule {
Auto(AutoPromotion),
Approval,
Canary {
fraction: f64,
auto: AutoPromotion,
},
}
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
pub struct PromotionEnvelope {
pub envelope_version: String,
pub prompt: EnvelopeRule,
pub policy: EnvelopeRule,
pub memory_set: EnvelopeRule,
pub tool_permission: EnvelopeRule,
#[serde(
default = "approval_envelope_rule",
skip_serializing_if = "is_approval_envelope_rule"
)]
pub tool_contract: EnvelopeRule,
#[serde(
default = "approval_envelope_rule",
skip_serializing_if = "is_approval_envelope_rule"
)]
pub model_settings: EnvelopeRule,
#[serde(
default = "approval_envelope_rule",
skip_serializing_if = "is_approval_envelope_rule"
)]
pub memory_configuration: EnvelopeRule,
#[serde(
default = "approval_envelope_rule",
skip_serializing_if = "is_approval_envelope_rule"
)]
pub middleware_composition: EnvelopeRule,
}
fn approval_envelope_rule() -> EnvelopeRule {
EnvelopeRule::Approval
}
fn is_approval_envelope_rule(rule: &EnvelopeRule) -> bool {
*rule == EnvelopeRule::Approval
}
impl PromotionEnvelope {
pub fn r08_default() -> Self {
Self {
envelope_version: "r0.8-default".to_owned(),
prompt: EnvelopeRule::Approval,
policy: EnvelopeRule::Approval,
memory_set: EnvelopeRule::Auto(AutoPromotion {
dataset_version: None,
min_improvement: 0.0,
scopes: vec![MemoryScope::Run, MemoryScope::Agent],
}),
tool_permission: EnvelopeRule::Approval,
tool_contract: EnvelopeRule::Approval,
model_settings: EnvelopeRule::Approval,
memory_configuration: EnvelopeRule::Approval,
middleware_composition: EnvelopeRule::Approval,
}
}
pub fn rule_for(&self, kind: CandidateKind) -> &EnvelopeRule {
match kind {
CandidateKind::Prompt => &self.prompt,
CandidateKind::Policy => &self.policy,
CandidateKind::MemorySet => &self.memory_set,
CandidateKind::ToolPermission => &self.tool_permission,
CandidateKind::ToolContract => &self.tool_contract,
CandidateKind::ModelSettings => &self.model_settings,
CandidateKind::MemoryConfiguration => &self.memory_configuration,
CandidateKind::MiddlewareComposition => &self.middleware_composition,
}
}
pub fn validate(&self) -> Result<()> {
for rule in [
&self.prompt,
&self.policy,
&self.memory_set,
&self.tool_permission,
&self.tool_contract,
&self.model_settings,
&self.memory_configuration,
&self.middleware_composition,
] {
match rule {
EnvelopeRule::Auto(auto) => check_auto(auto)?,
EnvelopeRule::Canary { fraction, auto } => {
if !(fraction.is_finite() && *fraction > 0.0 && *fraction <= 1.0) {
return Err(invalid(format!(
"canary fraction must be in (0, 1], got {fraction} — a fraction \
outside the interval binds nothing or everything, and both are \
silent full promotions or silent no-ops"
)));
}
check_auto(auto)?;
}
EnvelopeRule::Approval => {}
}
}
Ok(())
}
}
fn check_auto(auto: &AutoPromotion) -> Result<()> {
if !(auto.min_improvement.is_finite() && auto.min_improvement >= 0.0) {
return Err(invalid(format!(
"min_improvement must be finite and non-negative, got {} — a negative bar would \
auto-promote regressions, which is the one outcome the gate exists to refuse",
auto.min_improvement
)));
}
Ok(())
}
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
pub struct CanaryBinding {
pub candidate_id: CandidateId,
pub fraction: f64,
}
pub const CANARY_DRAW_DOMAIN: &str = "rusty/canary-draw/v1";
pub fn canary_admits(binding: &CanaryBinding, surface: &SurfaceKey, run_id: &str) -> bool {
if binding.fraction >= 1.0 {
return true;
}
let material = [
CANARY_DRAW_DOMAIN,
surface.as_str(),
binding.candidate_id.as_str(),
run_id,
]
.join("\n");
let mut hasher = Sha256::new();
hasher.update(material.as_bytes());
let digest = hasher.finalize();
let draw = u64::from_be_bytes(digest[..8].try_into().expect("eight bytes"));
(draw as f64) < binding.fraction * (u64::MAX as f64)
}
#[derive(Debug, Clone, PartialEq, Error)]
pub enum PromotionRefusal {
#[error(
"candidate `{candidate_id}` has no journaled evaluation: promotion is gated on \
evidence — a human approves reviewed evidence, never a blank check"
)]
NotEvaluated {
candidate_id: CandidateId,
},
#[error(
"evaluation names candidate `{evaluation}` but promotion was requested for \
`{candidate_id}` — the evidence must be the candidate's own"
)]
EvaluationMismatch {
candidate_id: CandidateId,
evaluation: CandidateId,
},
#[error(
"candidate `{candidate_id}` diverged under replay ({divergences} fixture(s)): a \
candidate that cannot reproduce its evidence never auto-promotes"
)]
ReplayDiverged {
candidate_id: CandidateId,
divergences: usize,
},
#[error(
"candidate `{candidate_id}` regressed against its baseline: the auto-promotion bar \
is no regression AND improvement — the evidence failed the first half"
)]
EvaluationRegressed {
candidate_id: CandidateId,
},
#[error(
"envelope requires a clean verdict over dataset `{required}` but the evaluation \
ran `{evaluated}` — the promotion bar names its evidence exactly"
)]
DatasetVersionMismatch {
required: String,
evaluated: String,
},
#[error(
"memory_set candidates at `{scope:?}` scope do not auto-promote under this envelope — \
the R0.8 default covers run and agent scope; wider scopes require an approval"
)]
ScopeNotAutoPromotable {
scope: MemoryScope,
},
#[error(
"target metric moved by {delta:?} but the envelope requires improvement past \
{required} — parity is not improvement, and the auto bar does not clear on it"
)]
InsufficientImprovement {
delta: Option<f64>,
required: f64,
},
#[error(
"this promotion is outside the envelope: present an approval token scoped to \
effect id {effect_id} — the scope check makes an approval for one candidate \
non-transferable to another"
)]
RequiresApproval {
effect_id: EffectId,
},
#[error(
"approval token for {presented} does not admit this promotion ({required}) — an \
approval is scoped to exactly one candidate's promotion and is not transferable"
)]
ApprovalMismatch {
required: EffectId,
presented: EffectId,
},
}
#[derive(Debug, Clone, PartialEq, Error)]
pub enum LearnError {
#[error(
"candidate address mismatch: id {claimed} does not re-derive from the content \
(derived {derived}) — identity is integrity, and a tampered candidate fails its \
own address"
)]
AddressMismatch {
claimed: CandidateId,
derived: CandidateId,
},
#[error("candidate content could not be addressed: {0}")]
UnaddressableContent(String),
#[error(
"invalid candidate transition: cannot {action} from status `{from:?}` — refused \
transitions are errors, not silent no-ops"
)]
InvalidTransition {
from: CandidateStatus,
action: &'static str,
},
#[error(
"receipt names candidate `{named}` but the record is `{candidate_id}` — a receipt \
applies to the candidate it was minted for, never to another"
)]
ReceiptMismatch {
candidate_id: CandidateId,
named: CandidateId,
},
#[error("{0}")]
Refused(#[from] PromotionRefusal),
}
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
pub struct PromotionDecision {
pub authority: PromotionAuthority,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub canary: Option<CanaryBinding>,
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
#[serde(tag = "authority", rename_all = "snake_case")]
pub enum PromotionAuthority {
Envelope {
envelope_version: String,
},
Approval {
approved_by: String,
},
}
pub fn admit_promotion(
envelope: &PromotionEnvelope,
candidate: &Candidate,
evaluation: Option<&CandidateEvaluation>,
approval: Option<&ApprovalToken>,
) -> std::result::Result<PromotionDecision, LearnError> {
candidate.verify_address()?;
let evaluation = evaluation.ok_or(PromotionRefusal::NotEvaluated {
candidate_id: candidate.candidate_id.clone(),
})?;
if evaluation.candidate_id != candidate.candidate_id {
return Err(PromotionRefusal::EvaluationMismatch {
candidate_id: candidate.candidate_id.clone(),
evaluation: evaluation.candidate_id.clone(),
}
.into());
}
match envelope.rule_for(candidate.kind()) {
EnvelopeRule::Auto(auto) => {
check_evidence_bar(auto, candidate, evaluation)?;
Ok(PromotionDecision {
authority: PromotionAuthority::Envelope {
envelope_version: envelope.envelope_version.clone(),
},
canary: None,
})
}
EnvelopeRule::Canary { fraction, auto } => {
check_evidence_bar(auto, candidate, evaluation)?;
Ok(PromotionDecision {
authority: PromotionAuthority::Envelope {
envelope_version: envelope.envelope_version.clone(),
},
canary: Some(CanaryBinding {
candidate_id: candidate.candidate_id.clone(),
fraction: *fraction,
}),
})
}
EnvelopeRule::Approval => {
let required = promotion_effect_id(candidate);
let token = approval.ok_or(PromotionRefusal::RequiresApproval {
effect_id: required.clone(),
})?;
if !token.admits(&required) {
return Err(PromotionRefusal::ApprovalMismatch {
required,
presented: token.effect_id().clone(),
}
.into());
}
Ok(PromotionDecision {
authority: PromotionAuthority::Approval {
approved_by: token.approved_by().to_owned(),
},
canary: None,
})
}
}
}
fn check_evidence_bar(
auto: &AutoPromotion,
candidate: &Candidate,
evaluation: &CandidateEvaluation,
) -> std::result::Result<(), LearnError> {
if !evaluation.replay.is_clean() {
return Err(PromotionRefusal::ReplayDiverged {
candidate_id: candidate.candidate_id.clone(),
divergences: evaluation.replay.divergences.len(),
}
.into());
}
if evaluation.verdict.regressed {
return Err(PromotionRefusal::EvaluationRegressed {
candidate_id: candidate.candidate_id.clone(),
}
.into());
}
if let Some(required) = &auto.dataset_version {
if required != &evaluation.dataset_version {
return Err(PromotionRefusal::DatasetVersionMismatch {
required: required.clone(),
evaluated: evaluation.dataset_version.clone(),
}
.into());
}
}
if let CandidateContent::MemorySet { scope, .. } = &candidate.content {
if !auto.scopes.is_empty() && !auto.scopes.contains(&scope.scope) {
return Err(PromotionRefusal::ScopeNotAutoPromotable { scope: scope.scope }.into());
}
}
let clears = evaluation
.verdict
.delta
.is_some_and(|delta| delta > auto.min_improvement);
if !clears {
return Err(PromotionRefusal::InsufficientImprovement {
delta: evaluation.verdict.delta,
required: auto.min_improvement,
}
.into());
}
Ok(())
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum CandidateStatus {
Created,
Evaluated,
Promoted,
RolledBack,
}
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
pub struct PromotionReceipt {
pub candidate_id: CandidateId,
pub surface: SurfaceKey,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub previous: Option<CandidateId>,
pub decision: PromotionDecision,
pub promoted_at: DateTime<Utc>,
}
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
pub struct RollbackReceipt {
pub surface: SurfaceKey,
pub from: CandidateId,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub to: Option<CandidateId>,
pub cause: String,
pub rolled_back_at: DateTime<Utc>,
}
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
pub struct CandidateRecord {
pub candidate: Candidate,
pub status: CandidateStatus,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub evaluation: Option<CandidateEvaluation>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub promotion: Option<PromotionReceipt>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub rollback: Option<RollbackReceipt>,
}
impl CandidateRecord {
pub fn new(candidate: Candidate) -> Self {
Self {
candidate,
status: CandidateStatus::Created,
evaluation: None,
promotion: None,
rollback: None,
}
}
pub fn candidate_id(&self) -> &CandidateId {
&self.candidate.candidate_id
}
pub fn apply_evaluation(
&mut self,
evaluation: CandidateEvaluation,
) -> std::result::Result<(), LearnError> {
match self.status {
CandidateStatus::Created | CandidateStatus::Evaluated => {}
other => {
return Err(LearnError::InvalidTransition {
from: other,
action: "evaluate",
})
}
}
if evaluation.candidate_id != self.candidate.candidate_id {
return Err(PromotionRefusal::EvaluationMismatch {
candidate_id: self.candidate.candidate_id.clone(),
evaluation: evaluation.candidate_id,
}
.into());
}
self.evaluation = Some(evaluation);
self.status = CandidateStatus::Evaluated;
Ok(())
}
pub fn apply_promotion(
&mut self,
receipt: PromotionReceipt,
) -> std::result::Result<(), LearnError> {
if self.status != CandidateStatus::Evaluated {
return Err(LearnError::InvalidTransition {
from: self.status,
action: "promote",
});
}
if receipt.candidate_id != self.candidate.candidate_id {
return Err(LearnError::ReceiptMismatch {
candidate_id: self.candidate.candidate_id.clone(),
named: receipt.candidate_id,
});
}
self.promotion = Some(receipt);
self.status = CandidateStatus::Promoted;
Ok(())
}
pub fn apply_rollback(
&mut self,
receipt: RollbackReceipt,
) -> std::result::Result<(), LearnError> {
if self.status != CandidateStatus::Promoted {
return Err(LearnError::InvalidTransition {
from: self.status,
action: "roll back",
});
}
if receipt.from != self.candidate.candidate_id {
return Err(LearnError::ReceiptMismatch {
candidate_id: self.candidate.candidate_id.clone(),
named: receipt.from,
});
}
self.rollback = Some(receipt);
self.status = CandidateStatus::RolledBack;
Ok(())
}
}
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
pub struct VersionPointer {
pub surface: SurfaceKey,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub active: Option<CandidateId>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub canary: Option<CanaryBinding>,
}
impl VersionPointer {
pub fn new(surface: SurfaceKey) -> Self {
Self {
surface,
active: None,
canary: None,
}
}
pub fn promoted(&self, receipt: &PromotionReceipt) -> VersionPointer {
let mut next = self.clone();
if let Some(binding) = &receipt.decision.canary {
next.canary = Some(binding.clone());
} else {
next.active = Some(receipt.candidate_id.clone());
next.canary = None;
}
next
}
pub fn rolled_back(&self, receipt: &RollbackReceipt) -> VersionPointer {
let mut next = self.clone();
if self.active.as_ref() == Some(&receipt.from) {
next.active = receipt.to.clone();
}
if self
.canary
.as_ref()
.is_some_and(|binding| binding.candidate_id == receipt.from)
{
next.canary = None;
}
next
}
}
use crate::durable::ErrorClass;
use crate::record::{
derive_policy_version, BackoffParameters, DecisionAction, DecisionEvent, DecisionOutcome,
Effect, EventStatus, ExecutorPolicy, PolicyVersion, RetryPolicyParameters, RunEvent,
TimeoutPolicyParameters,
};
use crate::twin::{
percentile, FaultSchedule, ParameterizedPolicy, Twin, TwinMetrics, TwinPolicy, TwinRunConfig,
DEFAULT_TIMEOUT_LADDER,
};
const DEAD_LETTER_PENALTY_MS: u64 = crate::durable::MAX_RETRY_DELAY_MS;
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
pub struct RetryLearningConfig {
pub min_samples: usize,
pub min_improvement_ms: u64,
pub base_grid_ms: Vec<u64>,
pub cap_grid_ms: Vec<u64>,
pub max_attempts_grid: Vec<u32>,
}
impl Default for RetryLearningConfig {
fn default() -> Self {
Self {
min_samples: 4,
min_improvement_ms: 2_000,
base_grid_ms: vec![100, 250, 500, 1_000, 2_000],
cap_grid_ms: vec![30_000, 120_000, 300_000],
max_attempts_grid: vec![1, 2, 3, 5],
}
}
}
#[derive(Debug, Default)]
struct ClassEvidence {
retries: u64,
successes: u64,
failures: u64,
retry_after_ms: Vec<u64>,
}
fn retry_schedule_score(
base_ms: u64,
cap_ms: u64,
max_attempts: u32,
p: f64,
retry_after: u64,
) -> f64 {
let mut wall = 0.0;
let mut survival = 1.0;
for k in 0..max_attempts {
let mean_delay =
((base_ms.saturating_mul(1u64 << k.min(20))).min(cap_ms) / 2).max(retry_after) as f64;
wall += survival * mean_delay;
survival *= 1.0 - p;
}
wall + survival * DEAD_LETTER_PENALTY_MS as f64
}
pub fn distill_retry_parameters(
events: &[DecisionEvent],
config: &RetryLearningConfig,
) -> RetryPolicyParameters {
let floor = ExecutorPolicy::static_v0().retry;
let mut classes: std::collections::BTreeMap<ErrorClass, ClassEvidence> =
std::collections::BTreeMap::new();
for event in events {
if event.family != crate::record::DecisionFamily::Retry {
continue;
}
let class = event
.features
.get("failure_class")
.and_then(|value| serde_json::from_value(value.clone()).ok())
.unwrap_or(ErrorClass::Unknown);
let evidence = classes.entry(class).or_default();
if matches!(event.selected, DecisionAction::Retry { .. }) {
evidence.retries += 1;
}
match event.outcome {
Some(DecisionOutcome::Success) => evidence.successes += 1,
Some(DecisionOutcome::Failure) => evidence.failures += 1,
_ => {}
}
if let Some(retry_after) = event
.features
.get("retry_after_ms")
.and_then(serde_json::Value::as_u64)
{
evidence.retry_after_ms.push(retry_after);
}
}
let mut table = std::collections::BTreeMap::new();
for (class, evidence) in &classes {
let total = evidence.retries + evidence.failures;
if (total as usize) < config.min_samples {
continue;
}
if evidence.successes == 0 && evidence.retries > 0 {
if evidence.failures >= config.min_samples as u64 {
table.insert(
*class,
BackoffParameters {
base_delay_ms: floor.base_delay_ms,
max_delay_ms: floor.max_delay_ms,
max_attempts: 1,
},
);
}
continue;
}
if evidence.retries == 0 || evidence.successes == 0 {
continue;
}
let p = evidence.successes as f64 / evidence.retries as f64;
let retry_after = percentile(&evidence.retry_after_ms, 50.0);
let floor_score = retry_schedule_score(
floor.base_delay_ms,
floor.max_delay_ms,
floor.max_attempts,
p,
retry_after,
);
let mut best: Option<(f64, BackoffParameters)> = None;
for &base in &config.base_grid_ms {
for &cap in &config.cap_grid_ms {
if base == 0 || base > cap {
continue;
}
for &max_attempts in &config.max_attempts_grid {
let score = retry_schedule_score(base, cap, max_attempts, p, retry_after);
let candidate = BackoffParameters {
base_delay_ms: base,
max_delay_ms: cap,
max_attempts,
};
if best
.as_ref()
.is_none_or(|(best_score, _)| score < *best_score)
{
best = Some((score, candidate));
}
}
}
}
if let Some((score, schedule)) = best {
if floor_score - score >= config.min_improvement_ms as f64 {
table.insert(*class, schedule);
}
}
}
RetryPolicyParameters {
per_class: if table.is_empty() { None } else { Some(table) },
..floor
}
}
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
pub struct TimeoutLearningConfig {
pub min_samples: usize,
pub abort_tolerance: f64,
pub ladder: Vec<u64>,
}
impl Default for TimeoutLearningConfig {
fn default() -> Self {
Self {
min_samples: 16,
abort_tolerance: 0.01,
ladder: DEFAULT_TIMEOUT_LADDER.to_vec(),
}
}
}
pub fn distill_timeout_parameters(
events: &[RunEvent],
config: &TimeoutLearningConfig,
) -> TimeoutPolicyParameters {
let mut latencies: std::collections::BTreeMap<String, Vec<u64>> =
std::collections::BTreeMap::new();
for event in events {
if event.status != EventStatus::Ok {
continue;
}
let (Some(callee), Some(latency)) = (event.node_id.clone(), event.latency_ms) else {
continue;
};
latencies.entry(callee).or_default().push(latency);
}
let mut table = std::collections::BTreeMap::new();
let top = config.ladder.last().copied().unwrap_or(u64::MAX);
for (callee, samples) in &latencies {
if samples.len() < config.min_samples {
continue;
}
let bound = config.ladder.iter().copied().find(|rung| {
let aborted = samples.iter().filter(|latency| *latency > rung).count();
(aborted as f64 / samples.len() as f64) <= config.abort_tolerance
});
match bound {
Some(rung) if rung < top => {
table.insert(callee.clone(), rung);
}
_ => {}
}
}
TimeoutPolicyParameters {
default_millis: None,
max_millis: if table.is_empty() { None } else { Some(top) },
per_callee: if table.is_empty() { None } else { Some(table) },
}
}
#[derive(Debug)]
pub struct TwinCandidateEvaluator {
seed: u64,
faults: FaultSchedule,
workers: Vec<String>,
max_attempts: u32,
completion_tolerance: f64,
evaluated_by: ProvenanceAuthor,
}
impl TwinCandidateEvaluator {
pub fn new(seed: u64, evaluated_by: ProvenanceAuthor) -> Self {
Self {
seed,
faults: FaultSchedule::new(seed),
workers: vec!["worker-0".to_owned()],
max_attempts: 3,
completion_tolerance: 0.0,
evaluated_by,
}
}
pub fn with_faults(mut self, faults: FaultSchedule) -> Self {
self.faults = faults;
self
}
pub fn with_workers(mut self, workers: impl IntoIterator<Item = impl Into<String>>) -> Self {
self.workers = workers.into_iter().map(Into::into).collect();
self
}
pub fn with_max_attempts(mut self, max_attempts: u32) -> Self {
self.max_attempts = max_attempts;
self
}
pub fn with_completion_tolerance(mut self, tolerance: f64) -> Self {
self.completion_tolerance = tolerance;
self
}
}
#[derive(Debug, Clone, Copy, Default)]
struct ArmAggregate {
items: usize,
completed: usize,
dead_lettered: usize,
attempts: u64,
cost_usd: f64,
wall_time_ms: u64,
latency_p95_ms: u64,
runs: usize,
}
impl ArmAggregate {
fn fold(&mut self, metrics: &TwinMetrics) {
self.items += metrics.items;
self.completed += metrics.completed;
self.dead_lettered += metrics.dead_lettered;
self.attempts += metrics.attempts;
self.cost_usd += metrics.cost_usd;
self.wall_time_ms += metrics.wall_time_ms;
self.latency_p95_ms = self.latency_p95_ms.max(metrics.item_latency_p95_ms);
self.runs += 1;
}
fn completion_rate(&self) -> f64 {
if self.items == 0 {
0.0
} else {
self.completed as f64 / self.items as f64
}
}
fn dead_letter_rate(&self) -> f64 {
if self.items == 0 {
0.0
} else {
self.dead_lettered as f64 / self.items as f64
}
}
fn mean_wall_ms(&self) -> f64 {
if self.runs == 0 {
0.0
} else {
self.wall_time_ms as f64 / self.runs as f64
}
}
fn as_value(&self) -> Value {
serde_json::json!({
"items": self.items,
"completed": self.completed,
"dead_lettered": self.dead_lettered,
"attempts": self.attempts,
"cost_usd": self.cost_usd,
"mean_wall_time_ms": self.mean_wall_ms(),
"item_latency_p95_ms": self.latency_p95_ms,
"completion_rate": self.completion_rate(),
"dead_letter_rate": self.dead_letter_rate(),
})
}
}
#[async_trait]
impl CandidateEvaluator for TwinCandidateEvaluator {
async fn evaluate(
&self,
candidate: &Candidate,
request: &EvaluationRequest,
) -> Result<CandidateEvaluation> {
let CandidateContent::Policy { family, parameters } = &candidate.content else {
return Err(invalid(format!(
"the twin-backed evaluator prices `policy` candidates; `{}` candidates \
evaluate through the application's own evaluator",
candidate.kind().as_str()
)));
};
let candidate_policy = ExecutorPolicy::static_v0()
.with_family_parameters(*family, parameters.clone())
.map_err(|e| {
invalid(format!(
"policy candidate cannot be evaluated: its parameters do not apply to \
the floor ({e}) — an out-of-envelope candidate never reaches evidence"
))
})?;
let version = derive_policy_version(&candidate_policy)?;
let acting: Arc<dyn TwinPolicy> =
Arc::new(ParameterizedPolicy::new(candidate_policy, version.clone())?);
let mut floor_aggregate = ArmAggregate::default();
let mut candidate_aggregate = ArmAggregate::default();
let mut fixture_ids = Vec::with_capacity(request.replay_evidence.len());
let mut matched = 0usize;
let mut divergences = Vec::new();
let mut floor_runs = Vec::new();
let mut candidate_runs = Vec::new();
let mut completion_breached = false;
for fixture in &request.replay_evidence {
fixture_ids.push(fixture.run_id.clone());
let outcome = (|| -> Result<_> {
let twin = Twin::from_snapshot(fixture.clone())?;
let config = TwinRunConfig::new(self.seed)
.with_faults(self.faults.clone())
.with_workers(self.workers.clone())
.with_max_attempts(self.max_attempts);
let floor_run = twin.run(&config)?;
let candidate_run = twin.run(&config.with_acting(acting.clone()))?;
Ok((floor_run, candidate_run))
})();
match outcome {
Ok((floor_run, candidate_run)) => {
matched += 1;
let floor_rate =
floor_run.metrics.completed as f64 / floor_run.metrics.items.max(1) as f64;
let candidate_rate = candidate_run.metrics.completed as f64
/ candidate_run.metrics.items.max(1) as f64;
if candidate_rate < floor_rate - self.completion_tolerance {
completion_breached = true;
}
floor_aggregate.fold(&floor_run.metrics);
candidate_aggregate.fold(&candidate_run.metrics);
floor_runs.push(serde_json::json!({
"fixture": fixture.run_id,
"metrics": floor_run.metrics,
"report": floor_run.report,
}));
candidate_runs.push(serde_json::json!({
"fixture": fixture.run_id,
"metrics": candidate_run.metrics,
"report": candidate_run.report,
}));
}
Err(error) => {
divergences.push(ReplayDivergence {
fixture_id: fixture.run_id.clone(),
detail: error.to_string(),
});
}
}
}
fixture_ids.sort();
divergences.sort_by(|a, b| a.fixture_id.cmp(&b.fixture_id));
let regressed = completion_breached
|| candidate_aggregate.completion_rate()
< floor_aggregate.completion_rate() - self.completion_tolerance;
let (baseline, candidate_value, delta) = match request.target_metric.as_str() {
"completion_rate" => (
Some(floor_aggregate.completion_rate()),
Some(candidate_aggregate.completion_rate()),
Some(candidate_aggregate.completion_rate() - floor_aggregate.completion_rate()),
),
"cost_usd" => (
Some(floor_aggregate.cost_usd),
Some(candidate_aggregate.cost_usd),
Some(floor_aggregate.cost_usd - candidate_aggregate.cost_usd),
),
"wall_time_ms" => (
Some(floor_aggregate.mean_wall_ms()),
Some(candidate_aggregate.mean_wall_ms()),
Some(floor_aggregate.mean_wall_ms() - candidate_aggregate.mean_wall_ms()),
),
"item_latency_p95_ms" => (
Some(floor_aggregate.latency_p95_ms as f64),
Some(candidate_aggregate.latency_p95_ms as f64),
Some(
floor_aggregate.latency_p95_ms as f64
- candidate_aggregate.latency_p95_ms as f64,
),
),
_ => (None, None, None),
};
Ok(CandidateEvaluation {
candidate_id: candidate.candidate_id.clone(),
dataset_version: request.dataset_version.clone(),
replay: ReplaySummary {
fixture_ids,
matched,
divergences,
},
baseline_report: serde_json::json!({
"arm": PolicyVersion::STATIC_V0,
"aggregate": floor_aggregate.as_value(),
"runs": floor_runs,
}),
candidate_report: serde_json::json!({
"arm": version,
"aggregate": candidate_aggregate.as_value(),
"runs": candidate_runs,
}),
verdict: EvaluationVerdict {
regressed,
target_metric: request.target_metric.clone(),
baseline,
candidate: candidate_value,
delta,
},
thresholds: request.thresholds,
evaluated_by: self.evaluated_by.clone(),
evaluated_at: Utc::now(),
})
}
}
#[derive(Debug, Clone, Copy, PartialEq, Serialize, Deserialize)]
pub struct DriftBaseline {
pub completion_rate: f64,
pub dead_letter_rate: f64,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub latency_p95_ms: Option<u64>,
}
impl DriftBaseline {
pub fn from_twin_report(report: &Value) -> Option<Self> {
let aggregate = report.get("aggregate")?;
Some(Self {
completion_rate: aggregate.get("completion_rate")?.as_f64()?,
dead_letter_rate: aggregate.get("dead_letter_rate")?.as_f64()?,
latency_p95_ms: aggregate.get("item_latency_p95_ms").and_then(Value::as_u64),
})
}
}
#[derive(Debug, Clone, Copy, PartialEq, Serialize, Deserialize)]
pub struct DriftThresholds {
pub max_completion_drop: f64,
pub max_dead_letter_growth: f64,
pub max_latency_p95_ratio: f64,
pub min_decisions: usize,
}
impl Default for DriftThresholds {
fn default() -> Self {
Self {
max_completion_drop: 0.05,
max_dead_letter_growth: 0.02,
max_latency_p95_ratio: 1.25,
min_decisions: 8,
}
}
}
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
pub struct PolicyDriftReport {
pub version: PolicyVersion,
pub decisions: usize,
pub terminal: usize,
pub completion_rate: f64,
pub dead_letter_rate: f64,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub latency_p95_ms: Option<u64>,
pub baseline: DriftBaseline,
pub drifted: bool,
pub reasons: Vec<String>,
}
pub fn detect_policy_drift(
decisions: &[DecisionEvent],
version: &PolicyVersion,
baseline: &DriftBaseline,
thresholds: &DriftThresholds,
) -> PolicyDriftReport {
let acting: Vec<&DecisionEvent> = decisions
.iter()
.filter(|event| {
&event.policy_version == version
&& event.role != Some(crate::record::DecisionRole::Shadow)
})
.collect();
let terminal: Vec<&DecisionEvent> = acting
.iter()
.filter(|event| {
matches!(
event.outcome,
Some(DecisionOutcome::Success) | Some(DecisionOutcome::Failure)
)
})
.copied()
.collect();
let successes = terminal
.iter()
.filter(|event| event.outcome == Some(DecisionOutcome::Success))
.count();
let dead_letters = terminal
.iter()
.filter(|event| {
if event.outcome != Some(DecisionOutcome::Failure)
|| event.selected != DecisionAction::Abort
{
return false;
}
let budget_spent = match (
event.features.get("attempt").and_then(Value::as_u64),
event.features.get("max_attempts").and_then(Value::as_u64),
) {
(Some(attempt), Some(max_attempts)) => attempt >= max_attempts,
_ => false,
};
let effect_open = event
.features
.get("effect")
.and_then(|value| serde_json::from_value::<Effect>(value.clone()).ok())
.is_some_and(|effect| effect.is_freely_repeatable());
let class_open = event
.features
.get("failure_class")
.and_then(|value| serde_json::from_value::<ErrorClass>(value.clone()).ok())
.is_some_and(|class| class.is_retryable());
budget_spent && effect_open && class_open
})
.count();
let latencies: Vec<u64> = acting
.iter()
.filter_map(|event| {
event
.features
.get("dependency_latency_ms")
.and_then(Value::as_u64)
})
.collect();
let completion_rate = if terminal.is_empty() {
0.0
} else {
successes as f64 / terminal.len() as f64
};
let dead_letter_rate = if terminal.is_empty() {
0.0
} else {
dead_letters as f64 / terminal.len() as f64
};
let latency_p95_ms = (!latencies.is_empty()).then(|| percentile(&latencies, 95.0));
let mut reasons = Vec::new();
let sufficient = terminal.len() >= thresholds.min_decisions;
let mut drifted = false;
if !sufficient {
reasons.push(format!(
"insufficient evidence: {} terminal decisions, fewer than the declared minimum {}",
terminal.len(),
thresholds.min_decisions
));
} else {
let completion_drop = baseline.completion_rate - completion_rate;
if completion_drop > thresholds.max_completion_drop {
drifted = true;
reasons.push(format!(
"completion rate {completion_rate:.3} is {completion_drop:.3} below the \
promotion-time baseline {:.3} (threshold {:.3})",
baseline.completion_rate, thresholds.max_completion_drop
));
}
let dead_letter_growth = dead_letter_rate - baseline.dead_letter_rate;
if dead_letter_growth > thresholds.max_dead_letter_growth {
drifted = true;
reasons.push(format!(
"dead-letter rate {dead_letter_rate:.3} grew {dead_letter_growth:.3} over the \
promotion-time baseline {:.3} (threshold {:.3})",
baseline.dead_letter_rate, thresholds.max_dead_letter_growth
));
}
if let (Some(current), Some(base)) = (latency_p95_ms, baseline.latency_p95_ms) {
if base > 0 && current as f64 / base as f64 > thresholds.max_latency_p95_ratio {
drifted = true;
reasons.push(format!(
"p95 decision latency {current} ms is {:.2}x the promotion-time baseline \
{base} ms (threshold {:.2}x)",
current as f64 / base as f64,
thresholds.max_latency_p95_ratio
));
}
}
}
PolicyDriftReport {
version: version.clone(),
decisions: acting.len(),
terminal: terminal.len(),
completion_rate,
dead_letter_rate,
latency_p95_ms,
baseline: *baseline,
drifted,
reasons,
}
}
#[cfg(test)]
mod tests {
use super::*;
use crate::memory::{InMemoryMemoryStore, MemoryKind, MemoryProvenance, ValidityWindow};
use serde_json::json;
fn ts(millis: i64) -> DateTime<Utc> {
DateTime::<Utc>::from_timestamp_millis(millis).unwrap()
}
fn distiller() -> ProvenanceAuthor {
ProvenanceAuthor::Distiller {
name: "correction-loop".into(),
}
}
fn memory_record(content: Value) -> MemoryRecord {
MemoryRecord::new(
MemoryKind::Fact,
ScopeAddress::new(MemoryScope::Agent, "support-1"),
MemoryProvenance {
author: ProvenanceAuthor::Human {
human_id: "amjad".into(),
},
evidence: Default::default(),
written_at: ts(1_000),
},
1.0,
ValidityWindow::starting(ts(500)),
ts(1_000),
content,
)
.unwrap()
}
fn memory_set_candidate(content: Value) -> Candidate {
Candidate::new(
CandidateContent::MemorySet {
scope: ScopeAddress::new(MemoryScope::Agent, "support-1"),
adds: vec![memory_record(content)],
supersedes: Vec::new(),
},
distiller(),
EvidenceSpan {
run_ids: vec!["run-1".into()],
..EvidenceSpan::default()
},
ts(2_000),
)
.unwrap()
}
fn clean_evaluation(candidate: &Candidate, delta: f64) -> CandidateEvaluation {
CandidateEvaluation {
candidate_id: candidate.candidate_id.clone(),
dataset_version: "v3".into(),
replay: ReplaySummary {
fixture_ids: vec!["run-1".into()],
matched: 1,
divergences: Vec::new(),
},
baseline_report: json!({"name": "support@v3", "summary": {"run_pass_rate": 0.5}}),
candidate_report: json!({"name": "support@v3", "summary": {"run_pass_rate": 0.9}}),
verdict: EvaluationVerdict {
regressed: false,
target_metric: "run_pass_rate".into(),
baseline: Some(0.5),
candidate: Some(0.5 + delta),
delta: Some(delta),
},
thresholds: EvaluationThresholds::default(),
evaluated_by: distiller(),
evaluated_at: ts(3_000),
}
}
#[test]
fn candidate_id_is_a_content_address() {
let a = memory_set_candidate(json!({"tone": "warm"}));
let b = memory_set_candidate(json!({"tone": "warm"}));
let c = memory_set_candidate(json!({"tone": "cold"}));
assert_eq!(a.candidate_id, b.candidate_id);
let d = Candidate::new(
a.content.clone(),
ProvenanceAuthor::System,
EvidenceSpan::default(),
ts(9_999),
)
.unwrap();
assert_eq!(a.candidate_id, d.candidate_id);
assert_ne!(a.candidate_id, c.candidate_id);
assert_eq!(a.candidate_id.as_str().len(), 64);
}
#[test]
fn tampered_candidate_fails_its_own_address() {
let mut candidate = memory_set_candidate(json!({"tone": "warm"}));
candidate.verify_address().unwrap();
candidate.candidate_id = CandidateId("f".repeat(64));
assert!(matches!(
candidate.verify_address(),
Err(LearnError::AddressMismatch { .. })
));
}
#[test]
fn surfaces_are_the_documented_keys() {
let memory = memory_set_candidate(json!({}));
assert_eq!(memory.surface().as_str(), "memory:agent:support-1");
let prompt = Candidate::new(
CandidateContent::Prompt {
name: "system".into(),
prompt: "You are careful.".into(),
},
distiller(),
EvidenceSpan::default(),
ts(2_000),
)
.unwrap();
assert_eq!(prompt.surface().as_str(), "prompt:system");
let policy = Candidate::new(
CandidateContent::Policy {
family: DecisionFamily::Retry,
parameters: json!({"max_attempts": 4}),
},
distiller(),
EvidenceSpan::default(),
ts(2_000),
)
.unwrap();
assert_eq!(policy.surface().as_str(), "policy:retry");
let tool = Candidate::new(
CandidateContent::ToolPermission {
tool: "shell".into(),
direction: GrantDirection::Narrow,
},
distiller(),
EvidenceSpan::default(),
ts(2_000),
)
.unwrap();
assert_eq!(tool.surface().as_str(), "tool:shell");
}
#[test]
fn lifecycle_refuses_illegal_transitions() {
let candidate = memory_set_candidate(json!({}));
let mut record = CandidateRecord::new(candidate.clone());
let receipt = PromotionReceipt {
candidate_id: candidate.candidate_id.clone(),
surface: candidate.surface(),
previous: None,
decision: PromotionDecision {
authority: PromotionAuthority::Envelope {
envelope_version: "r0.8-default".into(),
},
canary: None,
},
promoted_at: ts(4_000),
};
assert!(matches!(
record.apply_promotion(receipt.clone()),
Err(LearnError::InvalidTransition {
from: CandidateStatus::Created,
action: "promote",
})
));
record
.apply_evaluation(clean_evaluation(&candidate, 0.4))
.unwrap();
record.apply_promotion(receipt).unwrap();
assert!(matches!(
record.apply_evaluation(clean_evaluation(&candidate, 0.5)),
Err(LearnError::InvalidTransition {
from: CandidateStatus::Promoted,
action: "evaluate",
})
));
record
.apply_rollback(RollbackReceipt {
surface: candidate.surface(),
from: candidate.candidate_id.clone(),
to: None,
cause: "operator: drift on run-9".into(),
rolled_back_at: ts(5_000),
})
.unwrap();
assert_eq!(record.status, CandidateStatus::RolledBack);
}
#[test]
fn gate_admits_in_envelope_and_refuses_out_of_it() {
let envelope = PromotionEnvelope::r08_default();
let candidate = memory_set_candidate(json!({"tone": "warm"}));
let evaluation = clean_evaluation(&candidate, 0.4);
let decision = admit_promotion(&envelope, &candidate, Some(&evaluation), None).unwrap();
assert_eq!(
decision.authority,
PromotionAuthority::Envelope {
envelope_version: "r0.8-default".into()
}
);
let prompt = Candidate::new(
CandidateContent::Prompt {
name: "system".into(),
prompt: "You are careful.".into(),
},
distiller(),
EvidenceSpan::default(),
ts(2_000),
)
.unwrap();
let prompt_eval = clean_evaluation(&prompt, 0.4);
let refused = admit_promotion(&envelope, &prompt, Some(&prompt_eval), None).unwrap_err();
let effect_id = promotion_effect_id(&prompt);
assert_eq!(
refused,
LearnError::Refused(PromotionRefusal::RequiresApproval {
effect_id: effect_id.clone()
})
);
let wrong = ApprovalToken::approve(promotion_effect_id(&candidate), "ops:amjad");
assert!(matches!(
admit_promotion(&envelope, &prompt, Some(&prompt_eval), Some(&wrong)),
Err(LearnError::Refused(
PromotionRefusal::ApprovalMismatch { .. }
))
));
let token = ApprovalToken::approve(effect_id, "ops:amjad");
let decision =
admit_promotion(&envelope, &prompt, Some(&prompt_eval), Some(&token)).unwrap();
assert_eq!(
decision.authority,
PromotionAuthority::Approval {
approved_by: "ops:amjad".into()
}
);
}
#[test]
fn gate_enforces_the_evidence_bar() {
let envelope = PromotionEnvelope::r08_default();
let candidate = memory_set_candidate(json!({}));
let parity = clean_evaluation(&candidate, 0.0);
assert_eq!(
admit_promotion(&envelope, &candidate, Some(&parity), None).unwrap_err(),
LearnError::Refused(PromotionRefusal::InsufficientImprovement {
delta: Some(0.0),
required: 0.0,
})
);
let mut regressed = clean_evaluation(&candidate, 0.4);
regressed.verdict.regressed = true;
assert!(matches!(
admit_promotion(&envelope, &candidate, Some(®ressed), None),
Err(LearnError::Refused(
PromotionRefusal::EvaluationRegressed { .. }
))
));
let mut diverged = clean_evaluation(&candidate, 0.4);
diverged.replay.divergences = vec![ReplayDivergence {
fixture_id: "run-1".into(),
detail: "seq 4: request hash mismatch".into(),
}];
assert!(matches!(
admit_promotion(&envelope, &candidate, Some(&diverged), None),
Err(LearnError::Refused(PromotionRefusal::ReplayDiverged { .. }))
));
let tenant_scoped = Candidate::new(
CandidateContent::MemorySet {
scope: ScopeAddress::new(MemoryScope::Tenant, "default"),
adds: vec![],
supersedes: vec![],
},
distiller(),
EvidenceSpan::default(),
ts(2_000),
)
.unwrap();
let evaluation = clean_evaluation(&tenant_scoped, 0.4);
assert!(matches!(
admit_promotion(&envelope, &tenant_scoped, Some(&evaluation), None),
Err(LearnError::Refused(
PromotionRefusal::ScopeNotAutoPromotable {
scope: MemoryScope::Tenant
}
))
));
assert!(matches!(
admit_promotion(&envelope, &candidate, None, None),
Err(LearnError::Refused(PromotionRefusal::NotEvaluated { .. }))
));
}
#[test]
fn canary_draw_is_deterministic_uniform_and_scoped() {
let candidate = memory_set_candidate(json!({}));
let surface = candidate.surface();
let binding = CanaryBinding {
candidate_id: candidate.candidate_id.clone(),
fraction: 0.25,
};
assert_eq!(
canary_admits(&binding, &surface, "run-7"),
canary_admits(&binding, &surface, "run-7")
);
let full = CanaryBinding {
candidate_id: candidate.candidate_id.clone(),
fraction: 1.0,
};
for run in 0..32 {
assert!(canary_admits(&full, &surface, &format!("run-{run}")));
}
let admitted = (0..1_000)
.filter(|run| canary_admits(&binding, &surface, &format!("run-{run}")))
.count();
assert!(
(150..=350).contains(&admitted),
"fraction 0.25 admitted {admitted}/1000 runs"
);
}
#[test]
fn pointer_moves_on_promotion_and_reponts_on_rollback() {
let a = memory_set_candidate(json!({"v": "a"}));
let b = memory_set_candidate(json!({"v": "b"}));
let pointer = VersionPointer::new(a.surface());
assert_eq!(pointer.active, None);
let promote_a = PromotionReceipt {
candidate_id: a.candidate_id.clone(),
surface: a.surface(),
previous: None,
decision: PromotionDecision {
authority: PromotionAuthority::Envelope {
envelope_version: "r0.8-default".into(),
},
canary: None,
},
promoted_at: ts(4_000),
};
let pointer = pointer.promoted(&promote_a);
assert_eq!(pointer.active, Some(a.candidate_id.clone()));
let promote_b = PromotionReceipt {
candidate_id: b.candidate_id.clone(),
surface: a.surface(),
previous: Some(a.candidate_id.clone()),
decision: promote_a.decision.clone(),
promoted_at: ts(5_000),
};
let pointer = pointer.promoted(&promote_b);
assert_eq!(pointer.active, Some(b.candidate_id.clone()));
let pointer = pointer.rolled_back(&RollbackReceipt {
surface: a.surface(),
from: b.candidate_id.clone(),
to: promote_b.previous.clone(),
cause: "drift".into(),
rolled_back_at: ts(6_000),
});
assert_eq!(pointer.active, Some(a.candidate_id.clone()));
}
#[tokio::test]
async fn overlay_applies_the_candidate_as_a_read_lens() {
let base = Arc::new(InMemoryMemoryStore::new());
let existing = memory_record(json!({"tone": "flat"}));
base.put(&existing).await.unwrap();
let mut added = memory_record(json!({"tone": "warm"}));
added.supersedes = Some(existing.memory_id.clone());
let candidate = Candidate::new(
CandidateContent::MemorySet {
scope: ScopeAddress::new(MemoryScope::Agent, "support-1"),
adds: vec![added.clone()],
supersedes: Vec::new(),
},
distiller(),
EvidenceSpan::default(),
ts(2_000),
)
.unwrap();
let overlay = CandidateOverlay::new(base.clone(), &candidate).unwrap();
assert_eq!(overlay.get(&added.memory_id).await.unwrap(), Some(added));
assert_eq!(
overlay.get(&existing.memory_id).await.unwrap(),
Some(existing.clone())
);
let listed = overlay
.query(&MemoryQuery::default(), Utc::now())
.await
.unwrap();
assert_eq!(listed.len(), 1);
assert!(overlay.put(&existing).await.is_err());
}
}