use std::collections::BTreeMap;
use std::fmt;
use std::sync::Arc;
use serde::{Deserialize, Serialize};
use serde_json::{Map, Value};
use crate::durable::{
backoff_delay_ms, backoff_delay_ms_with, resolve_retry_parameters, resolve_timeout_bound_ms,
retry_legal_actions, ErrorClass, MAX_RETRY_DELAY_MS,
};
use crate::error::{Result, RustyError};
use crate::journal::{Clock, EventDraft, Journal, JournalSnapshot, RngSource};
use crate::llm::Usage;
use crate::record::{
DecisionAction, DecisionEvent, DecisionFamily, DecisionRole, Effect, EventStatus,
ExecutorPolicy, PayloadRef, PolicyVersion, RunEventKind,
};
use crate::replay::{BranchDiff, ExactReplay, LogicalClockParams, ReplayFixture, SERVABLE_KINDS};
fn twin_error(message: impl Into<String>) -> RustyError {
RustyError::Replay(format!("twin: {}", message.into()))
}
pub const TWIN_REPORT_BOUND: &str = "the twin evaluates only decisions that change when/whether \
effects execute — retry counts, backoff delays, timeout bounds, placement among equivalent \
workers, concurrency caps, checkpoint cadence — never what effects return; a decision that \
would change an effect's input is unevaluable and was excluded or refused";
pub const TWIN_FORK_POLICY_VERSION: &str = "twin-fork";
pub use crate::record::MIN_TIMEOUT_RUNG_MS;
pub const DEFAULT_TIMEOUT_LADDER: [u64; 7] =
[100, 500, 1_000, 5_000, 30_000, 120_000, MAX_RETRY_DELAY_MS];
pub const DEFAULT_CONCURRENCY_LADDER: [u32; 6] = [1, 2, 4, 8, 16, u32::MAX];
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
#[serde(tag = "fault", rename_all = "snake_case")]
pub enum InjectedFault {
WorkerCrash,
CalleeTimeout,
RateLimited {
retry_after_ms: u64,
},
ResourceExhausted,
}
impl InjectedFault {
pub fn error_class(&self) -> ErrorClass {
match self {
InjectedFault::WorkerCrash => ErrorClass::Unknown,
InjectedFault::CalleeTimeout => ErrorClass::Timeout,
InjectedFault::RateLimited { .. } => ErrorClass::RateLimited,
InjectedFault::ResourceExhausted => ErrorClass::ResourceExhausted,
}
}
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
#[serde(tag = "anchor", rename_all = "snake_case")]
pub enum FaultAnchor {
OnAttempt {
effect_seq: u64,
attempt: u32,
},
AtDecision {
decision_seq: u64,
},
Window {
from_seq: u64,
to_seq: u64,
},
OnWorker {
worker: String,
},
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct FaultInjection {
#[serde(flatten)]
pub anchor: FaultAnchor,
#[serde(flatten)]
pub fault: InjectedFault,
}
#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
pub struct FaultSchedule {
pub seed: u64,
#[serde(default, skip_serializing_if = "Vec::is_empty")]
pub injections: Vec<FaultInjection>,
}
impl FaultSchedule {
pub fn new(seed: u64) -> Self {
Self {
seed,
injections: Vec::new(),
}
}
pub fn with_injection(mut self, anchor: FaultAnchor, fault: InjectedFault) -> Self {
self.injections.push(FaultInjection { anchor, fault });
self
}
pub fn fault_for_attempt(
&self,
effect_seq: u64,
attempt: u32,
worker: &str,
) -> Option<&InjectedFault> {
self.injections
.iter()
.find(|injection| match &injection.anchor {
FaultAnchor::OnAttempt {
effect_seq: seq,
attempt: ord,
} => *seq == effect_seq && *ord == attempt,
FaultAnchor::Window { from_seq, to_seq } => {
*from_seq <= effect_seq && effect_seq <= *to_seq
}
FaultAnchor::OnWorker { worker: w } => w == worker,
FaultAnchor::AtDecision { .. } => false,
})
.map(|injection| &injection.fault)
}
pub fn fault_for_decision(&self, decision_seq: u64) -> Option<&InjectedFault> {
self.injections
.iter()
.find(|injection| match &injection.anchor {
FaultAnchor::AtDecision { decision_seq: seq } => *seq == decision_seq,
_ => false,
})
.map(|injection| &injection.fault)
}
}
#[derive(Debug)]
pub struct DecisionContext<'a> {
pub family: DecisionFamily,
pub features: &'a Map<String, Value>,
pub legal_actions: &'a [DecisionAction],
}
pub trait TwinPolicy: fmt::Debug + Send + Sync {
fn version(&self) -> PolicyVersion;
fn decide(&self, ctx: &DecisionContext<'_>, draw: f64) -> (DecisionAction, f64);
fn retry_delay_ms(&self, attempt: u32, draw: f64) -> u64 {
backoff_delay_ms(attempt, draw)
}
fn retry_delay_ms_for(&self, class: Option<ErrorClass>, attempt: u32, draw: f64) -> u64 {
let _ = class;
self.retry_delay_ms(attempt, draw)
}
}
#[derive(Debug, Default, Clone, Copy)]
pub struct StaticFloor;
impl TwinPolicy for StaticFloor {
fn version(&self) -> PolicyVersion {
PolicyVersion::new(PolicyVersion::STATIC_V0)
}
fn decide(&self, ctx: &DecisionContext<'_>, _draw: f64) -> (DecisionAction, f64) {
let selected = match ctx.family {
DecisionFamily::Retry => ctx
.legal_actions
.iter()
.find(|action| matches!(action, DecisionAction::Retry { .. }))
.cloned()
.unwrap_or(DecisionAction::Abort),
DecisionFamily::Timeout
| DecisionFamily::Concurrency
| DecisionFamily::CheckpointPlacement => ctx
.legal_actions
.last()
.cloned()
.unwrap_or(DecisionAction::Abort),
DecisionFamily::WorkerPlacement => ctx
.legal_actions
.first()
.cloned()
.unwrap_or(DecisionAction::Abort),
};
(selected, 1.0)
}
}
#[derive(Debug, Clone)]
pub struct ParameterizedPolicy {
policy: ExecutorPolicy,
version: PolicyVersion,
}
impl ParameterizedPolicy {
pub fn new(policy: ExecutorPolicy, version: PolicyVersion) -> Result<Self> {
policy.validate().map_err(|e| {
twin_error(format!(
"refusing to evaluate policy `{version}`: its parameters fall outside the \
envelope ({e}) — an invalid body cannot act, in evaluation or in production"
))
})?;
Ok(Self { policy, version })
}
pub fn policy(&self) -> &ExecutorPolicy {
&self.policy
}
fn failure_class(ctx: &DecisionContext<'_>) -> ErrorClass {
ctx.features
.get("failure_class")
.and_then(|value| serde_json::from_value(value.clone()).ok())
.unwrap_or(ErrorClass::Unknown)
}
fn u64_feature(ctx: &DecisionContext<'_>, key: &str) -> Option<u64> {
ctx.features.get(key).and_then(serde_json::Value::as_u64)
}
}
impl TwinPolicy for ParameterizedPolicy {
fn version(&self) -> PolicyVersion {
self.version.clone()
}
fn decide(&self, ctx: &DecisionContext<'_>, _draw: f64) -> (DecisionAction, f64) {
let selected = match ctx.family {
DecisionFamily::Retry => {
let class = Self::failure_class(ctx);
let attempt = Self::u64_feature(ctx, "attempt").unwrap_or(1) as u32;
let budget = Self::u64_feature(ctx, "max_attempts").unwrap_or(3) as u32;
let resolved = resolve_retry_parameters(&self.policy, class, budget);
if attempt < resolved.max_attempts {
ctx.legal_actions
.iter()
.find(|action| matches!(action, DecisionAction::Retry { .. }))
.cloned()
.unwrap_or(DecisionAction::Abort)
} else {
DecisionAction::Abort
}
}
DecisionFamily::Timeout => {
let callee = ctx
.features
.get("callee")
.or_else(|| ctx.features.get("node"))
.and_then(serde_json::Value::as_str);
let bound = resolve_timeout_bound_ms(&self.policy, callee);
match bound {
Some(bound) => ctx
.legal_actions
.iter()
.find(|action| matches!(action, DecisionAction::SetTimeout { millis } if *millis >= bound))
.cloned()
.or_else(|| ctx.legal_actions.last().cloned())
.unwrap_or(DecisionAction::Abort),
None => ctx.legal_actions.last().cloned().unwrap_or(DecisionAction::Abort),
}
}
DecisionFamily::Concurrency | DecisionFamily::CheckpointPlacement => ctx
.legal_actions
.last()
.cloned()
.unwrap_or(DecisionAction::Abort),
DecisionFamily::WorkerPlacement => ctx
.legal_actions
.first()
.cloned()
.unwrap_or(DecisionAction::Abort),
};
(selected, 1.0)
}
fn retry_delay_ms_for(&self, class: Option<ErrorClass>, attempt: u32, draw: f64) -> u64 {
let retry = &self.policy.retry;
if retry.validate().is_err() {
return backoff_delay_ms(attempt, draw);
}
let schedule = class
.and_then(|class| retry.per_class.as_ref()?.get(&class).copied())
.unwrap_or_else(|| retry.flat());
backoff_delay_ms_with(attempt, draw, schedule.base_delay_ms, schedule.max_delay_ms)
}
}
#[derive(Debug, Clone)]
pub struct RecordedAnswer {
pub status: EventStatus,
pub output: Option<Value>,
pub latency_ms: u64,
pub tokens: Option<Usage>,
pub cost_usd: Option<f64>,
}
#[derive(Debug, Clone)]
pub struct TwinWorkItem {
pub recorded_seq: u64,
pub kind: RunEventKind,
pub node_id: Option<String>,
pub effect: Effect,
pub request: Value,
pub answer: RecordedAnswer,
pub step: u64,
}
#[derive(Debug, Clone)]
pub struct TwinWorld {
items: Vec<TwinWorkItem>,
steps: Vec<u64>,
}
impl TwinWorld {
pub fn from_snapshot(snapshot: &JournalSnapshot) -> Result<Self> {
let resolve = |payload: &PayloadRef| -> Option<Value> {
match payload {
PayloadRef::Inline(value) => Some(value.clone()),
PayloadRef::Artifact(reference) => {
snapshot.artifacts.get(&reference.sha256).cloned()
}
}
};
let mut items = Vec::new();
let mut steps = Vec::new();
let mut current_step = 0u64;
for event in &snapshot.events {
if event.kind == RunEventKind::SuperStepStart {
current_step = event
.input
.as_ref()
.and_then(&resolve)
.and_then(|input| input.get("step").and_then(Value::as_u64))
.unwrap_or(current_step + 1);
steps.push(current_step);
continue;
}
if !SERVABLE_KINDS.contains(&event.kind) {
continue;
}
if event.status == EventStatus::Interrupted {
return Err(twin_error(format!(
"recorded {:?} at seq {} has status `interrupted`, which servable calls \
never produce — the journal is inconsistent",
event.kind, event.seq
)));
}
let request = event.input.as_ref().and_then(&resolve).ok_or_else(|| {
twin_error(format!(
"recorded {:?} at seq {} has no input payload; the twin needs the request \
the world answered",
event.kind, event.seq
))
})?;
let answer = RecordedAnswer {
status: event.status,
output: event.output.as_ref().and_then(&resolve),
latency_ms: event.latency_ms.unwrap_or(0),
tokens: event.tokens,
cost_usd: event.cost_usd,
};
items.push(TwinWorkItem {
recorded_seq: event.seq,
kind: event.kind,
node_id: event.node_id.clone(),
effect: event.effect,
request,
answer,
step: current_step,
});
}
if steps.is_empty() && !items.is_empty() {
steps.push(0);
}
Ok(Self { items, steps })
}
pub fn items(&self) -> &[TwinWorkItem] {
&self.items
}
pub fn steps(&self) -> &[u64] {
&self.steps
}
pub fn items_in(&self, step: u64) -> Vec<&TwinWorkItem> {
self.items.iter().filter(|item| item.step == step).collect()
}
}
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum Interleaving {
#[default]
Canonical,
Seeded,
}
pub struct TwinRunConfig {
pub seed: u64,
pub clock: LogicalClockParams,
pub faults: FaultSchedule,
pub acting: Arc<dyn TwinPolicy>,
pub shadow: Option<Arc<dyn TwinPolicy>>,
pub timeout_ladder: Vec<u64>,
pub concurrency_ladder: Vec<u32>,
pub workers: Vec<String>,
pub max_attempts: u32,
pub interleaving: Interleaving,
}
impl TwinRunConfig {
pub fn new(seed: u64) -> Self {
Self {
seed,
clock: LogicalClockParams {
start_ms: 1_700_000_000_000,
tick_ms: 10,
},
faults: FaultSchedule::new(seed),
acting: Arc::new(StaticFloor),
shadow: None,
timeout_ladder: DEFAULT_TIMEOUT_LADDER.to_vec(),
concurrency_ladder: DEFAULT_CONCURRENCY_LADDER.to_vec(),
workers: vec!["worker-0".to_owned()],
max_attempts: 3,
interleaving: Interleaving::Canonical,
}
}
pub fn with_faults(mut self, faults: FaultSchedule) -> Self {
self.faults = faults;
self
}
pub fn with_acting(mut self, policy: Arc<dyn TwinPolicy>) -> Self {
self.acting = policy;
self
}
pub fn with_shadow(mut self, candidate: Arc<dyn TwinPolicy>) -> Self {
self.shadow = Some(candidate);
self
}
pub fn with_workers(mut self, workers: impl IntoIterator<Item = impl Into<String>>) -> Self {
self.workers = workers.into_iter().map(Into::into).collect();
self
}
pub fn with_max_attempts(mut self, max_attempts: u32) -> Self {
self.max_attempts = max_attempts;
self
}
pub fn with_seeded_interleaving(mut self) -> Self {
self.interleaving = Interleaving::Seeded;
self
}
}
impl fmt::Debug for TwinRunConfig {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
f.debug_struct("TwinRunConfig")
.field("seed", &self.seed)
.field("clock", &self.clock)
.field("faults", &self.faults)
.field("acting", &self.acting.version())
.field("shadow", &self.shadow.as_ref().map(|p| p.version()))
.field("workers", &self.workers)
.field("max_attempts", &self.max_attempts)
.field("interleaving", &self.interleaving)
.finish()
}
}
#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)]
pub struct TwinMetrics {
pub items: usize,
pub completed: usize,
pub dead_lettered: usize,
pub failed: usize,
pub attempts: u64,
pub cost_usd: f64,
pub wall_time_ms: u64,
pub item_latency_p50_ms: u64,
pub item_latency_p95_ms: u64,
}
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
#[serde(tag = "case", rename_all = "snake_case")]
pub enum UnevaluableCase {
GatedEffect {
effect_seq: u64,
},
IllegalForkAction {
decision_seq: u64,
action: DecisionAction,
},
UnknownDecision {
decision_seq: u64,
},
}
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
pub struct TwinReport {
pub run_id: String,
pub seed: u64,
pub faults_declared: usize,
pub faults_fired: usize,
pub decisions: usize,
pub evaluable_decisions: usize,
pub shadow_divergences: Vec<u64>,
pub excluded: Vec<UnevaluableCase>,
pub bound: String,
}
#[derive(Debug)]
pub struct TwinOutcome {
pub journal: JournalSnapshot,
pub decisions: Vec<DecisionEvent>,
pub metrics: TwinMetrics,
pub report: TwinReport,
}
#[derive(Debug)]
pub struct CounterfactualFork {
pub decision_seq: u64,
pub action: DecisionAction,
pub then_act_with: Option<Arc<dyn TwinPolicy>>,
}
#[derive(Debug)]
pub struct CounterfactualBranch {
pub outcome: TwinOutcome,
pub baseline: TwinOutcome,
pub diff: BranchDiff,
}
#[derive(Debug)]
pub struct Twin {
world: TwinWorld,
run_id: String,
thread_id: String,
}
impl Twin {
pub fn from_snapshot(snapshot: JournalSnapshot) -> Result<Self> {
ExactReplay::new(snapshot.clone())?;
let world = TwinWorld::from_snapshot(&snapshot)?;
Ok(Self {
world,
run_id: snapshot.run_id,
thread_id: snapshot.thread_id,
})
}
pub fn from_fixture(fixture: &ReplayFixture) -> Result<Self> {
Self::from_snapshot(fixture.journal.clone())
}
pub fn world(&self) -> &TwinWorld {
&self.world
}
fn twin_run_id(&self, seed: u64) -> String {
format!("twin:{}:{seed:016x}", self.run_id)
}
pub fn run(&self, config: &TwinRunConfig) -> Result<TwinOutcome> {
self.run_inner(config, None)
}
pub fn run_interleavings(
&self,
runs: usize,
config: &TwinRunConfig,
) -> Result<Vec<TwinOutcome>> {
(0..runs)
.map(|k| {
let run_config = TwinRunConfig {
seed: config.seed ^ (k as u64).wrapping_mul(0x9E37_79B9_7F4A_7C15),
clock: config.clock,
faults: config.faults.clone(),
acting: config.acting.clone(),
shadow: config.shadow.clone(),
timeout_ladder: config.timeout_ladder.clone(),
concurrency_ladder: config.concurrency_ladder.clone(),
workers: config.workers.clone(),
max_attempts: config.max_attempts,
interleaving: Interleaving::Seeded,
};
self.run(&run_config)
})
.collect()
}
pub fn counterfactual(
&self,
baseline: &TwinRunConfig,
fork: CounterfactualFork,
) -> Result<CounterfactualBranch> {
let base_outcome = self.run(baseline)?;
let branch_outcome = self.run_inner(baseline, Some(&fork))?;
let diff = BranchDiff::between(&base_outcome.journal, &branch_outcome.journal);
Ok(CounterfactualBranch {
outcome: branch_outcome,
baseline: base_outcome,
diff,
})
}
fn run_inner(
&self,
config: &TwinRunConfig,
fork: Option<&CounterfactualFork>,
) -> Result<TwinOutcome> {
if config.workers.is_empty() {
return Err(twin_error(
"the worker pool is empty; placement needs at least one equivalent worker",
));
}
if config
.timeout_ladder
.iter()
.any(|rung| *rung < MIN_TIMEOUT_RUNG_MS)
{
return Err(twin_error(format!(
"timeout ladder carries a rung below the {MIN_TIMEOUT_RUNG_MS} ms minimum — \
below it ordinary work aborts early, a correctness hazard no policy may cross"
)));
}
let run_id = self.twin_run_id(config.seed);
let mut state = RunState {
journal: Journal::new(
run_id.clone(),
self.thread_id.clone(),
Clock::logical(config.clock.start_ms, config.clock.tick_ms),
),
rng: RngSource::seeded(config.seed),
run_id,
decisions: Vec::new(),
pending: Vec::new(),
next_decision_seq: 0,
faults_fired: 0,
divergences: Vec::new(),
excluded: Vec::new(),
forked: false,
};
let mut acting = config.acting.clone();
let mut lane_free = vec![0u64; config.workers.len()];
let mut item_elapsed: Vec<u64> = Vec::new();
let mut metrics = TwinMetrics {
items: self.world.items.len(),
..TwinMetrics::default()
};
for &step in self.world.steps() {
let items = self.world.items_in(step);
if items.is_empty() {
continue;
}
let step_start_time = lane_free.iter().max().copied().unwrap_or(0);
let step_event = state.journal.record(
EventDraft::new(RunEventKind::SuperStepStart, Effect::Pure).input(
serde_json::json!({
"step": step,
"active_nodes": items
.iter()
.map(|item| item.node_id.clone().unwrap_or_else(|| format!("seq:{}", item.recorded_seq)))
.collect::<Vec<_>>(),
}),
),
);
let mut features = Map::new();
features.insert("step".to_owned(), Value::from(step));
features.insert("queue_depth".to_owned(), Value::from(items.len()));
features.insert("workers".to_owned(), Value::from(config.workers.len()));
let legal: Vec<DecisionAction> = config
.concurrency_ladder
.iter()
.map(|limit| DecisionAction::SetConcurrency { limit: *limit })
.collect();
let action = decide(
&mut state,
fork,
&mut acting,
config,
DecisionFamily::Concurrency,
features,
legal,
&step_event,
)?;
let admitted = match action {
DecisionAction::SetConcurrency { limit } => {
(limit as usize).min(config.workers.len()).max(1)
}
_ => 1,
};
flush_pending(&mut state);
let mut order: Vec<usize> = (0..items.len()).collect();
if config.interleaving == Interleaving::Seeded {
for i in (1..order.len()).rev() {
let j = (state.rng.next_f64() * (i + 1) as f64) as usize;
order.swap(i, j.min(i));
}
}
let mut step_outcomes: BTreeMap<String, &str> = BTreeMap::new();
let mut buffers: Vec<Vec<EventDraft>> = vec![Vec::new(); items.len()];
for &position in &order {
let item = items[position];
let key = format!(
"{}#{}",
item.node_id.as_deref().unwrap_or("seq"),
item.recorded_seq
);
let outcome = run_item(
&mut state,
fork,
&mut acting,
config,
item,
admitted,
&mut lane_free,
&step_event,
step_start_time,
)?;
buffers[position] = std::mem::take(&mut state.pending);
metrics.attempts += outcome.attempts;
metrics.cost_usd += outcome.cost_usd;
item_elapsed.push(outcome.elapsed_ms);
let disposition = match outcome.terminal() {
Terminal::Completed => {
metrics.completed += 1;
"completed"
}
Terminal::DeadLettered => {
metrics.dead_lettered += 1;
"dead_lettered"
}
Terminal::Failed => {
metrics.failed += 1;
"failed"
}
};
step_outcomes.insert(key, disposition);
}
for buffer in &mut buffers {
for draft in buffer.drain(..) {
state.journal.record(draft);
}
}
state.journal.record(
EventDraft::new(RunEventKind::SuperStepEnd, Effect::Pure)
.output(serde_json::to_value(&step_outcomes)?)
.parent(step_event),
);
}
metrics.wall_time_ms = lane_free.iter().max().copied().unwrap_or(0);
metrics.item_latency_p50_ms = percentile(&item_elapsed, 50.0);
metrics.item_latency_p95_ms = percentile(&item_elapsed, 95.0);
if let Some(fork) = fork {
if !state.forked {
let case = UnevaluableCase::UnknownDecision {
decision_seq: fork.decision_seq,
};
return Err(twin_error(format!(
"counterfactual refused: the run never reaches decision d{} — {}",
fork.decision_seq,
serde_json::to_string(&case).unwrap_or_else(|_| format!("{case:?}"))
)));
}
}
let journal = state.journal.snapshot();
let report = TwinReport {
run_id: state.run_id.clone(),
seed: config.seed,
faults_declared: config.faults.injections.len(),
faults_fired: state.faults_fired,
decisions: state.next_decision_seq as usize,
evaluable_decisions: state.next_decision_seq as usize,
shadow_divergences: state.divergences,
excluded: state.excluded,
bound: TWIN_REPORT_BOUND.to_owned(),
};
Ok(TwinOutcome {
journal,
decisions: state.decisions,
metrics,
report,
})
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
enum Terminal {
Completed,
DeadLettered,
Failed,
}
#[derive(Debug, Default)]
struct ItemOutcome {
terminal: Option<Terminal>,
attempts: u64,
elapsed_ms: u64,
cost_usd: f64,
}
impl ItemOutcome {
fn terminal(&self) -> Terminal {
self.terminal.expect("run_item always settles its item")
}
}
struct RunState {
journal: Journal,
rng: RngSource,
run_id: String,
decisions: Vec<DecisionEvent>,
pending: Vec<EventDraft>,
next_decision_seq: u64,
faults_fired: usize,
divergences: Vec<u64>,
excluded: Vec<UnevaluableCase>,
forked: bool,
}
fn flush_pending(state: &mut RunState) {
for draft in std::mem::take(&mut state.pending) {
state.journal.record(draft);
}
}
#[allow(clippy::too_many_arguments)]
fn decide(
state: &mut RunState,
fork: Option<&CounterfactualFork>,
acting: &mut Arc<dyn TwinPolicy>,
config: &TwinRunConfig,
family: DecisionFamily,
features: Map<String, Value>,
legal_actions: Vec<DecisionAction>,
parent: &str,
) -> Result<DecisionAction> {
let seq = state.next_decision_seq;
let ctx = DecisionContext {
family,
features: &features,
legal_actions: &legal_actions,
};
let at_fork = fork.is_some_and(|fork| fork.decision_seq == seq);
let (selected, propensity, version) = if at_fork {
let fork = fork.expect("checked above");
if !legal_actions.contains(&fork.action) {
let case = UnevaluableCase::IllegalForkAction {
decision_seq: seq,
action: fork.action.clone(),
};
return Err(twin_error(format!(
"counterfactual refused: {:?} is not in the legal set at decision d{seq} \
({legal_actions:?}) — {}",
fork.action,
serde_json::to_string(&case).unwrap_or_else(|_| format!("{case:?}"))
)));
}
state.forked = true;
(
fork.action.clone(),
1.0,
PolicyVersion::new(TWIN_FORK_POLICY_VERSION),
)
} else {
let (selected, propensity) = acting.decide(&ctx, state.rng.next_f64());
validate_decision(&selected, propensity, &legal_actions, &acting.version())?;
(selected, propensity, acting.version())
};
let decided_at = state.journal.clock().now();
let acting_event = DecisionEvent {
id: format!("{}:d{seq}", state.run_id),
run_id: state.run_id.clone(),
thread_id: state.journal.thread_id().to_owned(),
seq,
family,
features: features.clone(),
legal_actions: legal_actions.clone(),
selected: selected.clone(),
propensity,
policy_version: version,
role: Some(DecisionRole::Acting),
outcome: None,
decided_at,
};
journal_decision(state, &acting_event, parent);
if let Some(shadow) = &config.shadow {
let (shadow_selected, shadow_propensity) = shadow.decide(&ctx, state.rng.next_f64());
validate_decision(
&shadow_selected,
shadow_propensity,
&legal_actions,
&shadow.version(),
)?;
let shadow_event = DecisionEvent {
id: format!("{}:d{seq}:shadow", state.run_id),
selected: shadow_selected.clone(),
propensity: shadow_propensity,
policy_version: shadow.version(),
role: Some(DecisionRole::Shadow),
decided_at: state.journal.clock().now(),
..acting_event.clone()
};
journal_decision(state, &shadow_event, parent);
if shadow_selected != selected {
state.divergences.push(seq);
}
}
state.next_decision_seq += 1;
if at_fork {
if let Some(next) = fork.and_then(|fork| fork.then_act_with.as_ref()) {
*acting = next.clone();
}
}
Ok(selected)
}
fn validate_decision(
selected: &DecisionAction,
propensity: f64,
legal_actions: &[DecisionAction],
version: &PolicyVersion,
) -> Result<()> {
if !legal_actions.contains(selected) {
return Err(twin_error(format!(
"policy `{}` selected {selected:?}, which is outside the legal set \
{legal_actions:?} — a policy chooses among declared members, never free-form \
outputs",
version.as_str()
)));
}
if !(0.0 < propensity && propensity <= 1.0) {
return Err(twin_error(format!(
"policy `{}` declared propensity {propensity}, outside (0, 1] — propensity is \
assigned at decision time and must be truthful, or off-policy evaluation over \
this evidence is meaningless",
version.as_str()
)));
}
Ok(())
}
fn journal_decision(state: &mut RunState, event: &DecisionEvent, parent: &str) {
state.pending.push(
EventDraft::new(RunEventKind::PolicyDecision, Effect::Pure)
.output(
serde_json::to_value(event)
.unwrap_or_else(|_| serde_json::json!({"id": event.id, "error": "decision payload failed to serialize"})),
)
.parent(parent),
);
state.decisions.push(event.clone());
}
struct AttemptObservation {
status: EventStatus,
class: Option<ErrorClass>,
retry_after_ms: Option<u64>,
latency_ms: u64,
output: Option<Value>,
tokens: Option<Usage>,
cost_usd: Option<f64>,
}
#[allow(clippy::too_many_arguments)]
fn run_item(
state: &mut RunState,
fork: Option<&CounterfactualFork>,
acting: &mut Arc<dyn TwinPolicy>,
config: &TwinRunConfig,
item: &TwinWorkItem,
admitted: usize,
lane_free: &mut [u64],
step_event: &str,
step_start_time: u64,
) -> Result<ItemOutcome> {
let mut outcome = ItemOutcome::default();
let item_ready = step_start_time;
let mut prev_attempt_end = item_ready;
let mut attempt = 0u32;
loop {
attempt += 1;
outcome.attempts += 1;
let mut features = Map::new();
features.insert("attempt".to_owned(), Value::from(attempt));
features.insert("effect".to_owned(), serde_json::to_value(item.effect)?);
features.insert(
"lane_free_ms".to_owned(),
serde_json::to_value(&*lane_free)?,
);
if let Some(node) = &item.node_id {
features.insert("node".to_owned(), Value::from(node.clone()));
}
let legal: Vec<DecisionAction> = config
.workers
.iter()
.map(|worker| DecisionAction::SelectWorker {
worker: worker.clone(),
})
.collect();
let action = decide(
state,
fork,
acting,
config,
DecisionFamily::WorkerPlacement,
features,
legal,
step_event,
)?;
let worker = match &action {
DecisionAction::SelectWorker { worker } => worker.clone(),
_ => config.workers[0].clone(),
};
let lane = config
.workers
.iter()
.position(|w| w == &worker)
.unwrap_or(0)
.min(admitted.saturating_sub(1))
.min(lane_free.len() - 1);
let mut features = Map::new();
features.insert(
"recorded_latency_ms".to_owned(),
Value::from(item.answer.latency_ms),
);
features.insert("attempt".to_owned(), Value::from(attempt));
features.insert("effect".to_owned(), serde_json::to_value(item.effect)?);
if let Some(node) = &item.node_id {
features.insert("node".to_owned(), Value::from(node.clone()));
}
let legal: Vec<DecisionAction> = config
.timeout_ladder
.iter()
.map(|millis| DecisionAction::SetTimeout { millis: *millis })
.collect();
let action = decide(
state,
fork,
acting,
config,
DecisionFamily::Timeout,
features,
legal,
step_event,
)?;
let bound = match action {
DecisionAction::SetTimeout { millis } => millis,
_ => MAX_RETRY_DELAY_MS,
};
let fault = config
.faults
.fault_for_attempt(item.recorded_seq, attempt, &worker)
.cloned();
let mut observation = match &fault {
Some(fault) => {
state.faults_fired += 1;
faulted_observation(fault, bound, item)
}
None => recorded_observation(item, bound),
};
if observation.status == EventStatus::Error {
if let Some(at_decision) = config
.faults
.fault_for_decision(state.next_decision_seq)
.cloned()
{
state.faults_fired += 1;
observation = faulted_observation(&at_decision, bound, item);
}
}
let ready = if attempt == 1 {
item_ready
} else {
prev_attempt_end
};
let start = ready.max(lane_free[lane]);
let wait = start.saturating_sub(ready);
let end = start + observation.latency_ms;
lane_free[lane] = end;
let journaled_latency = wait + observation.latency_ms;
let mut draft = EventDraft::new(item.kind, item.effect)
.input(item.request.clone())
.latency_ms(journaled_latency)
.status(observation.status)
.parent(step_event);
if let Some(node) = &item.node_id {
draft = draft.node(node.clone());
}
match observation.status {
EventStatus::Ok => {
if let Some(output) = &observation.output {
draft = draft.output(output.clone());
}
if let Some(tokens) = observation.tokens {
draft = draft.tokens(tokens);
}
}
EventStatus::Error => {
let class = observation.class.unwrap_or(ErrorClass::Unknown);
draft = draft.output(serde_json::json!({
"error": serde_json::to_value(class)?,
}));
}
EventStatus::Interrupted => unreachable!("servable calls never interrupt"),
}
if let Some(cost) = observation.cost_usd {
draft = draft.cost_usd(cost);
outcome.cost_usd += cost;
}
state.pending.push(draft);
prev_attempt_end = end;
if observation.status == EventStatus::Ok {
outcome.terminal = Some(Terminal::Completed);
outcome.elapsed_ms = end.saturating_sub(item_ready);
return Ok(outcome);
}
let class = observation.class.unwrap_or(ErrorClass::Unknown);
if fault.is_some() && !item.effect.is_freely_repeatable() {
let flag = UnevaluableCase::GatedEffect {
effect_seq: item.recorded_seq,
};
if !state.excluded.contains(&flag) {
state.excluded.push(flag);
}
}
let mut features = Map::new();
features.insert("failure_class".to_owned(), serde_json::to_value(class)?);
features.insert("attempt".to_owned(), Value::from(attempt));
features.insert("max_attempts".to_owned(), Value::from(config.max_attempts));
features.insert("effect".to_owned(), serde_json::to_value(item.effect)?);
features.insert(
"dependency_latency_ms".to_owned(),
Value::from(journaled_latency),
);
if let Some(retry_after) = observation.retry_after_ms {
features.insert("retry_after_ms".to_owned(), Value::from(retry_after));
}
let legal = retry_legal_actions(item.effect, class, attempt, config.max_attempts);
let budget_exhausted = item.effect.is_freely_repeatable()
&& class.is_retryable()
&& attempt >= config.max_attempts;
let action = decide(
state,
fork,
acting,
config,
DecisionFamily::Retry,
features,
legal,
step_event,
)?;
match action {
DecisionAction::Retry { .. } => {
let delay = acting
.retry_delay_ms_for(Some(class), attempt, state.rng.next_f64())
.max(observation.retry_after_ms.unwrap_or(0));
lane_free[lane] = lane_free[lane].max(prev_attempt_end + delay);
prev_attempt_end += delay;
}
_ => {
outcome.terminal = Some(if budget_exhausted {
Terminal::DeadLettered
} else {
Terminal::Failed
});
outcome.elapsed_ms = end.saturating_sub(item_ready);
return Ok(outcome);
}
}
}
}
fn recorded_observation(item: &TwinWorkItem, bound: u64) -> AttemptObservation {
match item.answer.status {
EventStatus::Ok if item.answer.latency_ms > bound => AttemptObservation {
status: EventStatus::Error,
class: Some(ErrorClass::Timeout),
retry_after_ms: None,
latency_ms: bound,
output: None,
tokens: None,
cost_usd: item.answer.cost_usd,
},
_ => AttemptObservation {
status: item.answer.status,
class: (item.answer.status == EventStatus::Error).then_some(ErrorClass::Unknown),
retry_after_ms: None,
latency_ms: item.answer.latency_ms,
output: item.answer.output.clone(),
tokens: item.answer.tokens,
cost_usd: item.answer.cost_usd,
},
}
}
fn faulted_observation(
fault: &InjectedFault,
bound: u64,
item: &TwinWorkItem,
) -> AttemptObservation {
let (latency_ms, retry_after_ms) = match fault {
InjectedFault::WorkerCrash => (MAX_RETRY_DELAY_MS, None),
InjectedFault::CalleeTimeout => (bound.min(MAX_RETRY_DELAY_MS), None),
InjectedFault::RateLimited { retry_after_ms } => {
(item.answer.latency_ms, Some(*retry_after_ms))
}
InjectedFault::ResourceExhausted => (item.answer.latency_ms, None),
};
AttemptObservation {
status: EventStatus::Error,
class: Some(fault.error_class()),
retry_after_ms,
latency_ms,
output: None,
tokens: None,
cost_usd: None,
}
}
pub(crate) fn percentile(samples: &[u64], p: f64) -> u64 {
if samples.is_empty() {
return 0;
}
let mut sorted = samples.to_vec();
sorted.sort_unstable();
let rank = ((p / 100.0) * sorted.len() as f64).ceil() as usize;
sorted[rank.max(1).min(sorted.len()) - 1]
}
#[cfg(test)]
mod tests {
use super::*;
use serde_json::json;
fn ctx<'a>(
family: DecisionFamily,
features: &'a Map<String, Value>,
legal: &'a [DecisionAction],
) -> DecisionContext<'a> {
DecisionContext {
family,
features,
legal_actions: legal,
}
}
#[test]
fn fault_schedule_matches_in_declaration_order() {
let schedule = FaultSchedule::new(7)
.with_injection(
FaultAnchor::Window {
from_seq: 0,
to_seq: 10,
},
InjectedFault::ResourceExhausted,
)
.with_injection(
FaultAnchor::OnAttempt {
effect_seq: 3,
attempt: 1,
},
InjectedFault::RateLimited {
retry_after_ms: 500,
},
)
.with_injection(
FaultAnchor::OnWorker {
worker: "w1".to_owned(),
},
InjectedFault::WorkerCrash,
);
assert_eq!(
schedule.fault_for_attempt(3, 1, "w0"),
Some(&InjectedFault::ResourceExhausted)
);
assert_eq!(
schedule.fault_for_attempt(11, 1, "w1"),
Some(&InjectedFault::WorkerCrash)
);
assert_eq!(schedule.fault_for_attempt(11, 1, "w0"), None);
assert_eq!(schedule.fault_for_decision(0), None);
let schedule = schedule.with_injection(
FaultAnchor::AtDecision { decision_seq: 2 },
InjectedFault::WorkerCrash,
);
assert_eq!(
schedule.fault_for_decision(2),
Some(&InjectedFault::WorkerCrash)
);
assert_eq!(schedule.fault_for_attempt(11, 2, "w0"), None);
}
#[test]
fn injected_faults_classify_into_the_retry_taxonomy() {
assert_eq!(
InjectedFault::WorkerCrash.error_class(),
ErrorClass::Unknown
);
assert_eq!(
InjectedFault::CalleeTimeout.error_class(),
ErrorClass::Timeout
);
assert_eq!(
InjectedFault::RateLimited { retry_after_ms: 50 }.error_class(),
ErrorClass::RateLimited
);
assert_eq!(
InjectedFault::ResourceExhausted.error_class(),
ErrorClass::ResourceExhausted
);
}
#[test]
fn floor_decides_the_static_v0_stance_per_family() {
let features = Map::new();
let floor = StaticFloor;
let legal = vec![DecisionAction::Retry { attempt: 2 }, DecisionAction::Abort];
let (action, propensity) =
floor.decide(&ctx(DecisionFamily::Retry, &features, &legal), 0.5);
assert_eq!(action, DecisionAction::Retry { attempt: 2 });
assert_eq!(propensity, 1.0);
let legal = vec![DecisionAction::Abort];
let (action, _) = floor.decide(&ctx(DecisionFamily::Retry, &features, &legal), 0.5);
assert_eq!(action, DecisionAction::Abort);
let legal: Vec<DecisionAction> = DEFAULT_TIMEOUT_LADDER
.iter()
.map(|millis| DecisionAction::SetTimeout { millis: *millis })
.collect();
let (action, _) = floor.decide(&ctx(DecisionFamily::Timeout, &features, &legal), 0.5);
assert_eq!(
action,
DecisionAction::SetTimeout {
millis: MAX_RETRY_DELAY_MS
}
);
let legal = vec![
DecisionAction::SetConcurrency { limit: 1 },
DecisionAction::SetConcurrency { limit: u32::MAX },
];
let (action, _) = floor.decide(&ctx(DecisionFamily::Concurrency, &features, &legal), 0.5);
assert_eq!(action, DecisionAction::SetConcurrency { limit: u32::MAX });
let legal = vec![
DecisionAction::SelectWorker {
worker: "w0".to_owned(),
},
DecisionAction::SelectWorker {
worker: "w1".to_owned(),
},
];
let (action, _) = floor.decide(
&ctx(DecisionFamily::WorkerPlacement, &features, &legal),
0.5,
);
assert_eq!(
action,
DecisionAction::SelectWorker {
worker: "w0".to_owned()
}
);
}
#[test]
fn validate_decision_rejects_illegal_actions_and_bad_propensities() {
let legal = vec![DecisionAction::Abort];
let version = PolicyVersion::new("policy-test");
assert!(validate_decision(&DecisionAction::Abort, 1.0, &legal, &version).is_ok());
let error = validate_decision(&DecisionAction::Retry { attempt: 2 }, 1.0, &legal, &version)
.unwrap_err();
assert!(error.to_string().contains("outside the legal set"));
for bad in [0.0, -0.5, 1.5] {
assert!(validate_decision(&DecisionAction::Abort, bad, &legal, &version).is_err());
}
}
#[test]
fn timeout_bound_truncates_the_observation_not_the_world() {
let item = TwinWorkItem {
recorded_seq: 0,
kind: RunEventKind::ToolCall,
node_id: Some("n".to_owned()),
effect: Effect::Idempotent,
request: json!({"tool": "t", "arguments": {}}),
answer: RecordedAnswer {
status: EventStatus::Ok,
output: Some(json!({"r": 1})),
latency_ms: 10_000,
tokens: None,
cost_usd: Some(0.01),
},
step: 0,
};
let observed = recorded_observation(&item, 5_000);
assert_eq!(observed.status, EventStatus::Error);
assert_eq!(observed.class, Some(ErrorClass::Timeout));
assert_eq!(observed.latency_ms, 5_000);
let observed = recorded_observation(&item, 30_000);
assert_eq!(observed.status, EventStatus::Ok);
assert_eq!(observed.latency_ms, 10_000);
assert_eq!(observed.output, Some(json!({"r": 1})));
}
#[test]
fn faulted_observations_follow_the_worlds_latency_rules() {
let item = TwinWorkItem {
recorded_seq: 0,
kind: RunEventKind::ToolCall,
node_id: None,
effect: Effect::Idempotent,
request: json!({}),
answer: RecordedAnswer {
status: EventStatus::Ok,
output: None,
latency_ms: 2_000,
tokens: None,
cost_usd: None,
},
step: 0,
};
let crash = faulted_observation(&InjectedFault::WorkerCrash, 5_000, &item);
assert_eq!(crash.latency_ms, MAX_RETRY_DELAY_MS);
let hang = faulted_observation(&InjectedFault::CalleeTimeout, 5_000, &item);
assert_eq!(hang.latency_ms, 5_000);
let hang = faulted_observation(&InjectedFault::CalleeTimeout, MAX_RETRY_DELAY_MS, &item);
assert_eq!(hang.latency_ms, MAX_RETRY_DELAY_MS);
let limited = faulted_observation(
&InjectedFault::RateLimited {
retry_after_ms: 800,
},
5_000,
&item,
);
assert_eq!(limited.retry_after_ms, Some(800));
assert_eq!(limited.latency_ms, 2_000);
}
#[test]
fn percentile_is_nearest_rank() {
assert_eq!(percentile(&[], 50.0), 0);
assert_eq!(percentile(&[10], 95.0), 10);
assert_eq!(percentile(&[1, 2, 3, 4], 50.0), 2);
assert_eq!(percentile(&[1, 2, 3, 4], 95.0), 4);
}
}