use std::collections::{BTreeMap, BTreeSet};
use serde::Serialize;
use crate::workflow_state::extractors::{ExtractorInput, extract_workflow_state};
use crate::workflow_state::fixture::WorkflowTraceFixture;
use crate::workflow_state::ir::{
CapabilityConstraints, EvidenceLevel, HarnessId, RequirementLevel, RouteProjection,
SessionConfidence, WorkflowStateKind,
};
use crate::workflow_state::predictive::{
NextActionClass, PredictiveRouteProjection, is_predictive_reason_code, predict_next_step,
};
#[derive(Debug, Default)]
pub struct ReplayEvaluator;
#[derive(Debug, Default, Serialize)]
pub struct ReplaySummary {
pub total: usize,
pub covered: usize,
pub coverage: f32,
pub baseline_bucket_count: usize,
pub ir_bucket_count: usize,
pub collision_count: usize,
pub visibility_gap_count: usize,
pub visibility_gaps_by_harness: BTreeMap<String, usize>,
pub session_confidence_distribution: BTreeMap<String, usize>,
pub baseline_midstream_count: usize,
pub ir_unknown_count: usize,
pub predictive_expectation_count: usize,
pub predictive_exact_count: usize,
pub records: Vec<ReplayRecord>,
pub model_ladder: ModelLadderConstraintSummary,
}
#[derive(Debug, Clone, Serialize)]
pub struct ReplayRecord {
pub fixture_id: String,
pub observed_projection: RouteProjection,
pub observed_route_key: String,
pub predictive_projection: PredictiveRouteProjection,
pub predictive_route_key: String,
pub next_action_class: NextActionClass,
pub predictor_contract_digest: String,
pub prediction_confidence_kind: String,
pub prediction_confidence_ppm: u32,
pub prediction_reason_codes: Vec<String>,
pub prediction_matches_expected: Option<bool>,
}
#[derive(Debug, Default, Serialize)]
pub struct ModelLadderConstraintSummary {
pub flagship: usize,
pub open_source_flagship: usize,
pub standard: usize,
pub cheap_tool_safe: usize,
pub cheap_fast: usize,
}
impl ReplayEvaluator {
pub fn run(&self, fixtures: &[WorkflowTraceFixture]) -> ReplaySummary {
let mut summary = ReplaySummary {
total: fixtures.len(),
..ReplaySummary::default()
};
let mut baseline_buckets = BTreeSet::new();
let mut ir_buckets = BTreeSet::new();
let mut labels_by_projection_key: BTreeMap<String, BTreeSet<String>> = BTreeMap::new();
for fixture in fixtures {
let baseline = fixture.baseline_fingerprint();
if baseline == "midstream" {
summary.baseline_midstream_count += 1;
}
baseline_buckets.insert(baseline);
let ir = extract_fixture_ir(fixture);
if ir.state_kind == WorkflowStateKind::Unknown {
summary.ir_unknown_count += 1;
}
if ir.state_kind != WorkflowStateKind::Unknown
&& ir.confidence >= fixture.expected.confidence_min
{
summary.covered += 1;
}
let observed_projection = ir.route_projection();
let observed_route_key = observed_projection.key();
ir_buckets.insert(observed_route_key.clone());
labels_by_projection_key
.entry(observed_route_key.clone())
.or_default()
.insert(fixture.expected.state_kind.to_string());
if ir
.evidence
.iter()
.any(|e| e.kind == "server_side_context_gap" && e.level == EvidenceLevel::Missing)
{
summary.visibility_gap_count += 1;
*summary
.visibility_gaps_by_harness
.entry(harness_key(&fixture.harness))
.or_insert(0) += 1;
}
*summary
.session_confidence_distribution
.entry(session_key(&ir.session.confidence))
.or_insert(0) += 1;
summary.model_ladder.observe(&ir.capability_constraints);
let prediction = predict_next_step(&ir, &fixture.prompt);
let predictive_projection = PredictiveRouteProjection::new(
prediction.task_family,
prediction.next_step_role,
prediction.route_risk,
);
let predictive_route_key = predictive_projection.key();
let prediction_reason_codes = prediction
.evidence
.iter()
.map(|evidence| evidence.code.as_str())
.filter(|code| is_predictive_reason_code(code))
.map(ToOwned::to_owned)
.collect::<BTreeSet<_>>()
.into_iter()
.collect::<Vec<_>>();
let prediction_matches_expected =
fixture.expected.prediction.as_ref().map(|expected| {
expected.next_step_role == prediction.next_step_role
&& expected.next_action_class == prediction.next_action_class
&& expected.route_risk == prediction.route_risk
&& expected
.task_family
.is_none_or(|family| family == prediction.task_family)
&& expected
.progress_state
.is_none_or(|progress| progress == prediction.progress_state)
});
if let Some(matches) = prediction_matches_expected {
summary.predictive_expectation_count += 1;
summary.predictive_exact_count += usize::from(matches);
}
summary.records.push(ReplayRecord {
fixture_id: fixture.id.clone(),
observed_projection,
observed_route_key,
predictive_projection,
predictive_route_key,
next_action_class: prediction.next_action_class,
predictor_contract_digest: prediction.predictor_contract_digest,
prediction_confidence_kind: prediction.confidence_kind,
prediction_confidence_ppm: confidence_ppm(prediction.confidence),
prediction_reason_codes,
prediction_matches_expected,
});
}
summary.coverage = if summary.total == 0 {
0.0
} else {
summary.covered as f32 / summary.total as f32
};
summary.baseline_bucket_count = baseline_buckets.len();
summary.ir_bucket_count = ir_buckets.len();
summary.collision_count = labels_by_projection_key
.values()
.filter(|labels| labels.len() > 1)
.count();
summary
}
}
fn confidence_ppm(confidence: f32) -> u32 {
if !confidence.is_finite() || confidence <= 0.0 {
0
} else if confidence >= 1.0 {
1_000_000
} else {
(confidence * 1_000_000.0).round() as u32
}
}
impl ModelLadderConstraintSummary {
fn observe(&mut self, constraints: &CapabilityConstraints) {
self.flagship += 1;
if constraints.context_pressure != RequirementLevel::High {
self.open_source_flagship += 1;
}
if constraints.context_pressure != RequirementLevel::High
&& constraints.expected_redo_penalty != RequirementLevel::High
{
self.standard += 1;
}
if constraints.context_pressure != RequirementLevel::High {
self.cheap_tool_safe += 1;
}
if constraints.tool_reliability == RequirementLevel::Low
&& constraints.context_pressure == RequirementLevel::Low
&& constraints.expected_redo_penalty != RequirementLevel::High
{
self.cheap_fast += 1;
}
}
}
pub fn extract_fixture_ir(
fixture: &WorkflowTraceFixture,
) -> crate::workflow_state::ir::WorkflowStateIR {
let input = ExtractorInput {
harness_hint: Some(fixture.harness.clone()),
protocol_hint: fixture.protocol.clone(),
headers: &fixture.headers,
raw_body: &fixture.raw_body,
prompt: &fixture.prompt,
};
extract_workflow_state(&input)
}
fn harness_key(harness: &HarnessId) -> String {
match harness {
HarnessId::Generic => "generic",
HarnessId::Hermes => "hermes",
HarnessId::ClaudeCode => "claude_code",
HarnessId::Codex => "codex",
HarnessId::Smithers => "smithers",
HarnessId::Terminus2 => "terminus_2",
HarnessId::OpenClaw => "openclaw",
HarnessId::Unknown => "unknown",
}
.to_string()
}
fn session_key(confidence: &SessionConfidence) -> String {
match confidence {
SessionConfidence::None => "none",
SessionConfidence::Low => "low",
SessionConfidence::Medium => "medium",
SessionConfidence::High => "high",
}
.to_string()
}
#[cfg(test)]
mod tests {
use serde_json::json;
use crate::workflow_state::fixture::WorkflowTraceFixture;
use crate::workflow_state::replay::ReplayEvaluator;
#[test]
fn task_aware_replay_uses_unified_v1_and_compares_expected_family() {
let fixture = WorkflowTraceFixture::from_value(json!({
"id": "task-aware-debugging",
"harness": "generic",
"protocol": "chat_completions",
"headers": {},
"raw_body": {
"model": "test",
"messages": [{
"role": "user",
"content": "Fix the parser panic in src/parser.rs after this regression failed."
}]
},
"expected": {
"state_kind": "opening",
"baseline_fingerprint": "opening",
"confidence_min": 0.0,
"prediction": {
"next_step_role": "implement",
"next_action_class": "mutate",
"route_risk": "normal",
"task_family": "code_debugging"
}
}
}))
.unwrap();
let summary = ReplayEvaluator.run(&[fixture]);
assert_eq!(
summary.records[0].predictive_route_key,
"semantic_route/v1|code:debugging|implement|normal"
);
assert_eq!(summary.records[0].prediction_matches_expected, Some(true));
}
}