Skip to main content

arabica_runtime/
scoring.rs

1//! Versioned scoring contracts and pure calculations for evaluation tasks.
2//!
3//! Implements weighted criterion scoring, cost/time efficiency penalties,
4//! budget enforcement, and explicit missing-data/evaluator-fault classification.
5
6use std::collections::{BTreeSet, HashMap};
7
8use arabica_protocol::{EventEnvelope, RunId, SessionId};
9use serde::{Deserialize, Serialize};
10
11use crate::accounting::{PriceCard, RunCostAccounting, account_run_cost_and_time};
12use crate::{
13    AcceptanceCriterion, AcceptanceSpec, AcceptanceStatus, ArtifactEvidence, EvaluationError,
14    RunAcceptance, RunEvidenceSnapshot,
15};
16
17/// Resource budget limits for a task trial.
18#[derive(Clone, Copy, Debug, Default, Deserialize, PartialEq, Serialize)]
19#[serde(deny_unknown_fields)]
20pub struct TaskResourceBudget {
21    /// Maximum allowed financial cost in the evaluation currency.
22    #[serde(default, skip_serializing_if = "Option::is_none")]
23    pub max_cost: Option<f64>,
24    /// Maximum allowed active agent time in milliseconds.
25    #[serde(default, skip_serializing_if = "Option::is_none")]
26    pub max_active_time_ms: Option<u64>,
27    /// Maximum allowed model steps / turns.
28    #[serde(default, skip_serializing_if = "Option::is_none")]
29    pub max_model_steps: Option<usize>,
30}
31
32impl TaskResourceBudget {
33    pub fn validate(&self) -> Result<(), EvaluationError> {
34        if self
35            .max_cost
36            .is_some_and(|cost| cost < 0.0 || !cost.is_finite())
37        {
38            return Err(EvaluationError(
39                "max_cost budget must be non-negative and finite".into(),
40            ));
41        }
42        Ok(())
43    }
44}
45
46/// Versioned task specification with criteria, weights, and resource budgets.
47#[derive(Clone, Debug, Deserialize, PartialEq, Serialize)]
48#[serde(deny_unknown_fields)]
49pub struct TaskSpec {
50    pub task_id: String,
51    pub version: u64,
52    #[serde(default, skip_serializing_if = "Option::is_none")]
53    pub environment_fingerprint: Option<String>,
54    pub criteria: Vec<AcceptanceCriterion>,
55    #[serde(default)]
56    pub budgets: TaskResourceBudget,
57}
58
59impl TaskSpec {
60    pub fn validate(&self) -> Result<(), EvaluationError> {
61        if self.task_id.trim().is_empty() || self.version == 0 || self.criteria.len() > 128 {
62            return Err(EvaluationError("invalid task specification".into()));
63        }
64        self.budgets.validate()?;
65        let mut ids = BTreeSet::new();
66        let mut total_weight = 0.0;
67        for criterion in &self.criteria {
68            if criterion.id.trim().is_empty() || !ids.insert(&criterion.id) {
69                return Err(EvaluationError(
70                    "acceptance criterion IDs must be unique and non-empty".into(),
71                ));
72            }
73            if criterion.weight < 0.0 || !criterion.weight.is_finite() {
74                return Err(EvaluationError(
75                    "acceptance criterion weight must be non-negative and finite".into(),
76                ));
77            }
78            total_weight += criterion.weight;
79            criterion.check.validate()?;
80        }
81        if !self.criteria.is_empty() && total_weight <= 0.0 {
82            return Err(EvaluationError(
83                "total criterion weight must be strictly positive".into(),
84            ));
85        }
86        Ok(())
87    }
88
89    pub fn fingerprint(&self) -> String {
90        crate::context::fingerprint(self)
91    }
92
93    pub fn acceptance_spec(&self) -> AcceptanceSpec {
94        AcceptanceSpec {
95            task_id: self.task_id.clone(),
96            version: self.version,
97            criteria: self.criteria.clone(),
98        }
99    }
100}
101
102impl From<AcceptanceSpec> for TaskSpec {
103    fn from(spec: AcceptanceSpec) -> Self {
104        Self {
105            task_id: spec.task_id,
106            version: spec.version,
107            environment_fingerprint: None,
108            criteria: spec.criteria,
109            budgets: TaskResourceBudget::default(),
110        }
111    }
112}
113
114/// Relative weights for scoring formula components summing to 1.0.
115#[derive(Clone, Copy, Debug, Deserialize, PartialEq, Serialize)]
116#[serde(deny_unknown_fields)]
117pub struct ScoringWeights {
118    pub quality: f64,
119    pub cost: f64,
120    pub time: f64,
121}
122
123impl Default for ScoringWeights {
124    fn default() -> Self {
125        Self {
126            quality: 0.70,
127            cost: 0.20,
128            time: 0.10,
129        }
130    }
131}
132
133impl ScoringWeights {
134    pub fn validate(&self) -> Result<(), EvaluationError> {
135        if self.quality < 0.0
136            || !self.quality.is_finite()
137            || self.cost < 0.0
138            || !self.cost.is_finite()
139            || self.time < 0.0
140            || !self.time.is_finite()
141        {
142            return Err(EvaluationError(
143                "scoring weights must be non-negative and finite".into(),
144            ));
145        }
146        let sum = self.quality + self.cost + self.time;
147        if (sum - 1.0).abs() > 1e-6 {
148            return Err(EvaluationError(format!(
149                "scoring weights must sum to 1.0 (got {sum:.6})"
150            )));
151        }
152        Ok(())
153    }
154}
155
156/// Frozen scoring profile with formula version, component weights, and references.
157#[derive(Clone, Debug, Deserialize, PartialEq, Serialize)]
158#[serde(deny_unknown_fields)]
159pub struct ScoringProfile {
160    pub formula_version: u64,
161    #[serde(default)]
162    pub weights: ScoringWeights,
163    pub reference_cost: f64,
164    pub reference_time_ms: u64,
165}
166
167impl ScoringProfile {
168    pub fn new(reference_cost: f64, reference_time_ms: u64) -> Result<Self, EvaluationError> {
169        let profile = Self {
170            formula_version: 1,
171            weights: ScoringWeights::default(),
172            reference_cost,
173            reference_time_ms,
174        };
175        profile.validate()?;
176        Ok(profile)
177    }
178
179    pub fn validate(&self) -> Result<(), EvaluationError> {
180        if self.formula_version == 0 {
181            return Err(EvaluationError("formula_version must be non-zero".into()));
182        }
183        if self.reference_cost <= 0.0 || !self.reference_cost.is_finite() {
184            return Err(EvaluationError(
185                "reference_cost must be positive and finite".into(),
186            ));
187        }
188        if self.reference_time_ms == 0 {
189            return Err(EvaluationError("reference_time_ms must be non-zero".into()));
190        }
191        self.weights.validate()?;
192        Ok(())
193    }
194
195    pub fn fingerprint(&self) -> String {
196        crate::context::fingerprint(self)
197    }
198}
199
200/// Composite evaluation identity tying checker version, task, and scoring profile.
201#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)]
202#[serde(deny_unknown_fields)]
203pub struct EvaluationIdentity {
204    pub checker_version: u64,
205    pub task_fingerprint: String,
206    pub scoring_fingerprint: String,
207}
208
209impl EvaluationIdentity {
210    pub fn new(
211        checker_version: u64,
212        task_fingerprint: impl Into<String>,
213        scoring_fingerprint: impl Into<String>,
214    ) -> Self {
215        Self {
216            checker_version,
217            task_fingerprint: task_fingerprint.into(),
218            scoring_fingerprint: scoring_fingerprint.into(),
219        }
220    }
221
222    pub fn fingerprint(&self) -> String {
223        crate::context::fingerprint(self)
224    }
225}
226
227/// Scoring outcome status distinguishing completed, pending, incomplete, and fault states.
228#[derive(Clone, Copy, Debug, Deserialize, Eq, PartialEq, Serialize)]
229#[serde(rename_all = "snake_case")]
230pub enum TaskScoreStatus {
231    PendingChecks,
232    IncompleteEvidence,
233    InvalidEvaluator,
234    Completed,
235}
236
237/// Outcome of evaluating an individual criterion.
238#[derive(Clone, Debug, Deserialize, PartialEq, Serialize)]
239#[serde(deny_unknown_fields)]
240pub struct CriterionEvaluation {
241    pub criterion_id: String,
242    pub status: AcceptanceStatus,
243    /// Score in `[0, 1]`. Binary checks return 1.0 or 0.0. Missing/unknown evidence has None.
244    #[serde(default, skip_serializing_if = "Option::is_none")]
245    pub score: Option<f64>,
246    #[serde(default, skip_serializing_if = "Option::is_none")]
247    pub evidence_id: Option<String>,
248}
249
250impl CriterionEvaluation {
251    pub fn from_status(
252        criterion_id: impl Into<String>,
253        status: AcceptanceStatus,
254        evidence_id: Option<String>,
255    ) -> Self {
256        let score = match status {
257            AcceptanceStatus::Pass => Some(1.0),
258            AcceptanceStatus::Fail => Some(0.0),
259            AcceptanceStatus::Unknown | AcceptanceStatus::NotApplicable => None,
260        };
261        Self {
262            criterion_id: criterion_id.into(),
263            status,
264            score,
265            evidence_id,
266        }
267    }
268
269    pub fn with_score(
270        criterion_id: impl Into<String>,
271        status: AcceptanceStatus,
272        score: Option<f64>,
273        evidence_id: Option<String>,
274    ) -> Self {
275        Self {
276            criterion_id: criterion_id.into(),
277            status,
278            score,
279            evidence_id,
280        }
281    }
282}
283
284/// Observed runtime metrics and evaluator conditions for scoring a task run.
285#[derive(Clone, Debug, Default, Deserialize, PartialEq, Serialize)]
286#[serde(deny_unknown_fields)]
287pub struct ScoringObservation {
288    #[serde(default, skip_serializing_if = "Option::is_none")]
289    pub actual_cost: Option<f64>,
290    #[serde(default, skip_serializing_if = "Option::is_none")]
291    pub active_time_ms: Option<u64>,
292    #[serde(default, skip_serializing_if = "Option::is_none")]
293    pub model_steps: Option<usize>,
294    #[serde(default, skip_serializing_if = "Option::is_none")]
295    pub evaluator_fault: Option<String>,
296    #[serde(default)]
297    pub pending: bool,
298}
299
300/// Evaluated task score and components.
301#[derive(Clone, Debug, Deserialize, PartialEq, Serialize)]
302#[serde(deny_unknown_fields)]
303pub struct TaskScore {
304    pub status: TaskScoreStatus,
305    pub quality: Option<f64>,
306    pub cost_efficiency: Option<f64>,
307    pub time_efficiency: Option<f64>,
308    pub total: Option<f64>,
309    pub required_passed: Option<bool>,
310    pub budget_exceeded: bool,
311    pub evidence_ids: Vec<String>,
312    pub evaluation_identity: EvaluationIdentity,
313    #[serde(default, skip_serializing_if = "Option::is_none")]
314    pub failure_reason: Option<String>,
315}
316
317/// Pure deterministic calculation of a versioned task score.
318pub fn calculate_task_score(
319    task_spec: &TaskSpec,
320    profile: &ScoringProfile,
321    checker_version: u64,
322    criterion_results: &[CriterionEvaluation],
323    observation: &ScoringObservation,
324) -> Result<TaskScore, EvaluationError> {
325    task_spec.validate()?;
326    profile.validate()?;
327
328    if observation
329        .actual_cost
330        .is_some_and(|cost| cost < 0.0 || !cost.is_finite())
331    {
332        return Err(EvaluationError(
333            "actual_cost must be non-negative and finite".into(),
334        ));
335    }
336
337    let evaluation_identity = EvaluationIdentity::new(
338        checker_version,
339        task_spec.fingerprint(),
340        profile.fingerprint(),
341    );
342
343    // 1. Evaluator infrastructure failure is not a task failure.
344    if let Some(fault) = &observation.evaluator_fault {
345        return Ok(TaskScore {
346            status: TaskScoreStatus::InvalidEvaluator,
347            quality: None,
348            cost_efficiency: None,
349            time_efficiency: None,
350            total: None,
351            required_passed: None,
352            budget_exceeded: false,
353            evidence_ids: vec![],
354            evaluation_identity,
355            failure_reason: Some(format!("evaluator infrastructure fault: {fault}")),
356        });
357    }
358
359    // 2. Pending checks
360    if observation.pending {
361        return Ok(TaskScore {
362            status: TaskScoreStatus::PendingChecks,
363            quality: None,
364            cost_efficiency: None,
365            time_efficiency: None,
366            total: None,
367            required_passed: None,
368            budget_exceeded: false,
369            evidence_ids: vec![],
370            evaluation_identity,
371            failure_reason: None,
372        });
373    }
374
375    // 3. Collect evidence IDs
376    let mut evidence_ids: Vec<String> = criterion_results
377        .iter()
378        .filter_map(|r| r.evidence_id.clone())
379        .filter(|id| !id.trim().is_empty())
380        .collect();
381    evidence_ids.sort();
382    evidence_ids.dedup();
383
384    // 4. Weight normalization & Quality calculation
385    // "Normalize criterion weights to sum to one."
386    // "Unknown evidence never passes a check or causes weight renormalization."
387    let total_declared_weight: f64 = task_spec.criteria.iter().map(|c| c.weight).sum();
388    let norm_factor = if total_declared_weight > 0.0 {
389        total_declared_weight
390    } else {
391        1.0
392    };
393
394    let results_by_id: HashMap<&str, &CriterionEvaluation> = criterion_results
395        .iter()
396        .map(|r| (r.criterion_id.as_str(), r))
397        .collect();
398
399    let mut quality_sum = 0.0;
400    let mut required_failure = false;
401    let mut required_unknown = false;
402    let mut missing_criterion_evidence = false;
403    let mut failure_reasons = Vec::new();
404
405    for criterion in &task_spec.criteria {
406        let norm_weight = criterion.weight / norm_factor;
407        match results_by_id.get(criterion.id.as_str()) {
408            Some(result) => match result.status {
409                AcceptanceStatus::Pass => {
410                    let score = result.score.unwrap_or(1.0).clamp(0.0, 1.0);
411                    quality_sum += norm_weight * score;
412                }
413                AcceptanceStatus::Fail => {
414                    if criterion.required {
415                        required_failure = true;
416                        failure_reasons
417                            .push(format!("required criterion '{}' failed", criterion.id));
418                    }
419                }
420                AcceptanceStatus::Unknown => {
421                    missing_criterion_evidence = true;
422                    if criterion.required {
423                        required_unknown = true;
424                    }
425                }
426                AcceptanceStatus::NotApplicable => {}
427            },
428            None => {
429                missing_criterion_evidence = true;
430                if criterion.required {
431                    required_unknown = true;
432                }
433            }
434        }
435    }
436
437    // 5. Budget checks
438    let mut budget_exceeded = false;
439    if task_spec
440        .budgets
441        .max_cost
442        .is_some_and(|max| observation.actual_cost.is_some_and(|actual| actual > max))
443    {
444        let actual = observation.actual_cost.unwrap();
445        let max = task_spec.budgets.max_cost.unwrap();
446        budget_exceeded = true;
447        failure_reasons.push(format!(
448            "declared cost budget exceeded: {actual:.4} > {max:.4}"
449        ));
450    }
451    if task_spec.budgets.max_active_time_ms.is_some_and(|max| {
452        observation
453            .active_time_ms
454            .is_some_and(|actual| actual > max)
455    }) {
456        let actual = observation.active_time_ms.unwrap();
457        let max = task_spec.budgets.max_active_time_ms.unwrap();
458        budget_exceeded = true;
459        failure_reasons.push(format!(
460            "declared active time budget exceeded: {actual}ms > {max}ms"
461        ));
462    }
463    if task_spec
464        .budgets
465        .max_model_steps
466        .is_some_and(|max| observation.model_steps.is_some_and(|actual| actual > max))
467    {
468        let actual = observation.model_steps.unwrap();
469        let max = task_spec.budgets.max_model_steps.unwrap();
470        budget_exceeded = true;
471        failure_reasons.push(format!(
472            "declared model steps budget exceeded: {actual} > {max}"
473        ));
474    }
475
476    // 6. Efficiency components
477    // E_cost = 1 / (1 + actual_cost / reference_cost)
478    let cost_efficiency = observation
479        .actual_cost
480        .map(|cost| 1.0 / (1.0 + cost / profile.reference_cost));
481    // E_time = 1 / (1 + active_time / reference_time)
482    let time_efficiency = observation
483        .active_time_ms
484        .map(|time| 1.0 / (1.0 + (time as f64) / (profile.reference_time_ms as f64)));
485
486    let required_passed = if required_failure {
487        Some(false)
488    } else if required_unknown {
489        None
490    } else {
491        Some(true)
492    };
493
494    // 7. Score determination
495    // "If any required criterion fails, or the task exceeds its declared budget:
496    //     Score = 0"
497    // "A known required failure is sufficient for a zero score even if other evidence is missing."
498    if required_failure || budget_exceeded {
499        return Ok(TaskScore {
500            status: TaskScoreStatus::Completed,
501            quality: Some(quality_sum),
502            cost_efficiency,
503            time_efficiency,
504            total: Some(0.0),
505            required_passed,
506            budget_exceeded,
507            evidence_ids,
508            evaluation_identity,
509            failure_reason: Some(failure_reasons.join("; ")),
510        });
511    }
512
513    // "Otherwise, when required criteria pass and all scoring evidence is available:
514    //     Score = 100 * Q * (0.70 + 0.20 * E_cost + 0.10 * E_time)"
515    // "Otherwise, missing evidence or accounting prevents a complete score; retain available quality and accounting components."
516    let all_evidence_available = !missing_criterion_evidence
517        && observation.actual_cost.is_some()
518        && observation.active_time_ms.is_some();
519
520    if !all_evidence_available {
521        let mut missing_items = Vec::new();
522        if missing_criterion_evidence {
523            missing_items.push("criterion evidence missing or unknown");
524        }
525        if observation.actual_cost.is_none() {
526            missing_items.push("actual cost accounting missing");
527        }
528        if observation.active_time_ms.is_none() {
529            missing_items.push("active time accounting missing");
530        }
531        return Ok(TaskScore {
532            status: TaskScoreStatus::IncompleteEvidence,
533            quality: Some(quality_sum),
534            cost_efficiency,
535            time_efficiency,
536            total: None,
537            required_passed,
538            budget_exceeded: false,
539            evidence_ids,
540            evaluation_identity,
541            failure_reason: Some(format!(
542                "incomplete scoring data: {}",
543                missing_items.join("; ")
544            )),
545        });
546    }
547
548    let e_cost = cost_efficiency.expect("cost efficiency verified available");
549    let e_time = time_efficiency.expect("time efficiency verified available");
550
551    let efficiency_multiplier =
552        profile.weights.quality + profile.weights.cost * e_cost + profile.weights.time * e_time;
553    let score = 100.0 * quality_sum * efficiency_multiplier;
554
555    Ok(TaskScore {
556        status: TaskScoreStatus::Completed,
557        quality: Some(quality_sum),
558        cost_efficiency,
559        time_efficiency,
560        total: Some(score.clamp(0.0, 100.0)),
561        required_passed: Some(true),
562        budget_exceeded: false,
563        evidence_ids,
564        evaluation_identity,
565        failure_reason: None,
566    })
567}
568
569/// Score a completed run evaluation against a task specification and profile.
570pub fn score_run_evaluation(
571    task_spec: &TaskSpec,
572    profile: &ScoringProfile,
573    checker_version: u64,
574    snapshot: &RunEvidenceSnapshot,
575    acceptance: Option<&RunAcceptance>,
576    actual_cost: Option<f64>,
577) -> Result<TaskScore, EvaluationError> {
578    let criterion_results = match acceptance {
579        Some(acc) => acc
580            .results
581            .iter()
582            .map(|r| CriterionEvaluation {
583                criterion_id: r.criterion_id.clone(),
584                status: r.status.clone(),
585                score: match r.status {
586                    AcceptanceStatus::Pass => Some(1.0),
587                    AcceptanceStatus::Fail => Some(0.0),
588                    _ => None,
589                },
590                evidence_id: (!r.evidence.is_empty()).then(|| r.evidence.clone()),
591            })
592            .collect(),
593        None => Vec::new(),
594    };
595
596    let observation = ScoringObservation {
597        actual_cost,
598        active_time_ms: snapshot.timing.active_ms,
599        model_steps: Some(snapshot.timing.model_calls),
600        evaluator_fault: None,
601        pending: false,
602    };
603
604    calculate_task_score(
605        task_spec,
606        profile,
607        checker_version,
608        &criterion_results,
609        &observation,
610    )
611}
612
613fn default_tool_health_weight() -> f64 {
614    0.40
615}
616
617fn default_require_clean_terminal_tools() -> bool {
618    true
619}
620
621fn default_gate_timeout_ms() -> u64 {
622    30_000
623}
624
625fn default_gate_output_limit() -> usize {
626    64 * 1024
627}
628
629/// Configuration for real live interactive session evaluation.
630#[derive(Clone, Debug, Deserialize, PartialEq, Serialize)]
631#[serde(deny_unknown_fields)]
632pub struct LiveSessionConfig {
633    pub session_id: SessionId,
634    #[serde(default)]
635    pub budgets: TaskResourceBudget,
636    #[serde(default, skip_serializing_if = "Option::is_none")]
637    pub workspace_gate: Option<WorkspaceGateConfig>,
638    /// Weight applied to tool health in quality score (0.0 .. 1.0, default 0.40).
639    #[serde(default = "default_tool_health_weight")]
640    pub tool_health_weight: f64,
641    /// If true, a session ending on an unrecovered failed tool call forces zero quality.
642    #[serde(default = "default_require_clean_terminal_tools")]
643    pub require_clean_terminal_tools: bool,
644    /// Optional price card for automatic cost accounting.
645    #[serde(default, skip_serializing_if = "Option::is_none")]
646    pub price_card: Option<PriceCard>,
647}
648
649impl LiveSessionConfig {
650    pub fn new(session_id: SessionId) -> Self {
651        Self {
652            session_id,
653            budgets: TaskResourceBudget::default(),
654            workspace_gate: None,
655            tool_health_weight: default_tool_health_weight(),
656            require_clean_terminal_tools: default_require_clean_terminal_tools(),
657            price_card: None,
658        }
659    }
660
661    pub fn with_workspace_gate(mut self, gate: WorkspaceGateConfig) -> Self {
662        self.workspace_gate = Some(gate);
663        self
664    }
665
666    pub fn with_budgets(mut self, budgets: TaskResourceBudget) -> Self {
667        self.budgets = budgets;
668        self
669    }
670
671    pub fn with_price_card(mut self, card: PriceCard) -> Self {
672        self.price_card = Some(card);
673        self
674    }
675}
676
677/// Configuration for workspace integrity and quality gates on real projects.
678#[derive(Clone, Debug, Deserialize, PartialEq, Serialize)]
679#[serde(deny_unknown_fields)]
680pub struct WorkspaceGateConfig {
681    /// Command to verify workspace compile/syntax (e.g. "cargo check").
682    pub check_command: String,
683    /// Optional command to run existing project tests for regressions (e.g. "cargo test").
684    #[serde(default, skip_serializing_if = "Option::is_none")]
685    pub test_command: Option<String>,
686    /// Optional whitelist of permitted modified files.
687    #[serde(default, skip_serializing_if = "Option::is_none")]
688    pub permitted_paths: Option<Vec<String>>,
689    /// Command timeout in milliseconds (default 30_000).
690    #[serde(default = "default_gate_timeout_ms")]
691    pub timeout_ms: u64,
692    /// Bounded stdout/stderr capture in bytes (default 64 KiB).
693    #[serde(default = "default_gate_output_limit")]
694    pub output_limit_bytes: usize,
695}
696
697impl WorkspaceGateConfig {
698    pub fn new(check_command: impl Into<String>) -> Self {
699        Self {
700            check_command: check_command.into(),
701            test_command: None,
702            permitted_paths: None,
703            timeout_ms: default_gate_timeout_ms(),
704            output_limit_bytes: default_gate_output_limit(),
705        }
706    }
707}
708
709/// Result of evaluating real workspace integrity gates.
710#[derive(Clone, Debug, Deserialize, PartialEq, Serialize)]
711#[serde(deny_unknown_fields)]
712pub struct WorkspaceGateResult {
713    pub check_passed: bool,
714    pub tests_passed: Option<bool>,
715    pub permitted_files_passed: Option<bool>,
716    pub overall_passed: bool,
717    pub failure_reasons: Vec<String>,
718}
719
720/// Configuration for historical offline trace rescoring.
721#[derive(Clone, Debug, Deserialize, PartialEq, Serialize)]
722#[serde(deny_unknown_fields)]
723pub struct HistoricalTraceConfig {
724    pub session_id: SessionId,
725    #[serde(default, skip_serializing_if = "Option::is_none")]
726    pub run_id: Option<RunId>,
727}
728
729/// Unified evaluation mode distinguishing live sessions, workspace gates, preset benchmarks, and historical audits.
730#[derive(Clone, Debug, Deserialize, PartialEq, Serialize)]
731#[serde(tag = "mode", rename_all = "snake_case")]
732pub enum EvaluationMode {
733    /// Mode 1: Live interactive session evaluated via trajectory health and efficiency.
734    LiveSession(LiveSessionConfig),
735    /// Mode 2: Real workspace integrity gate verification.
736    WorkspaceGate(WorkspaceGateConfig),
737    /// Mode 3: Preset benchmark task evaluation with explicit criteria.
738    PresetTask(TaskSpec),
739    /// Mode 4: Historical session trace offline rescoring and audit.
740    HistoricalTrace(HistoricalTraceConfig),
741}
742
743/// Execute workspace integrity checks on a real project workspace.
744pub fn execute_workspace_gate(
745    root: &std::path::Path,
746    config: &WorkspaceGateConfig,
747    modified_paths: Option<&[String]>,
748) -> WorkspaceGateResult {
749    let mut failure_reasons = Vec::new();
750
751    // 1. Check command (e.g. cargo check)
752    let check_evidence = crate::execute_controlled_test(
753        root,
754        &config.check_command,
755        config.timeout_ms,
756        config.output_limit_bytes,
757    );
758    let check_passed = match &check_evidence {
759        ArtifactEvidence::TestResult {
760            exit_code,
761            passed,
762            error,
763            ..
764        } => {
765            if let Some(err) = error {
766                failure_reasons.push(format!("workspace check execution failed: {err}"));
767                false
768            } else if *passed && *exit_code == Some(0) {
769                true
770            } else {
771                failure_reasons.push(format!("workspace check failed: exit code {:?}", exit_code));
772                false
773            }
774        }
775        _ => false,
776    };
777
778    // 2. Optional regression test command (e.g. cargo test)
779    let mut tests_passed = None;
780    if let Some(test_cmd) = &config.test_command {
781        let test_evidence = crate::execute_controlled_test(
782            root,
783            test_cmd,
784            config.timeout_ms,
785            config.output_limit_bytes,
786        );
787        let passed = match &test_evidence {
788            ArtifactEvidence::TestResult {
789                exit_code,
790                passed,
791                error,
792                ..
793            } => {
794                if let Some(err) = error {
795                    failure_reasons
796                        .push(format!("workspace regression test execution failed: {err}"));
797                    false
798                } else if *passed && *exit_code == Some(0) {
799                    true
800                } else {
801                    failure_reasons.push(format!(
802                        "workspace regression tests failed: exit code {:?}",
803                        exit_code
804                    ));
805                    false
806                }
807            }
808            _ => false,
809        };
810        tests_passed = Some(passed);
811    }
812
813    // 3. Optional permitted paths
814    let mut permitted_passed = None;
815    if let (Some(allowed), Some(modified)) = (&config.permitted_paths, modified_paths) {
816        let allowed_set: std::collections::BTreeSet<&str> =
817            allowed.iter().map(|s| s.as_str()).collect();
818        let mut violations = Vec::new();
819        for path in modified {
820            if !allowed_set.contains(path.as_str()) {
821                violations.push(path.clone());
822            }
823        }
824        if !violations.is_empty() {
825            failure_reasons.push(format!(
826                "unpermitted files modified: {}",
827                violations.join(", ")
828            ));
829            permitted_passed = Some(false);
830        } else {
831            permitted_passed = Some(true);
832        }
833    }
834
835    let overall_passed =
836        check_passed && tests_passed.unwrap_or(true) && permitted_passed.unwrap_or(true);
837
838    WorkspaceGateResult {
839        check_passed,
840        tests_passed,
841        permitted_files_passed: permitted_passed,
842        overall_passed,
843        failure_reasons,
844    }
845}
846
847/// Evaluate a real running session from its evidence snapshot and live configuration.
848pub fn evaluate_live_session(
849    config: &LiveSessionConfig,
850    profile: &ScoringProfile,
851    checker_version: u64,
852    snapshot: &RunEvidenceSnapshot,
853    actual_cost: Option<f64>,
854    workspace_gate_result: Option<&WorkspaceGateResult>,
855) -> Result<TaskScore, EvaluationError> {
856    profile.validate()?;
857    config.budgets.validate()?;
858
859    if snapshot.session_id != config.session_id {
860        return Err(EvaluationError(format!(
861            "session_id mismatch: expected '{}', got '{}'",
862            config.session_id, snapshot.session_id
863        )));
864    }
865
866    if actual_cost.is_some_and(|c| c < 0.0 || !c.is_finite()) {
867        return Err(EvaluationError(
868            "actual_cost must be non-negative and finite".into(),
869        ));
870    }
871
872    let evaluation_identity = EvaluationIdentity::new(
873        checker_version,
874        format!("live_session:{}", config.session_id),
875        profile.fingerprint(),
876    );
877
878    let mut failure_reasons = Vec::new();
879    let evidence_ids = vec![snapshot.fingerprint.clone()];
880
881    // 1. Terminal Outcome: Must be "completed"
882    let terminal_completed = snapshot.terminal_status == "completed";
883    if !terminal_completed {
884        failure_reasons.push(format!(
885            "session ended with non-completed terminal status: '{}'",
886            snapshot.terminal_status
887        ));
888    }
889
890    // 2. Tool Health Analysis
891    let total_tools = snapshot.tools.len();
892    let errored_tools = snapshot
893        .tools
894        .iter()
895        .filter(|t| t.is_error == Some(true))
896        .count();
897    let incomplete_tools = snapshot.timing.incomplete_tool_calls;
898
899    let tool_success_ratio = if total_tools > 0 {
900        (total_tools.saturating_sub(errored_tools)) as f64 / total_tools as f64
901    } else {
902        1.0
903    };
904
905    let last_tool_failed = snapshot
906        .tools
907        .last()
908        .and_then(|t| t.is_error)
909        .unwrap_or(false);
910
911    let mut tool_fatal_failure = false;
912    if config.require_clean_terminal_tools && last_tool_failed {
913        tool_fatal_failure = true;
914        failure_reasons.push("session ended with an unrecovered tool error".into());
915    }
916    if incomplete_tools > 0 {
917        failure_reasons.push(format!("{incomplete_tools} tool calls remained incomplete"));
918    }
919
920    // 3. Workspace Gate Analysis
921    let mut gate_passed = true;
922    if let Some(gate) = workspace_gate_result
923        && !gate.overall_passed
924    {
925        gate_passed = false;
926        failure_reasons.extend(gate.failure_reasons.clone());
927    }
928
929    // 4. Budget Checks
930    let mut budget_exceeded = false;
931    if config
932        .budgets
933        .max_cost
934        .is_some_and(|max| actual_cost.is_some_and(|actual| actual > max))
935    {
936        let actual = actual_cost.unwrap();
937        let max = config.budgets.max_cost.unwrap();
938        budget_exceeded = true;
939        failure_reasons.push(format!(
940            "declared cost budget exceeded: {actual:.4} > {max:.4}"
941        ));
942    }
943    if config
944        .budgets
945        .max_active_time_ms
946        .is_some_and(|max| snapshot.timing.active_ms.is_some_and(|actual| actual > max))
947    {
948        let actual = snapshot.timing.active_ms.unwrap();
949        let max = config.budgets.max_active_time_ms.unwrap();
950        budget_exceeded = true;
951        failure_reasons.push(format!(
952            "declared active time budget exceeded: {actual}ms > {max}ms"
953        ));
954    }
955    if config
956        .budgets
957        .max_model_steps
958        .is_some_and(|max| max < snapshot.timing.model_calls)
959    {
960        budget_exceeded = true;
961        failure_reasons.push(format!(
962            "declared model steps budget exceeded: {} > {}",
963            snapshot.timing.model_calls,
964            config.budgets.max_model_steps.unwrap()
965        ));
966    }
967
968    // 5. Efficiency Components
969    let cost_efficiency = actual_cost.map(|cost| 1.0 / (1.0 + cost / profile.reference_cost));
970    let time_efficiency = snapshot
971        .timing
972        .active_ms
973        .map(|time| 1.0 / (1.0 + (time as f64) / (profile.reference_time_ms as f64)));
974
975    // 6. Hard Failures Zero Score
976    if !terminal_completed || tool_fatal_failure || !gate_passed || budget_exceeded {
977        return Ok(TaskScore {
978            status: TaskScoreStatus::Completed,
979            quality: Some(0.0),
980            cost_efficiency,
981            time_efficiency,
982            total: Some(0.0),
983            required_passed: Some(false),
984            budget_exceeded,
985            evidence_ids,
986            evaluation_identity,
987            failure_reason: Some(failure_reasons.join("; ")),
988        });
989    }
990
991    // 7. Quality Calculation for successful session:
992    // Base completion = 1.0
993    // Tool health penalty applied according to tool_health_weight
994    let quality = 1.0 - (config.tool_health_weight * (1.0 - tool_success_ratio));
995    let quality = quality.clamp(0.0, 1.0);
996
997    // 8. Completeness of Accounting Data
998    let all_evidence_available = actual_cost.is_some() && snapshot.timing.active_ms.is_some();
999
1000    if !all_evidence_available {
1001        let mut missing = Vec::new();
1002        if actual_cost.is_none() {
1003            missing.push("actual cost accounting missing");
1004        }
1005        if snapshot.timing.active_ms.is_none() {
1006            missing.push("active time accounting missing");
1007        }
1008        return Ok(TaskScore {
1009            status: TaskScoreStatus::IncompleteEvidence,
1010            quality: Some(quality),
1011            cost_efficiency,
1012            time_efficiency,
1013            total: None,
1014            required_passed: Some(true),
1015            budget_exceeded: false,
1016            evidence_ids,
1017            evaluation_identity,
1018            failure_reason: Some(format!("incomplete scoring data: {}", missing.join("; "))),
1019        });
1020    }
1021
1022    let e_cost = cost_efficiency.expect("cost efficiency available");
1023    let e_time = time_efficiency.expect("time efficiency available");
1024
1025    let efficiency_multiplier =
1026        profile.weights.quality + profile.weights.cost * e_cost + profile.weights.time * e_time;
1027    let score = 100.0 * quality * efficiency_multiplier;
1028
1029    Ok(TaskScore {
1030        status: TaskScoreStatus::Completed,
1031        quality: Some(quality),
1032        cost_efficiency,
1033        time_efficiency,
1034        total: Some(score.clamp(0.0, 100.0)),
1035        required_passed: Some(true),
1036        budget_exceeded: false,
1037        evidence_ids,
1038        evaluation_identity,
1039        failure_reason: None,
1040    })
1041}
1042
1043/// Fully evaluate a live session directly from its event history, including
1044/// automatic snapshot extraction and price-card cost accounting.
1045pub fn evaluate_live_session_from_history(
1046    config: &LiveSessionConfig,
1047    profile: &ScoringProfile,
1048    checker_version: u64,
1049    history: &[EventEnvelope],
1050    workspace_gate_result: Option<&WorkspaceGateResult>,
1051) -> Result<(TaskScore, Option<RunCostAccounting>), EvaluationError> {
1052    let mut snapshots = crate::run_evidence_snapshots(history);
1053    let snapshot = snapshots.pop().ok_or_else(|| {
1054        EvaluationError("history does not contain any terminal run snapshot".into())
1055    })?;
1056
1057    let accounting = match &config.price_card {
1058        Some(card) => Some(
1059            account_run_cost_and_time(history, Some(&snapshot.run_id), card)
1060                .map_err(|e| EvaluationError(e.0))?,
1061        ),
1062        None => None,
1063    };
1064
1065    let actual_cost = accounting.as_ref().and_then(|a| a.total_agent_cost);
1066
1067    let score = evaluate_live_session(
1068        config,
1069        profile,
1070        checker_version,
1071        &snapshot,
1072        actual_cost,
1073        workspace_gate_result,
1074    )?;
1075
1076    Ok((score, accounting))
1077}
1078
1079#[cfg(test)]
1080mod tests {
1081    use super::*;
1082    use crate::AcceptanceCheck;
1083
1084    fn sample_spec() -> TaskSpec {
1085        TaskSpec {
1086            task_id: "test_task".into(),
1087            version: 1,
1088            environment_fingerprint: Some("env_123".into()),
1089            criteria: vec![
1090                AcceptanceCriterion {
1091                    id: "c1".into(),
1092                    check: AcceptanceCheck::OutputContains {
1093                        value: "pass".into(),
1094                    },
1095                    weight: 2.0,
1096                    required: true,
1097                },
1098                AcceptanceCriterion {
1099                    id: "c2".into(),
1100                    check: AcceptanceCheck::OutputContains {
1101                        value: "optional".into(),
1102                    },
1103                    weight: 2.0,
1104                    required: false,
1105                },
1106            ],
1107            budgets: TaskResourceBudget {
1108                max_cost: Some(1.0),
1109                max_active_time_ms: Some(10_000),
1110                max_model_steps: Some(5),
1111            },
1112        }
1113    }
1114
1115    fn sample_profile() -> ScoringProfile {
1116        ScoringProfile::new(0.50, 2_000).expect("valid profile")
1117    }
1118
1119    #[test]
1120    fn perfect_score_calculation() {
1121        let spec = sample_spec();
1122        let profile = sample_profile();
1123
1124        let results = vec![
1125            CriterionEvaluation::from_status("c1", AcceptanceStatus::Pass, Some("ev1".into())),
1126            CriterionEvaluation::from_status("c2", AcceptanceStatus::Pass, Some("ev2".into())),
1127        ];
1128        // Zero cost and zero time -> E_cost = 1.0, E_time = 1.0
1129        let obs = ScoringObservation {
1130            actual_cost: Some(0.0),
1131            active_time_ms: Some(0),
1132            model_steps: Some(1),
1133            evaluator_fault: None,
1134            pending: false,
1135        };
1136
1137        let score = calculate_task_score(&spec, &profile, 1, &results, &obs).unwrap();
1138        assert_eq!(score.status, TaskScoreStatus::Completed);
1139        assert_eq!(score.quality, Some(1.0));
1140        assert_eq!(score.cost_efficiency, Some(1.0));
1141        assert_eq!(score.time_efficiency, Some(1.0));
1142        assert!((score.total.unwrap() - 100.0).abs() < 1e-9);
1143        assert_eq!(score.required_passed, Some(true));
1144        assert!(!score.budget_exceeded);
1145        assert_eq!(score.evidence_ids, vec!["ev1", "ev2"]);
1146    }
1147
1148    #[test]
1149    fn reference_baseline_calculation() {
1150        let spec = sample_spec();
1151        let profile = sample_profile();
1152
1153        let results = vec![
1154            CriterionEvaluation::from_status("c1", AcceptanceStatus::Pass, Some("ev1".into())),
1155            CriterionEvaluation::from_status("c2", AcceptanceStatus::Pass, Some("ev2".into())),
1156        ];
1157        // actual == reference: E_cost = 0.5, E_time = 0.5
1158        // Multiplier = 0.70 + 0.20 * 0.5 + 0.10 * 0.5 = 0.85
1159        // Score = 100 * 1.0 * 0.85 = 85.0
1160        let obs = ScoringObservation {
1161            actual_cost: Some(0.50),
1162            active_time_ms: Some(2_000),
1163            model_steps: Some(2),
1164            evaluator_fault: None,
1165            pending: false,
1166        };
1167
1168        let score = calculate_task_score(&spec, &profile, 1, &results, &obs).unwrap();
1169        assert_eq!(score.status, TaskScoreStatus::Completed);
1170        assert_eq!(score.quality, Some(1.0));
1171        assert_eq!(score.cost_efficiency, Some(0.5));
1172        assert_eq!(score.time_efficiency, Some(0.5));
1173        assert!((score.total.unwrap() - 85.0).abs() < 1e-9);
1174    }
1175
1176    #[test]
1177    fn required_criterion_failure_forces_zero_score() {
1178        let spec = sample_spec();
1179        let profile = sample_profile();
1180
1181        let results = vec![
1182            CriterionEvaluation::from_status("c1", AcceptanceStatus::Fail, Some("ev1".into())),
1183            CriterionEvaluation::from_status("c2", AcceptanceStatus::Pass, Some("ev2".into())),
1184        ];
1185        let obs = ScoringObservation {
1186            actual_cost: Some(0.10),
1187            active_time_ms: Some(500),
1188            model_steps: Some(1),
1189            evaluator_fault: None,
1190            pending: false,
1191        };
1192
1193        let score = calculate_task_score(&spec, &profile, 1, &results, &obs).unwrap();
1194        assert_eq!(score.status, TaskScoreStatus::Completed);
1195        assert_eq!(score.total, Some(0.0));
1196        assert_eq!(score.required_passed, Some(false));
1197        assert!(
1198            score
1199                .failure_reason
1200                .unwrap()
1201                .contains("required criterion 'c1' failed")
1202        );
1203    }
1204
1205    #[test]
1206    fn required_failure_sufficient_even_if_other_evidence_missing() {
1207        let spec = sample_spec();
1208        let profile = sample_profile();
1209
1210        let results = vec![
1211            CriterionEvaluation::from_status("c1", AcceptanceStatus::Fail, None),
1212            CriterionEvaluation::from_status("c2", AcceptanceStatus::Unknown, None),
1213        ];
1214        // Cost and time missing
1215        let obs = ScoringObservation {
1216            actual_cost: None,
1217            active_time_ms: None,
1218            model_steps: None,
1219            evaluator_fault: None,
1220            pending: false,
1221        };
1222
1223        let score = calculate_task_score(&spec, &profile, 1, &results, &obs).unwrap();
1224        assert_eq!(score.status, TaskScoreStatus::Completed);
1225        assert_eq!(score.total, Some(0.0));
1226        assert_eq!(score.required_passed, Some(false));
1227    }
1228
1229    #[test]
1230    fn optional_criterion_failure_reduces_quality_proportionally() {
1231        let spec = sample_spec();
1232        let profile = sample_profile();
1233
1234        // c1 (weight 2, required) passes, c2 (weight 2, optional) fails
1235        // Normalized weights are 0.5 each. Q = 0.5 * 1.0 + 0.5 * 0.0 = 0.5
1236        let results = vec![
1237            CriterionEvaluation::from_status("c1", AcceptanceStatus::Pass, None),
1238            CriterionEvaluation::from_status("c2", AcceptanceStatus::Fail, None),
1239        ];
1240        let obs = ScoringObservation {
1241            actual_cost: Some(0.0),
1242            active_time_ms: Some(0),
1243            model_steps: Some(1),
1244            evaluator_fault: None,
1245            pending: false,
1246        };
1247
1248        let score = calculate_task_score(&spec, &profile, 1, &results, &obs).unwrap();
1249        assert_eq!(score.status, TaskScoreStatus::Completed);
1250        assert_eq!(score.quality, Some(0.5));
1251        assert!((score.total.unwrap() - 50.0).abs() < 1e-9);
1252        assert_eq!(score.required_passed, Some(true));
1253    }
1254
1255    #[test]
1256    fn declared_cost_budget_exceeded_forces_zero_score() {
1257        let spec = sample_spec();
1258        let profile = sample_profile();
1259
1260        let results = vec![
1261            CriterionEvaluation::from_status("c1", AcceptanceStatus::Pass, None),
1262            CriterionEvaluation::from_status("c2", AcceptanceStatus::Pass, None),
1263        ];
1264        // max_cost is 1.0, actual is 1.5
1265        let obs = ScoringObservation {
1266            actual_cost: Some(1.5),
1267            active_time_ms: Some(500),
1268            model_steps: Some(2),
1269            evaluator_fault: None,
1270            pending: false,
1271        };
1272
1273        let score = calculate_task_score(&spec, &profile, 1, &results, &obs).unwrap();
1274        assert_eq!(score.status, TaskScoreStatus::Completed);
1275        assert_eq!(score.total, Some(0.0));
1276        assert!(score.budget_exceeded);
1277        assert!(
1278            score
1279                .failure_reason
1280                .unwrap()
1281                .contains("cost budget exceeded")
1282        );
1283    }
1284
1285    #[test]
1286    fn declared_active_time_budget_exceeded_forces_zero_score() {
1287        let spec = sample_spec();
1288        let profile = sample_profile();
1289
1290        let results = vec![
1291            CriterionEvaluation::from_status("c1", AcceptanceStatus::Pass, None),
1292            CriterionEvaluation::from_status("c2", AcceptanceStatus::Pass, None),
1293        ];
1294        // max_active_time_ms is 10_000, actual is 12_000
1295        let obs = ScoringObservation {
1296            actual_cost: Some(0.1),
1297            active_time_ms: Some(12_000),
1298            model_steps: Some(2),
1299            evaluator_fault: None,
1300            pending: false,
1301        };
1302
1303        let score = calculate_task_score(&spec, &profile, 1, &results, &obs).unwrap();
1304        assert_eq!(score.status, TaskScoreStatus::Completed);
1305        assert_eq!(score.total, Some(0.0));
1306        assert!(score.budget_exceeded);
1307        assert!(
1308            score
1309                .failure_reason
1310                .unwrap()
1311                .contains("active time budget exceeded")
1312        );
1313    }
1314
1315    #[test]
1316    fn declared_model_steps_budget_exceeded_forces_zero_score() {
1317        let spec = sample_spec();
1318        let profile = sample_profile();
1319
1320        let results = vec![
1321            CriterionEvaluation::from_status("c1", AcceptanceStatus::Pass, None),
1322            CriterionEvaluation::from_status("c2", AcceptanceStatus::Pass, None),
1323        ];
1324        // max_model_steps is 5, actual is 6
1325        let obs = ScoringObservation {
1326            actual_cost: Some(0.1),
1327            active_time_ms: Some(500),
1328            model_steps: Some(6),
1329            evaluator_fault: None,
1330            pending: false,
1331        };
1332
1333        let score = calculate_task_score(&spec, &profile, 1, &results, &obs).unwrap();
1334        assert_eq!(score.status, TaskScoreStatus::Completed);
1335        assert_eq!(score.total, Some(0.0));
1336        assert!(score.budget_exceeded);
1337        assert!(
1338            score
1339                .failure_reason
1340                .unwrap()
1341                .contains("model steps budget exceeded")
1342        );
1343    }
1344
1345    #[test]
1346    fn missing_price_accounting_prevents_complete_score_retaining_quality() {
1347        let spec = sample_spec();
1348        let profile = sample_profile();
1349
1350        let results = vec![
1351            CriterionEvaluation::from_status("c1", AcceptanceStatus::Pass, None),
1352            CriterionEvaluation::from_status("c2", AcceptanceStatus::Pass, None),
1353        ];
1354        // actual_cost is None
1355        let obs = ScoringObservation {
1356            actual_cost: None,
1357            active_time_ms: Some(1_000),
1358            model_steps: Some(1),
1359            evaluator_fault: None,
1360            pending: false,
1361        };
1362
1363        let score = calculate_task_score(&spec, &profile, 1, &results, &obs).unwrap();
1364        assert_eq!(score.status, TaskScoreStatus::IncompleteEvidence);
1365        assert_eq!(score.total, None);
1366        assert_eq!(score.quality, Some(1.0));
1367        assert_eq!(score.cost_efficiency, None);
1368        assert!(score.time_efficiency.is_some());
1369    }
1370
1371    #[test]
1372    fn unknown_check_does_not_renormalize_weights_and_yields_incomplete() {
1373        let spec = sample_spec();
1374        let profile = sample_profile();
1375
1376        // c1 (weight 2, required) passes, c2 (weight 2, optional) is Unknown
1377        // Quality must be 0.5 * 1.0 = 0.5, NOT renormalized to 1.0!
1378        let results = vec![
1379            CriterionEvaluation::from_status("c1", AcceptanceStatus::Pass, None),
1380            CriterionEvaluation::from_status("c2", AcceptanceStatus::Unknown, None),
1381        ];
1382        let obs = ScoringObservation {
1383            actual_cost: Some(0.10),
1384            active_time_ms: Some(1_000),
1385            model_steps: Some(1),
1386            evaluator_fault: None,
1387            pending: false,
1388        };
1389
1390        let score = calculate_task_score(&spec, &profile, 1, &results, &obs).unwrap();
1391        assert_eq!(score.status, TaskScoreStatus::IncompleteEvidence);
1392        assert_eq!(score.total, None);
1393        assert_eq!(score.quality, Some(0.5));
1394    }
1395
1396    #[test]
1397    fn evaluator_infrastructure_fault_is_invalid_evaluator() {
1398        let spec = sample_spec();
1399        let profile = sample_profile();
1400
1401        let results = vec![];
1402        let obs = ScoringObservation {
1403            actual_cost: None,
1404            active_time_ms: None,
1405            model_steps: None,
1406            evaluator_fault: Some("connection timed out to docker daemon".into()),
1407            pending: false,
1408        };
1409
1410        let score = calculate_task_score(&spec, &profile, 1, &results, &obs).unwrap();
1411        assert_eq!(score.status, TaskScoreStatus::InvalidEvaluator);
1412        assert_eq!(score.total, None);
1413        assert!(
1414            score
1415                .failure_reason
1416                .unwrap()
1417                .contains("evaluator infrastructure fault")
1418        );
1419    }
1420
1421    #[test]
1422    fn pending_checks_status() {
1423        let spec = sample_spec();
1424        let profile = sample_profile();
1425
1426        let results = vec![];
1427        let obs = ScoringObservation {
1428            actual_cost: None,
1429            active_time_ms: None,
1430            model_steps: None,
1431            evaluator_fault: None,
1432            pending: true,
1433        };
1434
1435        let score = calculate_task_score(&spec, &profile, 1, &results, &obs).unwrap();
1436        assert_eq!(score.status, TaskScoreStatus::PendingChecks);
1437        assert_eq!(score.total, None);
1438    }
1439
1440    #[test]
1441    fn cost_and_time_penalties_are_monotonic() {
1442        let spec = sample_spec();
1443        let profile = sample_profile();
1444
1445        let results = vec![
1446            CriterionEvaluation::from_status("c1", AcceptanceStatus::Pass, None),
1447            CriterionEvaluation::from_status("c2", AcceptanceStatus::Pass, None),
1448        ];
1449
1450        let mut previous_score = 100.0;
1451        for cost in [0.0, 0.1, 0.25, 0.5, 0.75, 1.0] {
1452            let obs = ScoringObservation {
1453                actual_cost: Some(cost),
1454                active_time_ms: Some(100),
1455                model_steps: Some(1),
1456                evaluator_fault: None,
1457                pending: false,
1458            };
1459            let score = calculate_task_score(&spec, &profile, 1, &results, &obs)
1460                .unwrap()
1461                .total
1462                .unwrap();
1463            assert!(
1464                score <= previous_score,
1465                "score {score} must be <= previous {previous_score} as cost increases"
1466            );
1467            previous_score = score;
1468        }
1469
1470        previous_score = 100.0;
1471        for time in [0, 500, 1_000, 2_000, 5_000, 10_000] {
1472            let obs = ScoringObservation {
1473                actual_cost: Some(0.1),
1474                active_time_ms: Some(time),
1475                model_steps: Some(1),
1476                evaluator_fault: None,
1477                pending: false,
1478            };
1479            let score = calculate_task_score(&spec, &profile, 1, &results, &obs)
1480                .unwrap()
1481                .total
1482                .unwrap();
1483            assert!(
1484                score <= previous_score,
1485                "score {score} must be <= previous {previous_score} as time increases"
1486            );
1487            previous_score = score;
1488        }
1489    }
1490
1491    fn sample_snapshot(
1492        session_id: SessionId,
1493        terminal_status: &str,
1494        tools: Vec<crate::ObservedTool>,
1495        active_ms: Option<u64>,
1496    ) -> RunEvidenceSnapshot {
1497        use arabica_protocol::{
1498            CommandId, Event, EventEnvelope, EventId, EventMetadata, RunId, WorkspaceId,
1499        };
1500
1501        let event = |sequence: u64, ms: u64, ev: Event| {
1502            EventEnvelope::new(
1503                EventMetadata {
1504                    event_id: EventId::new(format!("ev-{sequence}")),
1505                    command_id: CommandId::new("cmd"),
1506                    workspace_id: WorkspaceId::new("ws1"),
1507                    session_id: session_id.clone(),
1508                    run_id: Some(RunId::new("run1")),
1509                    sequence,
1510                    occurred_at_ms: ms,
1511                },
1512                ev,
1513            )
1514        };
1515
1516        let mut snap = crate::run_evidence_snapshots(&[
1517            event(0, 0, Event::RunStarted),
1518            event(
1519                1,
1520                50,
1521                match terminal_status {
1522                    "completed" => Event::RunCompleted {
1523                        output: Some("done".into()),
1524                    },
1525                    "failed" => Event::RunFailed {
1526                        message: "fatal error".into(),
1527                    },
1528                    _ => Event::RunCancelled,
1529                },
1530            ),
1531        ])
1532        .pop()
1533        .unwrap();
1534
1535        snap.terminal_status = terminal_status.into();
1536        snap.timing.active_ms = active_ms;
1537        snap.tools = tools;
1538        snap
1539    }
1540
1541    #[test]
1542    fn live_session_clean_run_yields_full_quality() {
1543        use crate::ObservedTool;
1544        use arabica_protocol::SessionId;
1545
1546        let session_id = SessionId::new("live_sess_1");
1547        let config = LiveSessionConfig::new(session_id.clone());
1548        let profile = sample_profile();
1549
1550        let tools = vec![
1551            ObservedTool {
1552                call_id: "c1".into(),
1553                name: "read_file".into(),
1554                is_error: Some(false),
1555                runner_outcome: None,
1556            },
1557            ObservedTool {
1558                call_id: "c2".into(),
1559                name: "write_file".into(),
1560                is_error: Some(false),
1561                runner_outcome: None,
1562            },
1563        ];
1564        let snapshot = sample_snapshot(session_id, "completed", tools, Some(2000));
1565
1566        // actual == reference (0.50 cost, 2000 active ms) -> E_cost = 0.5, E_time = 0.5
1567        // multiplier = 0.70 + 0.20*0.5 + 0.10*0.5 = 0.85
1568        // Quality = 1.0 -> Score = 85.0
1569        let score =
1570            evaluate_live_session(&config, &profile, 1, &snapshot, Some(0.50), None).unwrap();
1571        assert_eq!(score.status, TaskScoreStatus::Completed);
1572        assert_eq!(score.quality, Some(1.0));
1573        assert!((score.total.unwrap() - 85.0).abs() < 1e-9);
1574        assert_eq!(score.required_passed, Some(true));
1575        assert!(!score.budget_exceeded);
1576    }
1577
1578    #[test]
1579    fn live_session_penalizes_errored_tools_proportionally() {
1580        use crate::ObservedTool;
1581        use arabica_protocol::SessionId;
1582
1583        let session_id = SessionId::new("live_sess_2");
1584        let mut config = LiveSessionConfig::new(session_id.clone());
1585        config.require_clean_terminal_tools = false;
1586        config.tool_health_weight = 0.40;
1587        let profile = sample_profile();
1588
1589        let tools = vec![
1590            ObservedTool {
1591                call_id: "c1".into(),
1592                name: "shell".into(),
1593                is_error: Some(true), // 1 of 2 tools errored
1594                runner_outcome: None,
1595            },
1596            ObservedTool {
1597                call_id: "c2".into(),
1598                name: "shell".into(),
1599                is_error: Some(false),
1600                runner_outcome: None,
1601            },
1602        ];
1603        let snapshot = sample_snapshot(session_id, "completed", tools, Some(0));
1604
1605        // Tool success ratio: 1/2 = 0.5.
1606        // Quality = 1.0 - (0.40 * (1 - 0.5)) = 1.0 - 0.20 = 0.80
1607        // Cost = 0, Time = 0 -> multiplier = 1.0
1608        // Score = 100 * 0.80 * 1.0 = 80.0
1609        let score =
1610            evaluate_live_session(&config, &profile, 1, &snapshot, Some(0.0), None).unwrap();
1611        assert_eq!(score.status, TaskScoreStatus::Completed);
1612        assert!((score.quality.unwrap() - 0.80).abs() < 1e-9);
1613        assert!((score.total.unwrap() - 80.0).abs() < 1e-9);
1614    }
1615
1616    #[test]
1617    fn live_session_fails_when_terminated_with_unrecovered_error() {
1618        use crate::ObservedTool;
1619        use arabica_protocol::SessionId;
1620
1621        let session_id = SessionId::new("live_sess_3");
1622        let config = LiveSessionConfig::new(session_id.clone());
1623        let profile = sample_profile();
1624
1625        let tools = vec![ObservedTool {
1626            call_id: "c1".into(),
1627            name: "shell".into(),
1628            is_error: Some(true), // Last tool failed and unrecovered!
1629            runner_outcome: None,
1630        }];
1631        let snapshot = sample_snapshot(session_id, "completed", tools, Some(100));
1632
1633        let score =
1634            evaluate_live_session(&config, &profile, 1, &snapshot, Some(0.0), None).unwrap();
1635        assert_eq!(score.status, TaskScoreStatus::Completed);
1636        assert_eq!(score.quality, Some(0.0));
1637        assert_eq!(score.total, Some(0.0));
1638        assert!(
1639            score
1640                .failure_reason
1641                .unwrap()
1642                .contains("unrecovered tool error")
1643        );
1644    }
1645
1646    #[test]
1647    fn live_session_fails_when_workspace_gate_fails() {
1648        use arabica_protocol::SessionId;
1649
1650        let session_id = SessionId::new("live_sess_4");
1651        let config = LiveSessionConfig::new(session_id.clone());
1652        let profile = sample_profile();
1653
1654        let snapshot = sample_snapshot(session_id, "completed", vec![], Some(100));
1655
1656        let gate_res = WorkspaceGateResult {
1657            check_passed: false,
1658            tests_passed: None,
1659            permitted_files_passed: None,
1660            overall_passed: false,
1661            failure_reasons: vec!["cargo check compilation error: undefined variable foo".into()],
1662        };
1663
1664        let score =
1665            evaluate_live_session(&config, &profile, 1, &snapshot, Some(0.01), Some(&gate_res))
1666                .unwrap();
1667        assert_eq!(score.status, TaskScoreStatus::Completed);
1668        assert_eq!(score.quality, Some(0.0));
1669        assert_eq!(score.total, Some(0.0));
1670        assert!(
1671            score
1672                .failure_reason
1673                .unwrap()
1674                .contains("cargo check compilation error")
1675        );
1676    }
1677
1678    #[test]
1679    fn workspace_gate_execution_checks_and_catches_unpermitted_files() {
1680        let temp_dir = std::env::temp_dir();
1681        let config = WorkspaceGateConfig {
1682            check_command: "echo 'build ok'".into(),
1683            test_command: Some("exit 0".into()),
1684            permitted_paths: Some(vec!["src/lib.rs".into()]),
1685            timeout_ms: 5000,
1686            output_limit_bytes: 4096,
1687        };
1688
1689        // Allowed path modified -> passes
1690        let res = execute_workspace_gate(&temp_dir, &config, Some(&["src/lib.rs".into()]));
1691        assert!(res.overall_passed);
1692        assert!(res.check_passed);
1693        assert_eq!(res.tests_passed, Some(true));
1694        assert_eq!(res.permitted_files_passed, Some(true));
1695
1696        // Unpermitted path modified -> fails gate!
1697        let res = execute_workspace_gate(&temp_dir, &config, Some(&["tests/test.rs".into()]));
1698        assert!(!res.overall_passed);
1699        assert_eq!(res.permitted_files_passed, Some(false));
1700        assert!(
1701            res.failure_reasons
1702                .join(" ")
1703                .contains("unpermitted files modified")
1704        );
1705    }
1706
1707    #[test]
1708    fn live_session_evaluation_from_history_computes_price_and_score() {
1709        use arabica_model::RuntimeUsage;
1710        use arabica_protocol::{
1711            CommandId, Event, EventEnvelope, EventId, EventMetadata, ModelCallOutcome, RunId,
1712            WorkspaceId,
1713        };
1714
1715        let session_id = SessionId::new("sess-live-pricing");
1716        let event = |sequence: u64, ms: u64, ev: Event| {
1717            EventEnvelope::new(
1718                EventMetadata {
1719                    event_id: EventId::new(format!("ev-{sequence}")),
1720                    command_id: CommandId::new("cmd"),
1721                    workspace_id: WorkspaceId::new("ws1"),
1722                    session_id: session_id.clone(),
1723                    run_id: Some(RunId::new("run1")),
1724                    sequence,
1725                    occurred_at_ms: ms,
1726                },
1727                ev,
1728            )
1729        };
1730
1731        let price_card = crate::accounting::PriceCard::new("test-card", "USD")
1732            .with_model("gpt-4o", crate::accounting::ModelPriceRule::new(2.5, 10.0));
1733
1734        let config = LiveSessionConfig::new(session_id.clone()).with_price_card(price_card);
1735        let profile = sample_profile();
1736
1737        let history = vec![
1738            event(0, 0, Event::RunStarted),
1739            event(
1740                1,
1741                50,
1742                Event::ModelRouteSelected {
1743                    model_step: 0,
1744                    decision_id: "dec-1".into(),
1745                    policy_id: "p".into(),
1746                    policy_version: 1,
1747                    policy_fingerprint: "f".into(),
1748                    model_registry_snapshot: "s".into(),
1749                    model_alias: Some("gpt-4o".into()),
1750                    reason: "main execution".into(),
1751                },
1752            ),
1753            event(
1754                2,
1755                150,
1756                Event::ModelCallObserved {
1757                    model_step: 0,
1758                    decision_id: "dec-1".into(),
1759                    elapsed_ms: 100,
1760                    provider_succeeded: true,
1761                    outcome: ModelCallOutcome::Succeeded,
1762                    usage: Some(RuntimeUsage {
1763                        input_tokens: 100_000,
1764                        output_tokens: 10_000,
1765                        cached_input_tokens: 0,
1766                        cache_creation_input_tokens: 0,
1767                        reasoning_output_tokens: 0,
1768                    }),
1769                },
1770            ),
1771            event(
1772                3,
1773                200,
1774                Event::RunCompleted {
1775                    output: Some("completed response".into()),
1776                },
1777            ),
1778        ];
1779
1780        let (score, accounting) =
1781            evaluate_live_session_from_history(&config, &profile, 1, &history, None).unwrap();
1782
1783        assert_eq!(score.status, TaskScoreStatus::Completed);
1784        assert_eq!(score.quality, Some(1.0));
1785        assert!(score.cost_efficiency.is_some());
1786        assert!(score.total.is_some());
1787
1788        assert!(accounting.is_some());
1789        let acc = accounting.unwrap();
1790        assert!(acc.coverage.is_complete);
1791        // Cost: 100k in * 2.5/1M + 10k out * 10/1M = 0.25 + 0.10 = 0.35
1792        let agent_cost = acc.total_agent_cost.unwrap();
1793        assert!((agent_cost - 0.35).abs() < 1e-9);
1794    }
1795}