1use std::collections::{BTreeSet, HashMap};
7
8use arabica_protocol::{EventEnvelope, RunId, SessionId};
9use serde::{Deserialize, Serialize};
10
11use crate::accounting::{PriceCard, RunCostAccounting, account_run_cost_and_time};
12use crate::{
13 AcceptanceCriterion, AcceptanceSpec, AcceptanceStatus, ArtifactEvidence, EvaluationError,
14 RunAcceptance, RunEvidenceSnapshot,
15};
16
17#[derive(Clone, Copy, Debug, Default, Deserialize, PartialEq, Serialize)]
19#[serde(deny_unknown_fields)]
20pub struct TaskResourceBudget {
21 #[serde(default, skip_serializing_if = "Option::is_none")]
23 pub max_cost: Option<f64>,
24 #[serde(default, skip_serializing_if = "Option::is_none")]
26 pub max_active_time_ms: Option<u64>,
27 #[serde(default, skip_serializing_if = "Option::is_none")]
29 pub max_model_steps: Option<usize>,
30}
31
32impl TaskResourceBudget {
33 pub fn validate(&self) -> Result<(), EvaluationError> {
34 if self
35 .max_cost
36 .is_some_and(|cost| cost < 0.0 || !cost.is_finite())
37 {
38 return Err(EvaluationError(
39 "max_cost budget must be non-negative and finite".into(),
40 ));
41 }
42 Ok(())
43 }
44}
45
46#[derive(Clone, Debug, Deserialize, PartialEq, Serialize)]
48#[serde(deny_unknown_fields)]
49pub struct TaskSpec {
50 pub task_id: String,
51 pub version: u64,
52 #[serde(default, skip_serializing_if = "Option::is_none")]
53 pub environment_fingerprint: Option<String>,
54 pub criteria: Vec<AcceptanceCriterion>,
55 #[serde(default)]
56 pub budgets: TaskResourceBudget,
57}
58
59impl TaskSpec {
60 pub fn validate(&self) -> Result<(), EvaluationError> {
61 if self.task_id.trim().is_empty() || self.version == 0 || self.criteria.len() > 128 {
62 return Err(EvaluationError("invalid task specification".into()));
63 }
64 self.budgets.validate()?;
65 let mut ids = BTreeSet::new();
66 let mut total_weight = 0.0;
67 for criterion in &self.criteria {
68 if criterion.id.trim().is_empty() || !ids.insert(&criterion.id) {
69 return Err(EvaluationError(
70 "acceptance criterion IDs must be unique and non-empty".into(),
71 ));
72 }
73 if criterion.weight < 0.0 || !criterion.weight.is_finite() {
74 return Err(EvaluationError(
75 "acceptance criterion weight must be non-negative and finite".into(),
76 ));
77 }
78 total_weight += criterion.weight;
79 criterion.check.validate()?;
80 }
81 if !self.criteria.is_empty() && total_weight <= 0.0 {
82 return Err(EvaluationError(
83 "total criterion weight must be strictly positive".into(),
84 ));
85 }
86 Ok(())
87 }
88
89 pub fn fingerprint(&self) -> String {
90 crate::context::fingerprint(self)
91 }
92
93 pub fn acceptance_spec(&self) -> AcceptanceSpec {
94 AcceptanceSpec {
95 task_id: self.task_id.clone(),
96 version: self.version,
97 criteria: self.criteria.clone(),
98 }
99 }
100}
101
102impl From<AcceptanceSpec> for TaskSpec {
103 fn from(spec: AcceptanceSpec) -> Self {
104 Self {
105 task_id: spec.task_id,
106 version: spec.version,
107 environment_fingerprint: None,
108 criteria: spec.criteria,
109 budgets: TaskResourceBudget::default(),
110 }
111 }
112}
113
114#[derive(Clone, Copy, Debug, Deserialize, PartialEq, Serialize)]
116#[serde(deny_unknown_fields)]
117pub struct ScoringWeights {
118 pub quality: f64,
119 pub cost: f64,
120 pub time: f64,
121}
122
123impl Default for ScoringWeights {
124 fn default() -> Self {
125 Self {
126 quality: 0.70,
127 cost: 0.20,
128 time: 0.10,
129 }
130 }
131}
132
133impl ScoringWeights {
134 pub fn validate(&self) -> Result<(), EvaluationError> {
135 if self.quality < 0.0
136 || !self.quality.is_finite()
137 || self.cost < 0.0
138 || !self.cost.is_finite()
139 || self.time < 0.0
140 || !self.time.is_finite()
141 {
142 return Err(EvaluationError(
143 "scoring weights must be non-negative and finite".into(),
144 ));
145 }
146 let sum = self.quality + self.cost + self.time;
147 if (sum - 1.0).abs() > 1e-6 {
148 return Err(EvaluationError(format!(
149 "scoring weights must sum to 1.0 (got {sum:.6})"
150 )));
151 }
152 Ok(())
153 }
154}
155
156#[derive(Clone, Debug, Deserialize, PartialEq, Serialize)]
158#[serde(deny_unknown_fields)]
159pub struct ScoringProfile {
160 pub formula_version: u64,
161 #[serde(default)]
162 pub weights: ScoringWeights,
163 pub reference_cost: f64,
164 pub reference_time_ms: u64,
165}
166
167impl ScoringProfile {
168 pub fn new(reference_cost: f64, reference_time_ms: u64) -> Result<Self, EvaluationError> {
169 let profile = Self {
170 formula_version: 1,
171 weights: ScoringWeights::default(),
172 reference_cost,
173 reference_time_ms,
174 };
175 profile.validate()?;
176 Ok(profile)
177 }
178
179 pub fn validate(&self) -> Result<(), EvaluationError> {
180 if self.formula_version == 0 {
181 return Err(EvaluationError("formula_version must be non-zero".into()));
182 }
183 if self.reference_cost <= 0.0 || !self.reference_cost.is_finite() {
184 return Err(EvaluationError(
185 "reference_cost must be positive and finite".into(),
186 ));
187 }
188 if self.reference_time_ms == 0 {
189 return Err(EvaluationError("reference_time_ms must be non-zero".into()));
190 }
191 self.weights.validate()?;
192 Ok(())
193 }
194
195 pub fn fingerprint(&self) -> String {
196 crate::context::fingerprint(self)
197 }
198}
199
200#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)]
202#[serde(deny_unknown_fields)]
203pub struct EvaluationIdentity {
204 pub checker_version: u64,
205 pub task_fingerprint: String,
206 pub scoring_fingerprint: String,
207}
208
209impl EvaluationIdentity {
210 pub fn new(
211 checker_version: u64,
212 task_fingerprint: impl Into<String>,
213 scoring_fingerprint: impl Into<String>,
214 ) -> Self {
215 Self {
216 checker_version,
217 task_fingerprint: task_fingerprint.into(),
218 scoring_fingerprint: scoring_fingerprint.into(),
219 }
220 }
221
222 pub fn fingerprint(&self) -> String {
223 crate::context::fingerprint(self)
224 }
225}
226
227#[derive(Clone, Copy, Debug, Deserialize, Eq, PartialEq, Serialize)]
229#[serde(rename_all = "snake_case")]
230pub enum TaskScoreStatus {
231 PendingChecks,
232 IncompleteEvidence,
233 InvalidEvaluator,
234 Completed,
235}
236
237#[derive(Clone, Debug, Deserialize, PartialEq, Serialize)]
239#[serde(deny_unknown_fields)]
240pub struct CriterionEvaluation {
241 pub criterion_id: String,
242 pub status: AcceptanceStatus,
243 #[serde(default, skip_serializing_if = "Option::is_none")]
245 pub score: Option<f64>,
246 #[serde(default, skip_serializing_if = "Option::is_none")]
247 pub evidence_id: Option<String>,
248}
249
250impl CriterionEvaluation {
251 pub fn from_status(
252 criterion_id: impl Into<String>,
253 status: AcceptanceStatus,
254 evidence_id: Option<String>,
255 ) -> Self {
256 let score = match status {
257 AcceptanceStatus::Pass => Some(1.0),
258 AcceptanceStatus::Fail => Some(0.0),
259 AcceptanceStatus::Unknown | AcceptanceStatus::NotApplicable => None,
260 };
261 Self {
262 criterion_id: criterion_id.into(),
263 status,
264 score,
265 evidence_id,
266 }
267 }
268
269 pub fn with_score(
270 criterion_id: impl Into<String>,
271 status: AcceptanceStatus,
272 score: Option<f64>,
273 evidence_id: Option<String>,
274 ) -> Self {
275 Self {
276 criterion_id: criterion_id.into(),
277 status,
278 score,
279 evidence_id,
280 }
281 }
282}
283
284#[derive(Clone, Debug, Default, Deserialize, PartialEq, Serialize)]
286#[serde(deny_unknown_fields)]
287pub struct ScoringObservation {
288 #[serde(default, skip_serializing_if = "Option::is_none")]
289 pub actual_cost: Option<f64>,
290 #[serde(default, skip_serializing_if = "Option::is_none")]
291 pub active_time_ms: Option<u64>,
292 #[serde(default, skip_serializing_if = "Option::is_none")]
293 pub model_steps: Option<usize>,
294 #[serde(default, skip_serializing_if = "Option::is_none")]
295 pub evaluator_fault: Option<String>,
296 #[serde(default)]
297 pub pending: bool,
298}
299
300#[derive(Clone, Debug, Deserialize, PartialEq, Serialize)]
302#[serde(deny_unknown_fields)]
303pub struct TaskScore {
304 pub status: TaskScoreStatus,
305 pub quality: Option<f64>,
306 pub cost_efficiency: Option<f64>,
307 pub time_efficiency: Option<f64>,
308 pub total: Option<f64>,
309 pub required_passed: Option<bool>,
310 pub budget_exceeded: bool,
311 pub evidence_ids: Vec<String>,
312 pub evaluation_identity: EvaluationIdentity,
313 #[serde(default, skip_serializing_if = "Option::is_none")]
314 pub failure_reason: Option<String>,
315}
316
317pub fn calculate_task_score(
319 task_spec: &TaskSpec,
320 profile: &ScoringProfile,
321 checker_version: u64,
322 criterion_results: &[CriterionEvaluation],
323 observation: &ScoringObservation,
324) -> Result<TaskScore, EvaluationError> {
325 task_spec.validate()?;
326 profile.validate()?;
327
328 if observation
329 .actual_cost
330 .is_some_and(|cost| cost < 0.0 || !cost.is_finite())
331 {
332 return Err(EvaluationError(
333 "actual_cost must be non-negative and finite".into(),
334 ));
335 }
336
337 let evaluation_identity = EvaluationIdentity::new(
338 checker_version,
339 task_spec.fingerprint(),
340 profile.fingerprint(),
341 );
342
343 if let Some(fault) = &observation.evaluator_fault {
345 return Ok(TaskScore {
346 status: TaskScoreStatus::InvalidEvaluator,
347 quality: None,
348 cost_efficiency: None,
349 time_efficiency: None,
350 total: None,
351 required_passed: None,
352 budget_exceeded: false,
353 evidence_ids: vec![],
354 evaluation_identity,
355 failure_reason: Some(format!("evaluator infrastructure fault: {fault}")),
356 });
357 }
358
359 if observation.pending {
361 return Ok(TaskScore {
362 status: TaskScoreStatus::PendingChecks,
363 quality: None,
364 cost_efficiency: None,
365 time_efficiency: None,
366 total: None,
367 required_passed: None,
368 budget_exceeded: false,
369 evidence_ids: vec![],
370 evaluation_identity,
371 failure_reason: None,
372 });
373 }
374
375 let mut evidence_ids: Vec<String> = criterion_results
377 .iter()
378 .filter_map(|r| r.evidence_id.clone())
379 .filter(|id| !id.trim().is_empty())
380 .collect();
381 evidence_ids.sort();
382 evidence_ids.dedup();
383
384 let total_declared_weight: f64 = task_spec.criteria.iter().map(|c| c.weight).sum();
388 let norm_factor = if total_declared_weight > 0.0 {
389 total_declared_weight
390 } else {
391 1.0
392 };
393
394 let results_by_id: HashMap<&str, &CriterionEvaluation> = criterion_results
395 .iter()
396 .map(|r| (r.criterion_id.as_str(), r))
397 .collect();
398
399 let mut quality_sum = 0.0;
400 let mut required_failure = false;
401 let mut required_unknown = false;
402 let mut missing_criterion_evidence = false;
403 let mut failure_reasons = Vec::new();
404
405 for criterion in &task_spec.criteria {
406 let norm_weight = criterion.weight / norm_factor;
407 match results_by_id.get(criterion.id.as_str()) {
408 Some(result) => match result.status {
409 AcceptanceStatus::Pass => {
410 let score = result.score.unwrap_or(1.0).clamp(0.0, 1.0);
411 quality_sum += norm_weight * score;
412 }
413 AcceptanceStatus::Fail => {
414 if criterion.required {
415 required_failure = true;
416 failure_reasons
417 .push(format!("required criterion '{}' failed", criterion.id));
418 }
419 }
420 AcceptanceStatus::Unknown => {
421 missing_criterion_evidence = true;
422 if criterion.required {
423 required_unknown = true;
424 }
425 }
426 AcceptanceStatus::NotApplicable => {}
427 },
428 None => {
429 missing_criterion_evidence = true;
430 if criterion.required {
431 required_unknown = true;
432 }
433 }
434 }
435 }
436
437 let mut budget_exceeded = false;
439 if task_spec
440 .budgets
441 .max_cost
442 .is_some_and(|max| observation.actual_cost.is_some_and(|actual| actual > max))
443 {
444 let actual = observation.actual_cost.unwrap();
445 let max = task_spec.budgets.max_cost.unwrap();
446 budget_exceeded = true;
447 failure_reasons.push(format!(
448 "declared cost budget exceeded: {actual:.4} > {max:.4}"
449 ));
450 }
451 if task_spec.budgets.max_active_time_ms.is_some_and(|max| {
452 observation
453 .active_time_ms
454 .is_some_and(|actual| actual > max)
455 }) {
456 let actual = observation.active_time_ms.unwrap();
457 let max = task_spec.budgets.max_active_time_ms.unwrap();
458 budget_exceeded = true;
459 failure_reasons.push(format!(
460 "declared active time budget exceeded: {actual}ms > {max}ms"
461 ));
462 }
463 if task_spec
464 .budgets
465 .max_model_steps
466 .is_some_and(|max| observation.model_steps.is_some_and(|actual| actual > max))
467 {
468 let actual = observation.model_steps.unwrap();
469 let max = task_spec.budgets.max_model_steps.unwrap();
470 budget_exceeded = true;
471 failure_reasons.push(format!(
472 "declared model steps budget exceeded: {actual} > {max}"
473 ));
474 }
475
476 let cost_efficiency = observation
479 .actual_cost
480 .map(|cost| 1.0 / (1.0 + cost / profile.reference_cost));
481 let time_efficiency = observation
483 .active_time_ms
484 .map(|time| 1.0 / (1.0 + (time as f64) / (profile.reference_time_ms as f64)));
485
486 let required_passed = if required_failure {
487 Some(false)
488 } else if required_unknown {
489 None
490 } else {
491 Some(true)
492 };
493
494 if required_failure || budget_exceeded {
499 return Ok(TaskScore {
500 status: TaskScoreStatus::Completed,
501 quality: Some(quality_sum),
502 cost_efficiency,
503 time_efficiency,
504 total: Some(0.0),
505 required_passed,
506 budget_exceeded,
507 evidence_ids,
508 evaluation_identity,
509 failure_reason: Some(failure_reasons.join("; ")),
510 });
511 }
512
513 let all_evidence_available = !missing_criterion_evidence
517 && observation.actual_cost.is_some()
518 && observation.active_time_ms.is_some();
519
520 if !all_evidence_available {
521 let mut missing_items = Vec::new();
522 if missing_criterion_evidence {
523 missing_items.push("criterion evidence missing or unknown");
524 }
525 if observation.actual_cost.is_none() {
526 missing_items.push("actual cost accounting missing");
527 }
528 if observation.active_time_ms.is_none() {
529 missing_items.push("active time accounting missing");
530 }
531 return Ok(TaskScore {
532 status: TaskScoreStatus::IncompleteEvidence,
533 quality: Some(quality_sum),
534 cost_efficiency,
535 time_efficiency,
536 total: None,
537 required_passed,
538 budget_exceeded: false,
539 evidence_ids,
540 evaluation_identity,
541 failure_reason: Some(format!(
542 "incomplete scoring data: {}",
543 missing_items.join("; ")
544 )),
545 });
546 }
547
548 let e_cost = cost_efficiency.expect("cost efficiency verified available");
549 let e_time = time_efficiency.expect("time efficiency verified available");
550
551 let efficiency_multiplier =
552 profile.weights.quality + profile.weights.cost * e_cost + profile.weights.time * e_time;
553 let score = 100.0 * quality_sum * efficiency_multiplier;
554
555 Ok(TaskScore {
556 status: TaskScoreStatus::Completed,
557 quality: Some(quality_sum),
558 cost_efficiency,
559 time_efficiency,
560 total: Some(score.clamp(0.0, 100.0)),
561 required_passed: Some(true),
562 budget_exceeded: false,
563 evidence_ids,
564 evaluation_identity,
565 failure_reason: None,
566 })
567}
568
569pub fn score_run_evaluation(
571 task_spec: &TaskSpec,
572 profile: &ScoringProfile,
573 checker_version: u64,
574 snapshot: &RunEvidenceSnapshot,
575 acceptance: Option<&RunAcceptance>,
576 actual_cost: Option<f64>,
577) -> Result<TaskScore, EvaluationError> {
578 let criterion_results = match acceptance {
579 Some(acc) => acc
580 .results
581 .iter()
582 .map(|r| CriterionEvaluation {
583 criterion_id: r.criterion_id.clone(),
584 status: r.status.clone(),
585 score: match r.status {
586 AcceptanceStatus::Pass => Some(1.0),
587 AcceptanceStatus::Fail => Some(0.0),
588 _ => None,
589 },
590 evidence_id: (!r.evidence.is_empty()).then(|| r.evidence.clone()),
591 })
592 .collect(),
593 None => Vec::new(),
594 };
595
596 let observation = ScoringObservation {
597 actual_cost,
598 active_time_ms: snapshot.timing.active_ms,
599 model_steps: Some(snapshot.timing.model_calls),
600 evaluator_fault: None,
601 pending: false,
602 };
603
604 calculate_task_score(
605 task_spec,
606 profile,
607 checker_version,
608 &criterion_results,
609 &observation,
610 )
611}
612
613fn default_tool_health_weight() -> f64 {
614 0.40
615}
616
617fn default_require_clean_terminal_tools() -> bool {
618 true
619}
620
621fn default_gate_timeout_ms() -> u64 {
622 30_000
623}
624
625fn default_gate_output_limit() -> usize {
626 64 * 1024
627}
628
629#[derive(Clone, Debug, Deserialize, PartialEq, Serialize)]
631#[serde(deny_unknown_fields)]
632pub struct LiveSessionConfig {
633 pub session_id: SessionId,
634 #[serde(default)]
635 pub budgets: TaskResourceBudget,
636 #[serde(default, skip_serializing_if = "Option::is_none")]
637 pub workspace_gate: Option<WorkspaceGateConfig>,
638 #[serde(default = "default_tool_health_weight")]
640 pub tool_health_weight: f64,
641 #[serde(default = "default_require_clean_terminal_tools")]
643 pub require_clean_terminal_tools: bool,
644 #[serde(default, skip_serializing_if = "Option::is_none")]
646 pub price_card: Option<PriceCard>,
647}
648
649impl LiveSessionConfig {
650 pub fn new(session_id: SessionId) -> Self {
651 Self {
652 session_id,
653 budgets: TaskResourceBudget::default(),
654 workspace_gate: None,
655 tool_health_weight: default_tool_health_weight(),
656 require_clean_terminal_tools: default_require_clean_terminal_tools(),
657 price_card: None,
658 }
659 }
660
661 pub fn with_workspace_gate(mut self, gate: WorkspaceGateConfig) -> Self {
662 self.workspace_gate = Some(gate);
663 self
664 }
665
666 pub fn with_budgets(mut self, budgets: TaskResourceBudget) -> Self {
667 self.budgets = budgets;
668 self
669 }
670
671 pub fn with_price_card(mut self, card: PriceCard) -> Self {
672 self.price_card = Some(card);
673 self
674 }
675}
676
677#[derive(Clone, Debug, Deserialize, PartialEq, Serialize)]
679#[serde(deny_unknown_fields)]
680pub struct WorkspaceGateConfig {
681 pub check_command: String,
683 #[serde(default, skip_serializing_if = "Option::is_none")]
685 pub test_command: Option<String>,
686 #[serde(default, skip_serializing_if = "Option::is_none")]
688 pub permitted_paths: Option<Vec<String>>,
689 #[serde(default = "default_gate_timeout_ms")]
691 pub timeout_ms: u64,
692 #[serde(default = "default_gate_output_limit")]
694 pub output_limit_bytes: usize,
695}
696
697impl WorkspaceGateConfig {
698 pub fn new(check_command: impl Into<String>) -> Self {
699 Self {
700 check_command: check_command.into(),
701 test_command: None,
702 permitted_paths: None,
703 timeout_ms: default_gate_timeout_ms(),
704 output_limit_bytes: default_gate_output_limit(),
705 }
706 }
707}
708
709#[derive(Clone, Debug, Deserialize, PartialEq, Serialize)]
711#[serde(deny_unknown_fields)]
712pub struct WorkspaceGateResult {
713 pub check_passed: bool,
714 pub tests_passed: Option<bool>,
715 pub permitted_files_passed: Option<bool>,
716 pub overall_passed: bool,
717 pub failure_reasons: Vec<String>,
718}
719
720#[derive(Clone, Debug, Deserialize, PartialEq, Serialize)]
722#[serde(deny_unknown_fields)]
723pub struct HistoricalTraceConfig {
724 pub session_id: SessionId,
725 #[serde(default, skip_serializing_if = "Option::is_none")]
726 pub run_id: Option<RunId>,
727}
728
729#[derive(Clone, Debug, Deserialize, PartialEq, Serialize)]
731#[serde(tag = "mode", rename_all = "snake_case")]
732pub enum EvaluationMode {
733 LiveSession(LiveSessionConfig),
735 WorkspaceGate(WorkspaceGateConfig),
737 PresetTask(TaskSpec),
739 HistoricalTrace(HistoricalTraceConfig),
741}
742
743pub fn execute_workspace_gate(
745 root: &std::path::Path,
746 config: &WorkspaceGateConfig,
747 modified_paths: Option<&[String]>,
748) -> WorkspaceGateResult {
749 let mut failure_reasons = Vec::new();
750
751 let check_evidence = crate::execute_controlled_test(
753 root,
754 &config.check_command,
755 config.timeout_ms,
756 config.output_limit_bytes,
757 );
758 let check_passed = match &check_evidence {
759 ArtifactEvidence::TestResult {
760 exit_code,
761 passed,
762 error,
763 ..
764 } => {
765 if let Some(err) = error {
766 failure_reasons.push(format!("workspace check execution failed: {err}"));
767 false
768 } else if *passed && *exit_code == Some(0) {
769 true
770 } else {
771 failure_reasons.push(format!("workspace check failed: exit code {:?}", exit_code));
772 false
773 }
774 }
775 _ => false,
776 };
777
778 let mut tests_passed = None;
780 if let Some(test_cmd) = &config.test_command {
781 let test_evidence = crate::execute_controlled_test(
782 root,
783 test_cmd,
784 config.timeout_ms,
785 config.output_limit_bytes,
786 );
787 let passed = match &test_evidence {
788 ArtifactEvidence::TestResult {
789 exit_code,
790 passed,
791 error,
792 ..
793 } => {
794 if let Some(err) = error {
795 failure_reasons
796 .push(format!("workspace regression test execution failed: {err}"));
797 false
798 } else if *passed && *exit_code == Some(0) {
799 true
800 } else {
801 failure_reasons.push(format!(
802 "workspace regression tests failed: exit code {:?}",
803 exit_code
804 ));
805 false
806 }
807 }
808 _ => false,
809 };
810 tests_passed = Some(passed);
811 }
812
813 let mut permitted_passed = None;
815 if let (Some(allowed), Some(modified)) = (&config.permitted_paths, modified_paths) {
816 let allowed_set: std::collections::BTreeSet<&str> =
817 allowed.iter().map(|s| s.as_str()).collect();
818 let mut violations = Vec::new();
819 for path in modified {
820 if !allowed_set.contains(path.as_str()) {
821 violations.push(path.clone());
822 }
823 }
824 if !violations.is_empty() {
825 failure_reasons.push(format!(
826 "unpermitted files modified: {}",
827 violations.join(", ")
828 ));
829 permitted_passed = Some(false);
830 } else {
831 permitted_passed = Some(true);
832 }
833 }
834
835 let overall_passed =
836 check_passed && tests_passed.unwrap_or(true) && permitted_passed.unwrap_or(true);
837
838 WorkspaceGateResult {
839 check_passed,
840 tests_passed,
841 permitted_files_passed: permitted_passed,
842 overall_passed,
843 failure_reasons,
844 }
845}
846
847pub fn evaluate_live_session(
849 config: &LiveSessionConfig,
850 profile: &ScoringProfile,
851 checker_version: u64,
852 snapshot: &RunEvidenceSnapshot,
853 actual_cost: Option<f64>,
854 workspace_gate_result: Option<&WorkspaceGateResult>,
855) -> Result<TaskScore, EvaluationError> {
856 profile.validate()?;
857 config.budgets.validate()?;
858
859 if snapshot.session_id != config.session_id {
860 return Err(EvaluationError(format!(
861 "session_id mismatch: expected '{}', got '{}'",
862 config.session_id, snapshot.session_id
863 )));
864 }
865
866 if actual_cost.is_some_and(|c| c < 0.0 || !c.is_finite()) {
867 return Err(EvaluationError(
868 "actual_cost must be non-negative and finite".into(),
869 ));
870 }
871
872 let evaluation_identity = EvaluationIdentity::new(
873 checker_version,
874 format!("live_session:{}", config.session_id),
875 profile.fingerprint(),
876 );
877
878 let mut failure_reasons = Vec::new();
879 let evidence_ids = vec![snapshot.fingerprint.clone()];
880
881 let terminal_completed = snapshot.terminal_status == "completed";
883 if !terminal_completed {
884 failure_reasons.push(format!(
885 "session ended with non-completed terminal status: '{}'",
886 snapshot.terminal_status
887 ));
888 }
889
890 let total_tools = snapshot.tools.len();
892 let errored_tools = snapshot
893 .tools
894 .iter()
895 .filter(|t| t.is_error == Some(true))
896 .count();
897 let incomplete_tools = snapshot.timing.incomplete_tool_calls;
898
899 let tool_success_ratio = if total_tools > 0 {
900 (total_tools.saturating_sub(errored_tools)) as f64 / total_tools as f64
901 } else {
902 1.0
903 };
904
905 let last_tool_failed = snapshot
906 .tools
907 .last()
908 .and_then(|t| t.is_error)
909 .unwrap_or(false);
910
911 let mut tool_fatal_failure = false;
912 if config.require_clean_terminal_tools && last_tool_failed {
913 tool_fatal_failure = true;
914 failure_reasons.push("session ended with an unrecovered tool error".into());
915 }
916 if incomplete_tools > 0 {
917 failure_reasons.push(format!("{incomplete_tools} tool calls remained incomplete"));
918 }
919
920 let mut gate_passed = true;
922 if let Some(gate) = workspace_gate_result
923 && !gate.overall_passed
924 {
925 gate_passed = false;
926 failure_reasons.extend(gate.failure_reasons.clone());
927 }
928
929 let mut budget_exceeded = false;
931 if config
932 .budgets
933 .max_cost
934 .is_some_and(|max| actual_cost.is_some_and(|actual| actual > max))
935 {
936 let actual = actual_cost.unwrap();
937 let max = config.budgets.max_cost.unwrap();
938 budget_exceeded = true;
939 failure_reasons.push(format!(
940 "declared cost budget exceeded: {actual:.4} > {max:.4}"
941 ));
942 }
943 if config
944 .budgets
945 .max_active_time_ms
946 .is_some_and(|max| snapshot.timing.active_ms.is_some_and(|actual| actual > max))
947 {
948 let actual = snapshot.timing.active_ms.unwrap();
949 let max = config.budgets.max_active_time_ms.unwrap();
950 budget_exceeded = true;
951 failure_reasons.push(format!(
952 "declared active time budget exceeded: {actual}ms > {max}ms"
953 ));
954 }
955 if config
956 .budgets
957 .max_model_steps
958 .is_some_and(|max| max < snapshot.timing.model_calls)
959 {
960 budget_exceeded = true;
961 failure_reasons.push(format!(
962 "declared model steps budget exceeded: {} > {}",
963 snapshot.timing.model_calls,
964 config.budgets.max_model_steps.unwrap()
965 ));
966 }
967
968 let cost_efficiency = actual_cost.map(|cost| 1.0 / (1.0 + cost / profile.reference_cost));
970 let time_efficiency = snapshot
971 .timing
972 .active_ms
973 .map(|time| 1.0 / (1.0 + (time as f64) / (profile.reference_time_ms as f64)));
974
975 if !terminal_completed || tool_fatal_failure || !gate_passed || budget_exceeded {
977 return Ok(TaskScore {
978 status: TaskScoreStatus::Completed,
979 quality: Some(0.0),
980 cost_efficiency,
981 time_efficiency,
982 total: Some(0.0),
983 required_passed: Some(false),
984 budget_exceeded,
985 evidence_ids,
986 evaluation_identity,
987 failure_reason: Some(failure_reasons.join("; ")),
988 });
989 }
990
991 let quality = 1.0 - (config.tool_health_weight * (1.0 - tool_success_ratio));
995 let quality = quality.clamp(0.0, 1.0);
996
997 let all_evidence_available = actual_cost.is_some() && snapshot.timing.active_ms.is_some();
999
1000 if !all_evidence_available {
1001 let mut missing = Vec::new();
1002 if actual_cost.is_none() {
1003 missing.push("actual cost accounting missing");
1004 }
1005 if snapshot.timing.active_ms.is_none() {
1006 missing.push("active time accounting missing");
1007 }
1008 return Ok(TaskScore {
1009 status: TaskScoreStatus::IncompleteEvidence,
1010 quality: Some(quality),
1011 cost_efficiency,
1012 time_efficiency,
1013 total: None,
1014 required_passed: Some(true),
1015 budget_exceeded: false,
1016 evidence_ids,
1017 evaluation_identity,
1018 failure_reason: Some(format!("incomplete scoring data: {}", missing.join("; "))),
1019 });
1020 }
1021
1022 let e_cost = cost_efficiency.expect("cost efficiency available");
1023 let e_time = time_efficiency.expect("time efficiency available");
1024
1025 let efficiency_multiplier =
1026 profile.weights.quality + profile.weights.cost * e_cost + profile.weights.time * e_time;
1027 let score = 100.0 * quality * efficiency_multiplier;
1028
1029 Ok(TaskScore {
1030 status: TaskScoreStatus::Completed,
1031 quality: Some(quality),
1032 cost_efficiency,
1033 time_efficiency,
1034 total: Some(score.clamp(0.0, 100.0)),
1035 required_passed: Some(true),
1036 budget_exceeded: false,
1037 evidence_ids,
1038 evaluation_identity,
1039 failure_reason: None,
1040 })
1041}
1042
1043pub fn evaluate_live_session_from_history(
1046 config: &LiveSessionConfig,
1047 profile: &ScoringProfile,
1048 checker_version: u64,
1049 history: &[EventEnvelope],
1050 workspace_gate_result: Option<&WorkspaceGateResult>,
1051) -> Result<(TaskScore, Option<RunCostAccounting>), EvaluationError> {
1052 let mut snapshots = crate::run_evidence_snapshots(history);
1053 let snapshot = snapshots.pop().ok_or_else(|| {
1054 EvaluationError("history does not contain any terminal run snapshot".into())
1055 })?;
1056
1057 let accounting = match &config.price_card {
1058 Some(card) => Some(
1059 account_run_cost_and_time(history, Some(&snapshot.run_id), card)
1060 .map_err(|e| EvaluationError(e.0))?,
1061 ),
1062 None => None,
1063 };
1064
1065 let actual_cost = accounting.as_ref().and_then(|a| a.total_agent_cost);
1066
1067 let score = evaluate_live_session(
1068 config,
1069 profile,
1070 checker_version,
1071 &snapshot,
1072 actual_cost,
1073 workspace_gate_result,
1074 )?;
1075
1076 Ok((score, accounting))
1077}
1078
1079#[cfg(test)]
1080mod tests {
1081 use super::*;
1082 use crate::AcceptanceCheck;
1083
1084 fn sample_spec() -> TaskSpec {
1085 TaskSpec {
1086 task_id: "test_task".into(),
1087 version: 1,
1088 environment_fingerprint: Some("env_123".into()),
1089 criteria: vec![
1090 AcceptanceCriterion {
1091 id: "c1".into(),
1092 check: AcceptanceCheck::OutputContains {
1093 value: "pass".into(),
1094 },
1095 weight: 2.0,
1096 required: true,
1097 },
1098 AcceptanceCriterion {
1099 id: "c2".into(),
1100 check: AcceptanceCheck::OutputContains {
1101 value: "optional".into(),
1102 },
1103 weight: 2.0,
1104 required: false,
1105 },
1106 ],
1107 budgets: TaskResourceBudget {
1108 max_cost: Some(1.0),
1109 max_active_time_ms: Some(10_000),
1110 max_model_steps: Some(5),
1111 },
1112 }
1113 }
1114
1115 fn sample_profile() -> ScoringProfile {
1116 ScoringProfile::new(0.50, 2_000).expect("valid profile")
1117 }
1118
1119 #[test]
1120 fn perfect_score_calculation() {
1121 let spec = sample_spec();
1122 let profile = sample_profile();
1123
1124 let results = vec![
1125 CriterionEvaluation::from_status("c1", AcceptanceStatus::Pass, Some("ev1".into())),
1126 CriterionEvaluation::from_status("c2", AcceptanceStatus::Pass, Some("ev2".into())),
1127 ];
1128 let obs = ScoringObservation {
1130 actual_cost: Some(0.0),
1131 active_time_ms: Some(0),
1132 model_steps: Some(1),
1133 evaluator_fault: None,
1134 pending: false,
1135 };
1136
1137 let score = calculate_task_score(&spec, &profile, 1, &results, &obs).unwrap();
1138 assert_eq!(score.status, TaskScoreStatus::Completed);
1139 assert_eq!(score.quality, Some(1.0));
1140 assert_eq!(score.cost_efficiency, Some(1.0));
1141 assert_eq!(score.time_efficiency, Some(1.0));
1142 assert!((score.total.unwrap() - 100.0).abs() < 1e-9);
1143 assert_eq!(score.required_passed, Some(true));
1144 assert!(!score.budget_exceeded);
1145 assert_eq!(score.evidence_ids, vec!["ev1", "ev2"]);
1146 }
1147
1148 #[test]
1149 fn reference_baseline_calculation() {
1150 let spec = sample_spec();
1151 let profile = sample_profile();
1152
1153 let results = vec![
1154 CriterionEvaluation::from_status("c1", AcceptanceStatus::Pass, Some("ev1".into())),
1155 CriterionEvaluation::from_status("c2", AcceptanceStatus::Pass, Some("ev2".into())),
1156 ];
1157 let obs = ScoringObservation {
1161 actual_cost: Some(0.50),
1162 active_time_ms: Some(2_000),
1163 model_steps: Some(2),
1164 evaluator_fault: None,
1165 pending: false,
1166 };
1167
1168 let score = calculate_task_score(&spec, &profile, 1, &results, &obs).unwrap();
1169 assert_eq!(score.status, TaskScoreStatus::Completed);
1170 assert_eq!(score.quality, Some(1.0));
1171 assert_eq!(score.cost_efficiency, Some(0.5));
1172 assert_eq!(score.time_efficiency, Some(0.5));
1173 assert!((score.total.unwrap() - 85.0).abs() < 1e-9);
1174 }
1175
1176 #[test]
1177 fn required_criterion_failure_forces_zero_score() {
1178 let spec = sample_spec();
1179 let profile = sample_profile();
1180
1181 let results = vec![
1182 CriterionEvaluation::from_status("c1", AcceptanceStatus::Fail, Some("ev1".into())),
1183 CriterionEvaluation::from_status("c2", AcceptanceStatus::Pass, Some("ev2".into())),
1184 ];
1185 let obs = ScoringObservation {
1186 actual_cost: Some(0.10),
1187 active_time_ms: Some(500),
1188 model_steps: Some(1),
1189 evaluator_fault: None,
1190 pending: false,
1191 };
1192
1193 let score = calculate_task_score(&spec, &profile, 1, &results, &obs).unwrap();
1194 assert_eq!(score.status, TaskScoreStatus::Completed);
1195 assert_eq!(score.total, Some(0.0));
1196 assert_eq!(score.required_passed, Some(false));
1197 assert!(
1198 score
1199 .failure_reason
1200 .unwrap()
1201 .contains("required criterion 'c1' failed")
1202 );
1203 }
1204
1205 #[test]
1206 fn required_failure_sufficient_even_if_other_evidence_missing() {
1207 let spec = sample_spec();
1208 let profile = sample_profile();
1209
1210 let results = vec![
1211 CriterionEvaluation::from_status("c1", AcceptanceStatus::Fail, None),
1212 CriterionEvaluation::from_status("c2", AcceptanceStatus::Unknown, None),
1213 ];
1214 let obs = ScoringObservation {
1216 actual_cost: None,
1217 active_time_ms: None,
1218 model_steps: None,
1219 evaluator_fault: None,
1220 pending: false,
1221 };
1222
1223 let score = calculate_task_score(&spec, &profile, 1, &results, &obs).unwrap();
1224 assert_eq!(score.status, TaskScoreStatus::Completed);
1225 assert_eq!(score.total, Some(0.0));
1226 assert_eq!(score.required_passed, Some(false));
1227 }
1228
1229 #[test]
1230 fn optional_criterion_failure_reduces_quality_proportionally() {
1231 let spec = sample_spec();
1232 let profile = sample_profile();
1233
1234 let results = vec![
1237 CriterionEvaluation::from_status("c1", AcceptanceStatus::Pass, None),
1238 CriterionEvaluation::from_status("c2", AcceptanceStatus::Fail, None),
1239 ];
1240 let obs = ScoringObservation {
1241 actual_cost: Some(0.0),
1242 active_time_ms: Some(0),
1243 model_steps: Some(1),
1244 evaluator_fault: None,
1245 pending: false,
1246 };
1247
1248 let score = calculate_task_score(&spec, &profile, 1, &results, &obs).unwrap();
1249 assert_eq!(score.status, TaskScoreStatus::Completed);
1250 assert_eq!(score.quality, Some(0.5));
1251 assert!((score.total.unwrap() - 50.0).abs() < 1e-9);
1252 assert_eq!(score.required_passed, Some(true));
1253 }
1254
1255 #[test]
1256 fn declared_cost_budget_exceeded_forces_zero_score() {
1257 let spec = sample_spec();
1258 let profile = sample_profile();
1259
1260 let results = vec![
1261 CriterionEvaluation::from_status("c1", AcceptanceStatus::Pass, None),
1262 CriterionEvaluation::from_status("c2", AcceptanceStatus::Pass, None),
1263 ];
1264 let obs = ScoringObservation {
1266 actual_cost: Some(1.5),
1267 active_time_ms: Some(500),
1268 model_steps: Some(2),
1269 evaluator_fault: None,
1270 pending: false,
1271 };
1272
1273 let score = calculate_task_score(&spec, &profile, 1, &results, &obs).unwrap();
1274 assert_eq!(score.status, TaskScoreStatus::Completed);
1275 assert_eq!(score.total, Some(0.0));
1276 assert!(score.budget_exceeded);
1277 assert!(
1278 score
1279 .failure_reason
1280 .unwrap()
1281 .contains("cost budget exceeded")
1282 );
1283 }
1284
1285 #[test]
1286 fn declared_active_time_budget_exceeded_forces_zero_score() {
1287 let spec = sample_spec();
1288 let profile = sample_profile();
1289
1290 let results = vec![
1291 CriterionEvaluation::from_status("c1", AcceptanceStatus::Pass, None),
1292 CriterionEvaluation::from_status("c2", AcceptanceStatus::Pass, None),
1293 ];
1294 let obs = ScoringObservation {
1296 actual_cost: Some(0.1),
1297 active_time_ms: Some(12_000),
1298 model_steps: Some(2),
1299 evaluator_fault: None,
1300 pending: false,
1301 };
1302
1303 let score = calculate_task_score(&spec, &profile, 1, &results, &obs).unwrap();
1304 assert_eq!(score.status, TaskScoreStatus::Completed);
1305 assert_eq!(score.total, Some(0.0));
1306 assert!(score.budget_exceeded);
1307 assert!(
1308 score
1309 .failure_reason
1310 .unwrap()
1311 .contains("active time budget exceeded")
1312 );
1313 }
1314
1315 #[test]
1316 fn declared_model_steps_budget_exceeded_forces_zero_score() {
1317 let spec = sample_spec();
1318 let profile = sample_profile();
1319
1320 let results = vec![
1321 CriterionEvaluation::from_status("c1", AcceptanceStatus::Pass, None),
1322 CriterionEvaluation::from_status("c2", AcceptanceStatus::Pass, None),
1323 ];
1324 let obs = ScoringObservation {
1326 actual_cost: Some(0.1),
1327 active_time_ms: Some(500),
1328 model_steps: Some(6),
1329 evaluator_fault: None,
1330 pending: false,
1331 };
1332
1333 let score = calculate_task_score(&spec, &profile, 1, &results, &obs).unwrap();
1334 assert_eq!(score.status, TaskScoreStatus::Completed);
1335 assert_eq!(score.total, Some(0.0));
1336 assert!(score.budget_exceeded);
1337 assert!(
1338 score
1339 .failure_reason
1340 .unwrap()
1341 .contains("model steps budget exceeded")
1342 );
1343 }
1344
1345 #[test]
1346 fn missing_price_accounting_prevents_complete_score_retaining_quality() {
1347 let spec = sample_spec();
1348 let profile = sample_profile();
1349
1350 let results = vec![
1351 CriterionEvaluation::from_status("c1", AcceptanceStatus::Pass, None),
1352 CriterionEvaluation::from_status("c2", AcceptanceStatus::Pass, None),
1353 ];
1354 let obs = ScoringObservation {
1356 actual_cost: None,
1357 active_time_ms: Some(1_000),
1358 model_steps: Some(1),
1359 evaluator_fault: None,
1360 pending: false,
1361 };
1362
1363 let score = calculate_task_score(&spec, &profile, 1, &results, &obs).unwrap();
1364 assert_eq!(score.status, TaskScoreStatus::IncompleteEvidence);
1365 assert_eq!(score.total, None);
1366 assert_eq!(score.quality, Some(1.0));
1367 assert_eq!(score.cost_efficiency, None);
1368 assert!(score.time_efficiency.is_some());
1369 }
1370
1371 #[test]
1372 fn unknown_check_does_not_renormalize_weights_and_yields_incomplete() {
1373 let spec = sample_spec();
1374 let profile = sample_profile();
1375
1376 let results = vec![
1379 CriterionEvaluation::from_status("c1", AcceptanceStatus::Pass, None),
1380 CriterionEvaluation::from_status("c2", AcceptanceStatus::Unknown, None),
1381 ];
1382 let obs = ScoringObservation {
1383 actual_cost: Some(0.10),
1384 active_time_ms: Some(1_000),
1385 model_steps: Some(1),
1386 evaluator_fault: None,
1387 pending: false,
1388 };
1389
1390 let score = calculate_task_score(&spec, &profile, 1, &results, &obs).unwrap();
1391 assert_eq!(score.status, TaskScoreStatus::IncompleteEvidence);
1392 assert_eq!(score.total, None);
1393 assert_eq!(score.quality, Some(0.5));
1394 }
1395
1396 #[test]
1397 fn evaluator_infrastructure_fault_is_invalid_evaluator() {
1398 let spec = sample_spec();
1399 let profile = sample_profile();
1400
1401 let results = vec![];
1402 let obs = ScoringObservation {
1403 actual_cost: None,
1404 active_time_ms: None,
1405 model_steps: None,
1406 evaluator_fault: Some("connection timed out to docker daemon".into()),
1407 pending: false,
1408 };
1409
1410 let score = calculate_task_score(&spec, &profile, 1, &results, &obs).unwrap();
1411 assert_eq!(score.status, TaskScoreStatus::InvalidEvaluator);
1412 assert_eq!(score.total, None);
1413 assert!(
1414 score
1415 .failure_reason
1416 .unwrap()
1417 .contains("evaluator infrastructure fault")
1418 );
1419 }
1420
1421 #[test]
1422 fn pending_checks_status() {
1423 let spec = sample_spec();
1424 let profile = sample_profile();
1425
1426 let results = vec![];
1427 let obs = ScoringObservation {
1428 actual_cost: None,
1429 active_time_ms: None,
1430 model_steps: None,
1431 evaluator_fault: None,
1432 pending: true,
1433 };
1434
1435 let score = calculate_task_score(&spec, &profile, 1, &results, &obs).unwrap();
1436 assert_eq!(score.status, TaskScoreStatus::PendingChecks);
1437 assert_eq!(score.total, None);
1438 }
1439
1440 #[test]
1441 fn cost_and_time_penalties_are_monotonic() {
1442 let spec = sample_spec();
1443 let profile = sample_profile();
1444
1445 let results = vec![
1446 CriterionEvaluation::from_status("c1", AcceptanceStatus::Pass, None),
1447 CriterionEvaluation::from_status("c2", AcceptanceStatus::Pass, None),
1448 ];
1449
1450 let mut previous_score = 100.0;
1451 for cost in [0.0, 0.1, 0.25, 0.5, 0.75, 1.0] {
1452 let obs = ScoringObservation {
1453 actual_cost: Some(cost),
1454 active_time_ms: Some(100),
1455 model_steps: Some(1),
1456 evaluator_fault: None,
1457 pending: false,
1458 };
1459 let score = calculate_task_score(&spec, &profile, 1, &results, &obs)
1460 .unwrap()
1461 .total
1462 .unwrap();
1463 assert!(
1464 score <= previous_score,
1465 "score {score} must be <= previous {previous_score} as cost increases"
1466 );
1467 previous_score = score;
1468 }
1469
1470 previous_score = 100.0;
1471 for time in [0, 500, 1_000, 2_000, 5_000, 10_000] {
1472 let obs = ScoringObservation {
1473 actual_cost: Some(0.1),
1474 active_time_ms: Some(time),
1475 model_steps: Some(1),
1476 evaluator_fault: None,
1477 pending: false,
1478 };
1479 let score = calculate_task_score(&spec, &profile, 1, &results, &obs)
1480 .unwrap()
1481 .total
1482 .unwrap();
1483 assert!(
1484 score <= previous_score,
1485 "score {score} must be <= previous {previous_score} as time increases"
1486 );
1487 previous_score = score;
1488 }
1489 }
1490
1491 fn sample_snapshot(
1492 session_id: SessionId,
1493 terminal_status: &str,
1494 tools: Vec<crate::ObservedTool>,
1495 active_ms: Option<u64>,
1496 ) -> RunEvidenceSnapshot {
1497 use arabica_protocol::{
1498 CommandId, Event, EventEnvelope, EventId, EventMetadata, RunId, WorkspaceId,
1499 };
1500
1501 let event = |sequence: u64, ms: u64, ev: Event| {
1502 EventEnvelope::new(
1503 EventMetadata {
1504 event_id: EventId::new(format!("ev-{sequence}")),
1505 command_id: CommandId::new("cmd"),
1506 workspace_id: WorkspaceId::new("ws1"),
1507 session_id: session_id.clone(),
1508 run_id: Some(RunId::new("run1")),
1509 sequence,
1510 occurred_at_ms: ms,
1511 },
1512 ev,
1513 )
1514 };
1515
1516 let mut snap = crate::run_evidence_snapshots(&[
1517 event(0, 0, Event::RunStarted),
1518 event(
1519 1,
1520 50,
1521 match terminal_status {
1522 "completed" => Event::RunCompleted {
1523 output: Some("done".into()),
1524 },
1525 "failed" => Event::RunFailed {
1526 message: "fatal error".into(),
1527 },
1528 _ => Event::RunCancelled,
1529 },
1530 ),
1531 ])
1532 .pop()
1533 .unwrap();
1534
1535 snap.terminal_status = terminal_status.into();
1536 snap.timing.active_ms = active_ms;
1537 snap.tools = tools;
1538 snap
1539 }
1540
1541 #[test]
1542 fn live_session_clean_run_yields_full_quality() {
1543 use crate::ObservedTool;
1544 use arabica_protocol::SessionId;
1545
1546 let session_id = SessionId::new("live_sess_1");
1547 let config = LiveSessionConfig::new(session_id.clone());
1548 let profile = sample_profile();
1549
1550 let tools = vec![
1551 ObservedTool {
1552 call_id: "c1".into(),
1553 name: "read_file".into(),
1554 is_error: Some(false),
1555 runner_outcome: None,
1556 },
1557 ObservedTool {
1558 call_id: "c2".into(),
1559 name: "write_file".into(),
1560 is_error: Some(false),
1561 runner_outcome: None,
1562 },
1563 ];
1564 let snapshot = sample_snapshot(session_id, "completed", tools, Some(2000));
1565
1566 let score =
1570 evaluate_live_session(&config, &profile, 1, &snapshot, Some(0.50), None).unwrap();
1571 assert_eq!(score.status, TaskScoreStatus::Completed);
1572 assert_eq!(score.quality, Some(1.0));
1573 assert!((score.total.unwrap() - 85.0).abs() < 1e-9);
1574 assert_eq!(score.required_passed, Some(true));
1575 assert!(!score.budget_exceeded);
1576 }
1577
1578 #[test]
1579 fn live_session_penalizes_errored_tools_proportionally() {
1580 use crate::ObservedTool;
1581 use arabica_protocol::SessionId;
1582
1583 let session_id = SessionId::new("live_sess_2");
1584 let mut config = LiveSessionConfig::new(session_id.clone());
1585 config.require_clean_terminal_tools = false;
1586 config.tool_health_weight = 0.40;
1587 let profile = sample_profile();
1588
1589 let tools = vec![
1590 ObservedTool {
1591 call_id: "c1".into(),
1592 name: "shell".into(),
1593 is_error: Some(true), runner_outcome: None,
1595 },
1596 ObservedTool {
1597 call_id: "c2".into(),
1598 name: "shell".into(),
1599 is_error: Some(false),
1600 runner_outcome: None,
1601 },
1602 ];
1603 let snapshot = sample_snapshot(session_id, "completed", tools, Some(0));
1604
1605 let score =
1610 evaluate_live_session(&config, &profile, 1, &snapshot, Some(0.0), None).unwrap();
1611 assert_eq!(score.status, TaskScoreStatus::Completed);
1612 assert!((score.quality.unwrap() - 0.80).abs() < 1e-9);
1613 assert!((score.total.unwrap() - 80.0).abs() < 1e-9);
1614 }
1615
1616 #[test]
1617 fn live_session_fails_when_terminated_with_unrecovered_error() {
1618 use crate::ObservedTool;
1619 use arabica_protocol::SessionId;
1620
1621 let session_id = SessionId::new("live_sess_3");
1622 let config = LiveSessionConfig::new(session_id.clone());
1623 let profile = sample_profile();
1624
1625 let tools = vec![ObservedTool {
1626 call_id: "c1".into(),
1627 name: "shell".into(),
1628 is_error: Some(true), runner_outcome: None,
1630 }];
1631 let snapshot = sample_snapshot(session_id, "completed", tools, Some(100));
1632
1633 let score =
1634 evaluate_live_session(&config, &profile, 1, &snapshot, Some(0.0), None).unwrap();
1635 assert_eq!(score.status, TaskScoreStatus::Completed);
1636 assert_eq!(score.quality, Some(0.0));
1637 assert_eq!(score.total, Some(0.0));
1638 assert!(
1639 score
1640 .failure_reason
1641 .unwrap()
1642 .contains("unrecovered tool error")
1643 );
1644 }
1645
1646 #[test]
1647 fn live_session_fails_when_workspace_gate_fails() {
1648 use arabica_protocol::SessionId;
1649
1650 let session_id = SessionId::new("live_sess_4");
1651 let config = LiveSessionConfig::new(session_id.clone());
1652 let profile = sample_profile();
1653
1654 let snapshot = sample_snapshot(session_id, "completed", vec![], Some(100));
1655
1656 let gate_res = WorkspaceGateResult {
1657 check_passed: false,
1658 tests_passed: None,
1659 permitted_files_passed: None,
1660 overall_passed: false,
1661 failure_reasons: vec!["cargo check compilation error: undefined variable foo".into()],
1662 };
1663
1664 let score =
1665 evaluate_live_session(&config, &profile, 1, &snapshot, Some(0.01), Some(&gate_res))
1666 .unwrap();
1667 assert_eq!(score.status, TaskScoreStatus::Completed);
1668 assert_eq!(score.quality, Some(0.0));
1669 assert_eq!(score.total, Some(0.0));
1670 assert!(
1671 score
1672 .failure_reason
1673 .unwrap()
1674 .contains("cargo check compilation error")
1675 );
1676 }
1677
1678 #[test]
1679 fn workspace_gate_execution_checks_and_catches_unpermitted_files() {
1680 let temp_dir = std::env::temp_dir();
1681 let config = WorkspaceGateConfig {
1682 check_command: "echo 'build ok'".into(),
1683 test_command: Some("exit 0".into()),
1684 permitted_paths: Some(vec!["src/lib.rs".into()]),
1685 timeout_ms: 5000,
1686 output_limit_bytes: 4096,
1687 };
1688
1689 let res = execute_workspace_gate(&temp_dir, &config, Some(&["src/lib.rs".into()]));
1691 assert!(res.overall_passed);
1692 assert!(res.check_passed);
1693 assert_eq!(res.tests_passed, Some(true));
1694 assert_eq!(res.permitted_files_passed, Some(true));
1695
1696 let res = execute_workspace_gate(&temp_dir, &config, Some(&["tests/test.rs".into()]));
1698 assert!(!res.overall_passed);
1699 assert_eq!(res.permitted_files_passed, Some(false));
1700 assert!(
1701 res.failure_reasons
1702 .join(" ")
1703 .contains("unpermitted files modified")
1704 );
1705 }
1706
1707 #[test]
1708 fn live_session_evaluation_from_history_computes_price_and_score() {
1709 use arabica_model::RuntimeUsage;
1710 use arabica_protocol::{
1711 CommandId, Event, EventEnvelope, EventId, EventMetadata, ModelCallOutcome, RunId,
1712 WorkspaceId,
1713 };
1714
1715 let session_id = SessionId::new("sess-live-pricing");
1716 let event = |sequence: u64, ms: u64, ev: Event| {
1717 EventEnvelope::new(
1718 EventMetadata {
1719 event_id: EventId::new(format!("ev-{sequence}")),
1720 command_id: CommandId::new("cmd"),
1721 workspace_id: WorkspaceId::new("ws1"),
1722 session_id: session_id.clone(),
1723 run_id: Some(RunId::new("run1")),
1724 sequence,
1725 occurred_at_ms: ms,
1726 },
1727 ev,
1728 )
1729 };
1730
1731 let price_card = crate::accounting::PriceCard::new("test-card", "USD")
1732 .with_model("gpt-4o", crate::accounting::ModelPriceRule::new(2.5, 10.0));
1733
1734 let config = LiveSessionConfig::new(session_id.clone()).with_price_card(price_card);
1735 let profile = sample_profile();
1736
1737 let history = vec![
1738 event(0, 0, Event::RunStarted),
1739 event(
1740 1,
1741 50,
1742 Event::ModelRouteSelected {
1743 model_step: 0,
1744 decision_id: "dec-1".into(),
1745 policy_id: "p".into(),
1746 policy_version: 1,
1747 policy_fingerprint: "f".into(),
1748 model_registry_snapshot: "s".into(),
1749 model_alias: Some("gpt-4o".into()),
1750 reason: "main execution".into(),
1751 },
1752 ),
1753 event(
1754 2,
1755 150,
1756 Event::ModelCallObserved {
1757 model_step: 0,
1758 decision_id: "dec-1".into(),
1759 elapsed_ms: 100,
1760 provider_succeeded: true,
1761 outcome: ModelCallOutcome::Succeeded,
1762 usage: Some(RuntimeUsage {
1763 input_tokens: 100_000,
1764 output_tokens: 10_000,
1765 cached_input_tokens: 0,
1766 cache_creation_input_tokens: 0,
1767 reasoning_output_tokens: 0,
1768 }),
1769 },
1770 ),
1771 event(
1772 3,
1773 200,
1774 Event::RunCompleted {
1775 output: Some("completed response".into()),
1776 },
1777 ),
1778 ];
1779
1780 let (score, accounting) =
1781 evaluate_live_session_from_history(&config, &profile, 1, &history, None).unwrap();
1782
1783 assert_eq!(score.status, TaskScoreStatus::Completed);
1784 assert_eq!(score.quality, Some(1.0));
1785 assert!(score.cost_efficiency.is_some());
1786 assert!(score.total.is_some());
1787
1788 assert!(accounting.is_some());
1789 let acc = accounting.unwrap();
1790 assert!(acc.coverage.is_complete);
1791 let agent_cost = acc.total_agent_cost.unwrap();
1793 assert!((agent_cost - 0.35).abs() < 1e-9);
1794 }
1795}