forge_engine/lab/
evaluate.rs

1use std::collections::BTreeSet;
2
3use crate::config::ForgeConfig;
4use crate::error::ForgeResult;
5use crate::exec::backend::CheckResult;
6use crate::lab::suite::{EvalTask, TaskExpected};
7use crate::runtime::novelty;
8use crate::runtime::patch::types::StructuredPatch;
9use crate::store::ForgeStore;
10
11/// Score vector for an evaluation run.
12#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)]
13pub struct ScoreVector {
14    pub correctness: f64,
15    pub novelty: f64,
16    pub stability: f64,
17    pub weighted_total: f64,
18    pub cea_confidence: Option<f64>,
19    pub cea_predicted_correctness: Option<f64>,
20}
21
22/// Result of a single evaluation run.
23#[derive(Debug)]
24pub struct EvalRunResult {
25    pub eval_id: String,
26    pub candidate_id: String,
27    pub task_id: String,
28    pub scores: ScoreVector,
29    pub check_result: Option<CheckResult>,
30    pub patch_hash: String,
31    pub structural_sig: String,
32    pub mindstate_hash: String,
33}
34
35/// Compute raw correctness score from check results.
36///
37/// Returns a raw score in [0.0, 1.0]. Sub-weights are fixed:
38/// fmt=0.10, clippy=0.30, test=0.60.
39/// If a check is not required by the task (`TaskExpected`), full credit
40/// is given for that sub-weight regardless of the actual result.
41/// Task weights are NOT applied here — they are applied in `compute_scores`.
42pub fn compute_correctness(check: &CheckResult, expected: &TaskExpected) -> f64 {
43    let mut score = 0.0_f64;
44    if !expected.require_fmt || check.fmt_pass {
45        score += 0.10;
46    }
47    if !expected.require_clippy || check.clippy_pass {
48        score += 0.30;
49    }
50    if !expected.require_tests || check.test_pass {
51        score += 0.60;
52    }
53    score // in [0.0, 1.0]
54}
55
56/// Compute raw novelty score from strategy tags and recent traces.
57///
58/// Returns a raw score in [0.0, 1.0]. Task weights are NOT applied here.
59pub fn compute_novelty_score(
60    patch: &StructuredPatch,
61    store: &ForgeStore,
62    question_sig: &str,
63    config: &ForgeConfig,
64) -> ForgeResult<f64> {
65    let tags = novelty::extract_strategy_tags(patch);
66
67    // Get recent traces
68    let traces = store.get_recent_traces_for_question(
69        question_sig,
70        config.novelty.min_traces_for_orthogonality,
71    )?;
72
73    let recent_tags_union: BTreeSet<String> = traces
74        .iter()
75        .flat_map(|t| {
76            serde_json::from_str::<Vec<String>>(&t.strategy_tags_json).unwrap_or_default()
77        })
78        .collect();
79
80    let tag_novelty = novelty::compute_tag_novelty(&tags, &recent_tags_union);
81
82    // Anti-fluff guard: if tags are identical to last trace AND patch structure
83    // matches, novelty is 0.
84    if !traces.is_empty() {
85        let last = &traces[0];
86        let last_tags: Vec<String> =
87            serde_json::from_str(&last.strategy_tags_json).unwrap_or_default();
88
89        if tags == last_tags {
90            // Compute a lightweight structural hash of this patch
91            let current_patch_sig = {
92                let mut h = blake3::Hasher::new();
93                for edit in &patch.edits {
94                    h.update(edit.path.to_string_lossy().as_bytes());
95                    h.update(&(edit.ops.len() as u64).to_le_bytes());
96                }
97                h.finalize().to_hex().to_string()
98            };
99            // Compare structural sig to the stored one
100            if current_patch_sig == last.structural_sig {
101                return Ok(0.0);
102            }
103        }
104    }
105
106    Ok(tag_novelty)
107}
108
109/// Compute full score vector.
110///
111/// `correctness`, `novelty`, and `stability` are raw metrics in [0.0, 1.0].
112/// `weighted_total` applies the task weights: w_c * correctness + w_n * novelty + w_s * stability.
113/// Gates (`correctness_gate`, `promotion_min_suite_pass_rate`) apply to the raw metrics.
114pub fn compute_scores(
115    check: &CheckResult,
116    patch: &StructuredPatch,
117    task: &EvalTask,
118    store: &ForgeStore,
119    question_sig: &str,
120    config: &ForgeConfig,
121) -> ForgeResult<ScoreVector> {
122    let correctness = compute_correctness(check, &task.expected);
123    let novelty_raw = compute_novelty_score(patch, store, question_sig, config).unwrap_or(0.0);
124    let stability = 0.0_f64; // Single-run default
125
126    let w = &task.weights;
127    let weighted_total =
128        w.correctness * correctness + w.novelty * novelty_raw + w.stability * stability;
129
130    Ok(ScoreVector {
131        correctness,
132        novelty: novelty_raw,
133        stability,
134        weighted_total,
135        cea_confidence: None,
136        cea_predicted_correctness: None,
137    })
138}
139
140/// Persist an evaluation run to the store.
141pub fn persist_eval_run(
142    store: &ForgeStore,
143    result: &EvalRunResult,
144    backend: &str,
145    violations_json: &str,
146    logs_ref: &str,
147    cea_run_hash: Option<&str>,
148) -> ForgeResult<()> {
149    let scores_json = serde_json::to_string(&result.scores)?;
150
151    store.insert_eval_run(
152        &result.eval_id,
153        &result.candidate_id,
154        &result.task_id,
155        backend,
156        0, // seed
157        &result.mindstate_hash,
158        &result.patch_hash,
159        &result.structural_sig,
160        &scores_json,
161        violations_json,
162        logs_ref,
163        cea_run_hash,
164    )
165}
forge_engine/lab/evaluate.rs

forge_engine/lab/
evaluate.rs
