selfware 0.6.0

Your personal AI workshop — software you own, software that lasts
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122

//! Candidate management for multi-candidate SWE-bench Pro generation.
//!
//! Each (quant, instance, trial) may produce multiple candidate patches.
//! `CandidatePool` provides honest selection and pass@k metrics.

use serde::{Deserialize, Serialize};

/// Result of an official SWE-bench Pro Docker evaluation.
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct OfficialEvalResult {
    pub resolved: bool,
}

/// Placeholder for test-run results (populated when a candidate is
/// evaluated against the instance's test suite).
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct TestResults {
    pub passed: bool,
}

/// A single generated patch candidate.
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct Candidate {
    pub trial: u32,
    pub patch: String,
    pub patch_bytes: usize,
    pub patch_lines: usize,
    pub has_source_edit: bool,
    pub has_test_edit: bool,
    pub syntax_check_passed: bool,
    pub test_results: Option<TestResults>,
    pub official_eval: Option<OfficialEvalResult>,
}

/// Collection of candidates for a single (quant, instance).
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct CandidatePool {
    pub candidates: Vec<Candidate>,
}

impl CandidatePool {
    pub fn new(candidates: Vec<Candidate>) -> Self {
        Self { candidates }
    }

    /// Select the best candidate using honest proxy metrics.
    ///
    /// Selection criteria (in order of priority):
    /// 1. Official eval `resolved=true` if available.
    /// 2. Non-empty source diff + no test edits.
    /// 3. Smaller diff (fewer lines changed).
    /// 4. Passes cheap syntax checks.
    pub fn select_best(&self) -> Option<&Candidate> {
        self.candidates.iter().max_by(|a, b| {
            // 1. Official eval resolved=true
            let a_official = a
                .official_eval
                .as_ref()
                .map(|e| e.resolved)
                .unwrap_or(false);
            let b_official = b
                .official_eval
                .as_ref()
                .map(|e| e.resolved)
                .unwrap_or(false);
            a_official
                .cmp(&b_official)
                // 2. Non-empty source diff + no test edits
                .then_with(|| {
                    let a_good = a.has_source_edit && !a.has_test_edit;
                    let b_good = b.has_source_edit && !b.has_test_edit;
                    a_good.cmp(&b_good)
                })
                // 3. Smaller diff (fewer lines changed)
                .then_with(|| b.patch_lines.cmp(&a.patch_lines))
                // 4. Passes cheap syntax checks
                .then_with(|| a.syntax_check_passed.cmp(&b.syntax_check_passed))
        })
    }

    /// `pass@1` — did the honestly-selected best candidate resolve?
    pub fn pass_at_1(&self) -> bool {
        self.select_best()
            .and_then(|c| c.official_eval.as_ref())
            .map(|e| e.resolved)
            .unwrap_or(false)
    }

    /// `pass@k` oracle — did *any* candidate resolve according to official
    /// eval?  When no candidate has official-eval data this falls back to
    /// proxy metrics (source edit, no test edits, syntax ok) and is therefore
    /// an **upper bound**.
    pub fn pass_at_k_oracle(&self) -> bool {
        if self.has_any_official_eval() {
            return self.candidates.iter().any(|c| {
                c.official_eval
                    .as_ref()
                    .map(|e| e.resolved)
                    .unwrap_or(false)
            });
        }
        // Proxy-based upper bound when official eval is not available.
        self.candidates
            .iter()
            .any(|c| c.has_source_edit && !c.has_test_edit && c.syntax_check_passed)
    }

    /// Returns `true` when at least one candidate has official-eval data.
    pub fn has_any_official_eval(&self) -> bool {
        self.candidates.iter().any(|c| c.official_eval.is_some())
    }

    /// Returns `true` when every candidate in the pool has official-eval
    /// data.  Used by reporting to label `pass@k_oracle` accurately.
    pub fn all_have_official_eval(&self) -> bool {
        !self.candidates.is_empty() && self.candidates.iter().all(|c| c.official_eval.is_some())
    }
}

#[cfg(test)]
#[path = "../../../tests/unit/bench_harness/swebench_pro/candidate/candidate_test.rs"]
mod tests;