selfware 0.6.1

Your personal AI workshop — software you own, software that lasts
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103

//! L2: Borrow Checker & Compiler Diagnostics (Medium)
//!
//! Input: Screenshots of `cargo check` errors (lifetime errors, type mismatches, trait bounds).
//! Tasks: Identify error type, extract error code, locate file:line, suggest fix category.
//! Scoring: Error code exact match, file path match, fix category BM25.
//! Pass threshold: 70% structured accuracy.

use async_trait::async_trait;
use std::path::PathBuf;

use crate::vlm_bench::scoring::{self, LevelScore, Rating};
use crate::vlm_bench::{BenchScenario, Difficulty, ExpectedAnswer, VlmBenchLevel};

const PASS_THRESHOLD: f64 = 0.70;

/// Level 2: Borrow Checker & Compiler Diagnostics.
pub struct L2Diagnostics {
    fixtures_dir: PathBuf,
}

impl Default for L2Diagnostics {
    fn default() -> Self {
        Self::new()
    }
}

impl L2Diagnostics {
    pub fn new() -> Self {
        Self {
            fixtures_dir: PathBuf::from("vlm_fixtures/l2_diagnostics"),
        }
    }
}

#[async_trait]
impl VlmBenchLevel for L2Diagnostics {
    fn name(&self) -> &str {
        "L2 Diagnostics"
    }

    fn difficulty(&self) -> Difficulty {
        Difficulty::Medium
    }

    fn description(&self) -> &str {
        "Compiler diagnostic reading: identify error types, extract error codes (E0xxx), \
         locate file:line references, and suggest fix categories from terminal screenshots."
    }

    fn scenarios(&self) -> Vec<BenchScenario> {
        vec![
            BenchScenario {
                id: "l2_lifetime_error".into(),
                description: "Identify a lifetime/borrow checker error".into(),
                image_path: self.fixtures_dir.join("lifetime_error.png"),
                prompt: "This screenshot shows a Rust compiler error. Analyze it and respond with JSON: \
                         {\"error_code\": \"E0xxx\", \"error_type\": \"<lifetime|borrow|type|trait>\", \
                         \"file\": \"<file path>\", \"line\": <line number>, \
                         \"fix_category\": \"<add lifetime annotation|clone|reference|restructure>\"}".into(),
                expected: ExpectedAnswer::JsonFields(serde_json::json!({
                    "error_type": "lifetime"
                })),
            },
            BenchScenario {
                id: "l2_type_mismatch".into(),
                description: "Identify a type mismatch error".into(),
                image_path: self.fixtures_dir.join("type_mismatch.png"),
                prompt: "This screenshot shows a Rust compiler error about type mismatches. \
                         Analyze it and respond with JSON: \
                         {\"error_code\": \"E0xxx\", \"error_type\": \"type\", \
                         \"expected_type\": \"<type>\", \"found_type\": \"<type>\", \
                         \"file\": \"<file path>\", \"line\": <line number>}".into(),
                expected: ExpectedAnswer::Keywords(vec![
                    "type".into(),
                    "mismatch".into(),
                    "expected".into(),
                ]),
            },
            BenchScenario {
                id: "l2_trait_bound".into(),
                description: "Identify a trait bound error".into(),
                image_path: self.fixtures_dir.join("trait_bound.png"),
                prompt: "This screenshot shows a Rust compiler error about trait bounds. \
                         What trait is missing? Respond with JSON: \
                         {\"error_code\": \"E0xxx\", \"error_type\": \"trait\", \
                         \"missing_trait\": \"<trait name>\", \"on_type\": \"<type>\", \
                         \"fix_category\": \"<derive|impl|bound|where clause>\"}".into(),
                expected: ExpectedAnswer::Keywords(vec![
                    "trait".into(),
                    "bound".into(),
                ]),
            },
        ]
    }

    fn evaluate(&self, scenario: &BenchScenario, response: &str) -> LevelScore {
        crate::vlm_bench::score_response(scenario, response, PASS_THRESHOLD)
    }
}

#[cfg(test)]
#[path = "../../../tests/unit/vlm_bench/levels/l2_diagnostics/l2_diagnostics_test.rs"]
mod tests;