swink-agent-eval 0.9.0

Evaluation framework for swink-agent: trajectory tracing, golden path verification, and cost governance
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59

//! Deterministic `cargo clippy` evaluator (T077 — clippy portion).
//!
//! Mirrors [`super::cargo_check::CargoCheckEvaluator`] but invokes
//! `cargo clippy -- -D warnings` so any warning fails the metric.

use std::sync::Arc;

use crate::evaluator::Evaluator;
use crate::evaluators::code::cargo_check::run_cargo;
use crate::evaluators::code::extractor::CodeExtractor;
use crate::evaluators::code::harness::CargoHarness;
use crate::types::{EvalCase, EvalMetricResult, Invocation};

/// Deterministic `cargo clippy` evaluator (FR-017).
pub struct ClippyEvaluator {
    name: &'static str,
    extractor: Arc<CodeExtractor>,
    harness: CargoHarness,
}

impl ClippyEvaluator {
    /// Create a new evaluator with the given extractor strategy.
    #[must_use]
    pub fn new(extractor: Arc<CodeExtractor>) -> Self {
        Self {
            name: "clippy",
            extractor,
            harness: CargoHarness::default(),
        }
    }

    /// Override the evaluator's reported name.
    #[must_use]
    pub const fn with_name(mut self, name: &'static str) -> Self {
        self.name = name;
        self
    }

    /// Override the harness used to scaffold the tempdir project.
    #[must_use]
    pub fn with_harness(mut self, harness: CargoHarness) -> Self {
        self.harness = harness;
        self
    }
}

impl Evaluator for ClippyEvaluator {
    fn name(&self) -> &'static str {
        self.name
    }

    fn evaluate(&self, _case: &EvalCase, invocation: &Invocation) -> Option<EvalMetricResult> {
        let response = invocation.final_response.as_ref()?;
        let code = crate::evaluators::block_on(self.extractor.extract(response))?;

        let outcome = run_cargo(&self.harness, &code, &["clippy", "--", "-D", "warnings"]);
        Some(outcome.into_metric_result(self.name))
    }
}