synaptic-eval 0.4.0

Evaluation: ExactMatch, JsonValidity, Regex, EmbeddingDistance, LLMJudge evaluators
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50

use async_trait::async_trait;
use synaptic_core::SynapticError;

use crate::evaluator::{EvalResult, Evaluator};

/// Evaluator that checks for exact string match between prediction and reference.
pub struct ExactMatchEvaluator {
    ignore_case: bool,
}

impl ExactMatchEvaluator {
    /// Create a case-sensitive exact match evaluator.
    pub fn new() -> Self {
        Self { ignore_case: false }
    }

    /// Create a case-insensitive exact match evaluator.
    pub fn case_insensitive() -> Self {
        Self { ignore_case: true }
    }
}

impl Default for ExactMatchEvaluator {
    fn default() -> Self {
        Self::new()
    }
}

#[async_trait]
impl Evaluator for ExactMatchEvaluator {
    async fn evaluate(
        &self,
        prediction: &str,
        reference: &str,
        _input: &str,
    ) -> Result<EvalResult, SynapticError> {
        let matches = if self.ignore_case {
            prediction.to_lowercase() == reference.to_lowercase()
        } else {
            prediction == reference
        };

        if matches {
            Ok(EvalResult::pass())
        } else {
            Ok(EvalResult::fail()
                .with_reasoning(format!("Expected {:?}, got {:?}", reference, prediction)))
        }
    }
}