synaptic-eval 0.4.0

Evaluation: ExactMatch, JsonValidity, Regex, EmbeddingDistance, LLMJudge evaluators
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41

use async_trait::async_trait;
use regex::Regex;
use synaptic_core::SynapticError;

use crate::evaluator::{EvalResult, Evaluator};

/// Evaluator that checks whether the prediction matches a regex pattern.
pub struct RegexMatchEvaluator {
    pattern: Regex,
}

impl RegexMatchEvaluator {
    /// Create a new regex match evaluator with the given pattern.
    ///
    /// Returns an error if the pattern is not a valid regex.
    pub fn new(pattern: &str) -> Result<Self, SynapticError> {
        let pattern = Regex::new(pattern)
            .map_err(|e| SynapticError::Validation(format!("Invalid regex pattern: {}", e)))?;
        Ok(Self { pattern })
    }
}

#[async_trait]
impl Evaluator for RegexMatchEvaluator {
    async fn evaluate(
        &self,
        prediction: &str,
        _reference: &str,
        _input: &str,
    ) -> Result<EvalResult, SynapticError> {
        if self.pattern.is_match(prediction) {
            Ok(EvalResult::pass())
        } else {
            Ok(EvalResult::fail().with_reasoning(format!(
                "Prediction {:?} does not match pattern {:?}",
                prediction,
                self.pattern.as_str()
            )))
        }
    }
}