synaptic-eval 0.4.0

Evaluation: ExactMatch, JsonValidity, Regex, EmbeddingDistance, LLMJudge evaluators
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34

use async_trait::async_trait;
use synaptic_core::SynapticError;

use crate::evaluator::{EvalResult, Evaluator};

/// Evaluator that checks whether the prediction is valid JSON.
pub struct JsonValidityEvaluator;

impl JsonValidityEvaluator {
    pub fn new() -> Self {
        Self
    }
}

impl Default for JsonValidityEvaluator {
    fn default() -> Self {
        Self::new()
    }
}

#[async_trait]
impl Evaluator for JsonValidityEvaluator {
    async fn evaluate(
        &self,
        prediction: &str,
        _reference: &str,
        _input: &str,
    ) -> Result<EvalResult, SynapticError> {
        match serde_json::from_str::<serde_json::Value>(prediction) {
            Ok(_) => Ok(EvalResult::pass()),
            Err(e) => Ok(EvalResult::fail().with_reasoning(format!("Invalid JSON: {}", e))),
        }
    }
}