scouter_types/eval/
mod.rs

1use crate::error::TypeError;
2use crate::PyHelperFuncs;
3use pyo3::prelude::*;
4use serde::{Deserialize, Serialize};
5
6use core::fmt::Debug;
7use potato_head::prompt::ResponseType;
8use potato_head::Prompt;
9
10#[pyclass]
11#[derive(Debug, Serialize, Deserialize, Clone, PartialEq)]
12pub struct LLMEvalMetric {
13    #[pyo3(get, set)]
14    pub name: String,
15
16    #[pyo3(get)]
17    pub prompt: Prompt,
18}
19
20#[pymethods]
21impl LLMEvalMetric {
22    #[new]
23    #[pyo3(signature = (name, prompt))]
24    pub fn new(name: &str, prompt: Prompt) -> Result<Self, TypeError> {
25        // assert that the prompt is a scoring prompt
26        if prompt.response_type != ResponseType::Score {
27            return Err(TypeError::InvalidResponseType);
28        }
29        Ok(Self {
30            name: name.to_lowercase(),
31            prompt,
32        })
33    }
34
35    pub fn __str__(&self) -> String {
36        // serialize the struct to a string
37        PyHelperFuncs::__str__(self)
38    }
39}
scouter_types/eval/mod.rs

scouter_types/eval/
mod.rs
