assay-core 2.17.0

High-performance evaluation framework for LLM agents (Core)
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56

use serde::{Deserialize, Serialize};

#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct ClosestMatch {
    pub prompt: String,
    pub similarity: f64,
}

pub fn closest_prompt<'a>(
    needle: &str,
    hay: impl Iterator<Item = &'a String>,
) -> Option<ClosestMatch> {
    let mut best: Option<ClosestMatch> = None;

    // Threshold for suggestion. 0.55 is a reasonable heuristic.
    const THRESHOLD: f64 = 0.55;

    for candidate in hay {
        let sim = strsim::normalized_levenshtein(needle, candidate);
        if sim >= THRESHOLD && best.as_ref().is_none_or(|b| sim > b.similarity) {
            best = Some(ClosestMatch {
                prompt: candidate.clone(),
                similarity: sim,
            });
        }
    }
    best
}

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn test_closest_prompt_exact() {
        let candidates = ["foo".to_string(), "bar".to_string()];
        let hit = closest_prompt("foo", candidates.iter()).unwrap();
        assert_eq!(hit.prompt, "foo");
        assert!((hit.similarity - 1.0).abs() < f64::EPSILON);
    }

    #[test]
    fn test_closest_prompt_typo() {
        let candidates = ["capitol".to_string(), "bar".to_string()];
        let hit = closest_prompt("capital", candidates.iter()).unwrap();
        assert_eq!(hit.prompt, "capitol");
        assert!(hit.similarity > 0.8);
    }

    #[test]
    fn test_closest_prompt_none() {
        let candidates = ["zulu".to_string(), "bar".to_string()];
        let hit = closest_prompt("alpha", candidates.iter());
        assert!(hit.is_none());
    }
}