use crate::algo;
use crate::preprocess::Preprocessor;
use crate::SimiError;
pub type Score = f64;
#[derive(Clone, Debug, PartialEq)]
pub enum Strategy {
Cascade,
}
#[derive(Clone, Debug)]
pub enum Threshold {
GreaterThan(f64),
LessThan(f64),
Between(f64, f64),
}
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub enum Intent {
Names,
Typos,
Codes,
Documents,
Deduplication,
Auto,
}
#[derive(Clone, Debug, PartialEq, Eq)]
pub enum Algo {
Levenshtein,
JaroWinkler,
Hamming,
Jaccard(usize), JaccardBigram,
JaccardTrigram,
JaccardWord,
MinHash(usize, usize), MinHashDefault,
SimHash(usize), SimHashDefault,
Bm25,
TfIdf,
}
#[derive(Clone, Debug)]
pub struct ComparisonResult {
pub score: Score,
pub tier: usize,
pub algorithm: String,
pub fallback_called: bool,
pub fallback_data: Option<String>,
}
pub type FallbackFn = Box<dyn Fn(&str, &str) -> (Score, Option<String>) + Send + Sync>;
pub struct SimiFlow {
strategy: Strategy,
preprocessor: Option<Preprocessor>,
tier_1: Option<(Algo, Threshold, Threshold)>, tier_2: Option<(Algo, Threshold)>,
fallback: Option<FallbackFn>,
auto_mode: bool,
}
impl Default for SimiFlow {
fn default() -> Self {
Self {
strategy: Strategy::Cascade,
preprocessor: Some(Preprocessor::default()),
tier_1: None,
tier_2: None,
fallback: None,
auto_mode: false,
}
}
}
impl SimiFlow {
#[inline]
pub fn new() -> Self {
Self::default()
}
#[inline]
pub fn for_intent(intent: Intent) -> Self {
let algo = resolve_intent(intent, "", "");
let match_thresh = match intent {
Intent::Names | Intent::Typos => Threshold::GreaterThan(0.95),
Intent::Codes => Threshold::GreaterThan(0.95),
Intent::Documents => Threshold::GreaterThan(0.90),
Intent::Deduplication => Threshold::GreaterThan(0.90),
Intent::Auto => Threshold::GreaterThan(0.95),
};
let mismatch_thresh = match intent {
Intent::Names | Intent::Typos => Threshold::LessThan(0.10),
Intent::Codes => Threshold::LessThan(0.10),
Intent::Documents => Threshold::LessThan(0.05),
Intent::Deduplication => Threshold::LessThan(0.10),
Intent::Auto => Threshold::LessThan(0.10),
};
Self {
strategy: Strategy::Cascade,
preprocessor: Some(Preprocessor::default()),
tier_1: Some((algo, match_thresh, mismatch_thresh)),
tier_2: None,
fallback: None,
auto_mode: false,
}
}
#[inline]
pub fn auto() -> Self {
Self {
strategy: Strategy::Cascade,
preprocessor: Some(Preprocessor::default()),
tier_1: None,
tier_2: None,
fallback: None,
auto_mode: true,
}
}
#[inline]
pub fn compare_with_intent(
&self,
intent: Intent,
a: &str,
b: &str,
) -> Result<ComparisonResult, SimiError> {
let a = if let Some(ref pre) = self.preprocessor {
pre.process(a)
} else {
a.to_string()
};
let b = if let Some(ref pre) = self.preprocessor {
pre.process(b)
} else {
b.to_string()
};
let algo = resolve_intent(intent, &a, &b);
let (score, name) = run_algorithm(&algo, &a, &b)?;
Ok(ComparisonResult {
score,
tier: 0,
algorithm: name,
fallback_called: false,
fallback_data: None,
})
}
#[inline]
pub fn preprocess(mut self, enable: bool) -> Self {
self.preprocessor = if enable {
Some(Preprocessor::default())
} else {
None
};
self
}
#[inline]
pub fn with_preprocessor(mut self, pre: Preprocessor) -> Self {
self.preprocessor = Some(pre);
self
}
#[inline]
pub fn strategy(mut self, s: Strategy) -> Self {
self.strategy = s;
self
}
#[inline]
pub fn tier_1(
mut self,
algo: Algo,
match_threshold: Threshold,
mismatch_threshold: Threshold,
) -> Self {
self.tier_1 = Some((algo, match_threshold, mismatch_threshold));
self
}
#[inline]
pub fn tier_2(mut self, algo: Algo, threshold: Threshold) -> Self {
self.tier_2 = Some((algo, threshold));
self
}
#[inline]
pub fn fallback<F>(mut self, f: F) -> Self
where
F: Fn(&str, &str) -> (Score, Option<String>) + Send + Sync + 'static,
{
self.fallback = Some(Box::new(f));
self
}
#[inline]
pub fn compare(&self, a: &str, b: &str) -> Result<ComparisonResult, SimiError> {
if self.auto_mode {
return self.compare_with_intent(Intent::Auto, a, b);
}
let a = if let Some(ref pre) = self.preprocessor {
pre.process(a)
} else {
a.to_string()
};
let b = if let Some(ref pre) = self.preprocessor {
pre.process(b)
} else {
b.to_string()
};
match self.strategy {
Strategy::Cascade => self.run_cascade(&a, &b),
}
}
fn run_cascade(&self, a: &str, b: &str) -> Result<ComparisonResult, SimiError> {
if let Some((ref algo, ref match_thresh, ref mismatch_thresh)) = self.tier_1 {
let (score, algo_name) = run_algorithm(algo, a, b)?;
if let Threshold::GreaterThan(t) = match_thresh {
if score > *t {
return Ok(ComparisonResult {
score,
tier: 1,
algorithm: algo_name,
fallback_called: false,
fallback_data: None,
});
}
}
if let Threshold::LessThan(t) = mismatch_thresh {
if score < *t {
return Ok(ComparisonResult {
score,
tier: 1,
algorithm: algo_name,
fallback_called: false,
fallback_data: None,
});
}
}
}
if let Some((ref algo, ref threshold)) = self.tier_2 {
let (score, algo_name) = run_algorithm(algo, a, b)?;
let in_range = match threshold {
Threshold::Between(lo, hi) => score >= *lo && score <= *hi,
Threshold::GreaterThan(t) => score > *t,
Threshold::LessThan(t) => score < *t,
};
if in_range {
return Ok(ComparisonResult {
score,
tier: 2,
algorithm: algo_name,
fallback_called: false,
fallback_data: None,
});
}
}
if let Some(ref fallback) = self.fallback {
let (score, data) = fallback(a, b);
return Ok(ComparisonResult {
score,
tier: 3,
algorithm: "fallback".to_string(),
fallback_called: true,
fallback_data: data,
});
}
if let Some((ref algo, _, _)) = self.tier_1 {
let (score, algo_name) = run_algorithm(algo, a, b)?;
return Ok(ComparisonResult {
score,
tier: 1,
algorithm: algo_name,
fallback_called: false,
fallback_data: None,
});
}
Err(SimiError::RouterError("No tiers configured".to_string()))
}
}
pub fn resolve_intent(intent: Intent, a: &str, b: &str) -> Algo {
match intent {
Intent::Names => Algo::JaroWinkler,
Intent::Typos => Algo::Levenshtein,
Intent::Codes => Algo::Hamming,
Intent::Documents => Algo::Bm25,
Intent::Deduplication => Algo::SimHashDefault,
Intent::Auto => auto_select(a, b),
}
}
fn auto_select(a: &str, b: &str) -> Algo {
let max_len = a.len().max(b.len());
if a.len() == b.len() && max_len <= 20 {
return Algo::Hamming;
}
if max_len <= 50 {
return Algo::JaroWinkler;
}
if max_len <= 500 {
return Algo::Bm25;
}
Algo::SimHashDefault
}
fn run_algorithm(algo: &Algo, a: &str, b: &str) -> Result<(f64, String), SimiError> {
match algo {
Algo::Levenshtein => Ok((algo::levenshtein::similarity(a, b), "levenshtein".into())),
Algo::JaroWinkler => Ok((algo::jaro_winkler::similarity(a, b), "jaro_winkler".into())),
Algo::Hamming => algo::hamming::similarity(a, b)
.map(|s| (s, "hamming".into()))
.ok_or_else(|| {
SimiError::AlgorithmError("Hamming requires equal-length strings".into())
}),
Algo::Jaccard(n) => Ok((algo::jaccard::similarity(a, b, *n), "jaccard".into())),
Algo::JaccardBigram => Ok((
algo::jaccard::bigram_similarity(a, b),
"jaccard_bigram".into(),
)),
Algo::JaccardTrigram => Ok((
algo::jaccard::trigram_similarity(a, b),
"jaccard_trigram".into(),
)),
Algo::JaccardWord => Ok((algo::jaccard::word_similarity(a, b), "jaccard_word".into())),
Algo::MinHash(shingle_size, num_hashes) => {
let s = algo::minhash::compare(a, b, *shingle_size, *num_hashes);
Ok((s, "minhash".into()))
}
Algo::MinHashDefault => {
let s = algo::minhash::compare_default(a, b);
Ok((s, "minhash".into()))
}
Algo::SimHash(shingle_size) => {
let s = algo::simhash::compare(a, b, *shingle_size);
Ok((s, "simhash".into()))
}
Algo::SimHashDefault => {
let s = algo::simhash::compare_default(a, b);
Ok((s, "simhash".into()))
}
Algo::Bm25 => Ok((algo::bm25::similarity(a, b), "bm25".into())),
Algo::TfIdf => Ok((algo::tfidf::similarity(a, b), "tfidf".into())),
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn basic_comparison() {
let result = SimiFlow::new()
.tier_1(
Algo::JaroWinkler,
Threshold::GreaterThan(0.95),
Threshold::LessThan(0.10),
)
.compare("MARTHA", "MARHTA")
.unwrap();
assert_eq!(result.tier, 1);
assert!((result.score - 0.961).abs() < 0.01);
}
#[test]
fn tier_2_fallback() {
let result = SimiFlow::new()
.tier_1(
Algo::Levenshtein,
Threshold::GreaterThan(0.95),
Threshold::LessThan(0.10),
)
.tier_2(Algo::Bm25, Threshold::Between(0.30, 0.95))
.compare("the quick brown fox", "the quick blue fox")
.unwrap();
assert_eq!(result.tier, 2);
assert_eq!(result.algorithm, "bm25");
assert!(result.score > 0.3 && result.score < 0.95);
}
#[test]
fn fallback_called() {
let result = SimiFlow::new()
.tier_1(
Algo::Levenshtein,
Threshold::GreaterThan(0.99),
Threshold::LessThan(0.01),
)
.fallback(|a, b| {
let score = if a == b { 1.0 } else { 0.5 };
(score, Some("llm_verified".to_string()))
})
.compare("hello", "world")
.unwrap();
assert_eq!(result.tier, 3);
assert!(result.fallback_called);
assert_eq!(result.fallback_data, Some("llm_verified".to_string()));
}
#[test]
fn tier_1_mismatch() {
let result = SimiFlow::new()
.tier_1(
Algo::Levenshtein,
Threshold::GreaterThan(0.95),
Threshold::LessThan(0.10),
)
.compare("abc", "xyz")
.unwrap();
assert_eq!(result.tier, 1);
assert!(result.score < 0.01);
}
#[test]
fn with_preprocessing() {
let result = SimiFlow::new()
.preprocess(true)
.tier_1(
Algo::Levenshtein,
Threshold::GreaterThan(0.95),
Threshold::LessThan(0.10),
)
.compare(" Hello World ", "hello world")
.unwrap();
assert!((result.score - 1.0).abs() < f64::EPSILON);
}
#[test]
fn no_fallback_returns_tier_1() {
let result = SimiFlow::new()
.tier_1(
Algo::Levenshtein,
Threshold::GreaterThan(0.99),
Threshold::LessThan(0.01),
)
.compare("kitten", "sitting")
.unwrap();
assert!(result.tier == 1 || result.tier == 2); }
#[test]
fn builder_pattern() {
let flow = SimiFlow::new()
.strategy(Strategy::Cascade)
.tier_1(
Algo::JaroWinkler,
Threshold::GreaterThan(0.95),
Threshold::LessThan(0.10),
)
.tier_2(Algo::Bm25, Threshold::Between(0.30, 0.95));
let result = flow.compare("hello world", "hello world").unwrap();
assert!((result.score - 1.0).abs() < 0.01);
assert_eq!(result.tier, 1);
}
#[test]
fn intent_names_uses_jaro_winkler() {
let flow = SimiFlow::for_intent(Intent::Names);
let result = flow.compare("MARTHA", "MARHTA").unwrap();
assert_eq!(result.algorithm, "jaro_winkler");
assert!((result.score - 0.961).abs() < 0.01);
}
#[test]
fn intent_typos_uses_levenshtein() {
let flow = SimiFlow::for_intent(Intent::Typos);
let result = flow.compare("kitten", "sitting").unwrap();
assert_eq!(result.algorithm, "levenshtein");
}
#[test]
fn intent_codes_uses_hamming() {
let flow = SimiFlow::for_intent(Intent::Codes);
let result = flow.compare("hello", "hello").unwrap();
assert_eq!(result.algorithm, "hamming");
}
#[test]
fn intent_documents_uses_bm25() {
let flow = SimiFlow::for_intent(Intent::Documents);
let result = flow
.compare("the quick brown fox", "the quick brown fox")
.unwrap();
assert_eq!(result.algorithm, "bm25");
}
#[test]
fn intent_deduplication_uses_simhash() {
let flow = SimiFlow::for_intent(Intent::Deduplication);
let result = flow
.compare("the quick brown fox", "the quick brown fox")
.unwrap();
assert_eq!(result.algorithm, "simhash");
}
#[test]
fn auto_select_short_picks_hamming() {
let flow = SimiFlow::auto();
let result = flow.compare("abc", "abc").unwrap();
assert_eq!(result.algorithm, "hamming");
}
#[test]
fn auto_select_medium_picks_jaro_winkler() {
let flow = SimiFlow::auto();
let result = flow
.compare("the quick brown fox", "the quick lazy dog")
.unwrap();
assert_eq!(result.algorithm, "jaro_winkler");
}
#[test]
fn auto_select_long_picks_bm25() {
let flow = SimiFlow::auto();
let a = "the quick brown fox jumps over the lazy dog near the river bank on a sunny day";
let b = "the quick brown fox jumps over the lazy cat near the river bank on a rainy day";
let result = flow.compare(a, b).unwrap();
assert_eq!(result.algorithm, "bm25");
}
#[test]
fn compare_with_intent_bypasses_tiers() {
let flow = SimiFlow::new().tier_1(
Algo::Levenshtein,
Threshold::GreaterThan(0.99),
Threshold::LessThan(0.01),
);
let result = flow
.compare_with_intent(Intent::Names, "MARTHA", "MARHTA")
.unwrap();
assert_eq!(result.algorithm, "jaro_winkler");
assert_eq!(result.tier, 0);
}
#[test]
fn compare_with_intent_all_variants() {
let flow = SimiFlow::new();
for (intent, expected_algo, a, b) in [
(Intent::Names, "jaro_winkler", "MARTHA", "MARHTA"),
(Intent::Typos, "levenshtein", "kitten", "sitting"),
(
Intent::Documents,
"bm25",
"the quick brown fox",
"the quick brown fox",
),
(
Intent::Deduplication,
"simhash",
"hello world",
"hello world",
),
] {
let r = flow.compare_with_intent(intent, a, b).unwrap();
assert_eq!(r.algorithm, expected_algo, "intent {intent:?}");
assert_normalized(r.score);
}
}
#[test]
fn auto_select_boundary_20_equal() {
let a = "x".repeat(20);
let b = "y".repeat(20);
assert_eq!(auto_select(&a, &b), Algo::Hamming);
}
#[test]
fn auto_select_boundary_21_jaro_winkler() {
let a = "x".repeat(21);
let b = "y".repeat(21);
assert_eq!(auto_select(&a, &b), Algo::JaroWinkler);
}
#[test]
fn auto_select_unequal_short_jaro_winkler() {
assert_eq!(auto_select("hello", "world!"), Algo::JaroWinkler);
}
#[test]
fn auto_select_boundary_50() {
let a = "x".repeat(50);
let b = "y".repeat(50);
assert_eq!(auto_select(&a, &b), Algo::JaroWinkler);
}
#[test]
fn auto_select_boundary_51_bm25() {
let a = "x".repeat(51);
let b = "y".repeat(51);
assert_eq!(auto_select(&a, &b), Algo::Bm25);
}
#[test]
fn auto_select_boundary_500() {
let a = "x".repeat(500);
let b = "y".repeat(500);
assert_eq!(auto_select(&a, &b), Algo::Bm25);
}
#[test]
fn auto_select_boundary_501_simhash() {
let a = "x".repeat(501);
let b = "y".repeat(501);
assert_eq!(auto_select(&a, &b), Algo::SimHashDefault);
}
#[test]
fn auto_select_empty_strings() {
assert_eq!(auto_select("", ""), Algo::Hamming); }
#[test]
fn auto_select_single_char_equal() {
assert_eq!(auto_select("a", "a"), Algo::Hamming);
}
#[test]
fn auto_select_single_char_unequal() {
assert_eq!(auto_select("a", "b"), Algo::Hamming); }
#[test]
fn for_intent_all_variants_work() {
for intent in [
Intent::Names,
Intent::Typos,
Intent::Codes,
Intent::Documents,
Intent::Deduplication,
] {
let flow = SimiFlow::for_intent(intent);
let result = flow.compare("hello", "hello").unwrap();
assert_normalized(result.score);
assert!(result.score > 0.9);
}
}
#[test]
fn for_intent_auto_loads_with_empty_heuristic() {
let flow = SimiFlow::for_intent(Intent::Auto);
let result = flow.compare("hello", "hello").unwrap();
assert_normalized(result.score);
}
#[test]
fn auto_mode_re_resolves_per_pair() {
let flow = SimiFlow::auto();
let r1 = flow.compare("abc", "abc").unwrap();
assert_eq!(r1.algorithm, "hamming");
let long = "x".repeat(600);
let r2 = flow.compare(&long, &long).unwrap();
assert_eq!(r2.algorithm, "simhash");
}
fn assert_normalized(score: f64) {
assert!(score.is_finite() && (0.0..=1.0).contains(&score));
}
}