1use crate::algo;
17use crate::preprocess::Preprocessor;
18use crate::SimiError;
19
20pub type Score = f64;
22
23#[derive(Clone, Debug, PartialEq)]
25pub enum Strategy {
26 Cascade,
28}
29
30#[derive(Clone, Debug)]
32pub enum Threshold {
33 GreaterThan(f64),
36 LessThan(f64),
39 Between(f64, f64),
41}
42
43#[derive(Clone, Copy, Debug, PartialEq, Eq)]
48pub enum Intent {
49 Names,
52 Typos,
55 Codes,
58 Documents,
61 Deduplication,
64 Auto,
66}
67
68#[derive(Clone, Debug, PartialEq, Eq)]
70pub enum Algo {
71 Levenshtein,
72 JaroWinkler,
73 Hamming,
74 Jaccard(usize), JaccardBigram,
76 JaccardTrigram,
77 JaccardWord,
78 MinHash(usize, usize), MinHashDefault,
80 SimHash(usize), SimHashDefault,
82 Bm25,
83 TfIdf,
84}
85
86#[derive(Clone, Debug)]
88pub struct ComparisonResult {
89 pub score: Score,
91 pub tier: usize,
93 pub algorithm: String,
95 pub fallback_called: bool,
97 pub fallback_data: Option<String>,
99}
100
101pub type FallbackFn = Box<dyn Fn(&str, &str) -> (Score, Option<String>) + Send + Sync>;
103
104pub struct SimiFlow {
117 strategy: Strategy,
118 preprocessor: Option<Preprocessor>,
119 tier_1: Option<(Algo, Threshold, Threshold)>, tier_2: Option<(Algo, Threshold)>,
121 fallback: Option<FallbackFn>,
122 auto_mode: bool,
124}
125
126impl Default for SimiFlow {
127 fn default() -> Self {
128 Self {
129 strategy: Strategy::Cascade,
130 preprocessor: Some(Preprocessor::default()),
131 tier_1: None,
132 tier_2: None,
133 fallback: None,
134 auto_mode: false,
135 }
136 }
137}
138
139impl SimiFlow {
140 #[inline]
142 pub fn new() -> Self {
143 Self::default()
144 }
145
146 #[inline]
157 pub fn for_intent(intent: Intent) -> Self {
158 let algo = resolve_intent(intent, "", "");
159 let match_thresh = match intent {
160 Intent::Names | Intent::Typos => Threshold::GreaterThan(0.95),
161 Intent::Codes => Threshold::GreaterThan(0.95),
162 Intent::Documents => Threshold::GreaterThan(0.90),
163 Intent::Deduplication => Threshold::GreaterThan(0.90),
164 Intent::Auto => Threshold::GreaterThan(0.95),
165 };
166 let mismatch_thresh = match intent {
167 Intent::Names | Intent::Typos => Threshold::LessThan(0.10),
168 Intent::Codes => Threshold::LessThan(0.10),
169 Intent::Documents => Threshold::LessThan(0.05),
170 Intent::Deduplication => Threshold::LessThan(0.10),
171 Intent::Auto => Threshold::LessThan(0.10),
172 };
173 Self {
174 strategy: Strategy::Cascade,
175 preprocessor: Some(Preprocessor::default()),
176 tier_1: Some((algo, match_thresh, mismatch_thresh)),
177 tier_2: None,
178 fallback: None,
179 auto_mode: false,
180 }
181 }
182
183 #[inline]
189 pub fn auto() -> Self {
190 Self {
191 strategy: Strategy::Cascade,
192 preprocessor: Some(Preprocessor::default()),
193 tier_1: None,
194 tier_2: None,
195 fallback: None,
196 auto_mode: true,
197 }
198 }
199
200 #[inline]
205 pub fn compare_with_intent(
206 &self,
207 intent: Intent,
208 a: &str,
209 b: &str,
210 ) -> Result<ComparisonResult, SimiError> {
211 let a = if let Some(ref pre) = self.preprocessor {
212 pre.process(a)
213 } else {
214 a.to_string()
215 };
216 let b = if let Some(ref pre) = self.preprocessor {
217 pre.process(b)
218 } else {
219 b.to_string()
220 };
221 let algo = resolve_intent(intent, &a, &b);
222 let (score, name) = run_algorithm(&algo, &a, &b)?;
223 Ok(ComparisonResult {
224 score,
225 tier: 0,
226 algorithm: name,
227 fallback_called: false,
228 fallback_data: None,
229 })
230 }
231
232 #[inline]
234 pub fn preprocess(mut self, enable: bool) -> Self {
235 self.preprocessor = if enable {
236 Some(Preprocessor::default())
237 } else {
238 None
239 };
240 self
241 }
242
243 #[inline]
245 pub fn with_preprocessor(mut self, pre: Preprocessor) -> Self {
246 self.preprocessor = Some(pre);
247 self
248 }
249
250 #[inline]
252 pub fn strategy(mut self, s: Strategy) -> Self {
253 self.strategy = s;
254 self
255 }
256
257 #[inline]
262 pub fn tier_1(
263 mut self,
264 algo: Algo,
265 match_threshold: Threshold,
266 mismatch_threshold: Threshold,
267 ) -> Self {
268 self.tier_1 = Some((algo, match_threshold, mismatch_threshold));
269 self
270 }
271
272 #[inline]
274 pub fn tier_2(mut self, algo: Algo, threshold: Threshold) -> Self {
275 self.tier_2 = Some((algo, threshold));
276 self
277 }
278
279 #[inline]
281 pub fn fallback<F>(mut self, f: F) -> Self
282 where
283 F: Fn(&str, &str) -> (Score, Option<String>) + Send + Sync + 'static,
284 {
285 self.fallback = Some(Box::new(f));
286 self
287 }
288
289 #[inline]
296 pub fn compare(&self, a: &str, b: &str) -> Result<ComparisonResult, SimiError> {
297 if self.auto_mode {
298 return self.compare_with_intent(Intent::Auto, a, b);
299 }
300
301 let a = if let Some(ref pre) = self.preprocessor {
302 pre.process(a)
303 } else {
304 a.to_string()
305 };
306
307 let b = if let Some(ref pre) = self.preprocessor {
308 pre.process(b)
309 } else {
310 b.to_string()
311 };
312
313 match self.strategy {
314 Strategy::Cascade => self.run_cascade(&a, &b),
315 }
316 }
317
318 fn run_cascade(&self, a: &str, b: &str) -> Result<ComparisonResult, SimiError> {
320 if let Some((ref algo, ref match_thresh, ref mismatch_thresh)) = self.tier_1 {
322 let (score, algo_name) = run_algorithm(algo, a, b)?;
323
324 if let Threshold::GreaterThan(t) = match_thresh {
326 if score > *t {
327 return Ok(ComparisonResult {
328 score,
329 tier: 1,
330 algorithm: algo_name,
331 fallback_called: false,
332 fallback_data: None,
333 });
334 }
335 }
336
337 if let Threshold::LessThan(t) = mismatch_thresh {
339 if score < *t {
340 return Ok(ComparisonResult {
341 score,
342 tier: 1,
343 algorithm: algo_name,
344 fallback_called: false,
345 fallback_data: None,
346 });
347 }
348 }
349 }
350
351 if let Some((ref algo, ref threshold)) = self.tier_2 {
353 let (score, algo_name) = run_algorithm(algo, a, b)?;
354
355 let in_range = match threshold {
356 Threshold::Between(lo, hi) => score >= *lo && score <= *hi,
357 Threshold::GreaterThan(t) => score > *t,
358 Threshold::LessThan(t) => score < *t,
359 };
360
361 if in_range {
362 return Ok(ComparisonResult {
363 score,
364 tier: 2,
365 algorithm: algo_name,
366 fallback_called: false,
367 fallback_data: None,
368 });
369 }
370 }
371
372 if let Some(ref fallback) = self.fallback {
374 let (score, data) = fallback(a, b);
375 return Ok(ComparisonResult {
376 score,
377 tier: 3,
378 algorithm: "fallback".to_string(),
379 fallback_called: true,
380 fallback_data: data,
381 });
382 }
383
384 if let Some((ref algo, _, _)) = self.tier_1 {
386 let (score, algo_name) = run_algorithm(algo, a, b)?;
387 return Ok(ComparisonResult {
388 score,
389 tier: 1,
390 algorithm: algo_name,
391 fallback_called: false,
392 fallback_data: None,
393 });
394 }
395
396 Err(SimiError::RouterError("No tiers configured".to_string()))
397 }
398}
399
400pub fn resolve_intent(intent: Intent, a: &str, b: &str) -> Algo {
404 match intent {
405 Intent::Names => Algo::JaroWinkler,
406 Intent::Typos => Algo::Levenshtein,
407 Intent::Codes => Algo::Hamming,
408 Intent::Documents => Algo::Bm25,
409 Intent::Deduplication => Algo::SimHashDefault,
410 Intent::Auto => auto_select(a, b),
411 }
412}
413
414fn auto_select(a: &str, b: &str) -> Algo {
416 let max_len = a.len().max(b.len());
417 if a.len() == b.len() && max_len <= 20 {
418 return Algo::Hamming;
419 }
420 if max_len <= 50 {
421 return Algo::JaroWinkler;
422 }
423 if max_len <= 500 {
424 return Algo::Bm25;
425 }
426 Algo::SimHashDefault
427}
428
429fn run_algorithm(algo: &Algo, a: &str, b: &str) -> Result<(f64, String), SimiError> {
431 match algo {
432 Algo::Levenshtein => Ok((algo::levenshtein::similarity(a, b), "levenshtein".into())),
433 Algo::JaroWinkler => Ok((algo::jaro_winkler::similarity(a, b), "jaro_winkler".into())),
434 Algo::Hamming => algo::hamming::similarity(a, b)
435 .map(|s| (s, "hamming".into()))
436 .ok_or_else(|| {
437 SimiError::AlgorithmError("Hamming requires equal-length strings".into())
438 }),
439 Algo::Jaccard(n) => Ok((algo::jaccard::similarity(a, b, *n), "jaccard".into())),
440 Algo::JaccardBigram => Ok((
441 algo::jaccard::bigram_similarity(a, b),
442 "jaccard_bigram".into(),
443 )),
444 Algo::JaccardTrigram => Ok((
445 algo::jaccard::trigram_similarity(a, b),
446 "jaccard_trigram".into(),
447 )),
448 Algo::JaccardWord => Ok((algo::jaccard::word_similarity(a, b), "jaccard_word".into())),
449 Algo::MinHash(shingle_size, num_hashes) => {
450 let s = algo::minhash::compare(a, b, *shingle_size, *num_hashes);
451 Ok((s, "minhash".into()))
452 }
453 Algo::MinHashDefault => {
454 let s = algo::minhash::compare_default(a, b);
455 Ok((s, "minhash".into()))
456 }
457 Algo::SimHash(shingle_size) => {
458 let s = algo::simhash::compare(a, b, *shingle_size);
459 Ok((s, "simhash".into()))
460 }
461 Algo::SimHashDefault => {
462 let s = algo::simhash::compare_default(a, b);
463 Ok((s, "simhash".into()))
464 }
465 Algo::Bm25 => Ok((algo::bm25::similarity(a, b), "bm25".into())),
466 Algo::TfIdf => Ok((algo::tfidf::similarity(a, b), "tfidf".into())),
467 }
468}
469
470#[cfg(test)]
471mod tests {
472 use super::*;
473
474 #[test]
475 fn basic_comparison() {
476 let result = SimiFlow::new()
477 .tier_1(
478 Algo::JaroWinkler,
479 Threshold::GreaterThan(0.95),
480 Threshold::LessThan(0.10),
481 )
482 .compare("MARTHA", "MARHTA")
483 .unwrap();
484 assert_eq!(result.tier, 1);
486 assert!((result.score - 0.961).abs() < 0.01);
487 }
488
489 #[test]
490 fn tier_2_fallback() {
491 let result = SimiFlow::new()
492 .tier_1(
493 Algo::Levenshtein,
494 Threshold::GreaterThan(0.95),
495 Threshold::LessThan(0.10),
496 )
497 .tier_2(Algo::Bm25, Threshold::Between(0.30, 0.95))
498 .compare("the quick brown fox", "the quick blue fox")
499 .unwrap();
500 assert_eq!(result.tier, 2);
503 assert_eq!(result.algorithm, "bm25");
504 assert!(result.score > 0.3 && result.score < 0.95);
505 }
506
507 #[test]
508 fn fallback_called() {
509 let result = SimiFlow::new()
510 .tier_1(
511 Algo::Levenshtein,
512 Threshold::GreaterThan(0.99),
513 Threshold::LessThan(0.01),
514 )
515 .fallback(|a, b| {
516 let score = if a == b { 1.0 } else { 0.5 };
518 (score, Some("llm_verified".to_string()))
519 })
520 .compare("hello", "world")
521 .unwrap();
522 assert_eq!(result.tier, 3);
523 assert!(result.fallback_called);
524 assert_eq!(result.fallback_data, Some("llm_verified".to_string()));
525 }
526
527 #[test]
528 fn tier_1_mismatch() {
529 let result = SimiFlow::new()
532 .tier_1(
533 Algo::Levenshtein,
534 Threshold::GreaterThan(0.95),
535 Threshold::LessThan(0.10),
536 )
537 .compare("abc", "xyz")
538 .unwrap();
539 assert_eq!(result.tier, 1);
540 assert!(result.score < 0.01);
541 }
542
543 #[test]
544 fn with_preprocessing() {
545 let result = SimiFlow::new()
546 .preprocess(true)
547 .tier_1(
548 Algo::Levenshtein,
549 Threshold::GreaterThan(0.95),
550 Threshold::LessThan(0.10),
551 )
552 .compare(" Hello World ", "hello world")
553 .unwrap();
554 assert!((result.score - 1.0).abs() < f64::EPSILON);
556 }
557
558 #[test]
559 fn no_fallback_returns_tier_1() {
560 let result = SimiFlow::new()
561 .tier_1(
562 Algo::Levenshtein,
563 Threshold::GreaterThan(0.99),
564 Threshold::LessThan(0.01),
565 )
566 .compare("kitten", "sitting")
567 .unwrap();
568 assert!(result.tier == 1 || result.tier == 2); }
572
573 #[test]
574 fn builder_pattern() {
575 let flow = SimiFlow::new()
576 .strategy(Strategy::Cascade)
577 .tier_1(
578 Algo::JaroWinkler,
579 Threshold::GreaterThan(0.95),
580 Threshold::LessThan(0.10),
581 )
582 .tier_2(Algo::Bm25, Threshold::Between(0.30, 0.95));
583
584 let result = flow.compare("hello world", "hello world").unwrap();
585 assert!((result.score - 1.0).abs() < 0.01);
586 assert_eq!(result.tier, 1);
587 }
588
589 #[test]
592 fn intent_names_uses_jaro_winkler() {
593 let flow = SimiFlow::for_intent(Intent::Names);
594 let result = flow.compare("MARTHA", "MARHTA").unwrap();
595 assert_eq!(result.algorithm, "jaro_winkler");
596 assert!((result.score - 0.961).abs() < 0.01);
597 }
598
599 #[test]
600 fn intent_typos_uses_levenshtein() {
601 let flow = SimiFlow::for_intent(Intent::Typos);
602 let result = flow.compare("kitten", "sitting").unwrap();
603 assert_eq!(result.algorithm, "levenshtein");
604 }
605
606 #[test]
607 fn intent_codes_uses_hamming() {
608 let flow = SimiFlow::for_intent(Intent::Codes);
609 let result = flow.compare("hello", "hello").unwrap();
610 assert_eq!(result.algorithm, "hamming");
611 }
612
613 #[test]
614 fn intent_documents_uses_bm25() {
615 let flow = SimiFlow::for_intent(Intent::Documents);
616 let result = flow
617 .compare("the quick brown fox", "the quick brown fox")
618 .unwrap();
619 assert_eq!(result.algorithm, "bm25");
620 }
621
622 #[test]
623 fn intent_deduplication_uses_simhash() {
624 let flow = SimiFlow::for_intent(Intent::Deduplication);
625 let result = flow
626 .compare("the quick brown fox", "the quick brown fox")
627 .unwrap();
628 assert_eq!(result.algorithm, "simhash");
629 }
630
631 #[test]
632 fn auto_select_short_picks_hamming() {
633 let flow = SimiFlow::auto();
634 let result = flow.compare("abc", "abc").unwrap();
635 assert_eq!(result.algorithm, "hamming");
637 }
638
639 #[test]
640 fn auto_select_medium_picks_jaro_winkler() {
641 let flow = SimiFlow::auto();
642 let result = flow
644 .compare("the quick brown fox", "the quick lazy dog")
645 .unwrap();
646 assert_eq!(result.algorithm, "jaro_winkler");
647 }
648
649 #[test]
650 fn auto_select_long_picks_bm25() {
651 let flow = SimiFlow::auto();
652 let a = "the quick brown fox jumps over the lazy dog near the river bank on a sunny day";
653 let b = "the quick brown fox jumps over the lazy cat near the river bank on a rainy day";
654 let result = flow.compare(a, b).unwrap();
656 assert_eq!(result.algorithm, "bm25");
657 }
658
659 #[test]
660 fn compare_with_intent_bypasses_tiers() {
661 let flow = SimiFlow::new().tier_1(
662 Algo::Levenshtein,
663 Threshold::GreaterThan(0.99),
664 Threshold::LessThan(0.01),
665 );
666 let result = flow
667 .compare_with_intent(Intent::Names, "MARTHA", "MARHTA")
668 .unwrap();
669 assert_eq!(result.algorithm, "jaro_winkler");
670 assert_eq!(result.tier, 0);
671 }
672
673 #[test]
674 fn compare_with_intent_all_variants() {
675 let flow = SimiFlow::new();
676 for (intent, expected_algo, a, b) in [
677 (Intent::Names, "jaro_winkler", "MARTHA", "MARHTA"),
678 (Intent::Typos, "levenshtein", "kitten", "sitting"),
679 (
680 Intent::Documents,
681 "bm25",
682 "the quick brown fox",
683 "the quick brown fox",
684 ),
685 (
686 Intent::Deduplication,
687 "simhash",
688 "hello world",
689 "hello world",
690 ),
691 ] {
692 let r = flow.compare_with_intent(intent, a, b).unwrap();
693 assert_eq!(r.algorithm, expected_algo, "intent {intent:?}");
694 assert_normalized(r.score);
695 }
696 }
697
698 #[test]
701 fn auto_select_boundary_20_equal() {
702 let a = "x".repeat(20);
703 let b = "y".repeat(20);
704 assert_eq!(auto_select(&a, &b), Algo::Hamming);
706 }
707
708 #[test]
709 fn auto_select_boundary_21_jaro_winkler() {
710 let a = "x".repeat(21);
711 let b = "y".repeat(21);
712 assert_eq!(auto_select(&a, &b), Algo::JaroWinkler);
714 }
715
716 #[test]
717 fn auto_select_unequal_short_jaro_winkler() {
718 assert_eq!(auto_select("hello", "world!"), Algo::JaroWinkler);
720 }
721
722 #[test]
723 fn auto_select_boundary_50() {
724 let a = "x".repeat(50);
725 let b = "y".repeat(50);
726 assert_eq!(auto_select(&a, &b), Algo::JaroWinkler);
727 }
728
729 #[test]
730 fn auto_select_boundary_51_bm25() {
731 let a = "x".repeat(51);
732 let b = "y".repeat(51);
733 assert_eq!(auto_select(&a, &b), Algo::Bm25);
734 }
735
736 #[test]
737 fn auto_select_boundary_500() {
738 let a = "x".repeat(500);
739 let b = "y".repeat(500);
740 assert_eq!(auto_select(&a, &b), Algo::Bm25);
741 }
742
743 #[test]
744 fn auto_select_boundary_501_simhash() {
745 let a = "x".repeat(501);
746 let b = "y".repeat(501);
747 assert_eq!(auto_select(&a, &b), Algo::SimHashDefault);
748 }
749
750 #[test]
751 fn auto_select_empty_strings() {
752 assert_eq!(auto_select("", ""), Algo::Hamming); }
754
755 #[test]
756 fn auto_select_single_char_equal() {
757 assert_eq!(auto_select("a", "a"), Algo::Hamming);
758 }
759
760 #[test]
761 fn auto_select_single_char_unequal() {
762 assert_eq!(auto_select("a", "b"), Algo::Hamming); }
764
765 #[test]
766 fn for_intent_all_variants_work() {
767 for intent in [
768 Intent::Names,
769 Intent::Typos,
770 Intent::Codes,
771 Intent::Documents,
772 Intent::Deduplication,
773 ] {
774 let flow = SimiFlow::for_intent(intent);
775 let result = flow.compare("hello", "hello").unwrap();
776 assert_normalized(result.score);
777 assert!(result.score > 0.9);
778 }
779 }
780
781 #[test]
782 fn for_intent_auto_loads_with_empty_heuristic() {
783 let flow = SimiFlow::for_intent(Intent::Auto);
785 let result = flow.compare("hello", "hello").unwrap();
786 assert_normalized(result.score);
790 }
791
792 #[test]
793 fn auto_mode_re_resolves_per_pair() {
794 let flow = SimiFlow::auto();
795 let r1 = flow.compare("abc", "abc").unwrap();
797 assert_eq!(r1.algorithm, "hamming");
798 let long = "x".repeat(600);
800 let r2 = flow.compare(&long, &long).unwrap();
801 assert_eq!(r2.algorithm, "simhash");
802 }
803
804 fn assert_normalized(score: f64) {
805 assert!(score.is_finite() && (0.0..=1.0).contains(&score));
806 }
807}