use crate::types::ChunkType;
use std::collections::HashSet;
pub(crate) fn canonicalize_text(s: &str) -> String {
s.to_lowercase()
.split_whitespace()
.collect::<Vec<_>>()
.join(" ")
}
pub(crate) fn canonicalize_for_type(text: &str, kind: ChunkType) -> String {
match kind {
ChunkType::Code => text.split_whitespace().collect::<Vec<_>>().join(" "),
_ => canonicalize_text(text),
}
}
#[allow(dead_code)] pub(crate) fn is_near_duplicate(a: &str, b: &str, threshold: f32) -> bool {
jaccard_trigram(a, b) >= threshold
}
#[allow(dead_code)] pub(crate) fn jaccard_trigram_score(a: &str, b: &str) -> f32 {
jaccard_trigram(a, b)
}
#[allow(dead_code)] fn jaccard_trigram(a: &str, b: &str) -> f32 {
let ta = trigram_set(a);
let tb = trigram_set(b);
if ta.is_empty() && tb.is_empty() {
return 1.0;
}
let inter = ta.intersection(&tb).count() as f32;
let union = ta.union(&tb).count() as f32;
if union == 0.0 {
0.0
} else {
inter / union
}
}
#[allow(dead_code)] fn trigram_set(s: &str) -> HashSet<[char; 3]> {
let lower: Vec<char> = s.to_lowercase().chars().collect();
if lower.is_empty() {
return HashSet::new();
}
let mut padded: Vec<char> = Vec::with_capacity(lower.len() + 3);
padded.push(' ');
padded.push(' ');
padded.extend(lower);
padded.push(' ');
let mut out = HashSet::with_capacity(padded.len().saturating_sub(2));
for w in padded.windows(3) {
out.insert([w[0], w[1], w[2]]);
}
out
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn canonicalize_trims_lowercases_and_collapses_whitespace() {
let out = canonicalize_for_type(" Hello World\n", ChunkType::Doc);
assert_eq!(out, "hello world");
}
#[test]
fn canonicalize_is_stable_on_idempotent_input() {
let once = canonicalize_for_type("already canonical", ChunkType::Code);
let twice = canonicalize_for_type(&once, ChunkType::Code);
assert_eq!(once, twice);
}
#[test]
fn canonicalize_handles_empty_and_whitespace_only() {
assert_eq!(canonicalize_for_type("", ChunkType::Doc), "");
assert_eq!(canonicalize_for_type(" \t\n ", ChunkType::Doc), "");
}
#[test]
fn canonicalize_text_strips_trivial_variations() {
assert_eq!(
canonicalize_text(" Release freeze begins Thursday. "),
canonicalize_text("release freeze begins thursday. "),
);
assert_eq!(
canonicalize_text("Release\tfreeze\nbegins Thursday."),
canonicalize_text("release freeze begins thursday."),
);
assert_ne!(
canonicalize_text("release freeze begins thursday"),
canonicalize_text("release freeze begins friday"),
);
}
#[test]
fn canonicalize_preserves_case_sensitive_tokens_in_code_chunks() {
assert_ne!(
canonicalize_for_type("fn Foo()", ChunkType::Code),
canonicalize_for_type("fn foo()", ChunkType::Code),
);
assert_eq!(
canonicalize_for_type(" fn Foo()\n", ChunkType::Code),
"fn Foo()",
);
assert_eq!(
canonicalize_for_type("fn Foo()", ChunkType::Doc),
"fn foo()",
);
}
#[test]
fn is_near_duplicate_trigram_jaccard() {
assert!(is_near_duplicate(
"Release freeze begins Thursday.",
"Release freeze begins on Thursday.",
0.85,
));
assert!(!is_near_duplicate(
"release freeze thursday",
"migration rolled back",
0.85,
));
assert!(!is_near_duplicate(
"Release freeze begins Thursday.",
"Release LOCK begins Thursday.",
0.92,
));
}
#[test]
fn jaccard_trigram_score_is_one_for_identical_inputs() {
assert!((jaccard_trigram_score("hello world", "hello world") - 1.0).abs() < 1e-6);
}
#[test]
fn jaccard_trigram_score_is_one_for_two_empty_inputs() {
assert!((jaccard_trigram_score("", "") - 1.0).abs() < 1e-6);
}
#[test]
fn is_near_duplicate_distinguishes_short_distinct_inputs() {
assert!(!is_near_duplicate("a", "b", 0.5));
assert!(!is_near_duplicate("ab", "cd", 0.5));
assert!(is_near_duplicate("a", "a", 0.99));
}
#[test]
fn is_near_duplicate_is_unicode_semantic_not_byte_semantic() {
assert!(!is_near_duplicate("é", "e", 0.5));
let s = jaccard_trigram_score("漢字仮", "漢字語");
assert!((0.0..0.6).contains(&s), "expected ~0.33, got {s}");
}
}