panproto-mig 0.48.6

//! Token-similarity alignment strategy.
//!
//! Splits identifier strings into a bag of word tokens (handling
//! `camelCase`, `snake_case`, `kebab-case`, and acronym boundaries),
//! then scores pairs by a convex combination of token Jaccard
//! similarity and character-bigram cosine similarity. The result is
//! independent of any specific protocol, alias table, or language, and
//! therefore serves as a general-purpose prior for the CSP solver.
//!
//! The output is validated downstream by the CSP's naturality check, so
//! false positives from the heuristic do not produce invalid morphisms.

use std::collections::HashMap;

use panproto_gat::Name;
use panproto_schema::Schema;

use super::{Anchor, StrategyTag, kinds_and_constraints_compatible};

/// Split an identifier into lowercase word tokens. Boundaries are
/// detected at:
///
/// * any of the separator characters `_ - . / ` and whitespace;
/// * camelCase transitions (lowercase letter or digit followed by an
///   uppercase letter);
/// * acronym→word transitions (uppercase letter followed by an uppercase
///   letter that is itself followed by a lowercase letter, e.g. `HTTPS`
///   in `HTTPServer` splits before `Server`);
/// * letter↔digit transitions in either direction.
///
/// Examples:
/// * `createdAt` → `["created", "at"]`
/// * `HTTPServer` → `["http", "server"]`
/// * `parseJSON` → `["parse", "json"]`
/// * `v2Endpoint` → `["v", "2", "endpoint"]`
#[must_use]
pub fn tokenize(s: &str) -> Vec<String> {
    let chars: Vec<char> = s.chars().collect();
    let mut out: Vec<String> = Vec::new();
    let mut buf = String::new();

    for (i, &ch) in chars.iter().enumerate() {
        let is_sep = ch == '_' || ch == '-' || ch == '.' || ch == '/' || ch.is_whitespace();
        if is_sep {
            if !buf.is_empty() {
                out.push(std::mem::take(&mut buf));
            }
            continue;
        }

        let prev = chars.get(i.wrapping_sub(1)).copied();
        let next = chars.get(i + 1).copied();

        let split_before = prev.is_some_and(|p| {
            // camelCase: lowercase|digit → uppercase
            let camel = (p.is_lowercase() || p.is_ascii_digit()) && ch.is_uppercase();
            // Acronym→word: prev upper, this upper, next lower (split before this).
            let acronym =
                p.is_uppercase() && ch.is_uppercase() && next.is_some_and(char::is_lowercase);
            // Letter↔digit
            let letter_digit = p.is_alphabetic() && ch.is_ascii_digit();
            let digit_letter = p.is_ascii_digit() && ch.is_alphabetic();
            camel || acronym || letter_digit || digit_letter
        });

        if split_before && !buf.is_empty() {
            out.push(std::mem::take(&mut buf));
        }
        for c in ch.to_lowercase() {
            buf.push(c);
        }
    }

    if !buf.is_empty() {
        out.push(buf);
    }

    out.into_iter().filter(|t| !t.is_empty()).collect()
}

/// Jaccard similarity between two token bags. Empty ∩ empty = 1.0.
#[must_use]
pub fn token_jaccard(a: &[String], b: &[String]) -> f64 {
    if a.is_empty() && b.is_empty() {
        return 1.0;
    }
    let set_a: std::collections::HashSet<&String> = a.iter().collect();
    let set_b: std::collections::HashSet<&String> = b.iter().collect();
    let intersection = set_a.intersection(&set_b).count();
    let union = set_a.union(&set_b).count();
    if union == 0 {
        1.0
    } else {
        let inter_f = f64::from(u32::try_from(intersection).unwrap_or(u32::MAX));
        let union_f = f64::from(u32::try_from(union).unwrap_or(u32::MAX));
        inter_f / union_f
    }
}

/// Character n-gram cosine similarity. Converts each string to a
/// multiset of character n-grams (n-length windows, padded with spaces),
/// then computes `cos(a, b) = a·b / (‖a‖ ‖b‖)`.
#[must_use]
pub fn char_ngram_cosine(a: &str, b: &str, n: usize) -> f64 {
    let grams_a = ngram_counts(a, n);
    let grams_b = ngram_counts(b, n);
    if grams_a.is_empty() || grams_b.is_empty() {
        return if a == b { 1.0 } else { 0.0 };
    }

    let count_to_f = |c: &usize| f64::from(u32::try_from(*c).unwrap_or(u32::MAX));
    let norm_a: f64 = grams_a
        .values()
        .map(|c| count_to_f(c).powi(2))
        .sum::<f64>()
        .sqrt();
    let norm_b: f64 = grams_b
        .values()
        .map(|c| count_to_f(c).powi(2))
        .sum::<f64>()
        .sqrt();
    if norm_a == 0.0 || norm_b == 0.0 {
        return 0.0;
    }

    let mut dot = 0.0;
    for (g, &ca) in &grams_a {
        if let Some(&cb) = grams_b.get(g) {
            let a_val = f64::from(u32::try_from(ca).unwrap_or(u32::MAX));
            let b_val = f64::from(u32::try_from(cb).unwrap_or(u32::MAX));
            dot += a_val * b_val;
        }
    }
    (dot / (norm_a * norm_b)).clamp(0.0, 1.0)
}

fn ngram_counts(s: &str, n: usize) -> HashMap<String, usize> {
    let normalized: String = s
        .chars()
        .filter_map(|c| {
            if c.is_alphanumeric() {
                Some(c.to_ascii_lowercase())
            } else {
                None
            }
        })
        .collect();
    let mut counts: HashMap<String, usize> = HashMap::new();
    // When normalization strips every character (e.g. `"!!!"`), the
    // string carries no alphanumeric signal. Returning an empty gram
    // multiset prevents two distinct punctuation-only strings from
    // scoring 1.0 on each other via shared padding grams.
    if normalized.is_empty() {
        return counts;
    }
    let padded: Vec<char> = std::iter::repeat_n(' ', n.saturating_sub(1))
        .chain(normalized.chars())
        .chain(std::iter::repeat_n(' ', n.saturating_sub(1)))
        .collect();
    if n == 0 || padded.len() < n {
        return counts;
    }
    for window in padded.windows(n) {
        let gram: String = window.iter().collect();
        *counts.entry(gram).or_insert(0) += 1;
    }
    counts
}

/// Compound token similarity.
///
/// Computes the larger of `0.6 · Jaccard(tokens) + 0.4 · cosine(bigrams)`
/// and `cosine(bigrams)` (keeping room for names with no token overlap
/// but high char similarity like typos). Exactly equal strings score
/// `1.0`.
#[must_use]
pub fn token_similarity(a: &str, b: &str) -> f64 {
    if a == b {
        return 1.0;
    }
    let ta = tokenize(a);
    let tb = tokenize(b);
    let jac = token_jaccard(&ta, &tb);
    let cos = char_ngram_cosine(a, b, 2);
    let combined = 0.6f64.mul_add(jac, 0.4 * cos);
    combined.max(cos).clamp(0.0, 1.0)
}

/// Emit token-similarity anchors. For each source vertex, find the
/// best-scoring kind-compatible target. If its score is above
/// `threshold`, emit an anchor with that score as confidence.
///
/// Priority is per-source (each source gets its best match), so ambiguous
/// many-to-one cases are resolved by [`super::resolve_anchors`] later.
#[must_use]
pub fn token_anchors(src: &Schema, tgt: &Schema, threshold: f64) -> Vec<Anchor> {
    let mut out = Vec::new();
    let mut src_ids: Vec<&Name> = src.vertices.keys().collect();
    src_ids.sort_by(|a, b| a.as_str().cmp(b.as_str()));
    let mut tgt_ids: Vec<&Name> = tgt.vertices.keys().collect();
    tgt_ids.sort_by(|a, b| a.as_str().cmp(b.as_str()));
    for src_id in src_ids.iter().copied() {
        let mut best: Option<(Name, f64)> = None;
        for tgt_id in tgt_ids.iter().copied() {
            if !kinds_and_constraints_compatible(src, src_id, tgt, tgt_id) {
                continue;
            }
            let score = token_similarity(src_id.as_str(), tgt_id.as_str());
            // Strict > ensures ties are resolved by sorted target order.
            if best.as_ref().is_none_or(|(_, bs)| score > *bs) {
                best = Some((tgt_id.clone(), score));
            }
        }
        if let Some((tgt_id, score)) = best {
            if score >= threshold && score < 1.0 {
                // skip exact matches (covered by exact strategy)
                out.push(Anchor {
                    src: src_id.clone(),
                    tgt: tgt_id.clone(),
                    confidence: score,
                    strategy: StrategyTag::TokenSimilarity,
                    explanation: format!(
                        "token similarity {:.2}: {} ↔ {}",
                        score,
                        src_id.as_str(),
                        tgt_id.as_str()
                    ),
                });
            }
        }
    }
    out
}

#[cfg(test)]
#[allow(clippy::unwrap_used, clippy::float_cmp)]
mod tests {
    use super::*;

    #[test]
    fn tokenize_splits_camel_snake_kebab() {
        assert_eq!(tokenize("createdAt"), vec!["created", "at"]);
        assert_eq!(tokenize("created_at"), vec!["created", "at"]);
        assert_eq!(tokenize("created-at"), vec!["created", "at"]);
        assert_eq!(tokenize("CreatedAt"), vec!["created", "at"]);
        assert_eq!(tokenize("created at"), vec!["created", "at"]);
    }

    #[test]
    fn tokenize_handles_acronyms() {
        assert_eq!(tokenize("HTTPServer"), vec!["http", "server"]);
        assert_eq!(tokenize("parseJSON"), vec!["parse", "json"]);
        assert_eq!(tokenize("URLParser"), vec!["url", "parser"]);
    }

    #[test]
    fn jaccard_identical_tokens() {
        let a = tokenize("createdAt");
        let b = tokenize("created_at");
        assert!((token_jaccard(&a, &b) - 1.0).abs() < 1e-9);
    }

    #[test]
    fn jaccard_disjoint() {
        let a = tokenize("hello");
        let b = tokenize("world");
        assert!((token_jaccard(&a, &b) - 0.0).abs() < 1e-9);
    }

    #[test]
    fn ngram_cosine_identical_strings() {
        assert!((char_ngram_cosine("hello", "hello", 2) - 1.0).abs() < 1e-9);
    }

    #[test]
    fn ngram_cosine_typo_variant_is_high() {
        let score = char_ngram_cosine("createdAt", "createAt", 2);
        assert!(score > 0.7, "typo variant should score high: {score}");
    }

    #[test]
    fn token_similarity_exact_is_one() {
        assert!((token_similarity("foo", "foo") - 1.0).abs() < 1e-9);
    }

    #[test]
    fn token_similarity_casing_equivalence_is_high() {
        let score = token_similarity("createdAt", "created_at");
        assert!(
            score > 0.85,
            "casing-equivalent strings should score near 1.0: {score}"
        );
    }

    #[test]
    fn tokenize_adversarial_inputs() {
        assert_eq!(tokenize(""), Vec::<String>::new());
        assert_eq!(tokenize("a"), vec!["a"]);
        assert_eq!(tokenize("A"), vec!["a"]);
        assert_eq!(tokenize("ABC"), vec!["abc"]);
        assert_eq!(tokenize("_abc_"), vec!["abc"]);
        assert_eq!(tokenize("-abc-"), vec!["abc"]);
        assert_eq!(tokenize("___"), Vec::<String>::new());
        assert_eq!(tokenize("v2"), vec!["v", "2"]);
        assert_eq!(tokenize("v2Endpoint"), vec!["v", "2", "endpoint"]);
        // letter->digit->letter
        assert_eq!(tokenize("a1b"), vec!["a", "1", "b"]);
        // unicode letter
        let toks = tokenize("αβγ");
        assert_eq!(toks.len(), 1);
    }

    #[test]
    fn token_jaccard_both_empty_is_one() {
        let empty: Vec<String> = vec![];
        assert!((token_jaccard(&empty, &empty) - 1.0).abs() < 1e-9);
    }

    #[test]
    fn char_ngram_cosine_degenerate_n() {
        // n=0: guard returns empty grams; strings are unequal so score is 0.
        assert_eq!(char_ngram_cosine("foo", "bar", 0), 0.0);
        // equal strings with n=0 map through the a==b shortcut.
        assert_eq!(char_ngram_cosine("foo", "foo", 0), 1.0);
        // n=1: works normally.
        let s = char_ngram_cosine("abc", "abc", 1);
        assert!((s - 1.0).abs() < 1e-9);
        // empty vs empty at n=2.
        assert_eq!(char_ngram_cosine("", "", 2), 1.0);
        // empty vs nonempty.
        assert_eq!(char_ngram_cosine("", "abc", 2), 0.0);
    }

    #[test]
    fn char_ngram_cosine_punctuation_only_strings_are_not_identical() {
        // Two distinct strings that normalize to empty (stripping
        // non-alphanumeric characters) have empty gram bags. They
        // must score 0.0 (not 1.0 from sharing a padding-only gram
        // multiset) because the strings differ.
        assert_eq!(char_ngram_cosine("!!!", "???", 2), 0.0);
        assert_eq!(char_ngram_cosine("!@#$", "%^&*", 2), 0.0);
        // Identical punctuation-only strings still compare equal via
        // the a == b shortcut at the top of char_ngram_cosine.
        assert_eq!(char_ngram_cosine("!!!", "!!!", 2), 1.0);
    }

    #[test]
    fn token_similarity_punctuation_only_disjoint_is_zero() {
        // Upstream of the fix above: token_similarity on disjoint
        // punctuation-only inputs must no longer return 1.0 via the
        // cosine branch.
        let score = token_similarity("!!!", "???");
        assert!(
            score < 0.5,
            "disjoint punctuation-only strings must not score 1.0: {score}"
        );
    }

    #[test]
    fn token_similarity_empty_strings() {
        // tokenize("") == []; Jaccard empty-empty == 1.0; cosine empty-empty == 1.0.
        let score = token_similarity("", "");
        assert!((score - 1.0).abs() < 1e-9);
        // one empty: cos=0, jac=0, combined=0.
        assert_eq!(token_similarity("", "foo"), 0.0);
    }

    #[test]
    fn token_anchors_minimal_disjoint_schema() {
        use panproto_schema::{Protocol, SchemaBuilder};
        let proto = Protocol {
            name: "t".into(),
            schema_theory: "ThTest".into(),
            instance_theory: "ThWType".into(),
            edge_rules: vec![],
            obj_kinds: vec!["string".into()],
            constraint_sorts: vec![],
            ..Protocol::default()
        };
        let s = SchemaBuilder::new(&proto)
            .vertex("alpha_beta_gamma", "string", None::<&str>)
            .unwrap()
            .build()
            .unwrap();
        let t = SchemaBuilder::new(&proto)
            .vertex("zzz_qqq_xxx", "string", None::<&str>)
            .unwrap()
            .build()
            .unwrap();
        // Only one vertex each, utterly dissimilar → no anchors at 0.5.
        assert!(token_anchors(&s, &t, 0.5).is_empty());
    }

    #[test]
    fn token_anchors_deterministic() {
        use panproto_schema::{Protocol, SchemaBuilder};
        let proto = Protocol {
            name: "t".into(),
            schema_theory: "ThTest".into(),
            instance_theory: "ThWType".into(),
            edge_rules: vec![],
            obj_kinds: vec!["string".into()],
            constraint_sorts: vec![],
            ..Protocol::default()
        };
        let build = |order: &[&str]| {
            let mut b = SchemaBuilder::new(&proto);
            for id in order {
                b = b.vertex(id, "string", None::<&str>).unwrap();
            }
            b.build().unwrap()
        };
        let s1 = build(&["createdAt", "sentAt", "updatedAt"]);
        let s2 = build(&["updatedAt", "createdAt", "sentAt"]);
        let t = build(&["created_at", "modified_at"]);
        let go = |s: &panproto_schema::Schema| {
            let mut pairs: Vec<_> = token_anchors(s, &t, 0.4)
                .iter()
                .map(|a| (a.src.as_str().to_owned(), a.tgt.as_str().to_owned()))
                .collect();
            pairs.sort();
            pairs
        };
        assert_eq!(go(&s1), go(&s2));
    }

    #[test]
    fn token_anchors_single_isolated_vertex() {
        use panproto_schema::{Protocol, SchemaBuilder};
        let proto = Protocol {
            name: "t".into(),
            schema_theory: "ThTest".into(),
            instance_theory: "ThWType".into(),
            edge_rules: vec![],
            obj_kinds: vec!["string".into()],
            constraint_sorts: vec![],
            ..Protocol::default()
        };
        // Smallest legal schema is one vertex; token_anchors returns empty
        // when there is no opposite-side match.
        let s = SchemaBuilder::new(&proto)
            .vertex("alpha", "string", None::<&str>)
            .unwrap()
            .build()
            .unwrap();
        let t = SchemaBuilder::new(&proto)
            .vertex("zzzzz", "string", None::<&str>)
            .unwrap()
            .build()
            .unwrap();
        assert!(token_anchors(&s, &t, 0.9).is_empty());
    }

    #[test]
    fn token_anchors_bit_identical_across_100_runs() {
        use panproto_schema::{Protocol, SchemaBuilder};
        let proto = Protocol {
            name: "t".into(),
            schema_theory: "ThTest".into(),
            instance_theory: "ThWType".into(),
            edge_rules: vec![],
            obj_kinds: vec!["string".into()],
            constraint_sorts: vec![],
            ..Protocol::default()
        };
        let build = |names: &[&str]| {
            let mut b = SchemaBuilder::new(&proto);
            for n in names {
                b = b.vertex(n, "string", None::<&str>).unwrap();
            }
            b.build().unwrap()
        };
        let s = build(&["createdAt", "sentAt", "updatedAt"]);
        let t = build(&["created_at", "modified_at"]);
        let baseline: Vec<(String, String, u64)> = token_anchors(&s, &t, 0.4)
            .iter()
            .map(|a| {
                (
                    a.src.as_str().into(),
                    a.tgt.as_str().into(),
                    a.confidence.to_bits(),
                )
            })
            .collect();
        for _ in 0..100 {
            let again: Vec<(String, String, u64)> = token_anchors(&s, &t, 0.4)
                .iter()
                .map(|a| {
                    (
                        a.src.as_str().into(),
                        a.tgt.as_str().into(),
                        a.confidence.to_bits(),
                    )
                })
                .collect();
            assert_eq!(again, baseline);
        }
    }

    proptest::proptest! {
        #[test]
        fn token_similarity_is_symmetric(a in "[a-zA-Z0-9_\\-]{0,20}", b in "[a-zA-Z0-9_\\-]{0,20}") {
            let ab = token_similarity(&a, &b);
            let ba = token_similarity(&b, &a);
            proptest::prop_assert!(
                (ab - ba).abs() < 1e-9,
                "token_similarity({a:?}, {b:?}) = {ab} != {ba} = token_similarity({b:?}, {a:?})"
            );
        }

        #[test]
        fn token_jaccard_is_symmetric(
            a in proptest::collection::vec("[a-z]{1,5}", 0..5),
            b in proptest::collection::vec("[a-z]{1,5}", 0..5),
        ) {
            let ja = token_jaccard(&a, &b);
            let jb = token_jaccard(&b, &a);
            proptest::prop_assert!((ja - jb).abs() < 1e-9);
        }

        #[test]
        fn char_ngram_cosine_is_symmetric(a in "[a-z0-9]{0,15}", b in "[a-z0-9]{0,15}") {
            let ab = char_ngram_cosine(&a, &b, 2);
            let ba = char_ngram_cosine(&b, &a, 2);
            proptest::prop_assert!((ab - ba).abs() < 1e-9);
        }
    }

    #[test]
    fn tokenize_preserves_non_letter_non_separator_runs() {
        // Pins current behaviour for characters that are neither letters,
        // digits, nor any of the configured separators (`_`, `-`, `.`, `/`,
        // whitespace). Emoji and other symbols do not trigger a token
        // boundary, so they are absorbed into the adjacent token. This is
        // consistent with the documented splitter rules but is not
        // obvious from the docstring; this test pins it so a future
        // refactor that changes splitting on symbols has to update the
        // pin deliberately.
        let toks = tokenize("\u{1F600}emoji");
        assert_eq!(toks, vec!["\u{1F600}emoji"]);
        let toks = tokenize("hello\u{1F600}world");
        // camelCase boundary does not fire because neither `o` nor `\u{1F600}`
        // is uppercase.
        assert_eq!(toks, vec!["hello\u{1F600}world"]);
    }

    #[test]
    fn tokenize_splits_on_tab_and_newline() {
        // The docstring lists `_ - . /` and whitespace as separators.
        // Tab and newline satisfy `char::is_whitespace`, so they act
        // as separators too. Pin the behaviour so a future narrowing
        // of the whitespace rule (e.g. restricting to ASCII space
        // only) has to update this test.
        assert_eq!(tokenize("foo\tbar"), vec!["foo", "bar"]);
        assert_eq!(tokenize("foo\nbar"), vec!["foo", "bar"]);
        assert_eq!(tokenize("foo\r\nbar"), vec!["foo", "bar"]);
        // Non-breaking space (U+00A0) also counts as whitespace.
        assert_eq!(tokenize("foo\u{00A0}bar"), vec!["foo", "bar"]);
    }

    #[test]
    fn char_ngram_cosine_single_char_inputs() {
        // When normalization is non-empty, a single surviving char
        // still pads to n-1 spaces on each side. For n=2 on "a" that
        // yields windows `" a"` and `"a "`. Two disjoint single-char
        // inputs share no grams (padding grams touch different
        // letters on the inside), so the cosine must be 0, not 1.
        assert_eq!(char_ngram_cosine("a", "b", 2), 0.0);
        // Identical single-char inputs use the a == b shortcut.
        assert!((char_ngram_cosine("a", "a", 2) - 1.0).abs() < 1e-9);
        // Two-of-same-char vs the char itself share the middle gram
        // `"aa"` plus the edges; cosine should be strictly between 0
        // and 1 (different bags, nonzero overlap).
        let s = char_ngram_cosine("a", "aa", 2);
        assert!(s > 0.0 && s < 1.0, "expected partial overlap: {s}");
    }

    #[test]
    fn tokenize_triple_underscore_is_empty() {
        // Pins the documented adversarial case: a string consisting
        // entirely of separator characters collapses to no tokens. The
        // position-1 byte-index check in split_prefix_suffix (upstream
        // helper) is unrelated; tokenize simply skips all separators and
        // emits no buffer.
        assert_eq!(tokenize("___"), Vec::<String>::new());
        assert_eq!(tokenize("---"), Vec::<String>::new());
        assert_eq!(tokenize(" / . _ - "), Vec::<String>::new());
    }

    #[test]
    fn token_similarity_treats_nfc_and_nfd_distinctly() {
        // Pins the current behaviour: `char_ngram_cosine` and
        // `tokenize` operate on raw `char`s without Unicode
        // normalization. The NFC "café" (single precomposed `é`) and
        // the NFD "cafe\u{0301}" (base `e` + combining acute) therefore
        // have DIFFERENT character-bigram multisets and DIFFERENT
        // token bags, so `token_similarity` returns a value strictly
        // less than `1.0`. Callers that need Unicode-equivalence must
        // normalize their inputs before invoking this function.
        let nfc = "caf\u{00E9}";
        let nfd = "cafe\u{0301}";
        assert_ne!(nfc, nfd);
        let score = token_similarity(nfc, nfd);
        assert!(
            score < 1.0,
            "NFC and NFD forms are not normalized, so they must not score exactly 1.0 (got {score})"
        );
        // Sanity: they still share most bigrams and should score
        // reasonably high (partial overlap).
        assert!(
            score > 0.4,
            "NFC/NFD should retain partial similarity: {score}"
        );
    }

    #[test]
    fn token_anchors_threshold_equal_one_emits_nothing() {
        // The gate inside `token_anchors` is
        // `score >= threshold && score < 1.0`. A caller passing
        // `threshold = 1.0` therefore asks for "strictly better than
        // any heuristic would yield" — the exact-match case is handled
        // by the `exact` strategy and intentionally excluded here, so
        // no anchor should ever be emitted at this threshold. Pin the
        // boundary so a future relaxation of the `< 1.0` clamp is a
        // deliberate, test-visible change.
        use panproto_schema::{Protocol, SchemaBuilder};
        let proto = Protocol {
            name: "t".into(),
            schema_theory: "ThTest".into(),
            instance_theory: "ThWType".into(),
            edge_rules: vec![],
            obj_kinds: vec!["string".into()],
            constraint_sorts: vec![],
            ..Protocol::default()
        };
        // Identical names across sides would yield score 1.0; the
        // strict `< 1.0` filter drops them. Near-identical names score
        // below 1.0; the `>= 1.0` filter drops them too.
        let s = SchemaBuilder::new(&proto)
            .vertex("createdAt", "string", None::<&str>)
            .unwrap()
            .vertex("sentAt", "string", None::<&str>)
            .unwrap()
            .build()
            .unwrap();
        let t = SchemaBuilder::new(&proto)
            .vertex("createdAt", "string", None::<&str>)
            .unwrap()
            .vertex("createdAtExt", "string", None::<&str>)
            .unwrap()
            .build()
            .unwrap();
        let anchors = token_anchors(&s, &t, 1.0);
        assert!(
            anchors.is_empty(),
            "threshold = 1.0 must emit no anchors (exact matches are handled by the exact strategy): {anchors:?}"
        );
    }

    #[test]
    fn tokenize_handles_pathological_unicode_scalars() {
        // Adversarial angle: tokenize must not panic on the
        // Unicode replacement character, non-BMP scalars, or NUL
        // bytes, and it must emit a stable result across repeated
        // invocations (the function is pure, but the assertion pins
        // that fact against future edits that might introduce
        // HashMap-iteration or caching shortcuts).
        //
        // U+FFFD REPLACEMENT CHARACTER: neither alphabetic, digit,
        // separator, uppercase, nor lowercase under Rust's char
        // classification. It is therefore absorbed into the adjacent
        // token without triggering a boundary (parallels the emoji
        // pin in tokenize_preserves_non_letter_non_separator_runs).
        let replacement = "\u{FFFD}";
        assert_eq!(tokenize(replacement), vec![replacement]);
        assert_eq!(tokenize(replacement), tokenize(replacement));
        // Non-BMP scalar (Rust crab emoji, U+1F980). Same reasoning as
        // above: it is neither a separator nor a letter nor a digit,
        // so it forms a single-char token. Importantly, slicing / iter
        // boundaries must not panic on 4-byte UTF-8.
        let crab = "\u{1F980}";
        assert_eq!(tokenize(crab), vec![crab]);
        assert_eq!(tokenize(&format!("a{crab}b")), vec![format!("a{crab}b")]);
        // NUL byte (U+0000). NUL is neither whitespace nor an ASCII
        // letter/digit, so it is absorbed into the token without a
        // boundary. It must not panic, and ngram-cosine must handle
        // the resulting string without producing NaN.
        let nul = "a\0b";
        assert_eq!(tokenize(nul), vec!["a\u{0}b"]);
        assert!(!char_ngram_cosine(nul, nul, 2).is_nan());
        // Empty and whitespace-only are already covered; pin one more
        // edge: a lone U+0085 (NEXT LINE, is_whitespace) must act as
        // a separator.
        assert_eq!(tokenize("a\u{0085}b"), vec!["a", "b"]);
    }

    #[test]
    fn token_similarity_never_returns_nan_on_unicode_inputs() {
        // Angle-2 guard: callers downstream (resolve_anchors) drop
        // NaN confidence anchors, but any strategy function that can
        // manufacture NaN from well-formed input would be a silent
        // data-loss bug. Spot-check a handful of Unicode corner cases
        // and assert the score is always finite in [0, 1].
        for pair in [
            ("\u{FFFD}", "\u{FFFD}"),
            ("\u{1F980}", "\u{1F980}crab"),
            ("a\0b", "ab"),
            ("", "\u{FFFD}"),
            ("caf\u{00E9}", "cafe\u{0301}"),
        ] {
            let s = token_similarity(pair.0, pair.1);
            assert!(
                s.is_finite() && (0.0..=1.0).contains(&s),
                "token_similarity({:?}, {:?}) = {s} must be finite in [0,1]",
                pair.0,
                pair.1
            );
        }
    }

    #[test]
    fn token_similarity_unrelated_is_low() {
        let score = token_similarity("createdAt", "authorId");
        assert!(
            score < 0.4,
            "unrelated identifiers should score low: {score}"
        );
    }
}