c2pa_text_binding/
normalize.rs

1// SPDX-License-Identifier: MIT OR Apache-2.0
2
3//! Canonical normalization shared by the text soft-binding family.
4//!
5//! Two streams are produced from the same source text:
6//!
7//! * [`canonical`] — the *surface* stream used by `text-fingerprint.1` and
8//!   `text-minhash.1`: NFC, strip zero-width/format characters, lowercase,
9//!   collapse whitespace, strip punctuation. Reformatting, re-encoding, case
10//!   and whitespace edits, and zero-width injection leave it unchanged.
11//! * [`structural`] — the *structure-preserving* stream used by
12//!   `text-structure.1`: NFC and strip zero-width/format only, keeping
13//!   sentence/paragraph boundaries and punctuation, which are the signal.
14
15use unicode_normalization::UnicodeNormalization;
16
17/// Zero-width and format characters removed by every algorithm.
18///
19/// U+200B ZERO WIDTH SPACE, U+200C ZWNJ, U+200D ZWJ, U+FEFF BOM,
20/// U+2060 WORD JOINER, and variation selectors U+FE00–U+FE0F.
21pub fn is_zero_width_format(c: char) -> bool {
22    matches!(
23        c,
24        '\u{200B}' | '\u{200C}' | '\u{200D}' | '\u{FEFF}' | '\u{2060}'
25    ) || matches!(c, '\u{FE00}'..='\u{FE0F}')
26}
27
28/// Surface canonical stream for the fingerprint and MinHash algorithms.
29///
30/// Alphanumeric characters (including CJK) are kept; every other character —
31/// whitespace, punctuation, and symbols — acts as a token separator that
32/// collapses to a single space, with no leading or trailing space.
33pub fn canonical(text: &str) -> String {
34    let mut out = String::with_capacity(text.len());
35    let mut pending_separator = false;
36    for c in text.nfc() {
37        if is_zero_width_format(c) {
38            continue;
39        }
40        for lc in c.to_lowercase() {
41            if lc.is_alphanumeric() {
42                if pending_separator && !out.is_empty() {
43                    out.push(' ');
44                }
45                pending_separator = false;
46                out.push(lc);
47            } else {
48                pending_separator = true;
49            }
50        }
51    }
52    out
53}
54
55/// Structure-preserving stream: NFC and strip zero-width/format only.
56///
57/// Case, punctuation, and sentence/paragraph boundaries are preserved so the
58/// structural fingerprint can read them.
59pub fn structural(text: &str) -> String {
60    let mut out = String::with_capacity(text.len());
61    for c in text.nfc() {
62        if is_zero_width_format(c) {
63            continue;
64        }
65        out.push(c);
66    }
67    out
68}
69
70/// Word tokens of the surface stream: the whitespace-separated pieces of
71/// [`canonical`]. Used by MinHash shingling and structural token counting.
72pub fn words(text: &str) -> Vec<String> {
73    canonical(text)
74        .split(' ')
75        .filter(|w| !w.is_empty())
76        .map(str::to_string)
77        .collect()
78}
79
80#[cfg(test)]
81mod tests {
82    use super::*;
83
84    #[test]
85    fn strips_zero_width_and_format() {
86        let dirty = "he\u{200B}llo\u{FEFF} wor\u{2060}ld";
87        assert_eq!(canonical(dirty), "hello world");
88    }
89
90    #[test]
91    fn lowercases_and_collapses_whitespace() {
92        assert_eq!(canonical("Hello   WORLD\t\nFoo"), "hello world foo");
93    }
94
95    #[test]
96    fn punctuation_separates_tokens() {
97        assert_eq!(canonical("foo, bar; baz."), "foo bar baz");
98        assert_eq!(canonical("foo,bar"), "foo bar");
99    }
100
101    #[test]
102    fn no_leading_or_trailing_space() {
103        assert_eq!(canonical("  ...hi!  "), "hi");
104    }
105
106    #[test]
107    fn nfc_equivalence() {
108        // "é" as precomposed vs. combining sequence must canonicalize equal.
109        let precomposed = "caf\u{00E9}";
110        let decomposed = "cafe\u{0301}";
111        assert_eq!(canonical(precomposed), canonical(decomposed));
112    }
113
114    #[test]
115    fn structural_keeps_punctuation_and_case() {
116        let s = "Hello, World! A test.";
117        assert_eq!(structural(s), s);
118    }
119
120    #[test]
121    fn structural_strips_zero_width() {
122        assert_eq!(structural("a\u{200D}b."), "ab.");
123    }
124
125    #[test]
126    fn words_tokenizes() {
127        assert_eq!(
128            words("The quick, brown fox."),
129            vec!["the", "quick", "brown", "fox"]
130        );
131    }
132
133    // Pinned test vector: input -> canonical value.
134    #[test]
135    fn vector_canonical() {
136        let input = "The Quick\u{200B} Brown Fox — jumps!";
137        assert_eq!(canonical(input), "the quick brown fox jumps");
138    }
139}
c2pa_text_binding/normalize.rs

c2pa_text_binding/
normalize.rs
