c2pa_text_binding/
structure.rs

1// SPDX-License-Identifier: MIT OR Apache-2.0
2
3//! `com.writerslogic.text-structure.1` — a deterministic, model-free 256-bit
4//! SimHash over a document's structural skeleton: sentence-length sequence,
5//! paragraph shape, punctuation-class profile, and function-word skeleton.
6//! Survives synonym-level paraphrase that defeats a surface fingerprint.
7//! Match when Hamming distance <= 24 / 256.
8
9use crate::normalize::structural;
10use crate::simhash::{simhash_weighted, Hash256};
11
12/// Structural match threshold in bits.
13pub const MATCH_THRESHOLD: u32 = 24;
14
15// Independent per-family weights so the four features combine without one
16// dominating; structural features are lower-entropy than lexical n-grams.
17const W_SENTENCE: i64 = 3;
18const W_PARAGRAPH: i64 = 2;
19const W_PUNCT: i64 = 1;
20const W_SKELETON: i64 = 2;
21
22/// English closed-class (function) words kept in the skeleton; content words
23/// are masked. Fixed and pinned as part of the `.1` algorithm version.
24const FUNCTION_WORDS: &[&str] = &[
25    "the", "a", "an", "and", "or", "but", "nor", "so", "yet", "for", "of", "to", "in", "on", "at",
26    "by", "as", "is", "are", "was", "were", "be", "been", "being", "am", "do", "does", "did",
27    "have", "has", "had", "will", "would", "shall", "should", "can", "could", "may", "might",
28    "must", "with", "from", "into", "onto", "upon", "over", "under", "above", "below", "between",
29    "through", "during", "before", "after", "about", "against", "among", "around", "because", "if",
30    "then", "than", "that", "this", "these", "those", "it", "its", "he", "she", "they", "them",
31    "his", "her", "their", "our", "your", "my", "we", "you", "i", "me", "us", "who", "whom",
32    "whose", "which", "what", "when", "where", "why", "how", "not", "no", "yes", "all", "any",
33    "some", "each", "every", "both", "few", "many", "more", "most", "other", "such", "only", "own",
34    "same", "too", "very", "just", "also", "here", "there", "up", "down", "out", "off", "again",
35    "once", "while", "until",
36];
37
38/// Compute the structural fingerprint of `text`.
39pub fn compute(text: &str) -> Hash256 {
40    let norm = structural(text);
41    let paragraphs = split_paragraphs(&norm);
42
43    let mut sentence_lengths: Vec<usize> = Vec::new();
44    let mut paragraph_lengths: Vec<usize> = Vec::new();
45    let mut skeleton_tokens: Vec<String> = Vec::new();
46
47    for para in &paragraphs {
48        let sentences = split_sentences(para);
49        paragraph_lengths.push(sentences.len());
50        for sent in &sentences {
51            let words = word_tokens(sent);
52            sentence_lengths.push(words.len());
53            for w in &words {
54                if FUNCTION_WORDS.contains(&w.as_str()) {
55                    skeleton_tokens.push(w.clone());
56                } else {
57                    skeleton_tokens.push("_".to_string());
58                }
59            }
60        }
61    }
62
63    // Own the feature key strings so their bytes outlive the SimHash borrow.
64    let mut features: Vec<(String, i64)> = Vec::new();
65
66    // Sentence-length sequence: unigrams and bigrams to keep order signal.
67    for len in &sentence_lengths {
68        features.push((format!("SL1:{len}"), W_SENTENCE));
69    }
70    for pair in sentence_lengths.windows(2) {
71        features.push((format!("SL2:{},{}", pair[0], pair[1]), W_SENTENCE));
72    }
73
74    // Paragraph-length sequence.
75    for len in &paragraph_lengths {
76        features.push((format!("PL:{len}"), W_PARAGRAPH));
77    }
78
79    // Punctuation-class profile: multiset of classes over the whole document.
80    for (class, count) in punctuation_profile(&norm) {
81        features.push((format!("PUNCT:{class}"), W_PUNCT * count as i64));
82    }
83
84    // Function-word skeleton: 3-gram shingles of the masked token sequence.
85    if skeleton_tokens.len() >= 3 {
86        for w in skeleton_tokens.windows(3) {
87            features.push((format!("FW:{} {} {}", w[0], w[1], w[2]), W_SKELETON));
88        }
89    } else if !skeleton_tokens.is_empty() {
90        features.push((format!("FW:{}", skeleton_tokens.join(" ")), W_SKELETON));
91    }
92
93    simhash_weighted(features.iter().map(|(k, w)| (k.as_bytes(), *w)))
94}
95
96/// Whether two structural fingerprints identify the same underlying structure.
97/// A structural hit is corroborating evidence, not a sole provenance decision.
98pub fn matches(a: &Hash256, b: &Hash256) -> bool {
99    a.hamming(b) <= MATCH_THRESHOLD
100}
101
102fn split_paragraphs(text: &str) -> Vec<String> {
103    let mut paras = Vec::new();
104    let mut current = String::new();
105    let mut blank_run = 0;
106    for line in text.lines() {
107        if line.trim().is_empty() {
108            blank_run += 1;
109            if blank_run >= 1 && !current.trim().is_empty() {
110                paras.push(std::mem::take(&mut current));
111            }
112        } else {
113            blank_run = 0;
114            current.push_str(line);
115            current.push(' ');
116        }
117    }
118    if !current.trim().is_empty() {
119        paras.push(current);
120    }
121    if paras.is_empty() {
122        paras.push(text.to_string());
123    }
124    paras
125}
126
127fn split_sentences(para: &str) -> Vec<String> {
128    let mut sentences = Vec::new();
129    let mut current = String::new();
130    for c in para.chars() {
131        current.push(c);
132        if matches!(c, '.' | '!' | '?') && current.trim().len() > 1 {
133            sentences.push(std::mem::take(&mut current));
134        }
135    }
136    if !current.trim().is_empty() {
137        sentences.push(current);
138    }
139    if sentences.is_empty() && !para.trim().is_empty() {
140        sentences.push(para.to_string());
141    }
142    sentences
143}
144
145fn word_tokens(sentence: &str) -> Vec<String> {
146    sentence
147        .split(|c: char| !c.is_alphanumeric())
148        .filter(|w| !w.is_empty())
149        .map(|w| w.to_lowercase())
150        .collect()
151}
152
153/// Ordered multiset of punctuation classes as `(class, count)` pairs.
154fn punctuation_profile(text: &str) -> Vec<(char, usize)> {
155    let mut counts: std::collections::BTreeMap<char, usize> = std::collections::BTreeMap::new();
156    for c in text.chars() {
157        if let Some(class) = punctuation_class(c) {
158            *counts.entry(class).or_insert(0) += 1;
159        }
160    }
161    counts.into_iter().collect()
162}
163
164/// Fold a character into a punctuation class, or `None` if not punctuation.
165fn punctuation_class(c: char) -> Option<char> {
166    match c {
167        '.' | '\u{2026}' => Some('.'), // period / ellipsis
168        ',' => Some(','),
169        ';' => Some(';'),
170        ':' => Some(':'),
171        '?' => Some('?'),
172        '!' => Some('!'),
173        '-' | '\u{2013}' | '\u{2014}' => Some('-'), // hyphen / en / em dash
174        '(' | ')' | '[' | ']' | '{' | '}' => Some('('),
175        '"' | '\'' | '\u{201C}' | '\u{201D}' | '\u{2018}' | '\u{2019}' => Some('"'),
176        _ => None,
177    }
178}
179
180#[cfg(test)]
181mod tests {
182    use super::*;
183
184    const SRC: &str = "Provenance must survive editing. A soft binding derives a durable value \
185        from the words themselves. When the embedded manifest is stripped, a resolver \
186        recomputes the value and finds the manifest again.\n\nText is mutable, so no single \
187        technique wins. A family of algorithms with different fragility profiles is combined \
188        by the verifier. Casual redistribution stays fully recoverable.";
189
190    #[test]
191    fn identical_text_zero_distance() {
192        let a = compute(SRC);
193        let b = compute(SRC);
194        assert_eq!(a.hamming(&b), 0);
195    }
196
197    #[test]
198    fn synonym_paraphrase_stays_within_threshold() {
199        let a = compute(SRC);
200        // Swap content words for synonyms; structure (sentence rhythm,
201        // punctuation, function words) is preserved.
202        let para = SRC
203            .replace("survive", "outlast")
204            .replace("durable", "lasting")
205            .replace("stripped", "removed")
206            .replace("recomputes", "recalculates")
207            .replace("mutable", "changeable")
208            .replace("technique", "method")
209            .replace("fragility", "brittleness")
210            .replace("redistribution", "resharing");
211        let b = compute(&para);
212        let d = a.hamming(&b);
213        assert!(
214            d <= MATCH_THRESHOLD,
215            "synonym-level paraphrase distance {d} exceeded structural threshold"
216        );
217    }
218
219    #[test]
220    fn reformatting_survives() {
221        let a = compute(SRC);
222        let b = compute(&SRC.replace(". ", ".  \u{200B}"));
223        assert!(matches(&a, &b));
224    }
225
226    #[test]
227    fn different_structure_diverges() {
228        let a = compute(SRC);
229        let b = compute(
230            "One short line. Another. And a third! Then a question? Yes. No. Maybe. \
231             Terse fragments everywhere, with commas, dashes — and semicolons; many of them.",
232        );
233        assert!(
234            a.hamming(&b) > MATCH_THRESHOLD,
235            "a structurally different document must exceed the threshold"
236        );
237    }
238
239    // Pinned test vector.
240    #[test]
241    fn vector_structure() {
242        let h = compute("First sentence here. Second one follows. Third to close it.");
243        assert_eq!(
244            h.to_hex(),
245            "1e7b9ab9dabce9dfe4461a259434f09be52147161a0228234838f86e7dc60a62",
246            "PIN: recompute and update on any intentional algorithm change"
247        );
248    }
249}
c2pa_text_binding/structure.rs

c2pa_text_binding/
structure.rs
