pith_text/reference.rs
1//! The canonical serialization the `reference.json` vectors are
2//! defined over — the single source of truth shared by the vector
3//! generator ([`crate::reference`] consumers, `tools/gen-reference`)
4//! and the FFI surface ([`crate::ffi`]).
5//!
6//! [`pipeline`] runs the documented pipeline stages over an input and
7//! reports the canonical UTF-8 bytes plus the tokenization/shingling
8//! counts; [`measure_signature`] folds a 128-word MinHash signature
9//! for compact transport (first two words verbatim, then FNV-1a 64
10//! and SHA-256 over all 128 little-endian words). Both were moved
11//! verbatim out of `tools/gen-reference/main.rs` so the generator and
12//! the SDKs cannot drift; `gen-reference verify` compares against the
13//! committed `reference.json` byte-for-byte on every run.
14//!
15//! [`canonical_stream`] packs both into the one byte stream the
16//! `pith_text_fingerprint` FFI hands out:
17//!
18//! ```text
19//! [0..4) word_count u32 big-endian
20//! [4..8) shingle_count u32 big-endian
21//! [8..8+C) canonical the canonical UTF-8 bytes (C bytes)
22//! [8+C..) signature 128 u64 little-endian words (1024 bytes)
23//! ```
24//!
25//! so `canonical_sha256 == sha256(stream[8..8+C])` and the signature
26//! folds of `reference.json` are folds over the 1024-byte tail.
27//! `canonical_stream("")` is the empty-input sentinel: zero counts,
28//! the one-byte canonical form `"\n"`, and a tail of 128 `u64::MAX`
29//! words.
30
31use alloc::format;
32use alloc::string::String;
33use alloc::vec::Vec;
34
35use pith_digest::{fnv1a64, sha256};
36
37use crate::{SIGNATURE_WORDS, canonicalize, signature};
38
39/// Shingle width of the pipeline (spec §4: three consecutive words). The
40/// signature vectors pin it implicitly: a width change drifts them and
41/// `verify` fails loudly.
42pub const SHINGLE_WORDS: usize = 3;
43
44/// One canonicalisation measurement: the canonical UTF-8 bytes plus the
45/// tokenization/shingling counts the pipeline derives from them.
46pub struct Canonicalized {
47 /// The canonical UTF-8 form ([`canonicalize`] output).
48 pub canonical: String,
49 /// Whitespace-separated word count of the canonical form.
50 pub word_count: usize,
51 /// Consecutive `k`-word window count, `k = min(3, n)` (spec §4).
52 pub shingle_count: usize,
53}
54
55/// Runs the documented pipeline stages over `input` and counts the
56/// tokens and `k`-word windows.
57pub fn pipeline(input: &str) -> Canonicalized {
58 let canonical = canonicalize(input);
59 let word_count = canonical.split_whitespace().count();
60 let shingle_count = if word_count == 0 {
61 0
62 } else {
63 word_count - word_count.min(SHINGLE_WORDS) + 1
64 };
65 Canonicalized {
66 canonical,
67 word_count,
68 shingle_count,
69 }
70}
71
72/// The 128-word MinHash signature folded for compact transport: first
73/// two words verbatim, then FNV-1a 64 and SHA-256 over all 128
74/// little-endian words.
75pub struct SigFold {
76 /// Signature word 0, verbatim.
77 pub first: u64,
78 /// Signature word 1, verbatim.
79 pub second: u64,
80 /// FNV-1a 64 over all 128 words as little-endian bytes.
81 pub fnv1a64: u64,
82 /// SHA-256 over all 128 words as little-endian bytes, hex.
83 pub sha256: String,
84 /// The full 128-word signature.
85 pub words: Vec<u64>,
86}
87
88/// Folds the 128-word MinHash signature of `input` for compact
89/// transport (see [`SigFold`]).
90pub fn measure_signature(input: &str) -> SigFold {
91 let words = signature(input);
92 assert_eq!(words.len(), 128, "signature length is part of the contract");
93 let mut le = Vec::with_capacity(words.len() * 8);
94 for w in &words {
95 le.extend_from_slice(&w.to_le_bytes());
96 }
97 SigFold {
98 first: words[0],
99 second: words[1],
100 fnv1a64: fnv1a64(&le),
101 sha256: hex(sha256(&le).expect("sha256 of signature words").as_bytes()),
102 words,
103 }
104}
105
106/// Lowercase hex of `bytes`.
107fn hex(bytes: &[u8]) -> String {
108 let mut s = String::with_capacity(bytes.len() * 2);
109 for b in bytes {
110 s.push_str(&format!("{b:02x}"));
111 }
112 s
113}
114
115/// Serializes the full fingerprint of `input` into the canonical byte
116/// stream the `reference.json` vectors and the `pith_text_fingerprint`
117/// FFI are defined over (layout in the [module docs](self)):
118/// big-endian counts header, the canonical UTF-8 bytes, then the
119/// 128 signature words little-endian. `canonical_stream("")` carries
120/// the empty-input sentinel: zero counts, no canonical bytes, and a
121/// tail of 128 `u64::MAX` words.
122#[must_use]
123pub fn canonical_stream(input: &str) -> Vec<u8> {
124 let d = pipeline(input);
125 let sig = measure_signature(&d.canonical);
126 let mut stream = Vec::with_capacity(8 + d.canonical.len() + SIGNATURE_WORDS * 8);
127 stream.extend_from_slice(&(d.word_count as u32).to_be_bytes());
128 stream.extend_from_slice(&(d.shingle_count as u32).to_be_bytes());
129 stream.extend_from_slice(d.canonical.as_bytes());
130 for w in &sig.words {
131 stream.extend_from_slice(&w.to_le_bytes());
132 }
133 stream
134}
135
136#[cfg(test)]
137mod tests {
138 use super::{SHINGLE_WORDS, canonical_stream, hex, measure_signature, pipeline};
139
140 /// Lowercase hex of `bytes` — the test-side re-derivation of the
141 /// private helper (the public folds are pinned instead).
142 #[test]
143 fn hex_encodes_lowercase() {
144 assert_eq!(hex(&[0x0f, 0xa0, 0x00]), "0fa000");
145 }
146
147 /// The stream over the golden-pin input carries the pinned counts,
148 /// the canonical bytes and the pinned signature words/folds.
149 #[test]
150 fn canonical_stream_alpha_beta_gamma() {
151 let stream = canonical_stream("alpha beta gamma");
152 assert_eq!(u32::from_be_bytes(stream[0..4].try_into().unwrap()), 3);
153 assert_eq!(u32::from_be_bytes(stream[4..8].try_into().unwrap()), 1);
154 assert_eq!(&stream[8..8 + 17], b"alpha beta gamma\n");
155 // 8-byte header + 17 canonical bytes + 1024-byte signature tail.
156 assert_eq!(stream.len(), 8 + 17 + 1024);
157 let word = |i: usize| {
158 u64::from_le_bytes(
159 stream[8 + 17 + i * 8..8 + 17 + (i + 1) * 8]
160 .try_into()
161 .unwrap(),
162 )
163 };
164 // Golden pins, byte-for-byte from the monorepo conformance suite
165 // (the same values `signature-alpha-beta-gamma` records).
166 assert_eq!(word(0), 0x5409_aadb_22bb_8479);
167 assert_eq!(word(1), 0xb2e6_ff4e_debf_53c6);
168 let tail = &stream[8 + 17..];
169 assert_eq!(tail.len(), 1024);
170 assert_eq!(pith_digest::fnv1a64(tail), 0x46f8_ab1a_50c6_1813);
171 }
172
173 /// The empty input pins the sentinel stream: zero counts, the
174 /// one-byte canonical form (`canonicalize("") == "\n"`), 128
175 /// `u64::MAX` words — the exact folds `signature-empty-sentinel`
176 /// records.
177 #[test]
178 fn canonical_stream_empty_input_is_the_sentinel() {
179 let stream = canonical_stream("");
180 assert_eq!(stream.len(), 8 + 1 + 1024);
181 assert_eq!(&stream[..8], &[0, 0, 0, 0, 0, 0, 0, 0]);
182 assert_eq!(stream[8], b'\n');
183 assert!(stream[9..].iter().all(|&b| b == 0xff));
184 assert_eq!(measure_signature("").fnv1a64, 0x89bf_a3a9_2853_9725);
185 assert_eq!(
186 measure_signature("").sha256,
187 "5f4ecdb7b71c3e403983fe405cddcdc2f2576b655fdb3e80d94a6f7c32e58bc2"
188 );
189 }
190
191 /// The stream header and [`pipeline`] always agree on both counts.
192 #[test]
193 fn stream_counts_agree_with_pipeline() {
194 for input in [
195 "",
196 " \n \t ",
197 "text\r\n",
198 "one two three",
199 "one two three four five",
200 "Ti\u{00EA}\u{0301}ng Vi\u{00EA}\u{0323}t",
201 ] {
202 let d = pipeline(input);
203 let stream = canonical_stream(input);
204 assert_eq!(
205 u32::from_be_bytes(stream[0..4].try_into().unwrap()) as usize,
206 d.word_count
207 );
208 assert_eq!(
209 u32::from_be_bytes(stream[4..8].try_into().unwrap()) as usize,
210 d.shingle_count
211 );
212 assert_eq!(&stream[8..8 + d.canonical.len()], d.canonical.as_bytes());
213 }
214 }
215
216 /// The shingle-count rule moved with `pipeline` unchanged
217 /// (`k = min(3, n)` windows, zero for the empty document).
218 #[test]
219 fn shingle_counts_follow_the_min_rule() {
220 assert_eq!(pipeline("").shingle_count, 0);
221 assert_eq!(pipeline("one").shingle_count, 1);
222 assert_eq!(pipeline("one two").shingle_count, 1);
223 assert_eq!(pipeline("one two three").shingle_count, 1);
224 assert_eq!(pipeline("one two three four").shingle_count, 2);
225 assert_eq!(pipeline("one two three four five").shingle_count, 3);
226 assert_eq!(pipeline("one two three four five").word_count, 5);
227 assert_eq!(SHINGLE_WORDS, 3);
228 }
229}