dag_ml_core/
canonical.rs

1//! DAG-ML Typed Canonical Value v1 (TCV1).
2//!
3//! TCV1 is deliberately distinct from the historical JSON fingerprints used
4//! elsewhere in this crate. It preserves the JSON token distinction between
5//! integers and binary64 values, normalizes text to Unicode NFC, orders object
6//! keys by normalized UTF-8 bytes, and hashes a domain-separated binary
7//! preimage. Existing callers of `stable_json_fingerprint` are intentionally
8//! unaffected.
9
10use std::collections::{HashMap, HashSet};
11use std::fmt::Write as _;
12
13use serde::de::DeserializeOwned;
14use serde::Serialize;
15use sha2::{Digest, Sha256};
16use thiserror::Error;
17use unicode_normalization::UnicodeNormalization;
18
19use crate::error::{DagMlError, Result as DagMlResult};
20
21/// Domain separator prepended to every TCV1 value before hashing.
22pub const TCV1_PREFIX: &[u8] = b"DAGML-TCV1\0";
23
24/// Unicode data version frozen by the TCV1 normalization contract.
25pub const TCV1_UNICODE_VERSION: (u8, u8, u8) = (17, 0, 0);
26
27const _: () = assert!(unicode_normalization::UNICODE_VERSION.0 == TCV1_UNICODE_VERSION.0);
28const _: () = assert!(unicode_normalization::UNICODE_VERSION.1 == TCV1_UNICODE_VERSION.1);
29const _: () = assert!(unicode_normalization::UNICODE_VERSION.2 == TCV1_UNICODE_VERSION.2);
30
31const MAX_NESTING_DEPTH: usize = 128;
32
33/// An integer parsed without passing through binary64.
34///
35/// During JSON parsing, tokens with a leading minus sign use the signed domain
36/// and all other integer tokens use the unsigned domain. The public variants
37/// also allow direct construction with any `i64` or `u64`; their common values
38/// intentionally have the same TCV1 decimal encoding. Lexical `-0` parses as
39/// signed zero and has the same TCV1 encoding as unsigned zero.
40#[derive(Clone, Debug, Eq, PartialEq)]
41pub enum CanonicalInteger {
42    Signed(i64),
43    Unsigned(u64),
44}
45
46impl CanonicalInteger {
47    fn decimal(&self) -> String {
48        match self {
49            Self::Signed(value) => value.to_string(),
50            Self::Unsigned(value) => value.to_string(),
51        }
52    }
53}
54
55/// A strict JSON value that retains integer-versus-binary64 token kind.
56///
57/// [`TypedCanonicalValue::Object`] retains input order for diagnostics and
58/// representation equality, but object order is not significant to TCV1.
59/// Encoding validates raw and NFC-normalized key uniqueness and then sorts keys
60/// by normalized UTF-8 bytes. Public variants permit construction without a
61/// JSON round-trip;
62/// [`tcv1_encode`] performs the same finite-number and object-key checks for
63/// such values as [`parse_typed_json`].
64#[derive(Clone, Debug, PartialEq)]
65pub enum TypedCanonicalValue {
66    Null,
67    Bool(bool),
68    Integer(CanonicalInteger),
69    Binary64(f64),
70    String(String),
71    Array(Vec<TypedCanonicalValue>),
72    Object(Vec<(String, TypedCanonicalValue)>),
73}
74
75impl TypedCanonicalValue {
76    /// Return the lowercase SHA-256 fingerprint of this value's TCV1 preimage.
77    pub fn fingerprint(&self) -> Result<String, Tcv1Error> {
78        tcv1_sha256(self)
79    }
80
81    /// Fingerprint an object after removing exactly one normalized key.
82    ///
83    /// This supports self-fingerprinted contracts without accepting ambiguous
84    /// NFC aliases. The receiver must be an object and the requested key must
85    /// occur exactly once after NFC normalization.
86    pub fn fingerprint_without(&self, key: &str) -> Result<String, Tcv1Error> {
87        let Self::Object(entries) = self else {
88            return Err(Tcv1Error::ExpectedObject);
89        };
90        let normalized_key = normalize(key);
91        let mut removed = 0_usize;
92        let mut filtered = Vec::with_capacity(entries.len().saturating_sub(1));
93        for (member_key, member_value) in entries {
94            if normalize(member_key) == normalized_key {
95                removed += 1;
96            } else {
97                filtered.push((member_key.clone(), member_value.clone()));
98            }
99        }
100        match removed {
101            0 => Err(Tcv1Error::MissingObjectKey(normalized_key)),
102            1 => tcv1_sha256(&Self::Object(filtered)),
103            _ => Err(Tcv1Error::AmbiguousObjectKey(normalized_key)),
104        }
105    }
106}
107
108/// Parse or encoding failure for the strict TCV1 domain.
109#[derive(Clone, Debug, Eq, PartialEq, Error)]
110pub enum Tcv1Error {
111    #[error("input is not valid UTF-8 at byte {valid_up_to}")]
112    InvalidUtf8 {
113        valid_up_to: usize,
114        error_len: Option<usize>,
115    },
116
117    #[error("invalid JSON at UTF-8 byte {offset}: {message}")]
118    InvalidJson {
119        offset: usize,
120        message: &'static str,
121    },
122
123    #[error("duplicate JSON object key `{key}` at UTF-8 byte {offset}")]
124    DuplicateObjectKey { key: String, offset: usize },
125
126    #[error("NFC-colliding JSON object keys `{first}` and `{second}` at UTF-8 byte {offset}")]
127    NfcKeyCollision {
128        first: String,
129        second: String,
130        offset: usize,
131    },
132
133    #[error("integer token at UTF-8 byte {offset} is outside the TCV1 {domain} range")]
134    IntegerOutOfRange { offset: usize, domain: &'static str },
135
136    #[error("number token at UTF-8 byte {offset} is outside finite binary64 range")]
137    Binary64OutOfRange { offset: usize },
138
139    #[error("TCV1 value nesting exceeds {MAX_NESTING_DEPTH} levels")]
140    NestingTooDeep,
141
142    #[error("TCV1 collection length does not fit u64")]
143    LengthOverflow,
144
145    #[error("programmatically constructed TCV1 binary64 must be finite")]
146    NonFiniteBinary64,
147
148    #[error("fingerprint_without requires a TCV1 object")]
149    ExpectedObject,
150
151    #[error("object does not contain normalized key `{0}`")]
152    MissingObjectKey(String),
153
154    #[error("object contains more than one key normalized as `{0}`")]
155    AmbiguousObjectKey(String),
156}
157
158/// Parse exactly one strict JSON document while retaining numeric token kind.
159///
160/// Only RFC 8259 whitespace is accepted. Duplicate decoded keys, NFC-colliding
161/// keys, unpaired UTF-16 surrogate escapes, out-of-range integer tokens,
162/// non-finite binary64 conversions, and trailing data are rejected.
163pub fn parse_typed_json(input: &str) -> Result<TypedCanonicalValue, Tcv1Error> {
164    let mut parser = Parser::new(input);
165    let value = parser.parse_value(0)?;
166    parser.skip_whitespace();
167    if parser.offset != input.len() {
168        return Err(parser.invalid("trailing data after the JSON value"));
169    }
170    Ok(value)
171}
172
173/// Parse exactly one strict UTF-8 JSON document while retaining numeric kind.
174pub fn parse_typed_json_bytes(input: &[u8]) -> Result<TypedCanonicalValue, Tcv1Error> {
175    let input = std::str::from_utf8(input).map_err(|error| Tcv1Error::InvalidUtf8 {
176        valid_up_to: error.valid_up_to(),
177        error_len: error.error_len(),
178    })?;
179    parse_typed_json(input)
180}
181
182/// Validate a structured `serde_json` value against the same finite-number,
183/// nesting and NFC-key rules as a raw TCV1 document.
184///
185/// A `serde_json::Value` has already lost duplicate raw object members, so raw
186/// JSON boundaries must still call [`parse_typed_json`] before decoding. This
187/// helper is for structured host boundaries (for example Python mappings),
188/// where distinct keys can still collide after Unicode normalization.
189pub fn validate_typed_serde_value(value: &serde_json::Value) -> Result<(), Tcv1Error> {
190    fn convert(value: &serde_json::Value) -> TypedCanonicalValue {
191        match value {
192            serde_json::Value::Null => TypedCanonicalValue::Null,
193            serde_json::Value::Bool(value) => TypedCanonicalValue::Bool(*value),
194            serde_json::Value::Number(value) => {
195                if let Some(value) = value.as_u64() {
196                    TypedCanonicalValue::Integer(CanonicalInteger::Unsigned(value))
197                } else if let Some(value) = value.as_i64() {
198                    TypedCanonicalValue::Integer(CanonicalInteger::Signed(value))
199                } else {
200                    TypedCanonicalValue::Binary64(
201                        value
202                            .as_f64()
203                            .expect("serde_json numbers are integer or finite binary64"),
204                    )
205                }
206            }
207            serde_json::Value::String(value) => TypedCanonicalValue::String(value.clone()),
208            serde_json::Value::Array(values) => {
209                TypedCanonicalValue::Array(values.iter().map(convert).collect())
210            }
211            serde_json::Value::Object(values) => TypedCanonicalValue::Object(
212                values
213                    .iter()
214                    .map(|(key, value)| (key.clone(), convert(value)))
215                    .collect(),
216            ),
217        }
218    }
219
220    tcv1_encode(&convert(value)).map(|_| ())
221}
222
223/// Encode one value using the DAG-ML TCV1 binary grammar.
224///
225/// The returned bytes begin with the value tag, not the domain separator. Use
226/// [`tcv1_preimage`] for bytes suitable for hashing.
227pub fn tcv1_encode(value: &TypedCanonicalValue) -> Result<Vec<u8>, Tcv1Error> {
228    let mut output = Vec::new();
229    encode_value(value, &mut output, 0)?;
230    Ok(output)
231}
232
233/// Return the domain-separated preimage `DAGML-TCV1\0 || encode(value)`.
234pub fn tcv1_preimage(value: &TypedCanonicalValue) -> Result<Vec<u8>, Tcv1Error> {
235    let mut output = Vec::from(TCV1_PREFIX);
236    encode_value(value, &mut output, 0)?;
237    Ok(output)
238}
239
240/// Return lowercase SHA-256 of [`tcv1_preimage`].
241pub fn tcv1_sha256(value: &TypedCanonicalValue) -> Result<String, Tcv1Error> {
242    let digest = Sha256::digest(tcv1_preimage(value)?);
243    let mut output = String::with_capacity(digest.len() * 2);
244    for byte in digest {
245        write!(&mut output, "{byte:02x}").expect("writing to String cannot fail");
246    }
247    Ok(output)
248}
249
250fn normalize(value: &str) -> String {
251    value.nfc().collect()
252}
253
254fn encode_value(
255    value: &TypedCanonicalValue,
256    output: &mut Vec<u8>,
257    depth: usize,
258) -> Result<(), Tcv1Error> {
259    if depth > MAX_NESTING_DEPTH {
260        return Err(Tcv1Error::NestingTooDeep);
261    }
262    match value {
263        TypedCanonicalValue::Null => output.push(b'N'),
264        TypedCanonicalValue::Bool(false) => output.push(b'F'),
265        TypedCanonicalValue::Bool(true) => output.push(b'T'),
266        TypedCanonicalValue::Integer(value) => {
267            output.push(b'I');
268            let payload = value.decimal();
269            encode_length(payload.len(), output)?;
270            output.extend_from_slice(payload.as_bytes());
271        }
272        TypedCanonicalValue::Binary64(value) => {
273            if !value.is_finite() {
274                return Err(Tcv1Error::NonFiniteBinary64);
275            }
276            output.push(b'D');
277            let normalized = if *value == 0.0 { 0.0 } else { *value };
278            output.extend_from_slice(&normalized.to_bits().to_be_bytes());
279        }
280        TypedCanonicalValue::String(value) => encode_string(value, output)?,
281        TypedCanonicalValue::Array(values) => {
282            output.push(b'A');
283            encode_length(values.len(), output)?;
284            for value in values {
285                encode_value(value, output, depth + 1)?;
286            }
287        }
288        TypedCanonicalValue::Object(entries) => {
289            output.push(b'O');
290            encode_length(entries.len(), output)?;
291
292            let mut raw_keys = HashSet::with_capacity(entries.len());
293            let mut normalized_keys = HashMap::with_capacity(entries.len());
294            let mut sorted = Vec::with_capacity(entries.len());
295            for (key, value) in entries {
296                if !raw_keys.insert(key.as_str()) {
297                    return Err(Tcv1Error::DuplicateObjectKey {
298                        key: key.clone(),
299                        offset: 0,
300                    });
301                }
302                let normalized = normalize(key);
303                if let Some(first) = normalized_keys.insert(normalized.clone(), key.as_str()) {
304                    return Err(Tcv1Error::NfcKeyCollision {
305                        first: first.to_string(),
306                        second: key.clone(),
307                        offset: 0,
308                    });
309                }
310                sorted.push((normalized.into_bytes(), value));
311            }
312            sorted.sort_by(|left, right| left.0.cmp(&right.0));
313            for (normalized_key, value) in sorted {
314                encode_normalized_string(&normalized_key, output)?;
315                encode_value(value, output, depth + 1)?;
316            }
317        }
318    }
319    Ok(())
320}
321
322fn encode_string(value: &str, output: &mut Vec<u8>) -> Result<(), Tcv1Error> {
323    let normalized = normalize(value);
324    encode_normalized_string(normalized.as_bytes(), output)
325}
326
327fn encode_normalized_string(payload: &[u8], output: &mut Vec<u8>) -> Result<(), Tcv1Error> {
328    output.push(b'S');
329    encode_length(payload.len(), output)?;
330    output.extend_from_slice(payload);
331    Ok(())
332}
333
334fn encode_length(length: usize, output: &mut Vec<u8>) -> Result<(), Tcv1Error> {
335    let length = u64::try_from(length).map_err(|_| Tcv1Error::LengthOverflow)?;
336    output.extend_from_slice(&length.to_be_bytes());
337    Ok(())
338}
339
340struct Parser<'a> {
341    input: &'a str,
342    bytes: &'a [u8],
343    offset: usize,
344}
345
346impl<'a> Parser<'a> {
347    fn new(input: &'a str) -> Self {
348        Self {
349            input,
350            bytes: input.as_bytes(),
351            offset: 0,
352        }
353    }
354
355    fn parse_value(&mut self, depth: usize) -> Result<TypedCanonicalValue, Tcv1Error> {
356        if depth > MAX_NESTING_DEPTH {
357            return Err(Tcv1Error::NestingTooDeep);
358        }
359        self.skip_whitespace();
360        match self.peek() {
361            Some(b'n') => {
362                self.consume_literal(b"null")?;
363                Ok(TypedCanonicalValue::Null)
364            }
365            Some(b'f') => {
366                self.consume_literal(b"false")?;
367                Ok(TypedCanonicalValue::Bool(false))
368            }
369            Some(b't') => {
370                self.consume_literal(b"true")?;
371                Ok(TypedCanonicalValue::Bool(true))
372            }
373            Some(b'"') => Ok(TypedCanonicalValue::String(self.parse_string()?)),
374            Some(b'[') => self.parse_array(depth),
375            Some(b'{') => self.parse_object(depth),
376            Some(b'-' | b'0'..=b'9') => self.parse_number(),
377            Some(_) => Err(self.invalid("expected a JSON value")),
378            None => Err(self.invalid("unexpected end of input")),
379        }
380    }
381
382    fn parse_array(&mut self, depth: usize) -> Result<TypedCanonicalValue, Tcv1Error> {
383        self.offset += 1;
384        self.skip_whitespace();
385        let mut values = Vec::new();
386        if self.consume_if(b']') {
387            return Ok(TypedCanonicalValue::Array(values));
388        }
389        loop {
390            values.push(self.parse_value(depth + 1)?);
391            self.skip_whitespace();
392            match self.peek() {
393                Some(b',') => self.offset += 1,
394                Some(b']') => {
395                    self.offset += 1;
396                    return Ok(TypedCanonicalValue::Array(values));
397                }
398                Some(_) => return Err(self.invalid("expected `,` or `]` in array")),
399                None => return Err(self.invalid("unterminated array")),
400            }
401        }
402    }
403
404    fn parse_object(&mut self, depth: usize) -> Result<TypedCanonicalValue, Tcv1Error> {
405        self.offset += 1;
406        self.skip_whitespace();
407        let mut entries = Vec::new();
408        let mut raw_keys = HashSet::new();
409        let mut normalized_keys: HashMap<String, String> = HashMap::new();
410        if self.consume_if(b'}') {
411            return Ok(TypedCanonicalValue::Object(entries));
412        }
413        loop {
414            self.skip_whitespace();
415            if self.peek() != Some(b'"') {
416                return Err(self.invalid("expected a string object key"));
417            }
418            let key_offset = self.offset;
419            let key = self.parse_string()?;
420            if !raw_keys.insert(key.clone()) {
421                return Err(Tcv1Error::DuplicateObjectKey {
422                    key,
423                    offset: key_offset,
424                });
425            }
426            let normalized = normalize(&key);
427            if let Some(first) = normalized_keys.insert(normalized, key.clone()) {
428                return Err(Tcv1Error::NfcKeyCollision {
429                    first,
430                    second: key,
431                    offset: key_offset,
432                });
433            }
434            self.skip_whitespace();
435            self.expect(b':', "expected `:` after object key")?;
436            let value = self.parse_value(depth + 1)?;
437            entries.push((key, value));
438            self.skip_whitespace();
439            match self.peek() {
440                Some(b',') => self.offset += 1,
441                Some(b'}') => {
442                    self.offset += 1;
443                    return Ok(TypedCanonicalValue::Object(entries));
444                }
445                Some(_) => return Err(self.invalid("expected `,` or `}` in object")),
446                None => return Err(self.invalid("unterminated object")),
447            }
448        }
449    }
450
451    fn parse_number(&mut self) -> Result<TypedCanonicalValue, Tcv1Error> {
452        let start = self.offset;
453        let negative = self.consume_if(b'-');
454
455        match self.peek() {
456            Some(b'0') => {
457                self.offset += 1;
458                if matches!(self.peek(), Some(b'0'..=b'9')) {
459                    return Err(self.invalid("leading zero in JSON number"));
460                }
461            }
462            Some(b'1'..=b'9') => {
463                self.offset += 1;
464                while matches!(self.peek(), Some(b'0'..=b'9')) {
465                    self.offset += 1;
466                }
467            }
468            _ => return Err(self.invalid("expected integer digits")),
469        }
470
471        let mut binary64 = false;
472        if self.consume_if(b'.') {
473            binary64 = true;
474            if !matches!(self.peek(), Some(b'0'..=b'9')) {
475                return Err(self.invalid("expected digit after decimal point"));
476            }
477            while matches!(self.peek(), Some(b'0'..=b'9')) {
478                self.offset += 1;
479            }
480        }
481        if matches!(self.peek(), Some(b'e' | b'E')) {
482            binary64 = true;
483            self.offset += 1;
484            if matches!(self.peek(), Some(b'+' | b'-')) {
485                self.offset += 1;
486            }
487            if !matches!(self.peek(), Some(b'0'..=b'9')) {
488                return Err(self.invalid("expected exponent digits"));
489            }
490            while matches!(self.peek(), Some(b'0'..=b'9')) {
491                self.offset += 1;
492            }
493        }
494
495        let token = &self.input[start..self.offset];
496        if binary64 {
497            let value = token
498                .parse::<f64>()
499                .map_err(|_| Tcv1Error::Binary64OutOfRange { offset: start })?;
500            if !value.is_finite() {
501                return Err(Tcv1Error::Binary64OutOfRange { offset: start });
502            }
503            Ok(TypedCanonicalValue::Binary64(value))
504        } else if negative {
505            token
506                .parse::<i64>()
507                .map(|value| TypedCanonicalValue::Integer(CanonicalInteger::Signed(value)))
508                .map_err(|_| Tcv1Error::IntegerOutOfRange {
509                    offset: start,
510                    domain: "i64",
511                })
512        } else {
513            token
514                .parse::<u64>()
515                .map(|value| TypedCanonicalValue::Integer(CanonicalInteger::Unsigned(value)))
516                .map_err(|_| Tcv1Error::IntegerOutOfRange {
517                    offset: start,
518                    domain: "u64",
519                })
520        }
521    }
522
523    fn parse_string(&mut self) -> Result<String, Tcv1Error> {
524        debug_assert_eq!(self.peek(), Some(b'"'));
525        self.offset += 1;
526        let mut output = String::new();
527        let mut chunk_start = self.offset;
528        loop {
529            match self.peek() {
530                Some(b'"') => {
531                    output.push_str(&self.input[chunk_start..self.offset]);
532                    self.offset += 1;
533                    return Ok(output);
534                }
535                Some(b'\\') => {
536                    output.push_str(&self.input[chunk_start..self.offset]);
537                    self.offset += 1;
538                    self.parse_escape(&mut output)?;
539                    chunk_start = self.offset;
540                }
541                Some(0x00..=0x1f) => {
542                    return Err(self.invalid("unescaped control character in JSON string"));
543                }
544                Some(byte) if byte.is_ascii() => self.offset += 1,
545                Some(_) => {
546                    let character = self.input[self.offset..]
547                        .chars()
548                        .next()
549                        .expect("offset is on a valid UTF-8 boundary");
550                    self.offset += character.len_utf8();
551                }
552                None => return Err(self.invalid("unterminated JSON string")),
553            }
554        }
555    }
556
557    fn parse_escape(&mut self, output: &mut String) -> Result<(), Tcv1Error> {
558        match self.peek() {
559            Some(b'"') => output.push('"'),
560            Some(b'\\') => output.push('\\'),
561            Some(b'/') => output.push('/'),
562            Some(b'b') => output.push('\u{0008}'),
563            Some(b'f') => output.push('\u{000c}'),
564            Some(b'n') => output.push('\n'),
565            Some(b'r') => output.push('\r'),
566            Some(b't') => output.push('\t'),
567            Some(b'u') => {
568                self.offset += 1;
569                return self.parse_unicode_escape(output);
570            }
571            Some(_) => return Err(self.invalid("invalid JSON string escape")),
572            None => return Err(self.invalid("unterminated JSON string escape")),
573        }
574        self.offset += 1;
575        Ok(())
576    }
577
578    fn parse_unicode_escape(&mut self, output: &mut String) -> Result<(), Tcv1Error> {
579        let first_offset = self.offset;
580        let first = self.parse_hex_quad()?;
581        let scalar = if (0xd800..=0xdbff).contains(&first) {
582            if self.peek() != Some(b'\\') || self.bytes.get(self.offset + 1) != Some(&b'u') {
583                return Err(Tcv1Error::InvalidJson {
584                    offset: first_offset,
585                    message: "high surrogate is not followed by a low surrogate",
586                });
587            }
588            self.offset += 2;
589            let low_offset = self.offset;
590            let low = self.parse_hex_quad()?;
591            if !(0xdc00..=0xdfff).contains(&low) {
592                return Err(Tcv1Error::InvalidJson {
593                    offset: low_offset,
594                    message: "high surrogate is not followed by a low surrogate",
595                });
596            }
597            0x10000 + ((u32::from(first) - 0xd800) << 10) + (u32::from(low) - 0xdc00)
598        } else if (0xdc00..=0xdfff).contains(&first) {
599            return Err(Tcv1Error::InvalidJson {
600                offset: first_offset,
601                message: "isolated low surrogate in JSON string",
602            });
603        } else {
604            u32::from(first)
605        };
606        output.push(char::from_u32(scalar).expect("validated Unicode scalar value"));
607        Ok(())
608    }
609
610    fn parse_hex_quad(&mut self) -> Result<u16, Tcv1Error> {
611        if self.offset + 4 > self.bytes.len() {
612            return Err(self.invalid("incomplete Unicode escape"));
613        }
614        let mut value = 0_u16;
615        for _ in 0..4 {
616            let digit = match self.peek() {
617                Some(b'0'..=b'9') => u16::from(self.bytes[self.offset] - b'0'),
618                Some(b'a'..=b'f') => u16::from(self.bytes[self.offset] - b'a' + 10),
619                Some(b'A'..=b'F') => u16::from(self.bytes[self.offset] - b'A' + 10),
620                _ => return Err(self.invalid("invalid hexadecimal digit in Unicode escape")),
621            };
622            value = (value << 4) | digit;
623            self.offset += 1;
624        }
625        Ok(value)
626    }
627
628    fn consume_literal(&mut self, literal: &[u8]) -> Result<(), Tcv1Error> {
629        if self.bytes.get(self.offset..self.offset + literal.len()) == Some(literal) {
630            self.offset += literal.len();
631            Ok(())
632        } else {
633            Err(self.invalid("invalid JSON literal"))
634        }
635    }
636
637    fn expect(&mut self, expected: u8, message: &'static str) -> Result<(), Tcv1Error> {
638        if self.consume_if(expected) {
639            Ok(())
640        } else {
641            Err(self.invalid(message))
642        }
643    }
644
645    fn consume_if(&mut self, expected: u8) -> bool {
646        if self.peek() == Some(expected) {
647            self.offset += 1;
648            true
649        } else {
650            false
651        }
652    }
653
654    fn skip_whitespace(&mut self) {
655        while matches!(self.peek(), Some(b' ' | b'\n' | b'\r' | b'\t')) {
656            self.offset += 1;
657        }
658    }
659
660    fn peek(&self) -> Option<u8> {
661        self.bytes.get(self.offset).copied()
662    }
663
664    fn invalid(&self, message: &'static str) -> Tcv1Error {
665        Tcv1Error::InvalidJson {
666            offset: self.offset,
667            message,
668        }
669    }
670}
671
672/// Require external JSON to use the same object/array container kinds as its
673/// typed serde representation, without requiring byte-canonical field presence
674/// or ordering. This closes serde's internal positional-sequence representation
675/// for structs while preserving defaults, skipped fields and BTree ordering.
676fn validate_external_container_shapes(
677    raw: &serde_json::Value,
678    typed: &serde_json::Value,
679    path: &str,
680) -> std::result::Result<(), String> {
681    match (raw, typed) {
682        (serde_json::Value::Object(raw), serde_json::Value::Object(typed)) => {
683            for (key, raw_value) in raw {
684                if let Some(typed_value) = typed.get(key) {
685                    validate_external_container_shapes(
686                        raw_value,
687                        typed_value,
688                        &format!("{path}.{key}"),
689                    )?;
690                }
691            }
692            Ok(())
693        }
694        (serde_json::Value::Array(raw), serde_json::Value::Array(typed)) => {
695            for (index, (raw_value, typed_value)) in raw.iter().zip(typed).enumerate() {
696                validate_external_container_shapes(
697                    raw_value,
698                    typed_value,
699                    &format!("{path}[{index}]"),
700                )?;
701            }
702            Ok(())
703        }
704        (_, serde_json::Value::Object(_)) => Err(format!(
705            "{path} must use a JSON object at the external contract boundary"
706        )),
707        (_, serde_json::Value::Array(_)) => Err(format!(
708            "{path} must use a JSON array at the external contract boundary"
709        )),
710        (serde_json::Value::Object(_) | serde_json::Value::Array(_), _) => Err(format!(
711            "{path} has the wrong JSON container kind at the external contract boundary"
712        )),
713        _ => Ok(()),
714    }
715}
716
717/// Deserialize one public JSON contract while rejecting serde's positional
718/// sequence representation for structs. Semantic validation remains the owning
719/// type's responsibility after this shape-preserving decode.
720pub fn deserialize_external_contract<T, F>(
721    json: &str,
722    label: &str,
723    shape_error: F,
724) -> DagMlResult<T>
725where
726    T: DeserializeOwned + Serialize,
727    F: Fn(String) -> DagMlError,
728{
729    parse_typed_json(json).map_err(|error| {
730        shape_error(format!(
731            "{label} is not a strict TCV1 JSON document: {error}"
732        ))
733    })?;
734    let raw: serde_json::Value = serde_json::from_str(json)?;
735    deserialize_external_value(raw, label, shape_error)
736}
737
738/// Deserialize a public contract that has already crossed a structured host
739/// boundary (for example a Python mapping), while preserving the same
740/// object/array rules as [`deserialize_external_contract`].
741pub fn deserialize_external_value<T, F>(
742    raw: serde_json::Value,
743    label: &str,
744    shape_error: F,
745) -> DagMlResult<T>
746where
747    T: DeserializeOwned + Serialize,
748    F: Fn(String) -> DagMlError,
749{
750    validate_typed_serde_value(&raw).map_err(|error| {
751        shape_error(format!(
752            "{label} is not a strict TCV1 structured value: {error}"
753        ))
754    })?;
755    let value: T = serde_json::from_value(raw.clone())?;
756    let typed = serde_json::to_value(&value)?;
757    validate_external_container_shapes(&raw, &typed, label).map_err(shape_error)?;
758    Ok(value)
759}
760
761#[cfg(test)]
762mod tests {
763    use serde::{Deserialize, Serialize};
764    use serde_json::json;
765
766    use super::*;
767
768    fn parse(input: &str) -> TypedCanonicalValue {
769        parse_typed_json(input).expect("valid strict JSON")
770    }
771
772    fn hex(bytes: &[u8]) -> String {
773        let mut output = String::with_capacity(bytes.len() * 2);
774        for byte in bytes {
775            write!(&mut output, "{byte:02x}").unwrap();
776        }
777        output
778    }
779
780    fn assert_vector(input: &str, expected_preimage: &str, expected_sha256: &str) {
781        let value = parse(input);
782        assert_eq!(hex(&tcv1_preimage(&value).unwrap()), expected_preimage);
783        assert_eq!(tcv1_sha256(&value).unwrap(), expected_sha256);
784    }
785
786    #[derive(Debug, Deserialize)]
787    struct GoldenFixture {
788        tcv1_vectors: Vec<GoldenVector>,
789    }
790
791    #[derive(Debug, Deserialize)]
792    struct GoldenVector {
793        id: String,
794        document_json: String,
795        equivalent_json: Option<String>,
796        expected_preimage_hex: String,
797        expected_sha256: String,
798    }
799
800    #[derive(Debug, Deserialize, PartialEq, Serialize)]
801    struct ExternalFixture {
802        schema_version: u64,
803        values: Vec<u64>,
804    }
805
806    #[test]
807    fn production_tcv1_matches_every_committed_golden_vector() {
808        let fixture: GoldenFixture = serde_json::from_str(include_str!(
809            "../../../parity/canonical/golden/tcv1_jcs_cross_language.v1.json"
810        ))
811        .expect("valid committed TCV1 golden fixture");
812
813        for vector in fixture.tcv1_vectors {
814            assert_vector(
815                &vector.document_json,
816                &vector.expected_preimage_hex,
817                &vector.expected_sha256,
818            );
819            if let Some(equivalent) = vector.equivalent_json {
820                assert_eq!(
821                    tcv1_preimage(&parse(&vector.document_json)).unwrap(),
822                    tcv1_preimage(&parse(&equivalent)).unwrap(),
823                    "{}",
824                    vector.id
825                );
826            }
827        }
828    }
829
830    #[test]
831    fn external_contract_boundary_is_strict_and_shape_preserving() {
832        let value: ExternalFixture = deserialize_external_contract(
833            r#"{"schema_version":1,"values":[2,3]}"#,
834            "fixture",
835            DagMlError::RuntimeValidation,
836        )
837        .expect("object-form contract is valid");
838        assert_eq!(
839            value,
840            ExternalFixture {
841                schema_version: 1,
842                values: vec![2, 3],
843            }
844        );
845
846        let duplicate = deserialize_external_contract::<ExternalFixture, _>(
847            r#"{"schema_version":1,"schema_version":2,"values":[]}"#,
848            "fixture",
849            DagMlError::RuntimeValidation,
850        )
851        .unwrap_err();
852        assert!(duplicate.to_string().contains("duplicate JSON object key"));
853
854        let positional = deserialize_external_contract::<ExternalFixture, _>(
855            r#"[1,[]]"#,
856            "fixture",
857            DagMlError::RuntimeValidation,
858        )
859        .unwrap_err();
860        assert!(positional.to_string().contains("must use a JSON object"));
861    }
862
863    #[test]
864    fn golden_map_order_preimage_and_digest() {
865        const PREIMAGE: &str = "4441474d4c2d54435631004f000000000000000253000000000000000161490000000000000001325300000000000000017a49000000000000000131";
866        const DIGEST: &str = "5441a8df23725b4a60e16316f3034a7ec8b25b853ce75600fa71dda19c8a16e1";
867        assert_vector(r#"{"z":1,"a":2}"#, PREIMAGE, DIGEST);
868        assert_vector(r#"{"a":2,"z":1}"#, PREIMAGE, DIGEST);
869    }
870
871    #[test]
872    fn object_order_uses_normalized_utf8_not_utf16() {
873        assert_vector(
874            r#"{"\ue000":1,"\ud800\udc00":2}"#,
875            "4441474d4c2d54435631004f0000000000000002530000000000000003ee808049000000000000000131530000000000000004f090808049000000000000000132",
876            "7c212789a6d362b8a34e8c271d5fe003c2026a828712eef6469610eedda79bc7",
877        );
878    }
879
880    #[test]
881    fn nfc_normalizes_strings_and_keys() {
882        let decomposed = parse(r#""e\u0301""#);
883        let composed = parse(r#""é""#);
884        // PartialEq deliberately retains source representation; TCV1 bytes are
885        // the canonical equivalence relation.
886        assert_ne!(decomposed, composed);
887        assert_eq!(
888            tcv1_preimage(&decomposed).unwrap(),
889            tcv1_preimage(&composed).unwrap()
890        );
891        assert_eq!(
892            tcv1_sha256(&decomposed).unwrap(),
893            "a4af538cebb2c18fed88a1ad4245509500d201e68802a217e4e8500ef61c0e86"
894        );
895
896        let error = parse_typed_json(r#"{"é":1,"e\u0301":2}"#).unwrap_err();
897        assert!(matches!(error, Tcv1Error::NfcKeyCollision { .. }));
898    }
899
900    #[test]
901    fn signed_zero_normalizes_only_within_binary64_kind() {
902        let negative = parse("-0.0");
903        let positive = parse("0.0");
904        assert_eq!(
905            tcv1_preimage(&negative).unwrap(),
906            tcv1_preimage(&positive).unwrap()
907        );
908        assert_vector(
909            "-0.0",
910            "4441474d4c2d5443563100440000000000000000",
911            "c01f83d2f6a8e96eb7f50c4794eef0dbae68ad4d20ed116af013ae5cd4ffa49d",
912        );
913        assert_ne!(
914            tcv1_preimage(&parse("-0")).unwrap(),
915            tcv1_preimage(&negative).unwrap()
916        );
917        assert_eq!(
918            tcv1_preimage(&parse("-0")).unwrap(),
919            tcv1_preimage(&parse("0")).unwrap()
920        );
921    }
922
923    #[test]
924    fn integer_and_binary64_tokens_remain_distinct() {
925        assert_eq!(
926            parse("2"),
927            TypedCanonicalValue::Integer(CanonicalInteger::Unsigned(2))
928        );
929        assert_eq!(parse("2.0"), TypedCanonicalValue::Binary64(2.0));
930        assert_ne!(
931            tcv1_preimage(&parse("2")).unwrap(),
932            tcv1_preimage(&parse("2.0")).unwrap()
933        );
934        assert_vector(
935            "2",
936            "4441474d4c2d544356310049000000000000000132",
937            "3940883272509c80c7bbff602794dce0f62dfa7850bc3041b37c56d36bc94701",
938        );
939        let float_preimage = tcv1_preimage(&parse("2.0")).unwrap();
940        assert_eq!(float_preimage, tcv1_preimage(&parse("2e0")).unwrap());
941        assert_eq!(
942            hex(&float_preimage),
943            "4441474d4c2d5443563100444000000000000000"
944        );
945    }
946
947    #[test]
948    fn frozen_binary64_boundary_vectors() {
949        let vectors = [
950            (
951                "5e-324",
952                "4441474d4c2d5443563100440000000000000001",
953                "88a7b6becacc6cf0bf2473332aa17d9f3ed513b4024d69684458a275b5c39c24",
954            ),
955            (
956                "2.2250738585072009e-308",
957                "4441474d4c2d544356310044000fffffffffffff",
958                "78c8b93679333797971ff7ef4dba4b284adb03da7a7379bf814f635b69164765",
959            ),
960            (
961                "2.2250738585072014e-308",
962                "4441474d4c2d5443563100440010000000000000",
963                "6ea1aeba7ec435fd15165511f602295eedd7de82832713f35891600a0f552702",
964            ),
965            (
966                "9007199254740992.0",
967                "4441474d4c2d5443563100444340000000000000",
968                "8e276db087fa6f18be879c6e32a034e44ff075c66c76ff8e43cbd3dc20e0673a",
969            ),
970            (
971                "1.7976931348623157e308",
972                "4441474d4c2d5443563100447fefffffffffffff",
973                "e9231aadbc74db0fd07f62e1b04c67ab93a73c30b756e85e36f80edd3766bf5a",
974            ),
975        ];
976        for (input, preimage, digest) in vectors {
977            assert_vector(input, preimage, digest);
978        }
979        assert_eq!(
980            tcv1_preimage(&parse("5e-324")).unwrap(),
981            tcv1_preimage(&parse("4.9406564584124654e-324")).unwrap()
982        );
983        assert_eq!(
984            tcv1_preimage(&parse("9007199254740992.0")).unwrap(),
985            tcv1_preimage(&parse("9.007199254740992e15")).unwrap()
986        );
987    }
988
989    #[test]
990    fn all_tags_and_big_endian_lengths_are_explicit() {
991        let value = parse(r#"[null,false,true,"x",-1,1.5]"#);
992        assert_eq!(
993            hex(&tcv1_encode(&value).unwrap()),
994            "4100000000000000064e4654530000000000000001784900000000000000022d31443ff8000000000000"
995        );
996        assert_vector(
997            "[]",
998            "4441474d4c2d5443563100410000000000000000",
999            "cea5f239e81001721b763cebf40cd71bca04972c51313fba335e0a96d7e81979",
1000        );
1001        assert_vector(
1002            "{}",
1003            "4441474d4c2d54435631004f0000000000000000",
1004            "05fb75f2c266555e97a65becbafc84f8dc52b9f4cb2da8f7b7c5bfc8073325f2",
1005        );
1006    }
1007
1008    #[test]
1009    fn integer_bounds_are_lexical_and_exact() {
1010        assert_eq!(
1011            parse("-9223372036854775808"),
1012            TypedCanonicalValue::Integer(CanonicalInteger::Signed(i64::MIN))
1013        );
1014        assert_eq!(
1015            parse("18446744073709551615"),
1016            TypedCanonicalValue::Integer(CanonicalInteger::Unsigned(u64::MAX))
1017        );
1018        assert!(matches!(
1019            parse_typed_json("-9223372036854775809"),
1020            Err(Tcv1Error::IntegerOutOfRange { domain: "i64", .. })
1021        ));
1022        assert!(matches!(
1023            parse_typed_json("18446744073709551616"),
1024            Err(Tcv1Error::IntegerOutOfRange { domain: "u64", .. })
1025        ));
1026    }
1027
1028    #[test]
1029    fn strict_parser_rejects_invalid_documents() {
1030        assert!(matches!(
1031            parse_typed_json(r#"{"a":1,"a":2}"#),
1032            Err(Tcv1Error::DuplicateObjectKey { .. })
1033        ));
1034        assert!(matches!(
1035            parse_typed_json(r#"{"a":1,"\u0061":2}"#),
1036            Err(Tcv1Error::DuplicateObjectKey { .. })
1037        ));
1038        for document in [r#""\ud800""#, r#""\udc00""#, r#""\ud800\u0061""#] {
1039            assert!(matches!(
1040                parse_typed_json(document),
1041                Err(Tcv1Error::InvalidJson { .. })
1042            ));
1043        }
1044        assert!(matches!(
1045            parse_typed_json("1e400"),
1046            Err(Tcv1Error::Binary64OutOfRange { .. })
1047        ));
1048        assert!(matches!(
1049            parse_typed_json_bytes(&[b'"', 0xff, b'"']),
1050            Err(Tcv1Error::InvalidUtf8 { .. })
1051        ));
1052        for document in ["null true", "01", "1.", "1e", "[1,]", r#"{"a":1,}"#] {
1053            assert!(parse_typed_json(document).is_err(), "accepted {document:?}");
1054        }
1055    }
1056
1057    #[test]
1058    fn programmatic_values_receive_the_same_safety_checks() {
1059        assert_eq!(
1060            tcv1_encode(&TypedCanonicalValue::Binary64(f64::INFINITY)),
1061            Err(Tcv1Error::NonFiniteBinary64)
1062        );
1063        assert_eq!(
1064            tcv1_encode(&TypedCanonicalValue::Binary64(f64::NAN)),
1065            Err(Tcv1Error::NonFiniteBinary64)
1066        );
1067        let collision = TypedCanonicalValue::Object(vec![
1068            ("é".to_string(), TypedCanonicalValue::Null),
1069            ("e\u{301}".to_string(), TypedCanonicalValue::Null),
1070        ]);
1071        assert!(matches!(
1072            tcv1_encode(&collision),
1073            Err(Tcv1Error::NfcKeyCollision { .. })
1074        ));
1075        let mut structured_collision = serde_json::Map::new();
1076        structured_collision.insert("é".to_string(), serde_json::Value::Null);
1077        structured_collision.insert("e\u{301}".to_string(), serde_json::Value::Null);
1078        assert!(matches!(
1079            validate_typed_serde_value(&serde_json::Value::Object(structured_collision)),
1080            Err(Tcv1Error::NfcKeyCollision { .. })
1081        ));
1082    }
1083
1084    #[test]
1085    fn self_fingerprint_omits_exactly_one_normalized_key() {
1086        let with_fingerprint = parse(r#"{"payload":2,"fingerprint":"pending"}"#);
1087        let payload_only = parse(r#"{"payload":2}"#);
1088        assert_eq!(
1089            with_fingerprint.fingerprint_without("fingerprint").unwrap(),
1090            payload_only.fingerprint().unwrap()
1091        );
1092        assert!(matches!(
1093            payload_only.fingerprint_without("fingerprint"),
1094            Err(Tcv1Error::MissingObjectKey(_))
1095        ));
1096        assert!(matches!(
1097            parse("[]").fingerprint_without("fingerprint"),
1098            Err(Tcv1Error::ExpectedObject)
1099        ));
1100
1101        let decomposed_key = parse(r#"{"empreinte\u0301":"pending","payload":2}"#);
1102        assert_eq!(
1103            decomposed_key.fingerprint_without("empreinté").unwrap(),
1104            payload_only.fingerprint().unwrap()
1105        );
1106        let ambiguous = TypedCanonicalValue::Object(vec![
1107            ("é".to_string(), TypedCanonicalValue::Null),
1108            ("e\u{301}".to_string(), TypedCanonicalValue::Null),
1109        ]);
1110        assert!(matches!(
1111            ambiguous.fingerprint_without("é"),
1112            Err(Tcv1Error::AmbiguousObjectKey(_))
1113        ));
1114    }
1115
1116    #[test]
1117    fn parser_and_encoder_enforce_the_nesting_limit() {
1118        let accepted = format!(
1119            "{}0{}",
1120            "[".repeat(MAX_NESTING_DEPTH),
1121            "]".repeat(MAX_NESTING_DEPTH)
1122        );
1123        let accepted = parse_typed_json(&accepted).expect("boundary depth is accepted");
1124        tcv1_encode(&accepted).expect("encoder accepts the same boundary depth");
1125
1126        let rejected = format!(
1127            "{}0{}",
1128            "[".repeat(MAX_NESTING_DEPTH + 1),
1129            "]".repeat(MAX_NESTING_DEPTH + 1)
1130        );
1131        assert_eq!(parse_typed_json(&rejected), Err(Tcv1Error::NestingTooDeep));
1132
1133        let mut programmatic = TypedCanonicalValue::Null;
1134        for _ in 0..=MAX_NESTING_DEPTH {
1135            programmatic = TypedCanonicalValue::Array(vec![programmatic]);
1136        }
1137        assert_eq!(tcv1_encode(&programmatic), Err(Tcv1Error::NestingTooDeep));
1138
1139        let accepted_objects = format!(
1140            "{}0{}",
1141            r#"{"a":"#.repeat(MAX_NESTING_DEPTH),
1142            "}".repeat(MAX_NESTING_DEPTH)
1143        );
1144        let accepted_objects =
1145            parse_typed_json(&accepted_objects).expect("object boundary depth is accepted");
1146        tcv1_encode(&accepted_objects).expect("encoder accepts object boundary depth");
1147
1148        let rejected_objects = format!(
1149            "{}0{}",
1150            r#"{"a":"#.repeat(MAX_NESTING_DEPTH + 1),
1151            "}".repeat(MAX_NESTING_DEPTH + 1)
1152        );
1153        assert_eq!(
1154            parse_typed_json(&rejected_objects),
1155            Err(Tcv1Error::NestingTooDeep)
1156        );
1157    }
1158
1159    #[test]
1160    fn historical_stable_json_fingerprint_does_not_drift() {
1161        let value = json!({"a": 2, "z": [true, null]});
1162        assert_eq!(
1163            crate::campaign::stable_json_fingerprint(&value).unwrap(),
1164            "b4f8d6fce8a1198ebca7d0206f8c229dfe7a0c663929b0df2d72053d3d34624a"
1165        );
1166    }
1167}
dag_ml_core/canonical.rs

dag_ml_core/
canonical.rs
