dag_ml_core/
canonical.rs

1//! DAG-ML Typed Canonical Value v1 (TCV1).
2//!
3//! TCV1 is deliberately distinct from the historical JSON fingerprints used
4//! elsewhere in this crate. It preserves the JSON token distinction between
5//! integers and binary64 values, normalizes text to Unicode NFC, orders object
6//! keys by normalized UTF-8 bytes, and hashes a domain-separated binary
7//! preimage. Existing callers of `stable_json_fingerprint` are intentionally
8//! unaffected.
9
10use std::collections::{HashMap, HashSet};
11use std::fmt::Write as _;
12
13use serde::de::DeserializeOwned;
14use serde::Serialize;
15use sha2::{Digest, Sha256};
16use thiserror::Error;
17use unicode_normalization::UnicodeNormalization;
18
19use crate::error::{DagMlError, Result as DagMlResult};
20
21/// Domain separator prepended to every TCV1 value before hashing.
22pub const TCV1_PREFIX: &[u8] = b"DAGML-TCV1\0";
23
24const MAX_NESTING_DEPTH: usize = 128;
25
26/// An integer parsed without passing through binary64.
27///
28/// During JSON parsing, tokens with a leading minus sign use the signed domain
29/// and all other integer tokens use the unsigned domain. The public variants
30/// also allow direct construction with any `i64` or `u64`; their common values
31/// intentionally have the same TCV1 decimal encoding. Lexical `-0` parses as
32/// signed zero and has the same TCV1 encoding as unsigned zero.
33#[derive(Clone, Debug, Eq, PartialEq)]
34pub enum CanonicalInteger {
35    Signed(i64),
36    Unsigned(u64),
37}
38
39impl CanonicalInteger {
40    fn decimal(&self) -> String {
41        match self {
42            Self::Signed(value) => value.to_string(),
43            Self::Unsigned(value) => value.to_string(),
44        }
45    }
46}
47
48/// A strict JSON value that retains integer-versus-binary64 token kind.
49///
50/// [`TypedCanonicalValue::Object`] retains input order for diagnostics and
51/// representation equality, but object order is not significant to TCV1.
52/// Encoding validates raw and NFC-normalized key uniqueness and then sorts keys
53/// by normalized UTF-8 bytes. Public variants permit construction without a
54/// JSON round-trip;
55/// [`tcv1_encode`] performs the same finite-number and object-key checks for
56/// such values as [`parse_typed_json`].
57#[derive(Clone, Debug, PartialEq)]
58pub enum TypedCanonicalValue {
59    Null,
60    Bool(bool),
61    Integer(CanonicalInteger),
62    Binary64(f64),
63    String(String),
64    Array(Vec<TypedCanonicalValue>),
65    Object(Vec<(String, TypedCanonicalValue)>),
66}
67
68impl TypedCanonicalValue {
69    /// Return the lowercase SHA-256 fingerprint of this value's TCV1 preimage.
70    pub fn fingerprint(&self) -> Result<String, Tcv1Error> {
71        tcv1_sha256(self)
72    }
73
74    /// Fingerprint an object after removing exactly one normalized key.
75    ///
76    /// This supports self-fingerprinted contracts without accepting ambiguous
77    /// NFC aliases. The receiver must be an object and the requested key must
78    /// occur exactly once after NFC normalization.
79    pub fn fingerprint_without(&self, key: &str) -> Result<String, Tcv1Error> {
80        let Self::Object(entries) = self else {
81            return Err(Tcv1Error::ExpectedObject);
82        };
83        let normalized_key = normalize(key);
84        let mut removed = 0_usize;
85        let mut filtered = Vec::with_capacity(entries.len().saturating_sub(1));
86        for (member_key, member_value) in entries {
87            if normalize(member_key) == normalized_key {
88                removed += 1;
89            } else {
90                filtered.push((member_key.clone(), member_value.clone()));
91            }
92        }
93        match removed {
94            0 => Err(Tcv1Error::MissingObjectKey(normalized_key)),
95            1 => tcv1_sha256(&Self::Object(filtered)),
96            _ => Err(Tcv1Error::AmbiguousObjectKey(normalized_key)),
97        }
98    }
99}
100
101/// Parse or encoding failure for the strict TCV1 domain.
102#[derive(Clone, Debug, Eq, PartialEq, Error)]
103pub enum Tcv1Error {
104    #[error("input is not valid UTF-8 at byte {valid_up_to}")]
105    InvalidUtf8 {
106        valid_up_to: usize,
107        error_len: Option<usize>,
108    },
109
110    #[error("invalid JSON at UTF-8 byte {offset}: {message}")]
111    InvalidJson {
112        offset: usize,
113        message: &'static str,
114    },
115
116    #[error("duplicate JSON object key `{key}` at UTF-8 byte {offset}")]
117    DuplicateObjectKey { key: String, offset: usize },
118
119    #[error("NFC-colliding JSON object keys `{first}` and `{second}` at UTF-8 byte {offset}")]
120    NfcKeyCollision {
121        first: String,
122        second: String,
123        offset: usize,
124    },
125
126    #[error("integer token at UTF-8 byte {offset} is outside the TCV1 {domain} range")]
127    IntegerOutOfRange { offset: usize, domain: &'static str },
128
129    #[error("number token at UTF-8 byte {offset} is outside finite binary64 range")]
130    Binary64OutOfRange { offset: usize },
131
132    #[error("TCV1 value nesting exceeds {MAX_NESTING_DEPTH} levels")]
133    NestingTooDeep,
134
135    #[error("TCV1 collection length does not fit u64")]
136    LengthOverflow,
137
138    #[error("programmatically constructed TCV1 binary64 must be finite")]
139    NonFiniteBinary64,
140
141    #[error("fingerprint_without requires a TCV1 object")]
142    ExpectedObject,
143
144    #[error("object does not contain normalized key `{0}`")]
145    MissingObjectKey(String),
146
147    #[error("object contains more than one key normalized as `{0}`")]
148    AmbiguousObjectKey(String),
149}
150
151/// Parse exactly one strict JSON document while retaining numeric token kind.
152///
153/// Only RFC 8259 whitespace is accepted. Duplicate decoded keys, NFC-colliding
154/// keys, unpaired UTF-16 surrogate escapes, out-of-range integer tokens,
155/// non-finite binary64 conversions, and trailing data are rejected.
156pub fn parse_typed_json(input: &str) -> Result<TypedCanonicalValue, Tcv1Error> {
157    let mut parser = Parser::new(input);
158    let value = parser.parse_value(0)?;
159    parser.skip_whitespace();
160    if parser.offset != input.len() {
161        return Err(parser.invalid("trailing data after the JSON value"));
162    }
163    Ok(value)
164}
165
166/// Parse exactly one strict UTF-8 JSON document while retaining numeric kind.
167pub fn parse_typed_json_bytes(input: &[u8]) -> Result<TypedCanonicalValue, Tcv1Error> {
168    let input = std::str::from_utf8(input).map_err(|error| Tcv1Error::InvalidUtf8 {
169        valid_up_to: error.valid_up_to(),
170        error_len: error.error_len(),
171    })?;
172    parse_typed_json(input)
173}
174
175/// Validate a structured `serde_json` value against the same finite-number,
176/// nesting and NFC-key rules as a raw TCV1 document.
177///
178/// A `serde_json::Value` has already lost duplicate raw object members, so raw
179/// JSON boundaries must still call [`parse_typed_json`] before decoding. This
180/// helper is for structured host boundaries (for example Python mappings),
181/// where distinct keys can still collide after Unicode normalization.
182pub fn validate_typed_serde_value(value: &serde_json::Value) -> Result<(), Tcv1Error> {
183    fn convert(value: &serde_json::Value) -> TypedCanonicalValue {
184        match value {
185            serde_json::Value::Null => TypedCanonicalValue::Null,
186            serde_json::Value::Bool(value) => TypedCanonicalValue::Bool(*value),
187            serde_json::Value::Number(value) => {
188                if let Some(value) = value.as_u64() {
189                    TypedCanonicalValue::Integer(CanonicalInteger::Unsigned(value))
190                } else if let Some(value) = value.as_i64() {
191                    TypedCanonicalValue::Integer(CanonicalInteger::Signed(value))
192                } else {
193                    TypedCanonicalValue::Binary64(
194                        value
195                            .as_f64()
196                            .expect("serde_json numbers are integer or finite binary64"),
197                    )
198                }
199            }
200            serde_json::Value::String(value) => TypedCanonicalValue::String(value.clone()),
201            serde_json::Value::Array(values) => {
202                TypedCanonicalValue::Array(values.iter().map(convert).collect())
203            }
204            serde_json::Value::Object(values) => TypedCanonicalValue::Object(
205                values
206                    .iter()
207                    .map(|(key, value)| (key.clone(), convert(value)))
208                    .collect(),
209            ),
210        }
211    }
212
213    tcv1_encode(&convert(value)).map(|_| ())
214}
215
216/// Encode one value using the DAG-ML TCV1 binary grammar.
217///
218/// The returned bytes begin with the value tag, not the domain separator. Use
219/// [`tcv1_preimage`] for bytes suitable for hashing.
220pub fn tcv1_encode(value: &TypedCanonicalValue) -> Result<Vec<u8>, Tcv1Error> {
221    let mut output = Vec::new();
222    encode_value(value, &mut output, 0)?;
223    Ok(output)
224}
225
226/// Return the domain-separated preimage `DAGML-TCV1\0 || encode(value)`.
227pub fn tcv1_preimage(value: &TypedCanonicalValue) -> Result<Vec<u8>, Tcv1Error> {
228    let mut output = Vec::from(TCV1_PREFIX);
229    encode_value(value, &mut output, 0)?;
230    Ok(output)
231}
232
233/// Return lowercase SHA-256 of [`tcv1_preimage`].
234pub fn tcv1_sha256(value: &TypedCanonicalValue) -> Result<String, Tcv1Error> {
235    let digest = Sha256::digest(tcv1_preimage(value)?);
236    let mut output = String::with_capacity(digest.len() * 2);
237    for byte in digest {
238        write!(&mut output, "{byte:02x}").expect("writing to String cannot fail");
239    }
240    Ok(output)
241}
242
243fn normalize(value: &str) -> String {
244    value.nfc().collect()
245}
246
247fn encode_value(
248    value: &TypedCanonicalValue,
249    output: &mut Vec<u8>,
250    depth: usize,
251) -> Result<(), Tcv1Error> {
252    if depth > MAX_NESTING_DEPTH {
253        return Err(Tcv1Error::NestingTooDeep);
254    }
255    match value {
256        TypedCanonicalValue::Null => output.push(b'N'),
257        TypedCanonicalValue::Bool(false) => output.push(b'F'),
258        TypedCanonicalValue::Bool(true) => output.push(b'T'),
259        TypedCanonicalValue::Integer(value) => {
260            output.push(b'I');
261            let payload = value.decimal();
262            encode_length(payload.len(), output)?;
263            output.extend_from_slice(payload.as_bytes());
264        }
265        TypedCanonicalValue::Binary64(value) => {
266            if !value.is_finite() {
267                return Err(Tcv1Error::NonFiniteBinary64);
268            }
269            output.push(b'D');
270            let normalized = if *value == 0.0 { 0.0 } else { *value };
271            output.extend_from_slice(&normalized.to_bits().to_be_bytes());
272        }
273        TypedCanonicalValue::String(value) => encode_string(value, output)?,
274        TypedCanonicalValue::Array(values) => {
275            output.push(b'A');
276            encode_length(values.len(), output)?;
277            for value in values {
278                encode_value(value, output, depth + 1)?;
279            }
280        }
281        TypedCanonicalValue::Object(entries) => {
282            output.push(b'O');
283            encode_length(entries.len(), output)?;
284
285            let mut raw_keys = HashSet::with_capacity(entries.len());
286            let mut normalized_keys = HashMap::with_capacity(entries.len());
287            let mut sorted = Vec::with_capacity(entries.len());
288            for (key, value) in entries {
289                if !raw_keys.insert(key.as_str()) {
290                    return Err(Tcv1Error::DuplicateObjectKey {
291                        key: key.clone(),
292                        offset: 0,
293                    });
294                }
295                let normalized = normalize(key);
296                if let Some(first) = normalized_keys.insert(normalized.clone(), key.as_str()) {
297                    return Err(Tcv1Error::NfcKeyCollision {
298                        first: first.to_string(),
299                        second: key.clone(),
300                        offset: 0,
301                    });
302                }
303                sorted.push((normalized.into_bytes(), value));
304            }
305            sorted.sort_by(|left, right| left.0.cmp(&right.0));
306            for (normalized_key, value) in sorted {
307                encode_normalized_string(&normalized_key, output)?;
308                encode_value(value, output, depth + 1)?;
309            }
310        }
311    }
312    Ok(())
313}
314
315fn encode_string(value: &str, output: &mut Vec<u8>) -> Result<(), Tcv1Error> {
316    let normalized = normalize(value);
317    encode_normalized_string(normalized.as_bytes(), output)
318}
319
320fn encode_normalized_string(payload: &[u8], output: &mut Vec<u8>) -> Result<(), Tcv1Error> {
321    output.push(b'S');
322    encode_length(payload.len(), output)?;
323    output.extend_from_slice(payload);
324    Ok(())
325}
326
327fn encode_length(length: usize, output: &mut Vec<u8>) -> Result<(), Tcv1Error> {
328    let length = u64::try_from(length).map_err(|_| Tcv1Error::LengthOverflow)?;
329    output.extend_from_slice(&length.to_be_bytes());
330    Ok(())
331}
332
333struct Parser<'a> {
334    input: &'a str,
335    bytes: &'a [u8],
336    offset: usize,
337}
338
339impl<'a> Parser<'a> {
340    fn new(input: &'a str) -> Self {
341        Self {
342            input,
343            bytes: input.as_bytes(),
344            offset: 0,
345        }
346    }
347
348    fn parse_value(&mut self, depth: usize) -> Result<TypedCanonicalValue, Tcv1Error> {
349        if depth > MAX_NESTING_DEPTH {
350            return Err(Tcv1Error::NestingTooDeep);
351        }
352        self.skip_whitespace();
353        match self.peek() {
354            Some(b'n') => {
355                self.consume_literal(b"null")?;
356                Ok(TypedCanonicalValue::Null)
357            }
358            Some(b'f') => {
359                self.consume_literal(b"false")?;
360                Ok(TypedCanonicalValue::Bool(false))
361            }
362            Some(b't') => {
363                self.consume_literal(b"true")?;
364                Ok(TypedCanonicalValue::Bool(true))
365            }
366            Some(b'"') => Ok(TypedCanonicalValue::String(self.parse_string()?)),
367            Some(b'[') => self.parse_array(depth),
368            Some(b'{') => self.parse_object(depth),
369            Some(b'-' | b'0'..=b'9') => self.parse_number(),
370            Some(_) => Err(self.invalid("expected a JSON value")),
371            None => Err(self.invalid("unexpected end of input")),
372        }
373    }
374
375    fn parse_array(&mut self, depth: usize) -> Result<TypedCanonicalValue, Tcv1Error> {
376        self.offset += 1;
377        self.skip_whitespace();
378        let mut values = Vec::new();
379        if self.consume_if(b']') {
380            return Ok(TypedCanonicalValue::Array(values));
381        }
382        loop {
383            values.push(self.parse_value(depth + 1)?);
384            self.skip_whitespace();
385            match self.peek() {
386                Some(b',') => self.offset += 1,
387                Some(b']') => {
388                    self.offset += 1;
389                    return Ok(TypedCanonicalValue::Array(values));
390                }
391                Some(_) => return Err(self.invalid("expected `,` or `]` in array")),
392                None => return Err(self.invalid("unterminated array")),
393            }
394        }
395    }
396
397    fn parse_object(&mut self, depth: usize) -> Result<TypedCanonicalValue, Tcv1Error> {
398        self.offset += 1;
399        self.skip_whitespace();
400        let mut entries = Vec::new();
401        let mut raw_keys = HashSet::new();
402        let mut normalized_keys: HashMap<String, String> = HashMap::new();
403        if self.consume_if(b'}') {
404            return Ok(TypedCanonicalValue::Object(entries));
405        }
406        loop {
407            self.skip_whitespace();
408            if self.peek() != Some(b'"') {
409                return Err(self.invalid("expected a string object key"));
410            }
411            let key_offset = self.offset;
412            let key = self.parse_string()?;
413            if !raw_keys.insert(key.clone()) {
414                return Err(Tcv1Error::DuplicateObjectKey {
415                    key,
416                    offset: key_offset,
417                });
418            }
419            let normalized = normalize(&key);
420            if let Some(first) = normalized_keys.insert(normalized, key.clone()) {
421                return Err(Tcv1Error::NfcKeyCollision {
422                    first,
423                    second: key,
424                    offset: key_offset,
425                });
426            }
427            self.skip_whitespace();
428            self.expect(b':', "expected `:` after object key")?;
429            let value = self.parse_value(depth + 1)?;
430            entries.push((key, value));
431            self.skip_whitespace();
432            match self.peek() {
433                Some(b',') => self.offset += 1,
434                Some(b'}') => {
435                    self.offset += 1;
436                    return Ok(TypedCanonicalValue::Object(entries));
437                }
438                Some(_) => return Err(self.invalid("expected `,` or `}` in object")),
439                None => return Err(self.invalid("unterminated object")),
440            }
441        }
442    }
443
444    fn parse_number(&mut self) -> Result<TypedCanonicalValue, Tcv1Error> {
445        let start = self.offset;
446        let negative = self.consume_if(b'-');
447
448        match self.peek() {
449            Some(b'0') => {
450                self.offset += 1;
451                if matches!(self.peek(), Some(b'0'..=b'9')) {
452                    return Err(self.invalid("leading zero in JSON number"));
453                }
454            }
455            Some(b'1'..=b'9') => {
456                self.offset += 1;
457                while matches!(self.peek(), Some(b'0'..=b'9')) {
458                    self.offset += 1;
459                }
460            }
461            _ => return Err(self.invalid("expected integer digits")),
462        }
463
464        let mut binary64 = false;
465        if self.consume_if(b'.') {
466            binary64 = true;
467            if !matches!(self.peek(), Some(b'0'..=b'9')) {
468                return Err(self.invalid("expected digit after decimal point"));
469            }
470            while matches!(self.peek(), Some(b'0'..=b'9')) {
471                self.offset += 1;
472            }
473        }
474        if matches!(self.peek(), Some(b'e' | b'E')) {
475            binary64 = true;
476            self.offset += 1;
477            if matches!(self.peek(), Some(b'+' | b'-')) {
478                self.offset += 1;
479            }
480            if !matches!(self.peek(), Some(b'0'..=b'9')) {
481                return Err(self.invalid("expected exponent digits"));
482            }
483            while matches!(self.peek(), Some(b'0'..=b'9')) {
484                self.offset += 1;
485            }
486        }
487
488        let token = &self.input[start..self.offset];
489        if binary64 {
490            let value = token
491                .parse::<f64>()
492                .map_err(|_| Tcv1Error::Binary64OutOfRange { offset: start })?;
493            if !value.is_finite() {
494                return Err(Tcv1Error::Binary64OutOfRange { offset: start });
495            }
496            Ok(TypedCanonicalValue::Binary64(value))
497        } else if negative {
498            token
499                .parse::<i64>()
500                .map(|value| TypedCanonicalValue::Integer(CanonicalInteger::Signed(value)))
501                .map_err(|_| Tcv1Error::IntegerOutOfRange {
502                    offset: start,
503                    domain: "i64",
504                })
505        } else {
506            token
507                .parse::<u64>()
508                .map(|value| TypedCanonicalValue::Integer(CanonicalInteger::Unsigned(value)))
509                .map_err(|_| Tcv1Error::IntegerOutOfRange {
510                    offset: start,
511                    domain: "u64",
512                })
513        }
514    }
515
516    fn parse_string(&mut self) -> Result<String, Tcv1Error> {
517        debug_assert_eq!(self.peek(), Some(b'"'));
518        self.offset += 1;
519        let mut output = String::new();
520        let mut chunk_start = self.offset;
521        loop {
522            match self.peek() {
523                Some(b'"') => {
524                    output.push_str(&self.input[chunk_start..self.offset]);
525                    self.offset += 1;
526                    return Ok(output);
527                }
528                Some(b'\\') => {
529                    output.push_str(&self.input[chunk_start..self.offset]);
530                    self.offset += 1;
531                    self.parse_escape(&mut output)?;
532                    chunk_start = self.offset;
533                }
534                Some(0x00..=0x1f) => {
535                    return Err(self.invalid("unescaped control character in JSON string"));
536                }
537                Some(byte) if byte.is_ascii() => self.offset += 1,
538                Some(_) => {
539                    let character = self.input[self.offset..]
540                        .chars()
541                        .next()
542                        .expect("offset is on a valid UTF-8 boundary");
543                    self.offset += character.len_utf8();
544                }
545                None => return Err(self.invalid("unterminated JSON string")),
546            }
547        }
548    }
549
550    fn parse_escape(&mut self, output: &mut String) -> Result<(), Tcv1Error> {
551        match self.peek() {
552            Some(b'"') => output.push('"'),
553            Some(b'\\') => output.push('\\'),
554            Some(b'/') => output.push('/'),
555            Some(b'b') => output.push('\u{0008}'),
556            Some(b'f') => output.push('\u{000c}'),
557            Some(b'n') => output.push('\n'),
558            Some(b'r') => output.push('\r'),
559            Some(b't') => output.push('\t'),
560            Some(b'u') => {
561                self.offset += 1;
562                return self.parse_unicode_escape(output);
563            }
564            Some(_) => return Err(self.invalid("invalid JSON string escape")),
565            None => return Err(self.invalid("unterminated JSON string escape")),
566        }
567        self.offset += 1;
568        Ok(())
569    }
570
571    fn parse_unicode_escape(&mut self, output: &mut String) -> Result<(), Tcv1Error> {
572        let first_offset = self.offset;
573        let first = self.parse_hex_quad()?;
574        let scalar = if (0xd800..=0xdbff).contains(&first) {
575            if self.peek() != Some(b'\\') || self.bytes.get(self.offset + 1) != Some(&b'u') {
576                return Err(Tcv1Error::InvalidJson {
577                    offset: first_offset,
578                    message: "high surrogate is not followed by a low surrogate",
579                });
580            }
581            self.offset += 2;
582            let low_offset = self.offset;
583            let low = self.parse_hex_quad()?;
584            if !(0xdc00..=0xdfff).contains(&low) {
585                return Err(Tcv1Error::InvalidJson {
586                    offset: low_offset,
587                    message: "high surrogate is not followed by a low surrogate",
588                });
589            }
590            0x10000 + ((u32::from(first) - 0xd800) << 10) + (u32::from(low) - 0xdc00)
591        } else if (0xdc00..=0xdfff).contains(&first) {
592            return Err(Tcv1Error::InvalidJson {
593                offset: first_offset,
594                message: "isolated low surrogate in JSON string",
595            });
596        } else {
597            u32::from(first)
598        };
599        output.push(char::from_u32(scalar).expect("validated Unicode scalar value"));
600        Ok(())
601    }
602
603    fn parse_hex_quad(&mut self) -> Result<u16, Tcv1Error> {
604        if self.offset + 4 > self.bytes.len() {
605            return Err(self.invalid("incomplete Unicode escape"));
606        }
607        let mut value = 0_u16;
608        for _ in 0..4 {
609            let digit = match self.peek() {
610                Some(b'0'..=b'9') => u16::from(self.bytes[self.offset] - b'0'),
611                Some(b'a'..=b'f') => u16::from(self.bytes[self.offset] - b'a' + 10),
612                Some(b'A'..=b'F') => u16::from(self.bytes[self.offset] - b'A' + 10),
613                _ => return Err(self.invalid("invalid hexadecimal digit in Unicode escape")),
614            };
615            value = (value << 4) | digit;
616            self.offset += 1;
617        }
618        Ok(value)
619    }
620
621    fn consume_literal(&mut self, literal: &[u8]) -> Result<(), Tcv1Error> {
622        if self.bytes.get(self.offset..self.offset + literal.len()) == Some(literal) {
623            self.offset += literal.len();
624            Ok(())
625        } else {
626            Err(self.invalid("invalid JSON literal"))
627        }
628    }
629
630    fn expect(&mut self, expected: u8, message: &'static str) -> Result<(), Tcv1Error> {
631        if self.consume_if(expected) {
632            Ok(())
633        } else {
634            Err(self.invalid(message))
635        }
636    }
637
638    fn consume_if(&mut self, expected: u8) -> bool {
639        if self.peek() == Some(expected) {
640            self.offset += 1;
641            true
642        } else {
643            false
644        }
645    }
646
647    fn skip_whitespace(&mut self) {
648        while matches!(self.peek(), Some(b' ' | b'\n' | b'\r' | b'\t')) {
649            self.offset += 1;
650        }
651    }
652
653    fn peek(&self) -> Option<u8> {
654        self.bytes.get(self.offset).copied()
655    }
656
657    fn invalid(&self, message: &'static str) -> Tcv1Error {
658        Tcv1Error::InvalidJson {
659            offset: self.offset,
660            message,
661        }
662    }
663}
664
665/// Require external JSON to use the same object/array container kinds as its
666/// typed serde representation, without requiring byte-canonical field presence
667/// or ordering. This closes serde's internal positional-sequence representation
668/// for structs while preserving defaults, skipped fields and BTree ordering.
669fn validate_external_container_shapes(
670    raw: &serde_json::Value,
671    typed: &serde_json::Value,
672    path: &str,
673) -> std::result::Result<(), String> {
674    match (raw, typed) {
675        (serde_json::Value::Object(raw), serde_json::Value::Object(typed)) => {
676            for (key, raw_value) in raw {
677                if let Some(typed_value) = typed.get(key) {
678                    validate_external_container_shapes(
679                        raw_value,
680                        typed_value,
681                        &format!("{path}.{key}"),
682                    )?;
683                }
684            }
685            Ok(())
686        }
687        (serde_json::Value::Array(raw), serde_json::Value::Array(typed)) => {
688            for (index, (raw_value, typed_value)) in raw.iter().zip(typed).enumerate() {
689                validate_external_container_shapes(
690                    raw_value,
691                    typed_value,
692                    &format!("{path}[{index}]"),
693                )?;
694            }
695            Ok(())
696        }
697        (_, serde_json::Value::Object(_)) => Err(format!(
698            "{path} must use a JSON object at the external contract boundary"
699        )),
700        (_, serde_json::Value::Array(_)) => Err(format!(
701            "{path} must use a JSON array at the external contract boundary"
702        )),
703        (serde_json::Value::Object(_) | serde_json::Value::Array(_), _) => Err(format!(
704            "{path} has the wrong JSON container kind at the external contract boundary"
705        )),
706        _ => Ok(()),
707    }
708}
709
710/// Deserialize one public JSON contract while rejecting serde's positional
711/// sequence representation for structs. Semantic validation remains the owning
712/// type's responsibility after this shape-preserving decode.
713pub fn deserialize_external_contract<T, F>(
714    json: &str,
715    label: &str,
716    shape_error: F,
717) -> DagMlResult<T>
718where
719    T: DeserializeOwned + Serialize,
720    F: Fn(String) -> DagMlError,
721{
722    parse_typed_json(json).map_err(|error| {
723        shape_error(format!(
724            "{label} is not a strict TCV1 JSON document: {error}"
725        ))
726    })?;
727    let raw: serde_json::Value = serde_json::from_str(json)?;
728    deserialize_external_value(raw, label, shape_error)
729}
730
731/// Deserialize a public contract that has already crossed a structured host
732/// boundary (for example a Python mapping), while preserving the same
733/// object/array rules as [`deserialize_external_contract`].
734pub fn deserialize_external_value<T, F>(
735    raw: serde_json::Value,
736    label: &str,
737    shape_error: F,
738) -> DagMlResult<T>
739where
740    T: DeserializeOwned + Serialize,
741    F: Fn(String) -> DagMlError,
742{
743    validate_typed_serde_value(&raw).map_err(|error| {
744        shape_error(format!(
745            "{label} is not a strict TCV1 structured value: {error}"
746        ))
747    })?;
748    let value: T = serde_json::from_value(raw.clone())?;
749    let typed = serde_json::to_value(&value)?;
750    validate_external_container_shapes(&raw, &typed, label).map_err(shape_error)?;
751    Ok(value)
752}
753
754#[cfg(test)]
755mod tests {
756    use serde_json::json;
757
758    use super::*;
759
760    fn parse(input: &str) -> TypedCanonicalValue {
761        parse_typed_json(input).expect("valid strict JSON")
762    }
763
764    fn hex(bytes: &[u8]) -> String {
765        let mut output = String::with_capacity(bytes.len() * 2);
766        for byte in bytes {
767            write!(&mut output, "{byte:02x}").unwrap();
768        }
769        output
770    }
771
772    fn assert_vector(input: &str, expected_preimage: &str, expected_sha256: &str) {
773        let value = parse(input);
774        assert_eq!(hex(&tcv1_preimage(&value).unwrap()), expected_preimage);
775        assert_eq!(tcv1_sha256(&value).unwrap(), expected_sha256);
776    }
777
778    #[test]
779    fn golden_map_order_preimage_and_digest() {
780        const PREIMAGE: &str = "4441474d4c2d54435631004f000000000000000253000000000000000161490000000000000001325300000000000000017a49000000000000000131";
781        const DIGEST: &str = "5441a8df23725b4a60e16316f3034a7ec8b25b853ce75600fa71dda19c8a16e1";
782        assert_vector(r#"{"z":1,"a":2}"#, PREIMAGE, DIGEST);
783        assert_vector(r#"{"a":2,"z":1}"#, PREIMAGE, DIGEST);
784    }
785
786    #[test]
787    fn object_order_uses_normalized_utf8_not_utf16() {
788        assert_vector(
789            r#"{"\ue000":1,"\ud800\udc00":2}"#,
790            "4441474d4c2d54435631004f0000000000000002530000000000000003ee808049000000000000000131530000000000000004f090808049000000000000000132",
791            "7c212789a6d362b8a34e8c271d5fe003c2026a828712eef6469610eedda79bc7",
792        );
793    }
794
795    #[test]
796    fn nfc_normalizes_strings_and_keys() {
797        let decomposed = parse(r#""e\u0301""#);
798        let composed = parse(r#""é""#);
799        // PartialEq deliberately retains source representation; TCV1 bytes are
800        // the canonical equivalence relation.
801        assert_ne!(decomposed, composed);
802        assert_eq!(
803            tcv1_preimage(&decomposed).unwrap(),
804            tcv1_preimage(&composed).unwrap()
805        );
806        assert_eq!(
807            tcv1_sha256(&decomposed).unwrap(),
808            "a4af538cebb2c18fed88a1ad4245509500d201e68802a217e4e8500ef61c0e86"
809        );
810
811        let error = parse_typed_json(r#"{"é":1,"e\u0301":2}"#).unwrap_err();
812        assert!(matches!(error, Tcv1Error::NfcKeyCollision { .. }));
813    }
814
815    #[test]
816    fn signed_zero_normalizes_only_within_binary64_kind() {
817        let negative = parse("-0.0");
818        let positive = parse("0.0");
819        assert_eq!(
820            tcv1_preimage(&negative).unwrap(),
821            tcv1_preimage(&positive).unwrap()
822        );
823        assert_vector(
824            "-0.0",
825            "4441474d4c2d5443563100440000000000000000",
826            "c01f83d2f6a8e96eb7f50c4794eef0dbae68ad4d20ed116af013ae5cd4ffa49d",
827        );
828        assert_ne!(
829            tcv1_preimage(&parse("-0")).unwrap(),
830            tcv1_preimage(&negative).unwrap()
831        );
832        assert_eq!(
833            tcv1_preimage(&parse("-0")).unwrap(),
834            tcv1_preimage(&parse("0")).unwrap()
835        );
836    }
837
838    #[test]
839    fn integer_and_binary64_tokens_remain_distinct() {
840        assert_eq!(
841            parse("2"),
842            TypedCanonicalValue::Integer(CanonicalInteger::Unsigned(2))
843        );
844        assert_eq!(parse("2.0"), TypedCanonicalValue::Binary64(2.0));
845        assert_ne!(
846            tcv1_preimage(&parse("2")).unwrap(),
847            tcv1_preimage(&parse("2.0")).unwrap()
848        );
849        assert_vector(
850            "2",
851            "4441474d4c2d544356310049000000000000000132",
852            "3940883272509c80c7bbff602794dce0f62dfa7850bc3041b37c56d36bc94701",
853        );
854        let float_preimage = tcv1_preimage(&parse("2.0")).unwrap();
855        assert_eq!(float_preimage, tcv1_preimage(&parse("2e0")).unwrap());
856        assert_eq!(
857            hex(&float_preimage),
858            "4441474d4c2d5443563100444000000000000000"
859        );
860    }
861
862    #[test]
863    fn frozen_binary64_boundary_vectors() {
864        let vectors = [
865            (
866                "5e-324",
867                "4441474d4c2d5443563100440000000000000001",
868                "88a7b6becacc6cf0bf2473332aa17d9f3ed513b4024d69684458a275b5c39c24",
869            ),
870            (
871                "2.2250738585072009e-308",
872                "4441474d4c2d544356310044000fffffffffffff",
873                "78c8b93679333797971ff7ef4dba4b284adb03da7a7379bf814f635b69164765",
874            ),
875            (
876                "2.2250738585072014e-308",
877                "4441474d4c2d5443563100440010000000000000",
878                "6ea1aeba7ec435fd15165511f602295eedd7de82832713f35891600a0f552702",
879            ),
880            (
881                "9007199254740992.0",
882                "4441474d4c2d5443563100444340000000000000",
883                "8e276db087fa6f18be879c6e32a034e44ff075c66c76ff8e43cbd3dc20e0673a",
884            ),
885            (
886                "1.7976931348623157e308",
887                "4441474d4c2d5443563100447fefffffffffffff",
888                "e9231aadbc74db0fd07f62e1b04c67ab93a73c30b756e85e36f80edd3766bf5a",
889            ),
890        ];
891        for (input, preimage, digest) in vectors {
892            assert_vector(input, preimage, digest);
893        }
894        assert_eq!(
895            tcv1_preimage(&parse("5e-324")).unwrap(),
896            tcv1_preimage(&parse("4.9406564584124654e-324")).unwrap()
897        );
898        assert_eq!(
899            tcv1_preimage(&parse("9007199254740992.0")).unwrap(),
900            tcv1_preimage(&parse("9.007199254740992e15")).unwrap()
901        );
902    }
903
904    #[test]
905    fn all_tags_and_big_endian_lengths_are_explicit() {
906        let value = parse(r#"[null,false,true,"x",-1,1.5]"#);
907        assert_eq!(
908            hex(&tcv1_encode(&value).unwrap()),
909            "4100000000000000064e4654530000000000000001784900000000000000022d31443ff8000000000000"
910        );
911        assert_vector(
912            "[]",
913            "4441474d4c2d5443563100410000000000000000",
914            "cea5f239e81001721b763cebf40cd71bca04972c51313fba335e0a96d7e81979",
915        );
916        assert_vector(
917            "{}",
918            "4441474d4c2d54435631004f0000000000000000",
919            "05fb75f2c266555e97a65becbafc84f8dc52b9f4cb2da8f7b7c5bfc8073325f2",
920        );
921    }
922
923    #[test]
924    fn integer_bounds_are_lexical_and_exact() {
925        assert_eq!(
926            parse("-9223372036854775808"),
927            TypedCanonicalValue::Integer(CanonicalInteger::Signed(i64::MIN))
928        );
929        assert_eq!(
930            parse("18446744073709551615"),
931            TypedCanonicalValue::Integer(CanonicalInteger::Unsigned(u64::MAX))
932        );
933        assert!(matches!(
934            parse_typed_json("-9223372036854775809"),
935            Err(Tcv1Error::IntegerOutOfRange { domain: "i64", .. })
936        ));
937        assert!(matches!(
938            parse_typed_json("18446744073709551616"),
939            Err(Tcv1Error::IntegerOutOfRange { domain: "u64", .. })
940        ));
941    }
942
943    #[test]
944    fn strict_parser_rejects_invalid_documents() {
945        assert!(matches!(
946            parse_typed_json(r#"{"a":1,"a":2}"#),
947            Err(Tcv1Error::DuplicateObjectKey { .. })
948        ));
949        assert!(matches!(
950            parse_typed_json(r#"{"a":1,"\u0061":2}"#),
951            Err(Tcv1Error::DuplicateObjectKey { .. })
952        ));
953        for document in [r#""\ud800""#, r#""\udc00""#, r#""\ud800\u0061""#] {
954            assert!(matches!(
955                parse_typed_json(document),
956                Err(Tcv1Error::InvalidJson { .. })
957            ));
958        }
959        assert!(matches!(
960            parse_typed_json("1e400"),
961            Err(Tcv1Error::Binary64OutOfRange { .. })
962        ));
963        assert!(matches!(
964            parse_typed_json_bytes(&[b'"', 0xff, b'"']),
965            Err(Tcv1Error::InvalidUtf8 { .. })
966        ));
967        for document in ["null true", "01", "1.", "1e", "[1,]", r#"{"a":1,}"#] {
968            assert!(parse_typed_json(document).is_err(), "accepted {document:?}");
969        }
970    }
971
972    #[test]
973    fn programmatic_values_receive_the_same_safety_checks() {
974        assert_eq!(
975            tcv1_encode(&TypedCanonicalValue::Binary64(f64::INFINITY)),
976            Err(Tcv1Error::NonFiniteBinary64)
977        );
978        assert_eq!(
979            tcv1_encode(&TypedCanonicalValue::Binary64(f64::NAN)),
980            Err(Tcv1Error::NonFiniteBinary64)
981        );
982        let collision = TypedCanonicalValue::Object(vec![
983            ("é".to_string(), TypedCanonicalValue::Null),
984            ("e\u{301}".to_string(), TypedCanonicalValue::Null),
985        ]);
986        assert!(matches!(
987            tcv1_encode(&collision),
988            Err(Tcv1Error::NfcKeyCollision { .. })
989        ));
990        let mut structured_collision = serde_json::Map::new();
991        structured_collision.insert("é".to_string(), serde_json::Value::Null);
992        structured_collision.insert("e\u{301}".to_string(), serde_json::Value::Null);
993        assert!(matches!(
994            validate_typed_serde_value(&serde_json::Value::Object(structured_collision)),
995            Err(Tcv1Error::NfcKeyCollision { .. })
996        ));
997    }
998
999    #[test]
1000    fn self_fingerprint_omits_exactly_one_normalized_key() {
1001        let with_fingerprint = parse(r#"{"payload":2,"fingerprint":"pending"}"#);
1002        let payload_only = parse(r#"{"payload":2}"#);
1003        assert_eq!(
1004            with_fingerprint.fingerprint_without("fingerprint").unwrap(),
1005            payload_only.fingerprint().unwrap()
1006        );
1007        assert!(matches!(
1008            payload_only.fingerprint_without("fingerprint"),
1009            Err(Tcv1Error::MissingObjectKey(_))
1010        ));
1011        assert!(matches!(
1012            parse("[]").fingerprint_without("fingerprint"),
1013            Err(Tcv1Error::ExpectedObject)
1014        ));
1015
1016        let decomposed_key = parse(r#"{"empreinte\u0301":"pending","payload":2}"#);
1017        assert_eq!(
1018            decomposed_key.fingerprint_without("empreinté").unwrap(),
1019            payload_only.fingerprint().unwrap()
1020        );
1021        let ambiguous = TypedCanonicalValue::Object(vec![
1022            ("é".to_string(), TypedCanonicalValue::Null),
1023            ("e\u{301}".to_string(), TypedCanonicalValue::Null),
1024        ]);
1025        assert!(matches!(
1026            ambiguous.fingerprint_without("é"),
1027            Err(Tcv1Error::AmbiguousObjectKey(_))
1028        ));
1029    }
1030
1031    #[test]
1032    fn parser_and_encoder_enforce_the_nesting_limit() {
1033        let accepted = format!(
1034            "{}0{}",
1035            "[".repeat(MAX_NESTING_DEPTH),
1036            "]".repeat(MAX_NESTING_DEPTH)
1037        );
1038        let accepted = parse_typed_json(&accepted).expect("boundary depth is accepted");
1039        tcv1_encode(&accepted).expect("encoder accepts the same boundary depth");
1040
1041        let rejected = format!(
1042            "{}0{}",
1043            "[".repeat(MAX_NESTING_DEPTH + 1),
1044            "]".repeat(MAX_NESTING_DEPTH + 1)
1045        );
1046        assert_eq!(parse_typed_json(&rejected), Err(Tcv1Error::NestingTooDeep));
1047
1048        let mut programmatic = TypedCanonicalValue::Null;
1049        for _ in 0..=MAX_NESTING_DEPTH {
1050            programmatic = TypedCanonicalValue::Array(vec![programmatic]);
1051        }
1052        assert_eq!(tcv1_encode(&programmatic), Err(Tcv1Error::NestingTooDeep));
1053
1054        let accepted_objects = format!(
1055            "{}0{}",
1056            r#"{"a":"#.repeat(MAX_NESTING_DEPTH),
1057            "}".repeat(MAX_NESTING_DEPTH)
1058        );
1059        let accepted_objects =
1060            parse_typed_json(&accepted_objects).expect("object boundary depth is accepted");
1061        tcv1_encode(&accepted_objects).expect("encoder accepts object boundary depth");
1062
1063        let rejected_objects = format!(
1064            "{}0{}",
1065            r#"{"a":"#.repeat(MAX_NESTING_DEPTH + 1),
1066            "}".repeat(MAX_NESTING_DEPTH + 1)
1067        );
1068        assert_eq!(
1069            parse_typed_json(&rejected_objects),
1070            Err(Tcv1Error::NestingTooDeep)
1071        );
1072    }
1073
1074    #[test]
1075    fn historical_stable_json_fingerprint_does_not_drift() {
1076        let value = json!({"a": 2, "z": [true, null]});
1077        assert_eq!(
1078            crate::campaign::stable_json_fingerprint(&value).unwrap(),
1079            "b4f8d6fce8a1198ebca7d0206f8c229dfe7a0c663929b0df2d72053d3d34624a"
1080        );
1081    }
1082}
dag_ml_core/canonical.rs

dag_ml_core/
canonical.rs
