Skip to main content

eidetic/
schema.rs

1// Copyright 2026 Mark AB (markik)
2// SPDX-License-Identifier: MIT OR Apache-2.0
3
4//! Schema, identity, and three-axis classification types for Layer 2.
5//!
6//! These types are the vocabulary of the manifest layer:
7//!
8//! - [`Hash`] / [`ManifestId`] — content-addressed identity (BLAKE3, 32 bytes)
9//! - [`SchemaRef`] — a content-addressed pointer to a schema engram
10//! - [`Timestamp`] — Unix milliseconds since epoch (portable, no chrono dep)
11//! - [`PrivacyClass`] / [`ProvenanceRecord`] / [`TrustEnvelope`] — the three
12//!   orthogonal classification axes (privacy / provenance / trust). Kept
13//!   separate, never bundled — see the eidetic design pass §8.
14
15use serde::{Deserialize, Deserializer, Serialize, Serializer};
16
17/// Multicodec-tagged hash function.
18///
19/// Per the event-DAG substrate brief §13, every digest field carries its hash
20/// function identifier alongside the digest, so a future hash migration
21/// (BLAKE4, post-quantum, ...) needs no flag day: new writes carry a new tag
22/// and old reads still parse. BLAKE3 is the only function today; adding one is
23/// the only change a migration requires here.
24#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]
25pub enum HashFn {
26    /// BLAKE3-256. Multicodec `0x1e`, 32-byte digest.
27    Blake3,
28}
29
30impl HashFn {
31    /// Multicodec code (encoded as an unsigned-varint in the binary multihash).
32    pub const fn multicodec(self) -> u64 {
33        match self {
34            HashFn::Blake3 => 0x1e,
35        }
36    }
37
38    /// Short tag used in the self-describing string form `<fn>:<hex>`.
39    pub const fn name(self) -> &'static str {
40        match self {
41            HashFn::Blake3 => "blake3",
42        }
43    }
44
45    /// Digest length in bytes.
46    pub const fn digest_len(self) -> usize {
47        match self {
48            HashFn::Blake3 => 32,
49        }
50    }
51
52    fn from_name(name: &str) -> Option<Self> {
53        match name {
54            "blake3" => Some(HashFn::Blake3),
55            _ => None,
56        }
57    }
58
59    fn from_multicodec(code: u64) -> Option<Self> {
60        match code {
61            0x1e => Some(HashFn::Blake3),
62            _ => None,
63        }
64    }
65}
66
67/// Error parsing a [`Hash`] from its string or binary multihash form.
68#[derive(Clone, Debug, PartialEq, Eq)]
69pub enum HashError {
70    /// Hash-function tag / multicodec is not one this build understands.
71    UnknownFunction,
72    /// Digest length does not match the function's expected length.
73    BadLength,
74    /// Hex string contained a non-hex byte.
75    BadHex,
76    /// Binary multihash ended before a full code/length/digest could be read.
77    Truncated,
78}
79
80impl std::fmt::Display for HashError {
81    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
82        f.write_str(match self {
83            HashError::UnknownFunction => "unknown hash function",
84            HashError::BadLength => "digest length mismatch",
85            HashError::BadHex => "invalid hex in digest",
86            HashError::Truncated => "truncated multihash",
87        })
88    }
89}
90
91impl std::error::Error for HashError {}
92
93/// Content hash, multihash-aware: a digest tagged with the function that
94/// produced it. BLAKE3-256 today (32 bytes). The string form is `<fn>:<hex>`
95/// (human-inspectable, the [`Display`](std::fmt::Display) form); the binary
96/// form ([`to_multihash_bytes`](Hash::to_multihash_bytes)) is a self-describing
97/// multihash (`uvarint(code) ++ uvarint(len) ++ digest`).
98#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]
99pub struct Hash {
100    func: HashFn,
101    digest: [u8; 32],
102}
103
104impl Hash {
105    /// Compute the BLAKE3 hash of a byte slice.
106    pub fn of(bytes: &[u8]) -> Self {
107        Self {
108            func: HashFn::Blake3,
109            digest: *blake3::hash(bytes).as_bytes(),
110        }
111    }
112
113    /// Compute the BLAKE3 hash by streaming chunks through the incremental
114    /// hasher. Avoids materialising the full payload as a single contiguous
115    /// slice — useful for verifying large model weights or other multi-MB
116    /// blobs that arrive in chunks.
117    ///
118    /// ```ignore
119    /// let hash = Hash::from_chunks([b"first ", b"second"].into_iter());
120    /// assert_eq!(hash, Hash::of(b"first second"));
121    /// ```
122    pub fn from_chunks<'a, I: IntoIterator<Item = &'a [u8]>>(chunks: I) -> Self {
123        let mut hasher = blake3::Hasher::new();
124        for chunk in chunks {
125            hasher.update(chunk);
126        }
127        Self {
128            func: HashFn::Blake3,
129            digest: *hasher.finalize().as_bytes(),
130        }
131    }
132
133    /// Compute the BLAKE3 hash by streaming bytes from a `Read` source —
134    /// e.g. a file handle or HTTP response body. Reads in 64 KiB chunks
135    /// until EOF.
136    ///
137    /// This is the path that lets eidetic-side verification scale to
138    /// multi-GiB blobs without holding the full payload in memory at
139    /// once: pair it with a `BlobFetcher` that streams.
140    pub fn from_reader<R: std::io::Read>(mut reader: R) -> std::io::Result<Self> {
141        let mut hasher = blake3::Hasher::new();
142        let mut buffer = [0u8; 65_536];
143        loop {
144            let n = reader.read(&mut buffer)?;
145            if n == 0 {
146                break;
147            }
148            hasher.update(&buffer[..n]);
149        }
150        Ok(Self {
151            func: HashFn::Blake3,
152            digest: *hasher.finalize().as_bytes(),
153        })
154    }
155
156    /// The hash function that produced this digest.
157    pub fn func(&self) -> HashFn {
158        self.func
159    }
160
161    /// The raw digest bytes (no function tag). BLAKE3 is 32 bytes.
162    pub fn as_bytes(&self) -> &[u8; 32] {
163        &self.digest
164    }
165
166    /// Hex-encoded digest (no algorithm prefix). For the self-describing form
167    /// use [`Display`](std::fmt::Display) (`<fn>:<hex>`).
168    pub fn to_hex(&self) -> String {
169        let mut out = String::with_capacity(self.digest.len() * 2);
170        for byte in &self.digest {
171            out.push_str(&format!("{byte:02x}"));
172        }
173        out
174    }
175
176    /// Self-describing binary multihash: `uvarint(multicodec) ++ uvarint(len)
177    /// ++ digest`. Digest fields on the wire, in the event DAG, and in
178    /// capability scopes should use this form so a hash migration never needs a
179    /// flag day.
180    pub fn to_multihash_bytes(&self) -> Vec<u8> {
181        let mut out = Vec::with_capacity(2 + self.digest.len());
182        write_uvarint(self.func.multicodec(), &mut out);
183        write_uvarint(self.func.digest_len() as u64, &mut out);
184        out.extend_from_slice(&self.digest);
185        out
186    }
187
188    /// Parse a [`Hash`] from its self-describing binary multihash form.
189    pub fn from_multihash_bytes(bytes: &[u8]) -> Result<Self, HashError> {
190        let (code, n1) = read_uvarint(bytes).ok_or(HashError::Truncated)?;
191        let func = HashFn::from_multicodec(code).ok_or(HashError::UnknownFunction)?;
192        let rest = &bytes[n1..];
193        let (len, n2) = read_uvarint(rest).ok_or(HashError::Truncated)?;
194        let digest_bytes = &rest[n2..];
195        if len as usize != func.digest_len() || digest_bytes.len() != func.digest_len() {
196            return Err(HashError::BadLength);
197        }
198        let mut digest = [0u8; 32];
199        digest.copy_from_slice(digest_bytes);
200        Ok(Self { func, digest })
201    }
202
203    /// Parse a [`Hash`] from its string form `<fn>:<hex>`. A bare hex string
204    /// (no `<fn>:` prefix) is accepted as legacy BLAKE3 for backward reads.
205    pub fn parse(s: &str) -> Result<Self, HashError> {
206        let (name, hex) = match s.split_once(':') {
207            Some((name, hex)) => (name, hex),
208            None => ("blake3", s),
209        };
210        let func = HashFn::from_name(name).ok_or(HashError::UnknownFunction)?;
211        if hex.len() != func.digest_len() * 2 {
212            return Err(HashError::BadLength);
213        }
214        let mut digest = [0u8; 32];
215        for (idx, pair) in hex.as_bytes().chunks(2).enumerate() {
216            let byte_str = std::str::from_utf8(pair).map_err(|_| HashError::BadHex)?;
217            digest[idx] = u8::from_str_radix(byte_str, 16).map_err(|_| HashError::BadHex)?;
218        }
219        Ok(Self { func, digest })
220    }
221}
222
223impl std::fmt::Display for Hash {
224    fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
225        write!(formatter, "{}:{}", self.func.name(), self.to_hex())
226    }
227}
228
229impl Serialize for Hash {
230    /// Serialize as the self-describing string `<fn>:<hex>` so manifests stay
231    /// human-inspectable and the hash function travels with the digest.
232    fn serialize<S: Serializer>(&self, serializer: S) -> Result<S::Ok, S::Error> {
233        serializer.serialize_str(&self.to_string())
234    }
235}
236
237impl<'de> Deserialize<'de> for Hash {
238    fn deserialize<D: Deserializer<'de>>(deserializer: D) -> Result<Self, D::Error> {
239        let s = String::deserialize(deserializer)?;
240        Hash::parse(&s).map_err(serde::de::Error::custom)
241    }
242}
243
244/// Unsigned-varint (LEB128) encoder for multihash code/length prefixes.
245fn write_uvarint(mut value: u64, out: &mut Vec<u8>) {
246    loop {
247        let mut byte = (value & 0x7f) as u8;
248        value >>= 7;
249        if value != 0 {
250            byte |= 0x80;
251        }
252        out.push(byte);
253        if value == 0 {
254            break;
255        }
256    }
257}
258
259/// Unsigned-varint decoder; returns the value and the number of bytes read.
260fn read_uvarint(bytes: &[u8]) -> Option<(u64, usize)> {
261    let mut value = 0u64;
262    let mut shift = 0u32;
263    for (idx, &byte) in bytes.iter().enumerate() {
264        if shift >= 64 {
265            return None;
266        }
267        value |= ((byte & 0x7f) as u64) << shift;
268        if byte & 0x80 == 0 {
269            return Some((value, idx + 1));
270        }
271        shift += 7;
272    }
273    None
274}
275
276/// Stable identifier for a manifest. Equals the BLAKE3 hash of the blob the
277/// manifest describes — two blobs with identical bytes share a manifest id.
278#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, Serialize, Deserialize)]
279#[serde(transparent)]
280pub struct ManifestId(pub Hash);
281
282impl ManifestId {
283    /// Construct from a content hash (typically the blob's BLAKE3 hash).
284    pub fn from_hash(hash: Hash) -> Self {
285        Self(hash)
286    }
287
288    /// Compute the manifest id for a given blob.
289    pub fn of_blob(bytes: &[u8]) -> Self {
290        Self(Hash::of(bytes))
291    }
292
293    /// The Store key under which the manifest itself is persisted.
294    pub fn store_key(&self) -> String {
295        format!("manifest:{}", self.0.to_hex())
296    }
297}
298
299impl std::fmt::Display for ManifestId {
300    fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
301        self.0.fmt(formatter)
302    }
303}
304
305/// Content-addressed pointer to a schema engram. A schema is itself an engram
306/// (recursive); this is just a typed wrapper around a [`ManifestId`] that
307/// signals "the thing at this id is a schema definition."
308#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, Serialize, Deserialize)]
309#[serde(transparent)]
310pub struct SchemaRef(pub ManifestId);
311
312impl SchemaRef {
313    pub fn from_id(id: ManifestId) -> Self {
314        Self(id)
315    }
316}
317
318impl std::fmt::Display for SchemaRef {
319    fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
320        self.0.fmt(formatter)
321    }
322}
323
324/// Unix milliseconds since epoch. Portable across native + wasm; no chrono
325/// dependency.
326#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)]
327#[serde(transparent)]
328pub struct Timestamp(pub u64);
329
330impl Timestamp {
331    pub const ZERO: Self = Self(0);
332}
333
334/// Privacy axis — who is allowed to see this artifact. Default is `LocalOnly`;
335/// promotion to a wider audience is always an explicit operation, never
336/// automatic.
337#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, Serialize, Deserialize)]
338pub enum PrivacyClass {
339    LocalOnly,
340    TrustedPeersOnly,
341    MootScoped,
342    PublicPortable,
343}
344
345impl Default for PrivacyClass {
346    fn default() -> Self {
347        Self::LocalOnly
348    }
349}
350
351/// Where an artifact came from. The shape stays loose for Phase 2 — concrete
352/// origin enrichment lands when `identity` and the Distillery aspect
353/// surface.
354#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
355pub enum ProvenanceOrigin {
356    /// Imported from outside the local system (HF Hub, peer, file). The
357    /// inner string identifies the source informally (e.g. an HF model URL).
358    Imported { source: String },
359    /// Derived from one or more upstream engrams via a Distillery transform
360    /// or merge.
361    Derived,
362    /// Generated by a local tool / pipeline (e.g. a vector index built from
363    /// existing engrams).
364    Generated,
365}
366
367/// Provenance axis — origin, ancestry, generation context. Independent of
368/// privacy and trust; provenance enables payout/audit/lineage tracking.
369#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
370pub struct ProvenanceRecord {
371    pub origin: ProvenanceOrigin,
372    /// Ancestor engrams (manifest ids) — for derivations / merges.
373    #[serde(default)]
374    pub upstream: Vec<ManifestId>,
375    /// Tooling identifier (e.g. crate name + version) that produced the
376    /// artifact, when known.
377    #[serde(default)]
378    pub tooling: Option<String>,
379    pub generated_at: Timestamp,
380}
381
382impl ProvenanceRecord {
383    /// Convenience for "user explicitly imported this from <source>."
384    pub fn self_imported(source: impl Into<String>) -> Self {
385        Self {
386            origin: ProvenanceOrigin::Imported {
387                source: source.into(),
388            },
389            upstream: Vec::new(),
390            tooling: None,
391            generated_at: Timestamp::ZERO,
392        }
393    }
394}
395
396/// Trust level — how confident the local node is in the artifact's contents.
397/// Independent of privacy: an engram can be `LocalOnly` + `CheckpointAccepted`
398/// (validated upstream contribution kept private) or `PublicPortable` +
399/// `SelfAsserted` (something the user shares without external review).
400#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, Serialize, Deserialize)]
401pub enum TrustLevel {
402    SelfAsserted,
403    PeerAttested,
404    CommunityReviewed,
405    CheckpointAccepted,
406}
407
408impl Default for TrustLevel {
409    fn default() -> Self {
410        Self::SelfAsserted
411    }
412}
413
414/// Reference to a signature — opaque string for Phase 2; concrete shape
415/// (likely DID / VC envelope per the engram_spec) lands with `identity`.
416#[derive(Clone, Debug, PartialEq, Eq, Hash, Serialize, Deserialize)]
417#[serde(transparent)]
418pub struct SignatureRef(pub String);
419
420#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, Serialize, Deserialize)]
421pub enum ModerationState {
422    Unreviewed,
423    Quarantined,
424    Accepted,
425    Rejected,
426    Superseded,
427}
428
429impl Default for ModerationState {
430    fn default() -> Self {
431        Self::Unreviewed
432    }
433}
434
435/// Trust axis — confidence level + signatures + moderation state.
436#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
437pub struct TrustEnvelope {
438    pub level: TrustLevel,
439    #[serde(default)]
440    pub signatures: Vec<SignatureRef>,
441    pub moderation_state: ModerationState,
442}
443
444impl TrustEnvelope {
445    /// Convenience — the trust level a user's own freshly-imported artifact
446    /// starts at: self-asserted, no signatures, unreviewed.
447    pub fn self_asserted() -> Self {
448        Self {
449            level: TrustLevel::SelfAsserted,
450            signatures: Vec::new(),
451            moderation_state: ModerationState::Unreviewed,
452        }
453    }
454}
455
456#[cfg(test)]
457mod tests {
458    use super::*;
459
460    #[test]
461    fn hash_of_known_input_matches_blake3_reference() {
462        // BLAKE3 of empty input is a fixed value; compare via to_hex stability.
463        let h = Hash::of(b"");
464        assert_eq!(h.to_hex().len(), 64);
465        // Re-hashing identical bytes returns identical digest.
466        assert_eq!(Hash::of(b"hello"), Hash::of(b"hello"));
467        assert_ne!(Hash::of(b"hello"), Hash::of(b"world"));
468    }
469
470    #[test]
471    fn hash_from_chunks_matches_full_slice() {
472        // Streaming via from_chunks is bit-for-bit identical to a single
473        // call against the concatenated bytes — that's the property
474        // streaming verification depends on.
475        let full = b"first chunk + second chunk + third";
476        let chunks: Vec<&[u8]> = vec![b"first chunk ", b"+ second chunk ", b"+ third"];
477        assert_eq!(Hash::from_chunks(chunks.into_iter()), Hash::of(full));
478    }
479
480    #[test]
481    fn hash_from_reader_matches_full_slice() {
482        let payload: Vec<u8> = (0..8192u32).map(|i| (i % 256) as u8).collect();
483        let from_slice = Hash::of(&payload);
484        let from_reader = Hash::from_reader(payload.as_slice()).unwrap();
485        assert_eq!(from_slice, from_reader);
486    }
487
488    #[test]
489    fn hash_serializes_as_self_describing_string() {
490        let h = Hash::of(b"abc");
491        let json = serde_json::to_string(&h).unwrap();
492        // Quoted "blake3:<64 hex>" — the function identifier travels with the
493        // digest, so a future hash migration needs no flag day (§13).
494        assert_eq!(json, format!("\"blake3:{}\"", h.to_hex()));
495        let round: Hash = serde_json::from_str(&json).unwrap();
496        assert_eq!(round, h);
497    }
498
499    #[test]
500    fn hash_multihash_bytes_round_trip() {
501        let h = Hash::of(b"payload");
502        let mh = h.to_multihash_bytes();
503        // BLAKE3 multihash = code 0x1e, length 0x20 (32), then 32 digest bytes.
504        assert_eq!(mh[0], 0x1e);
505        assert_eq!(mh[1], 0x20);
506        assert_eq!(mh.len(), 2 + 32);
507        assert_eq!(Hash::from_multihash_bytes(&mh).unwrap(), h);
508    }
509
510    #[test]
511    fn hash_parse_accepts_tagged_and_legacy_bare_hex() {
512        let h = Hash::of(b"legacy");
513        // Tagged form round-trips.
514        assert_eq!(Hash::parse(&h.to_string()).unwrap(), h);
515        // The old serialized form was bare hex with no function prefix; still
516        // readable so pre-migration fixtures load.
517        assert_eq!(Hash::parse(&h.to_hex()).unwrap(), h);
518        // An unknown function tag is rejected, not silently treated as BLAKE3.
519        assert_eq!(
520            Hash::parse(&format!("blake4:{}", h.to_hex())),
521            Err(HashError::UnknownFunction)
522        );
523    }
524
525    #[test]
526    fn manifest_id_store_key_is_prefix_plus_hex() {
527        let id = ManifestId::of_blob(b"payload");
528        let key = id.store_key();
529        assert!(key.starts_with("manifest:"));
530        assert_eq!(key.len(), "manifest:".len() + 64);
531    }
532
533    #[test]
534    fn privacy_class_defaults_to_local_only() {
535        assert_eq!(PrivacyClass::default(), PrivacyClass::LocalOnly);
536    }
537
538    #[test]
539    fn trust_envelope_self_asserted_starts_unreviewed() {
540        let trust = TrustEnvelope::self_asserted();
541        assert_eq!(trust.level, TrustLevel::SelfAsserted);
542        assert!(trust.signatures.is_empty());
543        assert_eq!(trust.moderation_state, ModerationState::Unreviewed);
544    }
545
546    #[test]
547    fn provenance_self_imported_records_source() {
548        let prov = ProvenanceRecord::self_imported("hf:test-model");
549        match prov.origin {
550            ProvenanceOrigin::Imported { source } => {
551                assert_eq!(source, "hf:test-model");
552            }
553            _ => panic!("expected Imported origin"),
554        }
555        assert!(prov.upstream.is_empty());
556    }
557}