Skip to main content

eidetic/
typed.rs

1// Copyright 2026 Mark AB (markik)
2// SPDX-License-Identifier: MIT OR Apache-2.0
3
4//! Layer 3 — Rust-side bindings for schema-conformant typed payloads.
5//!
6//! Layer 3 is **not** the schema. The schema lives one level up as a
7//! content-addressed engram (Phase 4 work). Layer 3 is the language-specific
8//! binding to a schema — Rust structs that serialize/deserialize bytes that
9//! conform to a known schema, given a [`SchemaRef`] declared by the type.
10//! Other implementations in other languages would bind the same schemas to
11//! their own native types.
12//!
13//! ## Save / load semantics
14//!
15//! [`save_typed`] serializes a typed value to bytes, computes the BLAKE3
16//! hash, builds a [`BlobManifest`] with the type's [`SchemaRef`], saves the
17//! blob bytes under `"blob:<hash>"`, saves the manifest under
18//! `"manifest:<hash>"`, and returns the manifest id.
19//!
20//! [`load_typed`] looks up the manifest, verifies its `schema` field matches
21//! `T::schema_ref()`, resolves the blob via [`resolve_blob`], hash-verifies
22//! the bytes, then deserializes them into `T`.
23
24use serde::Serialize;
25use serde::de::DeserializeOwned;
26
27use crate::manifest::{
28    BlobFetcher, BlobManifest, BlobSource, list_manifests, load_manifest, save_manifest,
29};
30use crate::schema::{
31    Hash, ManifestId, PrivacyClass, ProvenanceRecord, SchemaRef, Timestamp, TrustEnvelope,
32};
33use crate::seal::{PayloadSealer, resolve_sealed_blob, seal_payload_for_store};
34use crate::{Error, Result, Store};
35
36/// A Rust type that binds to a known schema.
37///
38/// Implementors declare which schema they conform to via [`schema_ref`]. The
39/// per-schema serializer is chosen by the impl — JSON is the default offered
40/// by the trait's default methods, but implementations are free to override
41/// (rkyv for [`VectorIndex`], raw bytes for `ModelWeights`, JSON-LD for
42/// schema.org-shaped payloads, etc.).
43///
44/// [`schema_ref`]: TypedPayload::schema_ref
45/// [`VectorIndex`]: https://crates.io/crates/embed
46pub trait TypedPayload: Serialize + DeserializeOwned + Sized {
47    /// Content-addressed reference to the schema this type conforms to.
48    /// In practice this is typically a `const fn`-computed `SchemaRef` or a
49    /// lazily-computed one keyed by the schema engram's known content hash.
50    fn schema_ref() -> SchemaRef;
51
52    /// Serialize the value to bytes. Default implementation uses
53    /// `serde_json`; impls override to pick a per-schema serializer.
54    fn serialize_to_bytes(&self) -> Result<Vec<u8>> {
55        serde_json::to_vec(self).map_err(|e| Error::new(format!("typed serialize: {e}")))
56    }
57
58    /// Deserialize bytes into a value. Default uses `serde_json`.
59    fn deserialize_from_bytes(bytes: &[u8]) -> Result<Self> {
60        serde_json::from_slice(bytes).map_err(|e| Error::new(format!("typed deserialize: {e}")))
61    }
62}
63
64/// Save a typed payload to the Store, producing a manifest id.
65///
66/// The blob bytes are saved under `"blob:<hash>"`; the manifest is saved
67/// under `"manifest:<hash>"`. The returned id is the BLAKE3 hash of the
68/// serialized bytes.
69///
70/// `additional_sources` lets callers attach Iroh tickets / HTTPS URLs / etc.
71/// alongside the implicit `Local` source. Pass an empty `Vec` for purely
72/// local-cached artifacts.
73pub async fn save_typed<T: TypedPayload>(
74    store: &mut dyn Store,
75    value: &T,
76    additional_sources: Vec<BlobSource>,
77    privacy: PrivacyClass,
78    provenance: ProvenanceRecord,
79    trust: TrustEnvelope,
80    created_at: Timestamp,
81) -> Result<ManifestId> {
82    save_typed_sealed(
83        store,
84        None,
85        value,
86        additional_sources,
87        privacy,
88        provenance,
89        trust,
90        created_at,
91    )
92    .await
93}
94
95/// Save a typed payload, sealing the private lane at rest.
96///
97/// As [`save_typed`], but with a [`PayloadSealer`]: when `sealer` is `Some` and
98/// the payload's `privacy` is a private-lane class (`LocalOnly` /
99/// `TrustedPeersOnly`), the stored blob bytes are sealed under the sealer's epoch
100/// and the manifest is stamped with the seal marker. The public lane and the
101/// no-sealer case store cleartext. The blob key and `content_hash` stay the
102/// cleartext hash, so addressing and integrity are unchanged; only the bytes on
103/// disk differ. Read back with [`load_typed_sealed`].
104#[allow(clippy::too_many_arguments)] // mirrors save_typed + the sealer.
105pub async fn save_typed_sealed<T: TypedPayload>(
106    store: &mut dyn Store,
107    sealer: Option<&dyn PayloadSealer>,
108    value: &T,
109    additional_sources: Vec<BlobSource>,
110    privacy: PrivacyClass,
111    provenance: ProvenanceRecord,
112    trust: TrustEnvelope,
113    created_at: Timestamp,
114) -> Result<ManifestId> {
115    let bytes = value.serialize_to_bytes()?;
116    let hash = Hash::of(&bytes);
117    let id = ManifestId::from_hash(hash);
118    let local_key = format!("blob:{}", hash.to_hex());
119
120    let mut sources = Vec::with_capacity(1 + additional_sources.len());
121    sources.push(BlobSource::Local {
122        key: local_key.clone(),
123    });
124    sources.extend(additional_sources);
125
126    let mut manifest = BlobManifest {
127        id,
128        schema: T::schema_ref(),
129        content_hash: hash,
130        byte_size: bytes.len() as u64,
131        created_at,
132        last_accessed: None,
133        sources,
134        privacy,
135        provenance,
136        trust,
137        schema_metadata: serde_json::Value::Null,
138        manifest_version: BlobManifest::CURRENT_VERSION,
139    };
140
141    // Seals + stamps the manifest marker for the private lane; passthrough
142    // otherwise. The stored bytes may be sealed; the manifest's content_hash
143    // stays the cleartext hash for the post-unseal integrity check.
144    let stored = seal_payload_for_store(sealer, &mut manifest, &bytes)?;
145    store.put(&local_key, &stored).await?;
146    save_manifest(store, &manifest).await?;
147    Ok(id)
148}
149
150/// Enumerate all manifests in the Store whose schema matches
151/// `T::schema_ref()`. Returns the manifest list (without resolving the
152/// payload bytes); use [`load_typed`] on each id to fetch.
153pub async fn list_typed<T: TypedPayload>(store: &mut dyn Store) -> Result<Vec<BlobManifest>> {
154    list_manifests(store, Some(T::schema_ref())).await
155}
156
157/// Load a typed payload by manifest id.
158///
159/// Verifies the manifest's `schema` field matches `T::schema_ref()` before
160/// deserializing — a manifest with a different schema is a load error
161/// (returns `Err`, not `None`) because mistyped reads are bugs.
162///
163/// Returns `Ok(None)` if no manifest is stored under `id`.
164pub async fn load_typed<T: TypedPayload>(
165    store: &mut dyn Store,
166    fetcher: &mut dyn BlobFetcher,
167    id: ManifestId,
168) -> Result<Option<T>> {
169    load_typed_sealed(store, fetcher, None, id).await
170}
171
172/// Load a typed payload, unsealing the private lane at rest.
173///
174/// As [`load_typed`], but with a [`PayloadSealer`]: a manifest whose blob is
175/// sealed (written via [`save_typed_sealed`]) is unsealed with `sealer` before
176/// the hash check. A sealed blob loaded with `sealer = None` is a hard error
177/// (never a silent hash mismatch); an unsealed blob reads cleartext either way.
178pub async fn load_typed_sealed<T: TypedPayload>(
179    store: &mut dyn Store,
180    fetcher: &mut dyn BlobFetcher,
181    sealer: Option<&dyn PayloadSealer>,
182    id: ManifestId,
183) -> Result<Option<T>> {
184    let Some(manifest) = load_manifest(store, id).await? else {
185        return Ok(None);
186    };
187
188    if manifest.schema != T::schema_ref() {
189        return Err(Error::new(format!(
190            "schema mismatch loading manifest {}: declared {}, requested {}",
191            id,
192            manifest.schema,
193            T::schema_ref()
194        )));
195    }
196
197    let bytes = resolve_sealed_blob(store, fetcher, sealer, &manifest).await?;
198    let value = T::deserialize_from_bytes(&bytes)?;
199    Ok(Some(value))
200}
201
202#[cfg(test)]
203mod tests {
204    use super::*;
205    use crate::manifest::NoFetcher;
206    use crate::schema::{
207        ManifestId, ModerationState, ProvenanceOrigin, ProvenanceRecord, SchemaRef, Timestamp,
208        TrustLevel,
209    };
210    use async_trait::async_trait;
211    use serde::{Deserialize, Serialize};
212    use std::collections::HashMap;
213
214    // The in-memory test store is muniment's (2026-07-12): eidetic's
215    // hand-rolled one was the same map behind the same seam.
216    use muniment::MemoryBackend as InMemoryStore;
217
218    /// First test schema: a simple greeting record.
219    #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
220    struct Greeting {
221        salutation: String,
222        recipient: String,
223    }
224
225    impl TypedPayload for Greeting {
226        fn schema_ref() -> SchemaRef {
227            SchemaRef::from_id(ManifestId::of_blob(b"schema:test-greeting/v1"))
228        }
229    }
230
231    /// Second test schema: a numeric record. Ensures the schema-mismatch
232    /// guard rejects mistyped loads.
233    #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
234    struct Counter {
235        ticks: u64,
236    }
237
238    impl TypedPayload for Counter {
239        fn schema_ref() -> SchemaRef {
240            SchemaRef::from_id(ManifestId::of_blob(b"schema:test-counter/v1"))
241        }
242    }
243
244    fn test_provenance() -> ProvenanceRecord {
245        ProvenanceRecord {
246            origin: ProvenanceOrigin::Generated,
247            upstream: Vec::new(),
248            tooling: Some("eidetic-test/0.0.1".to_string()),
249            generated_at: Timestamp(1_700_000_000_000),
250        }
251    }
252
253    fn test_trust() -> TrustEnvelope {
254        TrustEnvelope {
255            level: TrustLevel::SelfAsserted,
256            signatures: Vec::new(),
257            moderation_state: ModerationState::Unreviewed,
258        }
259    }
260
261    #[test]
262    fn save_then_load_round_trips_value() {
263        pollster::block_on(async {
264            let mut store = InMemoryStore::default();
265            let original = Greeting {
266                salutation: "Hello".to_string(),
267                recipient: "world".to_string(),
268            };
269
270            let id = save_typed(
271                &mut store,
272                &original,
273                Vec::new(),
274                PrivacyClass::LocalOnly,
275                test_provenance(),
276                test_trust(),
277                Timestamp(1_700_000_000_000),
278            )
279            .await
280            .unwrap();
281
282            let mut fetcher = NoFetcher;
283            let loaded: Greeting = load_typed(&mut store, &mut fetcher, id)
284                .await
285                .unwrap()
286                .expect("manifest present after save");
287            assert_eq!(loaded, original);
288        });
289    }
290
291    #[test]
292    fn load_typed_returns_none_for_unknown_id() {
293        pollster::block_on(async {
294            let mut store = InMemoryStore::default();
295            let mut fetcher = NoFetcher;
296            let unknown = ManifestId::of_blob(b"never-saved");
297            let result: Option<Greeting> =
298                load_typed(&mut store, &mut fetcher, unknown).await.unwrap();
299            assert!(result.is_none());
300        });
301    }
302
303    #[test]
304    fn schema_mismatch_load_returns_error() {
305        pollster::block_on(async {
306            // Save a Greeting, then try to load it as a Counter. Should
307            // error with an explicit schema mismatch message rather than
308            // silently producing garbage.
309            let mut store = InMemoryStore::default();
310            let id = save_typed(
311                &mut store,
312                &Greeting {
313                    salutation: "Hi".to_string(),
314                    recipient: "you".to_string(),
315                },
316                Vec::new(),
317                PrivacyClass::LocalOnly,
318                test_provenance(),
319                test_trust(),
320                Timestamp(0),
321            )
322            .await
323            .unwrap();
324
325            let mut fetcher = NoFetcher;
326            let result: Result<Option<Counter>> = load_typed(&mut store, &mut fetcher, id).await;
327            assert!(result.is_err());
328            assert!(format!("{}", result.unwrap_err()).contains("schema mismatch"));
329        });
330    }
331
332    #[test]
333    fn multiple_schemas_coexist_in_one_store() {
334        pollster::block_on(async {
335            let mut store = InMemoryStore::default();
336            let mut fetcher = NoFetcher;
337
338            let greeting_id = save_typed(
339                &mut store,
340                &Greeting {
341                    salutation: "Yo".to_string(),
342                    recipient: "everyone".to_string(),
343                },
344                Vec::new(),
345                PrivacyClass::LocalOnly,
346                test_provenance(),
347                test_trust(),
348                Timestamp(0),
349            )
350            .await
351            .unwrap();
352
353            let counter_id = save_typed(
354                &mut store,
355                &Counter { ticks: 42 },
356                Vec::new(),
357                PrivacyClass::LocalOnly,
358                test_provenance(),
359                test_trust(),
360                Timestamp(0),
361            )
362            .await
363            .unwrap();
364
365            assert_ne!(
366                greeting_id, counter_id,
367                "different content -> different ids"
368            );
369
370            let g: Greeting = load_typed(&mut store, &mut fetcher, greeting_id)
371                .await
372                .unwrap()
373                .unwrap();
374            assert_eq!(g.salutation, "Yo");
375
376            let c: Counter = load_typed(&mut store, &mut fetcher, counter_id)
377                .await
378                .unwrap()
379                .unwrap();
380            assert_eq!(c.ticks, 42);
381        });
382    }
383
384    #[test]
385    fn list_typed_returns_only_matching_schema_manifests() {
386        pollster::block_on(async {
387            let mut store = InMemoryStore::default();
388
389            // Save two Greetings and one Counter.
390            save_typed(
391                &mut store,
392                &Greeting {
393                    salutation: "Yo".to_string(),
394                    recipient: "alice".to_string(),
395                },
396                Vec::new(),
397                PrivacyClass::LocalOnly,
398                test_provenance(),
399                test_trust(),
400                Timestamp(0),
401            )
402            .await
403            .unwrap();
404            save_typed(
405                &mut store,
406                &Greeting {
407                    salutation: "Hi".to_string(),
408                    recipient: "bob".to_string(),
409                },
410                Vec::new(),
411                PrivacyClass::LocalOnly,
412                test_provenance(),
413                test_trust(),
414                Timestamp(0),
415            )
416            .await
417            .unwrap();
418            save_typed(
419                &mut store,
420                &Counter { ticks: 7 },
421                Vec::new(),
422                PrivacyClass::LocalOnly,
423                test_provenance(),
424                test_trust(),
425                Timestamp(0),
426            )
427            .await
428            .unwrap();
429
430            let greetings = list_typed::<Greeting>(&mut store).await.unwrap();
431            assert_eq!(greetings.len(), 2);
432            for m in &greetings {
433                assert_eq!(m.schema, Greeting::schema_ref());
434            }
435
436            let counters = list_typed::<Counter>(&mut store).await.unwrap();
437            assert_eq!(counters.len(), 1);
438            assert_eq!(counters[0].schema, Counter::schema_ref());
439        });
440    }
441
442    #[test]
443    fn load_typed_rejects_blob_whose_hash_no_longer_matches_manifest() {
444        pollster::block_on(async {
445            // Save a value, then corrupt the underlying blob bytes (simulate
446            // disk corruption / tampering). load_typed should fail at the
447            // resolve_blob hash-check step, not deserialize garbage.
448            let mut store = InMemoryStore::default();
449            let original = Greeting {
450                salutation: "Hi".to_string(),
451                recipient: "you".to_string(),
452            };
453            let id = save_typed(
454                &mut store,
455                &original,
456                Vec::new(),
457                PrivacyClass::LocalOnly,
458                test_provenance(),
459                test_trust(),
460                Timestamp(0),
461            )
462            .await
463            .unwrap();
464
465            let blob_key = format!("blob:{}", id.0.to_hex());
466            store
467                .put(&blob_key, b"not the original bytes")
468                .await
469                .unwrap();
470
471            let mut fetcher = NoFetcher;
472            let result: Result<Option<Greeting>> = load_typed(&mut store, &mut fetcher, id).await;
473            assert!(result.is_err());
474            assert!(format!("{}", result.unwrap_err()).contains("content hash mismatch"));
475        });
476    }
477
478    /// Trivial reversible sealer for wiring tests (XOR). Real AEAD crypto is
479    /// tested in the `seal` module and in pandect's `WalletEpochSealer`;
480    /// here we only prove the `save_typed_sealed` / `load_typed_sealed` plumbing.
481    struct XorSealer;
482
483    impl PayloadSealer for XorSealer {
484        fn seal(
485            &self,
486            _content_hash: &Hash,
487            cleartext: &[u8],
488        ) -> Result<(Vec<u8>, crate::seal::SealedBlobRef)> {
489            Ok((
490                cleartext.iter().map(|b| b ^ 0xAA).collect(),
491                crate::seal::SealedBlobRef {
492                    epoch: crate::seal::SealEpochId([1u8; 16]),
493                    format: "xor-test".to_string(),
494                },
495            ))
496        }
497
498        fn unseal(
499            &self,
500            _content_hash: &Hash,
501            _marker: &crate::seal::SealedBlobRef,
502            sealed: &[u8],
503        ) -> Result<Vec<u8>> {
504            Ok(sealed.iter().map(|b| b ^ 0xAA).collect())
505        }
506    }
507
508    #[test]
509    fn sealed_typed_round_trips_and_stores_non_cleartext() {
510        pollster::block_on(async {
511            let mut store = InMemoryStore::default();
512            let original = Greeting {
513                salutation: "secret".to_string(),
514                recipient: "you".to_string(),
515            };
516            let id = save_typed_sealed(
517                &mut store,
518                Some(&XorSealer),
519                &original,
520                Vec::new(),
521                PrivacyClass::LocalOnly,
522                test_provenance(),
523                test_trust(),
524                Timestamp(0),
525            )
526            .await
527            .unwrap();
528
529            // On disk the bytes are sealed, not the cleartext serialization.
530            let blob_key = format!("blob:{}", id.0.to_hex());
531            let stored = store.get(&blob_key).await.unwrap().unwrap();
532            assert_ne!(stored, original.serialize_to_bytes().unwrap());
533
534            // The sealed read round-trips through the same sealer.
535            let mut fetcher = NoFetcher;
536            let got: Greeting = load_typed_sealed(&mut store, &mut fetcher, Some(&XorSealer), id)
537                .await
538                .unwrap()
539                .unwrap();
540            assert_eq!(got, original);
541
542            // A keyless read is a hard error, not a silent hash mismatch.
543            let err = load_typed::<Greeting>(&mut store, &mut fetcher, id)
544                .await
545                .unwrap_err();
546            assert!(err.to_string().contains("sealed"), "got: {err}");
547        });
548    }
549}