Skip to main content

eidetic/
schema_def.rs

1// Copyright 2026 Mark AB (markik)
2// SPDX-License-Identifier: MIT OR Apache-2.0
3
4//! Polyglot schema definitions for schema engrams.
5//!
6//! A schema engram's payload is a [`SchemaDefinition`]. The definition
7//! declares its format (Mere-native / JSON Schema / JSON-LD) and carries the
8//! format-specific schema body. [`validate_payload`] dispatches to the
9//! matching validator.
10//!
11//! ## The three formats
12//!
13//! - **`SchemaFormat::MereNative`** — eidetic's own JSON shape, simplest
14//!   viable. Full structural validation: required fields, field types,
15//!   allowed-values enums.
16//! - **`SchemaFormat::JsonSchema`** — JSON Schema (Draft 7+), validated via
17//!   the `jsonschema` crate. Heavier, established, ecosystem-friendly.
18//! - **`SchemaFormat::JsonLd`** — JSON-LD with optional schema.org `@type`.
19//!   Phase 4 ships parse-only validation (verify the document is well-formed
20//!   and has a `@context` or `@type`); full SHACL / vocab-checking lands
21//!   when a payload pulls on it.
22//!
23//! New formats can be added without rearchitecting — the [`SchemaValidator`]
24//! trait + format dispatch make it nematic-shaped extensible.
25//!
26//! ## Recursion and the meta-schema
27//!
28//! A schema engram is itself an engram, so it has a schema reference of its
29//! own — the **meta-schema**, identified by [`META_SCHEMA_REF`]. The
30//! meta-schema describes "this is a [`SchemaDefinition`]." Its own `schema`
31//! field is `META_SCHEMA_REF` — self-referential, terminating the recursion.
32
33use serde::{Deserialize, Serialize};
34
35use crate::engram::{Engram, TimeBounds};
36use crate::manifest::{BlobFetcher, BlobSource};
37use crate::schema::{
38    Hash, ManifestId, ModerationState, PrivacyClass, ProvenanceOrigin, ProvenanceRecord, SchemaRef,
39    Timestamp, TrustEnvelope, TrustLevel,
40};
41use crate::typed::{TypedPayload, load_typed, save_typed};
42use crate::{Error, Result, Store};
43
44/// The schema-definition format used by a schema engram. Open-ended
45/// architecturally; new variants can be added without breaking existing
46/// schema engrams (other consumers see an unknown variant and either skip
47/// validation or surface a "validator not available" error per their own
48/// policy).
49#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)]
50pub enum SchemaFormat {
51    /// Eidetic's native JSON shape — primary format.
52    #[serde(rename = "mere-native")]
53    MereNative,
54    /// JSON Schema (Draft 7+).
55    #[serde(rename = "json-schema")]
56    JsonSchema,
57    /// JSON-LD with optional schema.org `@type`.
58    #[serde(rename = "json-ld")]
59    JsonLd,
60}
61
62/// Schema-definition payload for a schema engram.
63///
64/// `body` carries the format-specific schema document. For Mere-native, this
65/// is a [`MereNativeSchemaBody`]-shaped JSON value. For JSON Schema, it is
66/// the JSON Schema document itself. For JSON-LD, it is a JSON-LD context /
67/// document.
68#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
69pub struct SchemaDefinition {
70    pub format: SchemaFormat,
71    /// Human-readable identifier (e.g. `"VectorIndex/v1"`). Not load-bearing
72    /// for identity — content hash of the engram is. Used for diagnostics
73    /// and authoring tooling.
74    pub schema_id: String,
75    /// Format-specific schema body. Interpreted by the validator that
76    /// matches `format`.
77    pub body: serde_json::Value,
78}
79
80impl TypedPayload for SchemaDefinition {
81    fn schema_ref() -> SchemaRef {
82        *META_SCHEMA_REF
83    }
84}
85
86// ---------------------------------------------------------------------------
87// Meta-schema bootstrap
88// ---------------------------------------------------------------------------
89
90/// Canonical bytes of the meta-schema engram's payload.
91///
92/// These bytes describe the [`SchemaDefinition`] shape itself, in
93/// Mere-native format. They are constant across all eidetic instances and
94/// versions; their BLAKE3 hash is [`META_SCHEMA_REF`]'s id.
95const META_SCHEMA_PAYLOAD: &[u8] = br#"{"format":"mere-native","schema_id":"eidetic.meta-schema/v1","body":{"version":1,"description":"Schema engram payload describing another schema in one of: mere-native, json-schema, json-ld.","required":["format","schema_id","body"],"fields":{"format":{"type":"enum","values":["mere-native","json-schema","json-ld"]},"schema_id":{"type":"string"},"body":{"type":"object"}}}}"#;
96
97/// The well-known schema reference for schema engrams.
98///
99/// Equal to `SchemaRef::from_id(ManifestId::of_blob(META_SCHEMA_PAYLOAD))`.
100/// Computed lazily because BLAKE3 hashing isn't const-fn yet.
101pub fn meta_schema_ref() -> SchemaRef {
102    SchemaRef::from_id(ManifestId::from_hash(Hash::of(META_SCHEMA_PAYLOAD)))
103}
104
105/// The well-known schema reference for schema engrams.
106///
107/// Computed lazily at first use from `META_SCHEMA_PAYLOAD`; stable for the
108/// process lifetime. Callers that need a value rather than a static can use
109/// [`meta_schema_ref`].
110pub static META_SCHEMA_REF: std::sync::LazyLock<SchemaRef> =
111    std::sync::LazyLock::new(meta_schema_ref);
112
113/// Idempotently seed the meta-schema engram into a Store.
114///
115/// On first call against a fresh Store, writes the meta-schema's payload
116/// bytes (BLAKE3-anchored to [`META_SCHEMA_REF`]) and its manifest. On
117/// subsequent calls, returns immediately if the manifest is already
118/// present.
119///
120/// **Call this once during init** for any consumer that will use schema
121/// engrams. Without it, [`validate_against_schema`] silently tolerates
122/// missing schemas (per the design pass's "tolerant on read" rule), so
123/// forgetting bootstrap masks bugs rather than surfacing them.
124pub async fn bootstrap_meta_schema(store: &mut dyn Store) -> Result<()> {
125    let meta_id = META_SCHEMA_REF.0;
126    if crate::manifest::load_manifest(store, meta_id)
127        .await?
128        .is_some()
129    {
130        return Ok(());
131    }
132
133    // Write META_SCHEMA_PAYLOAD bytes verbatim (don't re-serialize the
134    // SchemaDefinition — re-serialization can reorder JSON object keys
135    // and produce a different hash than META_SCHEMA_REF anchors to).
136    let local_key = format!("blob:{}", Hash::of(META_SCHEMA_PAYLOAD).to_hex());
137    store.put(&local_key, META_SCHEMA_PAYLOAD).await?;
138
139    let manifest = crate::manifest::BlobManifest {
140        id: meta_id,
141        schema: *META_SCHEMA_REF,
142        content_hash: Hash::of(META_SCHEMA_PAYLOAD),
143        byte_size: META_SCHEMA_PAYLOAD.len() as u64,
144        created_at: Timestamp::ZERO,
145        last_accessed: None,
146        sources: vec![crate::manifest::BlobSource::Local { key: local_key }],
147        privacy: PrivacyClass::PublicPortable,
148        provenance: ProvenanceRecord {
149            origin: ProvenanceOrigin::Generated,
150            upstream: Vec::new(),
151            tooling: Some(format!("eidetic/{}", env!("CARGO_PKG_VERSION"))),
152            generated_at: Timestamp::ZERO,
153        },
154        trust: TrustEnvelope {
155            level: TrustLevel::CheckpointAccepted,
156            signatures: Vec::new(),
157            moderation_state: ModerationState::Accepted,
158        },
159        schema_metadata: serde_json::Value::Null,
160        manifest_version: crate::manifest::BlobManifest::CURRENT_VERSION,
161    };
162    crate::manifest::save_manifest(store, &manifest).await?;
163    Ok(())
164}
165
166/// Build the canonical meta-schema engram. Call this once at first init to
167/// populate the Store with the bootstrap engram, so subsequent schema
168/// engrams resolve their `schema` field correctly. Most consumers should
169/// use [`bootstrap_meta_schema`] instead, which writes the engram into
170/// the Store idempotently.
171pub fn meta_schema_engram() -> Engram {
172    Engram::new(
173        *META_SCHEMA_REF,
174        META_SCHEMA_PAYLOAD.to_vec(),
175        PrivacyClass::PublicPortable,
176        ProvenanceRecord {
177            origin: ProvenanceOrigin::Generated,
178            upstream: Vec::new(),
179            tooling: Some(format!("eidetic/{}", env!("CARGO_PKG_VERSION"))),
180            generated_at: Timestamp::ZERO,
181        },
182        TrustEnvelope {
183            level: TrustLevel::CheckpointAccepted,
184            signatures: Vec::new(),
185            moderation_state: ModerationState::Accepted,
186        },
187        TimeBounds::at(Timestamp::ZERO),
188    )
189}
190
191// ---------------------------------------------------------------------------
192// Validator trait + dispatch
193// ---------------------------------------------------------------------------
194
195/// Validate raw payload bytes against a parsed [`SchemaDefinition`].
196///
197/// Implementations interpret `definition.body` according to their format and
198/// check that `payload_bytes` conforms.
199pub trait SchemaValidator {
200    fn format(&self) -> SchemaFormat;
201    fn validate(&self, definition: &SchemaDefinition, payload_bytes: &[u8]) -> Result<()>;
202}
203
204/// Validate `payload_bytes` against `definition`, dispatching to the right
205/// validator by format.
206pub fn validate_payload(definition: &SchemaDefinition, payload_bytes: &[u8]) -> Result<()> {
207    match definition.format {
208        SchemaFormat::MereNative => MereNativeValidator.validate(definition, payload_bytes),
209        SchemaFormat::JsonSchema => JsonSchemaValidator.validate(definition, payload_bytes),
210        SchemaFormat::JsonLd => JsonLdValidator.validate(definition, payload_bytes),
211    }
212}
213
214/// Save a schema engram to the Store. Returns its manifest id.
215///
216/// The schema engram's content is a [`SchemaDefinition`] serialized as JSON;
217/// its own `schema` field is [`META_SCHEMA_REF`].
218pub async fn save_schema(
219    store: &mut dyn Store,
220    definition: &SchemaDefinition,
221    privacy: PrivacyClass,
222    provenance: ProvenanceRecord,
223    trust: TrustEnvelope,
224    created_at: Timestamp,
225) -> Result<ManifestId> {
226    save_typed(
227        store,
228        definition,
229        Vec::<BlobSource>::new(),
230        privacy,
231        provenance,
232        trust,
233        created_at,
234    )
235    .await
236}
237
238/// Load a schema engram by its manifest id. Returns `Ok(None)` if no such
239/// engram is stored.
240pub async fn load_schema(
241    store: &mut dyn Store,
242    fetcher: &mut dyn BlobFetcher,
243    schema_id: ManifestId,
244) -> Result<Option<SchemaDefinition>> {
245    load_typed::<SchemaDefinition>(store, fetcher, schema_id).await
246}
247
248/// Recursive resolution: load the schema engram referenced by `schema_ref`,
249/// then validate `payload_bytes` against it.
250///
251/// Forward-compatibility: if the schema engram is missing, returns
252/// `Ok(())` rather than erroring (per the design pass's "tolerant on read,
253/// strict on required_for_application" rule). Callers that need strict
254/// validation can check schema presence themselves before calling.
255pub async fn validate_against_schema(
256    store: &mut dyn Store,
257    fetcher: &mut dyn BlobFetcher,
258    schema_ref: SchemaRef,
259    payload_bytes: &[u8],
260) -> Result<()> {
261    let Some(definition) = load_schema(store, fetcher, schema_ref.0).await? else {
262        // Tolerant default: unknown schema is not a fatal load failure.
263        return Ok(());
264    };
265    validate_payload(&definition, payload_bytes)
266}
267
268// ---------------------------------------------------------------------------
269// Mere-native validator
270// ---------------------------------------------------------------------------
271
272/// Builder for ergonomic Mere-native [`SchemaDefinition`] construction.
273///
274/// ```ignore
275/// let definition = MereNativeSchemaBuilder::new("Counter/v1")
276///     .description("Tick counter")
277///     .field("ticks", MereNativeFieldSpec::U64, true)
278///     .field("notes", MereNativeFieldSpec::String, false)
279///     .build();
280/// ```
281///
282/// Fields added as `required: true` go into `body.required`; all fields
283/// are added to `body.fields`.
284pub struct MereNativeSchemaBuilder {
285    schema_id: String,
286    description: String,
287    fields: Vec<(String, MereNativeFieldSpec, bool)>,
288    version: u32,
289}
290
291impl MereNativeSchemaBuilder {
292    pub fn new(schema_id: impl Into<String>) -> Self {
293        Self {
294            schema_id: schema_id.into(),
295            description: String::new(),
296            fields: Vec::new(),
297            version: 1,
298        }
299    }
300
301    pub fn description(mut self, description: impl Into<String>) -> Self {
302        self.description = description.into();
303        self
304    }
305
306    pub fn version(mut self, version: u32) -> Self {
307        self.version = version;
308        self
309    }
310
311    /// Add a field. `required = true` includes the field in `body.required`.
312    pub fn field(
313        mut self,
314        name: impl Into<String>,
315        spec: MereNativeFieldSpec,
316        required: bool,
317    ) -> Self {
318        self.fields.push((name.into(), spec, required));
319        self
320    }
321
322    pub fn build(self) -> SchemaDefinition {
323        let mut required = Vec::new();
324        let mut fields = serde_json::Map::new();
325        for (name, spec, is_required) in self.fields {
326            if is_required {
327                required.push(name.clone());
328            }
329            fields.insert(
330                name,
331                serde_json::to_value(&spec).expect("field spec serializes"),
332            );
333        }
334        let body = serde_json::json!({
335            "version": self.version,
336            "description": self.description,
337            "required": required,
338            "fields": fields,
339        });
340        SchemaDefinition {
341            format: SchemaFormat::MereNative,
342            schema_id: self.schema_id,
343            body,
344        }
345    }
346}
347
348/// Find a schema engram by its human-readable `schema_id`.
349///
350/// Walks every schema engram in the Store (i.e. manifests with
351/// `schema == META_SCHEMA_REF`), loads each as a [`SchemaDefinition`],
352/// and returns the first match. Useful when consumers want to find a
353/// schema by name rather than by content hash.
354///
355/// Returns `Ok(None)` if no matching schema is present. Returns the
356/// matching schema's manifest id along with its definition so callers
357/// can build a `SchemaRef` for downstream use.
358pub async fn find_schema_by_id(
359    store: &mut dyn Store,
360    fetcher: &mut dyn BlobFetcher,
361    schema_id: &str,
362) -> Result<Option<(ManifestId, SchemaDefinition)>> {
363    let manifests = crate::manifest::list_manifests(store, Some(*META_SCHEMA_REF)).await?;
364    for manifest in manifests {
365        let definition: SchemaDefinition =
366            match load_typed::<SchemaDefinition>(store, fetcher, manifest.id).await {
367                Ok(Some(d)) => d,
368                _ => continue,
369            };
370        if definition.schema_id == schema_id {
371            return Ok(Some((manifest.id, definition)));
372        }
373    }
374    Ok(None)
375}
376
377/// Mere-native schema body shape. Stored under `SchemaDefinition::body`
378/// when `format == MereNative`.
379#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, Default)]
380pub struct MereNativeSchemaBody {
381    #[serde(default)]
382    pub version: u32,
383    #[serde(default)]
384    pub description: String,
385    #[serde(default)]
386    pub required: Vec<String>,
387    #[serde(default)]
388    pub fields: std::collections::BTreeMap<String, MereNativeFieldSpec>,
389}
390
391/// Per-field schema spec. Loose intentionally — Mere-native is meant to
392/// stay simple. Richer constraints belong in JSON Schema bodies.
393#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
394#[serde(tag = "type", rename_all = "snake_case")]
395pub enum MereNativeFieldSpec {
396    String,
397    U64,
398    I64,
399    F64,
400    Bool,
401    Object,
402    Array,
403    Enum { values: Vec<String> },
404}
405
406/// Validator for `SchemaFormat::MereNative` — full structural check.
407pub struct MereNativeValidator;
408
409impl SchemaValidator for MereNativeValidator {
410    fn format(&self) -> SchemaFormat {
411        SchemaFormat::MereNative
412    }
413
414    fn validate(&self, definition: &SchemaDefinition, payload_bytes: &[u8]) -> Result<()> {
415        if definition.format != SchemaFormat::MereNative {
416            return Err(Error::new(format!(
417                "MereNativeValidator received non-MereNative format: {:?}",
418                definition.format
419            )));
420        }
421        let body: MereNativeSchemaBody = serde_json::from_value(definition.body.clone())
422            .map_err(|e| Error::new(format!("mere-native schema body parse: {e}")))?;
423
424        let payload_value: serde_json::Value = serde_json::from_slice(payload_bytes)
425            .map_err(|e| Error::new(format!("payload parse (mere-native): {e}")))?;
426        let object = payload_value
427            .as_object()
428            .ok_or_else(|| Error::new("mere-native payloads must be JSON objects".to_string()))?;
429
430        for required in &body.required {
431            if !object.contains_key(required) {
432                return Err(Error::new(format!(
433                    "mere-native schema {}: missing required field `{}`",
434                    definition.schema_id, required
435                )));
436            }
437        }
438
439        for (name, spec) in &body.fields {
440            if let Some(value) = object.get(name) {
441                check_field_spec(&definition.schema_id, name, spec, value)?;
442            }
443        }
444
445        Ok(())
446    }
447}
448
449fn check_field_spec(
450    schema_id: &str,
451    name: &str,
452    spec: &MereNativeFieldSpec,
453    value: &serde_json::Value,
454) -> Result<()> {
455    let kind_ok = match spec {
456        MereNativeFieldSpec::String => value.is_string(),
457        MereNativeFieldSpec::U64 => value.as_u64().is_some(),
458        MereNativeFieldSpec::I64 => value.as_i64().is_some(),
459        MereNativeFieldSpec::F64 => value.as_f64().is_some(),
460        MereNativeFieldSpec::Bool => value.is_boolean(),
461        MereNativeFieldSpec::Object => value.is_object(),
462        MereNativeFieldSpec::Array => value.is_array(),
463        MereNativeFieldSpec::Enum { values } => match value.as_str() {
464            Some(s) => values.iter().any(|v| v == s),
465            None => false,
466        },
467    };
468    if !kind_ok {
469        return Err(Error::new(format!(
470            "schema {}: field `{}` failed type check (expected {:?}, got {})",
471            schema_id, name, spec, value
472        )));
473    }
474    Ok(())
475}
476
477mod validators;
478pub use validators::{JsonLdValidator, JsonSchemaValidator};
479#[cfg(test)]
480mod tests;