dag_ml_data_core/
representation_registry.rs

1//! Frozen, published catalogue of the built-in representation IDs (`B-014` /
2//! `DMD-001`).
3//!
4//! `B-014` is a freeze/publish problem, not an invention problem: the stable
5//! representation-ID vocabulary already lives in [`crate::builtin_models`] as
6//! `pub const` strings, a [`BuiltinDataModel`] enum and its constructors. This
7//! module does not add a new vocabulary; it *publishes* the existing one as a
8//! single, deterministic, machine-readable registry so cross-repo consumers
9//! (notably `ControllerManifest.data_requirements` ports on `L16` and the
10//! `nirs4all-io` emit on `L7`) can reference the frozen strings, and so a drift
11//! test can freeze them against [`crate::builtin_models`].
12//!
13//! The companion artifact `docs/contracts/representation_registry.v1.json` is
14//! generated from [`representation_registry`] (via the
15//! `dag-ml-data-cli representation-registry` command) and pinned by the
16//! `published_registry_matches_builtin_models` drift test, which fails the
17//! moment a built-in representation changes without the published manifest
18//! being regenerated.
19
20use std::collections::{BTreeMap, BTreeSet};
21
22use serde::{Deserialize, Serialize};
23
24use crate::adapter::ModelInputSpec;
25use crate::builtin_models::{
26    BuiltinDataModel, BUILTIN_DATA_MODELS, REPRESENTATION_FEATURE_BLOCK_SET,
27    REPRESENTATION_GRAY_IMAGE, REPRESENTATION_MC_IMAGE, REPRESENTATION_MULTISPECTRAL_IMAGE,
28    REPRESENTATION_RGB_IMAGE, REPRESENTATION_SAMPLE_METADATA, REPRESENTATION_SIGNAL_1D,
29    REPRESENTATION_SIGNAL_WITH_PROCESSINGS, REPRESENTATION_TARGET_CATEGORICAL,
30    REPRESENTATION_TARGET_CATEGORICAL_MATRIX, REPRESENTATION_TARGET_NUMERIC,
31    REPRESENTATION_TARGET_NUMERIC_MATRIX,
32};
33use crate::error::{DataError, Result};
34use crate::model::RepresentationSpec;
35
36/// Wire-shape version of the published representation registry manifest.
37pub const REPRESENTATION_REGISTRY_SCHEMA_VERSION: u32 = 1;
38
39/// Stable identifier of the representation registry contract.
40pub const REPRESENTATION_REGISTRY_ID: &str = "dag-ml-data.representation_registry.v1";
41
42/// Repo-relative provenance of the frozen representation IDs.
43pub const REPRESENTATION_REGISTRY_SOURCE: &str = "crates/dag-ml-data-core/src/builtin_models.rs";
44
45/// Name of the spectra+image MVP profile defined by `B-014` / `IO_spec.md` §5.
46pub const MVP_PROFILE_SPECTRA_IMAGE: &str = "spectra_image";
47
48/// The 8 spectra/target/metadata representation IDs that the `nirs4all-io`
49/// bridge already emits for the spectra+image MVP (`IO_spec.md` §5).
50const MVP_SPECTRA_IMAGE_EMITTED: &[&str] = &[
51    REPRESENTATION_SIGNAL_1D,
52    REPRESENTATION_SIGNAL_WITH_PROCESSINGS,
53    REPRESENTATION_FEATURE_BLOCK_SET,
54    REPRESENTATION_TARGET_NUMERIC,
55    REPRESENTATION_TARGET_CATEGORICAL,
56    REPRESENTATION_TARGET_NUMERIC_MATRIX,
57    REPRESENTATION_TARGET_CATEGORICAL_MATRIX,
58    REPRESENTATION_SAMPLE_METADATA,
59];
60
61/// The 4 image representation IDs that are landed in this crate but not yet
62/// emitted by `nirs4all-io` (`IO-010`).
63const MVP_SPECTRA_IMAGE_PENDING: &[&str] = &[
64    REPRESENTATION_GRAY_IMAGE,
65    REPRESENTATION_RGB_IMAGE,
66    REPRESENTATION_MC_IMAGE,
67    REPRESENTATION_MULTISPECTRAL_IMAGE,
68];
69
70/// Cross-repo publication status of a representation ID inside an MVP profile.
71///
72/// This annotates *downstream emission readiness only*; every representation in
73/// the registry is fully landed in this crate (constructor, axes and dtype all
74/// present). The status is sourced from `IO_spec.md` §5, not verified at runtime
75/// by `dag-ml-data`.
76#[derive(Clone, Copy, Debug, Eq, PartialEq, Serialize, Deserialize)]
77#[serde(rename_all = "snake_case")]
78pub enum MvpEmissionStatus {
79    /// Landed here and already emitted by the `nirs4all-io` → `dag-ml-data`
80    /// bridge (the 8 spectra/target/metadata IDs of `IO_spec.md` §5).
81    Emitted,
82    /// Landed here (representation, axes, dtype and constructor present) but not
83    /// yet emitted by `nirs4all-io`; emission is tracked by `IO-010`.
84    LandedPendingEmit,
85}
86
87/// MVP profile membership for a published representation ID.
88#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)]
89pub struct MvpStatus {
90    /// Named MVP profile this ID belongs to (currently only
91    /// [`MVP_PROFILE_SPECTRA_IMAGE`]).
92    pub profile: String,
93    /// Cross-repo emission status of the ID within that profile.
94    pub emission: MvpEmissionStatus,
95}
96
97/// One frozen, published entry of the built-in representation registry.
98#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
99pub struct RegisteredRepresentation {
100    /// Stable representation-ID string (e.g. `signal_1d`) — the value that
101    /// `ControllerManifest.data_requirements` ports reference by string.
102    pub representation_id: String,
103    /// [`BuiltinDataModel`] catalogue key (e.g. `nirs.signal_1d`).
104    pub builtin_key: String,
105    /// Generic modality label (e.g. `nirs`, `image`, `target`).
106    pub modality: String,
107    /// Optional spectra+image MVP annotation; `None` for post-MVP IDs.
108    #[serde(default, skip_serializing_if = "Option::is_none")]
109    pub mvp: Option<MvpStatus>,
110    /// The complete frozen representation spec (axes, rank, dtype, container).
111    pub representation: RepresentationSpec,
112}
113
114/// The frozen, published catalogue of built-in representation IDs.
115///
116/// Build it with [`representation_registry`]. The serialized form is the
117/// `docs/contracts/representation_registry.v1.json` contract artifact.
118#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
119pub struct RepresentationRegistry {
120    /// Wire-shape version of this manifest
121    /// ([`REPRESENTATION_REGISTRY_SCHEMA_VERSION`]).
122    pub schema_version: u32,
123    /// Stable identifier of this registry contract
124    /// ([`REPRESENTATION_REGISTRY_ID`]).
125    pub registry_id: String,
126    /// Repo-relative provenance of the frozen IDs
127    /// ([`REPRESENTATION_REGISTRY_SOURCE`]).
128    pub source: String,
129    /// Every built-in representation, in [`BUILTIN_DATA_MODELS`] order.
130    pub representations: Vec<RegisteredRepresentation>,
131}
132
133/// Build the frozen, published catalogue of built-in representation IDs from
134/// [`BUILTIN_DATA_MODELS`]. The result is deterministic and source-order stable.
135pub fn representation_registry() -> RepresentationRegistry {
136    RepresentationRegistry {
137        schema_version: REPRESENTATION_REGISTRY_SCHEMA_VERSION,
138        registry_id: REPRESENTATION_REGISTRY_ID.to_string(),
139        source: REPRESENTATION_REGISTRY_SOURCE.to_string(),
140        representations: BUILTIN_DATA_MODELS
141            .iter()
142            .copied()
143            .map(registered_representation)
144            .collect(),
145    }
146}
147
148impl RepresentationRegistry {
149    /// Validate the published registry manifest shape and the embedded
150    /// representation specs before a consumer uses it for compatibility checks.
151    pub fn validate(&self) -> Result<()> {
152        if self.schema_version != REPRESENTATION_REGISTRY_SCHEMA_VERSION {
153            return Err(DataError::Validation(format!(
154                "representation registry uses unsupported schema_version {}, expected {}",
155                self.schema_version, REPRESENTATION_REGISTRY_SCHEMA_VERSION
156            )));
157        }
158        if self.registry_id != REPRESENTATION_REGISTRY_ID {
159            return Err(DataError::Validation(format!(
160                "representation registry id `{}` does not match `{}`",
161                self.registry_id, REPRESENTATION_REGISTRY_ID
162            )));
163        }
164        if self.representations.is_empty() {
165            return Err(DataError::Validation(
166                "representation registry contains no representations".to_string(),
167            ));
168        }
169        let mut ids = BTreeSet::new();
170        for entry in &self.representations {
171            if entry.representation_id.trim().is_empty() {
172                return Err(DataError::Validation(
173                    "representation registry contains an empty representation_id".to_string(),
174                ));
175            }
176            if !ids.insert(entry.representation_id.as_str()) {
177                return Err(DataError::Validation(format!(
178                    "representation registry contains duplicate representation `{}`",
179                    entry.representation_id
180                )));
181            }
182            if entry.representation_id != entry.representation.id.as_str() {
183                return Err(DataError::Validation(format!(
184                    "representation registry entry `{}` does not match embedded representation id `{}`",
185                    entry.representation_id, entry.representation.id
186                )));
187            }
188            entry.representation.validate()?;
189        }
190        Ok(())
191    }
192}
193
194/// Validate that a model-input/data-requirements contract consumes only
195/// published representation IDs, and that each accepted representation's frozen
196/// `type_id` and rank are allowed by the port.
197pub fn validate_model_input_spec_against_registry(
198    model_input: &ModelInputSpec,
199    registry: &RepresentationRegistry,
200) -> Result<()> {
201    registry.validate()?;
202    model_input.validate()?;
203    let by_id = registry
204        .representations
205        .iter()
206        .map(|entry| (entry.representation_id.as_str(), entry))
207        .collect::<BTreeMap<_, _>>();
208    for port in &model_input.ports {
209        for representation_id in &port.accepted_representations {
210            let representation_key = representation_id.as_str();
211            let Some(entry) = by_id.get(representation_key) else {
212                return Err(DataError::Validation(format!(
213                    "model input port `{}` accepts unknown representation `{representation_key}`",
214                    port.name
215                )));
216            };
217            if !port
218                .accepted_types
219                .iter()
220                .any(|type_id| type_id == &entry.representation.type_id)
221            {
222                return Err(DataError::Validation(format!(
223                    "model input port `{}` representation `{}` has type `{}` not listed in accepted_types",
224                    port.name, representation_key, entry.representation.type_id
225                )));
226            }
227            if let Some(rank) = port.rank {
228                if entry.representation.rank != Some(rank) {
229                    return Err(DataError::Validation(format!(
230                        "model input port `{}` requires rank {rank} but representation `{}` has {:?}",
231                        port.name, representation_key, entry.representation.rank
232                    )));
233                }
234            }
235        }
236    }
237    Ok(())
238}
239
240fn registered_representation(model: BuiltinDataModel) -> RegisteredRepresentation {
241    let representation = model.representation();
242    let representation_id = representation.id.as_str().to_string();
243    let mvp = mvp_status(&representation_id);
244    RegisteredRepresentation {
245        representation_id,
246        builtin_key: model.key().to_string(),
247        modality: model.modality().to_string(),
248        mvp,
249        representation,
250    }
251}
252
253fn mvp_status(representation_id: &str) -> Option<MvpStatus> {
254    if MVP_SPECTRA_IMAGE_EMITTED.contains(&representation_id) {
255        Some(MvpStatus {
256            profile: MVP_PROFILE_SPECTRA_IMAGE.to_string(),
257            emission: MvpEmissionStatus::Emitted,
258        })
259    } else if MVP_SPECTRA_IMAGE_PENDING.contains(&representation_id) {
260        Some(MvpStatus {
261            profile: MVP_PROFILE_SPECTRA_IMAGE.to_string(),
262            emission: MvpEmissionStatus::LandedPendingEmit,
263        })
264    } else {
265        None
266    }
267}
268
269#[cfg(test)]
270mod tests {
271    use std::collections::BTreeSet;
272
273    use crate::adapter::InputPortSpec;
274    use crate::ids::{RepresentationId, TypeId};
275
276    use super::*;
277
278    const PUBLISHED: &str = include_str!("../../../docs/contracts/representation_registry.v1.json");
279
280    #[test]
281    fn published_registry_matches_builtin_models() {
282        let generated = serde_json::to_value(representation_registry()).unwrap();
283        let published: serde_json::Value = serde_json::from_str(PUBLISHED).unwrap();
284        assert_eq!(
285            published, generated,
286            "docs/contracts/representation_registry.v1.json drifted from builtin_models.rs; \
287             regenerate with `cargo run -p dag-ml-data-cli -- representation-registry`"
288        );
289    }
290
291    #[test]
292    fn registry_header_is_stable() {
293        let registry = representation_registry();
294        assert_eq!(
295            registry.schema_version,
296            REPRESENTATION_REGISTRY_SCHEMA_VERSION
297        );
298        assert_eq!(registry.registry_id, REPRESENTATION_REGISTRY_ID);
299        assert_eq!(registry.source, REPRESENTATION_REGISTRY_SOURCE);
300    }
301
302    #[test]
303    fn registry_covers_every_builtin_with_unique_ids() {
304        let registry = representation_registry();
305        assert_eq!(registry.representations.len(), BUILTIN_DATA_MODELS.len());
306
307        let mut ids = BTreeSet::new();
308        let mut keys = BTreeSet::new();
309        for entry in &registry.representations {
310            assert!(
311                ids.insert(entry.representation_id.clone()),
312                "duplicate representation id {}",
313                entry.representation_id
314            );
315            assert!(
316                keys.insert(entry.builtin_key.clone()),
317                "duplicate builtin key {}",
318                entry.builtin_key
319            );
320            // The denormalized id must equal the embedded representation spec id.
321            assert_eq!(entry.representation_id, entry.representation.id.as_str());
322            // The embedded spec must validate (it is the frozen contract object).
323            entry.representation.validate().unwrap();
324        }
325    }
326
327    #[test]
328    fn spectra_image_mvp_profile_is_consistent() {
329        let registry = representation_registry();
330        let frozen: BTreeSet<&str> = registry
331            .representations
332            .iter()
333            .map(|entry| entry.representation_id.as_str())
334            .collect();
335
336        let emitted: BTreeSet<&str> = registry
337            .representations
338            .iter()
339            .filter(|entry| {
340                matches!(&entry.mvp, Some(status) if status.emission == MvpEmissionStatus::Emitted)
341            })
342            .map(|entry| entry.representation_id.as_str())
343            .collect();
344        let pending: BTreeSet<&str> = registry
345            .representations
346            .iter()
347            .filter(|entry| {
348                matches!(
349                    &entry.mvp,
350                    Some(status) if status.emission == MvpEmissionStatus::LandedPendingEmit
351                )
352            })
353            .map(|entry| entry.representation_id.as_str())
354            .collect();
355
356        // B-014 spectra+image MVP = 12 IDs (8 emitted + 4 image landed-pending-emit).
357        assert_eq!(emitted.len(), 8, "spectra MVP must publish 8 emitted IDs");
358        assert_eq!(pending.len(), 4, "image MVP must publish 4 pending IDs");
359        assert!(emitted.is_disjoint(&pending), "MVP groups must be disjoint");
360        for id in emitted.iter().chain(pending.iter()) {
361            assert!(
362                frozen.contains(id),
363                "MVP id {id} is not a frozen representation"
364            );
365        }
366
367        // The four pending IDs are exactly the image set.
368        assert_eq!(
369            pending,
370            BTreeSet::from([
371                REPRESENTATION_GRAY_IMAGE,
372                REPRESENTATION_RGB_IMAGE,
373                REPRESENTATION_MC_IMAGE,
374                REPRESENTATION_MULTISPECTRAL_IMAGE,
375            ])
376        );
377
378        // Every MVP annotation uses the published profile name.
379        for entry in &registry.representations {
380            if let Some(status) = &entry.mvp {
381                assert_eq!(status.profile, MVP_PROFILE_SPECTRA_IMAGE);
382            }
383        }
384    }
385
386    #[test]
387    fn registry_round_trips_through_json() {
388        let registry = representation_registry();
389        let json = serde_json::to_string(&registry).unwrap();
390        let decoded: RepresentationRegistry = serde_json::from_str(&json).unwrap();
391        assert_eq!(decoded, registry);
392    }
393
394    #[test]
395    fn model_input_spec_registry_validation_accepts_published_tabular_requirement() {
396        let spec = crate::tabular_numeric_model_input_spec();
397        validate_model_input_spec_against_registry(&spec, &representation_registry()).unwrap();
398    }
399
400    #[test]
401    fn model_input_spec_registry_validation_rejects_unknown_representation() {
402        let spec = ModelInputSpec {
403            ports: vec![InputPortSpec {
404                name: "x".to_string(),
405                accepted_representations: vec![RepresentationId::new("columnar_f64").unwrap()],
406                accepted_types: vec![TypeId::new("table").unwrap()],
407                rank: Some(2),
408                multi_source: false,
409                optional: false,
410            }],
411            default_fusion: None,
412        };
413        let error = validate_model_input_spec_against_registry(&spec, &representation_registry())
414            .unwrap_err();
415        assert!(
416            error.to_string().contains("unknown representation"),
417            "unexpected error: {error}"
418        );
419    }
420
421    #[test]
422    fn model_input_spec_registry_validation_rejects_type_drift() {
423        let mut spec = crate::tabular_numeric_model_input_spec();
424        spec.ports[0].accepted_types = vec![TypeId::new("f64").unwrap()];
425        let error = validate_model_input_spec_against_registry(&spec, &representation_registry())
426            .unwrap_err();
427        assert!(
428            error.to_string().contains("not listed in accepted_types"),
429            "unexpected error: {error}"
430        );
431    }
432}
dag_ml_data_core/representation_registry.rs

dag_ml_data_core/
representation_registry.rs
