graphforge_storage/
embedding_manifest.rs

1//! Durable source-state and complete-generation metadata for embeddings.
2
3use std::fmt;
4use std::path::{Path, PathBuf};
5
6use serde::Deserialize;
7use serde_json::Value;
8use sha2::{Digest, Sha256};
9
10use crate::{
11    EmbeddingCompatibilityId, EmbeddingContentDigest, EmbeddingGenerationId,
12    EmbeddingSourceFingerprint, SearchArtifactError,
13};
14
15/// Generation manifest schema implemented by this release.
16pub const EMBEDDING_GENERATION_MANIFEST_VERSION: u32 = 1;
17/// Maximum accepted generation manifest bytes.
18pub const MAX_EMBEDDING_GENERATION_MANIFEST_BYTES: usize = 64 * 1024;
19const SOURCE_FINGERPRINT_DOMAIN: &[u8] = b"graphforge_embedding_source_v1";
20
21/// SHA-256 fingerprint of the exact published vector file bytes.
22#[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)]
23pub struct EmbeddingPublicationFingerprint([u8; 32]);
24
25impl EmbeddingPublicationFingerprint {
26    /// Digest exact publication-file bytes.
27    #[must_use]
28    pub fn digest(bytes: &[u8]) -> Self {
29        Self(Sha256::digest(bytes).into())
30    }
31
32    /// Parse exactly 64 lowercase hexadecimal digits.
33    ///
34    /// # Errors
35    /// Rejects uppercase, non-hexadecimal, and wrong-width input.
36    pub fn from_hex(value: &str) -> Result<Self, SearchArtifactError> {
37        parse_digest("publication_fingerprint", value).map(Self)
38    }
39
40    /// Lowercase hexadecimal representation.
41    #[must_use]
42    pub fn to_hex(self) -> String {
43        encode_digest(self.0)
44    }
45}
46
47impl fmt::Debug for EmbeddingPublicationFingerprint {
48    fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
49        formatter
50            .debug_tuple("EmbeddingPublicationFingerprint")
51            .field(&self.to_hex())
52            .finish()
53    }
54}
55
56impl fmt::Display for EmbeddingPublicationFingerprint {
57    fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
58        formatter.write_str(&self.to_hex())
59    }
60}
61
62/// Exact committed graph inputs used to build one complete generation.
63#[derive(Clone, Copy, Debug, PartialEq, Eq)]
64pub struct EmbeddingSourceState {
65    graph_generation: u64,
66    label_membership_digest: [u8; 32],
67    dependency_input_digest: [u8; 32],
68    eligible_uuid_count: u64,
69    fingerprint: EmbeddingSourceFingerprint,
70}
71
72impl EmbeddingSourceState {
73    /// Freeze source components and derive their versioned SHA-256 identity.
74    #[must_use]
75    pub fn new(
76        graph_generation: u64,
77        label_membership_digest: [u8; 32],
78        dependency_input_digest: [u8; 32],
79        eligible_uuid_count: u64,
80    ) -> Self {
81        let mut hasher = Sha256::new();
82        hasher.update(SOURCE_FINGERPRINT_DOMAIN);
83        hasher.update(graph_generation.to_le_bytes());
84        hasher.update(label_membership_digest);
85        hasher.update(dependency_input_digest);
86        hasher.update(eligible_uuid_count.to_le_bytes());
87        let fingerprint = EmbeddingSourceFingerprint::from_hex(&format!("{:x}", hasher.finalize()))
88            .expect("SHA-256 output is always a valid lowercase source fingerprint");
89        Self {
90            graph_generation,
91            label_membership_digest,
92            dependency_input_digest,
93            eligible_uuid_count,
94            fingerprint,
95        }
96    }
97
98    /// Committed graph mutation generation.
99    #[must_use]
100    pub const fn graph_generation(self) -> u64 {
101        self.graph_generation
102    }
103
104    /// Digest of canonical selected-label UUID membership.
105    #[must_use]
106    pub const fn label_membership_digest(self) -> [u8; 32] {
107        self.label_membership_digest
108    }
109
110    /// Digest of canonical topology/property dependencies.
111    #[must_use]
112    pub const fn dependency_input_digest(self) -> [u8; 32] {
113        self.dependency_input_digest
114    }
115
116    /// Complete eligible UUID count.
117    #[must_use]
118    pub const fn eligible_uuid_count(self) -> u64 {
119        self.eligible_uuid_count
120    }
121
122    /// Versioned source fingerprint used in generation identity.
123    #[must_use]
124    pub const fn fingerprint(self) -> EmbeddingSourceFingerprint {
125        self.fingerprint
126    }
127}
128
129/// Validated input for one completed generation manifest.
130#[derive(Clone, Copy, Debug)]
131pub struct EmbeddingGenerationManifestInput {
132    /// Validated space compatibility identity.
133    pub compatibility_id: EmbeddingCompatibilityId,
134    /// Exact committed source state.
135    pub source: EmbeddingSourceState,
136    /// Canonical UUID/vector content digest.
137    pub content_digest: EmbeddingContentDigest,
138    /// Rows in the complete generation.
139    pub vector_count: u64,
140    /// Fixed Float32 vector width.
141    pub dimension: u32,
142    /// Producer completion time in UTC microseconds since Unix epoch.
143    pub generated_at_micros: i64,
144    /// Durable publication time in UTC microseconds since Unix epoch.
145    pub committed_at_micros: i64,
146    /// Digest of exact persisted vector-file bytes.
147    pub publication_fingerprint: EmbeddingPublicationFingerprint,
148}
149
150/// Durable metadata for one validated, complete embedding generation.
151#[derive(Clone, Debug, PartialEq, Eq)]
152pub struct EmbeddingGenerationManifest {
153    compatibility_id: EmbeddingCompatibilityId,
154    generation_id: EmbeddingGenerationId,
155    source: EmbeddingSourceState,
156    content_digest: EmbeddingContentDigest,
157    vector_count: u64,
158    dimension: u32,
159    generated_at_micros: i64,
160    committed_at_micros: i64,
161    publication_fingerprint: EmbeddingPublicationFingerprint,
162}
163
164impl EmbeddingGenerationManifest {
165    /// Validate a complete generation and derive its content-idempotent ID.
166    ///
167    /// # Errors
168    /// Rejects incomplete coverage, zero dimensions, and invalid timestamps.
169    pub fn new(input: EmbeddingGenerationManifestInput) -> Result<Self, SearchArtifactError> {
170        if input.vector_count != input.source.eligible_uuid_count {
171            return Err(invalid(
172                "embedding generation",
173                "vector_count must equal eligible_uuid_count",
174            ));
175        }
176        if input.dimension == 0 {
177            return Err(invalid(
178                "embedding generation dimension",
179                "must be greater than zero",
180            ));
181        }
182        if input.generated_at_micros < 0 || input.committed_at_micros < 0 {
183            return Err(invalid(
184                "embedding generation timestamp",
185                "must be non-negative UTC microseconds",
186            ));
187        }
188        if input.committed_at_micros < input.generated_at_micros {
189            return Err(invalid(
190                "embedding generation timestamp",
191                "committed_at_micros must not precede generated_at_micros",
192            ));
193        }
194        let generation_id = EmbeddingGenerationId::for_generation(
195            input.compatibility_id,
196            input.source.fingerprint,
197            input.content_digest,
198        );
199        Ok(Self {
200            compatibility_id: input.compatibility_id,
201            generation_id,
202            source: input.source,
203            content_digest: input.content_digest,
204            vector_count: input.vector_count,
205            dimension: input.dimension,
206            generated_at_micros: input.generated_at_micros,
207            committed_at_micros: input.committed_at_micros,
208            publication_fingerprint: input.publication_fingerprint,
209        })
210    }
211
212    /// Serialize exact compact deterministic JSON.
213    ///
214    /// # Errors
215    /// Returns a structured corruption error only for unexpected serialization failure.
216    pub fn to_canonical_json(&self) -> Result<Vec<u8>, SearchArtifactError> {
217        serde_json::to_vec(&serde_json::json!({
218            "committed_at_micros": self.committed_at_micros,
219            "compatibility_id": self.compatibility_id.to_hex(),
220            "content_digest": self.content_digest.to_hex(),
221            "dependency_input_digest": encode_digest(self.source.dependency_input_digest),
222            "dimension": self.dimension,
223            "eligible_uuid_count": self.source.eligible_uuid_count,
224            "generated_at_micros": self.generated_at_micros,
225            "generation_id": self.generation_id.to_hex(),
226            "graph_generation": self.source.graph_generation,
227            "label_membership_digest": encode_digest(self.source.label_membership_digest),
228            "manifest_version": EMBEDDING_GENERATION_MANIFEST_VERSION,
229            "publication_fingerprint": self.publication_fingerprint.to_hex(),
230            "source_fingerprint": self.source.fingerprint.to_hex(),
231            "vector_count": self.vector_count,
232        }))
233        .map_err(|error| corrupt(Path::new("<memory>"), error.to_string()))
234    }
235
236    /// Parse exact canonical manifest bytes and re-run every invariant.
237    ///
238    /// # Errors
239    /// Distinguishes oversized, corrupt, and incompatible persisted metadata.
240    pub fn from_json(path: &Path, bytes: &[u8]) -> Result<Self, SearchArtifactError> {
241        if bytes.len() > MAX_EMBEDDING_GENERATION_MANIFEST_BYTES {
242            return Err(SearchArtifactError::ResourceExhausted {
243                resource: "embedding_generation_manifest_bytes",
244                limit: MAX_EMBEDDING_GENERATION_MANIFEST_BYTES as u64,
245            });
246        }
247        let value: Value =
248            serde_json::from_slice(bytes).map_err(|error| corrupt(path, error.to_string()))?;
249        let object = value
250            .as_object()
251            .ok_or_else(|| corrupt(path, "expected a JSON object"))?;
252        let version = object
253            .get("manifest_version")
254            .and_then(Value::as_u64)
255            .ok_or_else(|| corrupt(path, "manifest_version must be an unsigned integer"))?;
256        if version != u64::from(EMBEDDING_GENERATION_MANIFEST_VERSION) {
257            return Err(SearchArtifactError::IncompatibleManifest {
258                path: path.to_path_buf(),
259                found: version,
260                supported: EMBEDDING_GENERATION_MANIFEST_VERSION,
261            });
262        }
263        let raw: RawEmbeddingGenerationManifest =
264            serde_json::from_value(value).map_err(|error| corrupt(path, error.to_string()))?;
265        debug_assert_eq!(raw.manifest_version, EMBEDDING_GENERATION_MANIFEST_VERSION);
266        let compatibility_id = EmbeddingCompatibilityId::from_hex(&raw.compatibility_id)
267            .map_err(|error| corrupt(path, error.to_string()))?;
268        let content_digest = EmbeddingContentDigest::from_hex(&raw.content_digest)
269            .map_err(|error| corrupt(path, error.to_string()))?;
270        let expected_generation = EmbeddingGenerationId::from_hex(&raw.generation_id)
271            .map_err(|error| corrupt(path, error.to_string()))?;
272        let expected_source = EmbeddingSourceFingerprint::from_hex(&raw.source_fingerprint)
273            .map_err(|error| corrupt(path, error.to_string()))?;
274        let source = EmbeddingSourceState::new(
275            raw.graph_generation,
276            parse_digest("label_membership_digest", &raw.label_membership_digest)
277                .map_err(|error| corrupt(path, error.to_string()))?,
278            parse_digest("dependency_input_digest", &raw.dependency_input_digest)
279                .map_err(|error| corrupt(path, error.to_string()))?,
280            raw.eligible_uuid_count,
281        );
282        if source.fingerprint != expected_source {
283            return Err(corrupt(
284                path,
285                "source_fingerprint does not match source state",
286            ));
287        }
288        let manifest = Self::new(EmbeddingGenerationManifestInput {
289            compatibility_id,
290            source,
291            content_digest,
292            vector_count: raw.vector_count,
293            dimension: raw.dimension,
294            generated_at_micros: raw.generated_at_micros,
295            committed_at_micros: raw.committed_at_micros,
296            publication_fingerprint: EmbeddingPublicationFingerprint::from_hex(
297                &raw.publication_fingerprint,
298            )
299            .map_err(|error| corrupt(path, error.to_string()))?,
300        })
301        .map_err(|error| corrupt(path, error.to_string()))?;
302        if manifest.generation_id != expected_generation {
303            return Err(corrupt(
304                path,
305                "generation_id does not match compatibility, source, and content",
306            ));
307        }
308        if manifest
309            .to_canonical_json()
310            .map_err(|error| corrupt(path, error.to_string()))?
311            != bytes
312        {
313            return Err(corrupt(path, "manifest bytes are not exact canonical JSON"));
314        }
315        Ok(manifest)
316    }
317
318    /// Content-idempotent generation identity.
319    #[must_use]
320    pub const fn generation_id(&self) -> EmbeddingGenerationId {
321        self.generation_id
322    }
323
324    /// Compatibility lineage identity.
325    #[must_use]
326    pub const fn compatibility_id(&self) -> EmbeddingCompatibilityId {
327        self.compatibility_id
328    }
329
330    /// Exact source state.
331    #[must_use]
332    pub const fn source(&self) -> EmbeddingSourceState {
333        self.source
334    }
335
336    /// Canonical content digest.
337    #[must_use]
338    pub const fn content_digest(&self) -> EmbeddingContentDigest {
339        self.content_digest
340    }
341
342    /// Complete vector row count.
343    #[must_use]
344    pub const fn vector_count(&self) -> u64 {
345        self.vector_count
346    }
347
348    /// Fixed vector width.
349    #[must_use]
350    pub const fn dimension(&self) -> u32 {
351        self.dimension
352    }
353
354    /// Producer completion time in UTC microseconds since Unix epoch.
355    #[must_use]
356    pub const fn generated_at_micros(&self) -> i64 {
357        self.generated_at_micros
358    }
359
360    /// Durable publication time in UTC microseconds since Unix epoch.
361    #[must_use]
362    pub const fn committed_at_micros(&self) -> i64 {
363        self.committed_at_micros
364    }
365
366    /// Exact persisted vector-file fingerprint.
367    #[must_use]
368    pub const fn publication_fingerprint(&self) -> EmbeddingPublicationFingerprint {
369        self.publication_fingerprint
370    }
371}
372
373#[derive(Deserialize)]
374#[serde(deny_unknown_fields)]
375struct RawEmbeddingGenerationManifest {
376    manifest_version: u32,
377    compatibility_id: String,
378    generation_id: String,
379    source_fingerprint: String,
380    graph_generation: u64,
381    label_membership_digest: String,
382    dependency_input_digest: String,
383    eligible_uuid_count: u64,
384    content_digest: String,
385    vector_count: u64,
386    dimension: u32,
387    generated_at_micros: i64,
388    committed_at_micros: i64,
389    publication_fingerprint: String,
390}
391
392fn parse_digest(field: &'static str, value: &str) -> Result<[u8; 32], SearchArtifactError> {
393    if value.len() != 64
394        || !value
395            .bytes()
396            .all(|byte| byte.is_ascii_hexdigit() && !byte.is_ascii_uppercase())
397    {
398        return Err(invalid(
399            field,
400            "must be exactly 64 lowercase hexadecimal digits",
401        ));
402    }
403    let mut output = [0_u8; 32];
404    for (index, pair) in value.as_bytes().chunks_exact(2).enumerate() {
405        output[index] = (hex_nibble(pair[0]) << 4) | hex_nibble(pair[1]);
406    }
407    Ok(output)
408}
409
410fn encode_digest(value: [u8; 32]) -> String {
411    const HEX: &[u8; 16] = b"0123456789abcdef";
412    let mut output = String::with_capacity(64);
413    for byte in value {
414        output.push(char::from(HEX[usize::from(byte >> 4)]));
415        output.push(char::from(HEX[usize::from(byte & 0x0f)]));
416    }
417    output
418}
419
420fn hex_nibble(byte: u8) -> u8 {
421    match byte {
422        b'0'..=b'9' => byte - b'0',
423        b'a'..=b'f' => byte - b'a' + 10,
424        _ => unreachable!("parse_digest validates lowercase hexadecimal input"),
425    }
426}
427
428fn invalid(field: &'static str, reason: impl Into<String>) -> SearchArtifactError {
429    SearchArtifactError::InvalidSelector {
430        field,
431        reason: reason.into(),
432    }
433}
434
435fn corrupt(path: &Path, reason: impl Into<String>) -> SearchArtifactError {
436    SearchArtifactError::CorruptManifest {
437        path: PathBuf::from(path),
438        reason: reason.into(),
439    }
440}
441
442#[cfg(test)]
443mod tests {
444    use super::*;
445
446    fn manifest_input() -> EmbeddingGenerationManifestInput {
447        EmbeddingGenerationManifestInput {
448            compatibility_id: EmbeddingCompatibilityId::from_hex(&"11".repeat(32)).unwrap(),
449            source: EmbeddingSourceState::new(7, [2; 32], [3; 32], 2),
450            content_digest: EmbeddingContentDigest::digest(b"canonical rows"),
451            vector_count: 2,
452            dimension: 4,
453            generated_at_micros: 100,
454            committed_at_micros: 101,
455            publication_fingerprint: EmbeddingPublicationFingerprint::digest(b"parquet"),
456        }
457    }
458
459    #[test]
460    fn every_source_component_changes_fingerprint() {
461        let base = EmbeddingSourceState::new(1, [2; 32], [3; 32], 4).fingerprint();
462        let changed = [
463            EmbeddingSourceState::new(2, [2; 32], [3; 32], 4).fingerprint(),
464            EmbeddingSourceState::new(1, [9; 32], [3; 32], 4).fingerprint(),
465            EmbeddingSourceState::new(1, [2; 32], [9; 32], 4).fingerprint(),
466            EmbeddingSourceState::new(1, [2; 32], [3; 32], 5).fingerprint(),
467        ];
468        assert!(changed.into_iter().all(|value| value != base));
469    }
470
471    #[test]
472    fn complete_manifest_round_trips_exactly() {
473        let manifest = EmbeddingGenerationManifest::new(manifest_input()).unwrap();
474        let bytes = manifest.to_canonical_json().unwrap();
475        let reopened =
476            EmbeddingGenerationManifest::from_json(Path::new("manifest.json"), &bytes).unwrap();
477        assert_eq!(reopened, manifest);
478        assert_eq!(reopened.to_canonical_json().unwrap(), bytes);
479        assert_eq!(
480            reopened.generation_id(),
481            EmbeddingGenerationId::for_generation(
482                reopened.compatibility_id(),
483                reopened.source().fingerprint(),
484                reopened.content_digest()
485            )
486        );
487    }
488
489    #[test]
490    fn constructor_rejects_incomplete_dimensions_and_timestamps() {
491        let mut incomplete = manifest_input();
492        incomplete.vector_count = 1;
493        assert!(EmbeddingGenerationManifest::new(incomplete).is_err());
494
495        let mut zero_dimension = manifest_input();
496        zero_dimension.dimension = 0;
497        assert!(EmbeddingGenerationManifest::new(zero_dimension).is_err());
498
499        let mut negative = manifest_input();
500        negative.generated_at_micros = -1;
501        assert!(EmbeddingGenerationManifest::new(negative).is_err());
502
503        let mut reversed = manifest_input();
504        reversed.committed_at_micros = 99;
505        assert!(EmbeddingGenerationManifest::new(reversed).is_err());
506    }
507
508    #[test]
509    fn reopen_rejects_identity_digest_and_shape_corruption() {
510        let manifest = EmbeddingGenerationManifest::new(manifest_input()).unwrap();
511        let mut value: Value =
512            serde_json::from_slice(&manifest.to_canonical_json().unwrap()).unwrap();
513        value["generation_id"] = Value::String("22".repeat(32));
514        assert!(matches!(
515            EmbeddingGenerationManifest::from_json(
516                Path::new("manifest.json"),
517                &serde_json::to_vec(&value).unwrap()
518            ),
519            Err(SearchArtifactError::CorruptManifest { .. })
520        ));
521
522        value["generation_id"] = Value::String(manifest.generation_id().to_hex());
523        value["content_digest"] = Value::String("not-a-digest".to_owned());
524        assert!(matches!(
525            EmbeddingGenerationManifest::from_json(
526                Path::new("manifest.json"),
527                &serde_json::to_vec(&value).unwrap()
528            ),
529            Err(SearchArtifactError::CorruptManifest { .. })
530        ));
531
532        value["content_digest"] = Value::String(manifest.content_digest().to_hex());
533        value["unknown"] = Value::Bool(true);
534        assert!(matches!(
535            EmbeddingGenerationManifest::from_json(
536                Path::new("manifest.json"),
537                &serde_json::to_vec(&value).unwrap()
538            ),
539            Err(SearchArtifactError::CorruptManifest { .. })
540        ));
541    }
542
543    #[test]
544    fn reopen_rejects_duplicate_noncanonical_version_and_size() {
545        let manifest = EmbeddingGenerationManifest::new(manifest_input()).unwrap();
546        let canonical = String::from_utf8(manifest.to_canonical_json().unwrap()).unwrap();
547        let duplicate = canonical.replacen(r#""dimension":4"#, r#""dimension":4,"dimension":4"#, 1);
548        assert_ne!(duplicate, canonical);
549        let padded = format!(" {canonical}");
550        for bytes in [duplicate.as_bytes(), padded.as_bytes()] {
551            assert!(matches!(
552                EmbeddingGenerationManifest::from_json(Path::new("manifest.json"), bytes),
553                Err(SearchArtifactError::CorruptManifest { .. })
554            ));
555        }
556
557        let mut value: Value = serde_json::from_str(&canonical).unwrap();
558        value["manifest_version"] = Value::from(99);
559        assert!(matches!(
560            EmbeddingGenerationManifest::from_json(
561                Path::new("manifest.json"),
562                &serde_json::to_vec(&value).unwrap()
563            ),
564            Err(SearchArtifactError::IncompatibleManifest {
565                found: 99,
566                supported: EMBEDDING_GENERATION_MANIFEST_VERSION,
567                ..
568            })
569        ));
570
571        let oversized = vec![b' '; MAX_EMBEDDING_GENERATION_MANIFEST_BYTES + 1];
572        assert!(matches!(
573            EmbeddingGenerationManifest::from_json(Path::new("manifest.json"), &oversized),
574            Err(SearchArtifactError::ResourceExhausted {
575                resource: "embedding_generation_manifest_bytes",
576                ..
577            })
578        ));
579    }
580}
graphforge_storage/embedding_manifest.rs

graphforge_storage/
embedding_manifest.rs
