zeppelin_embed/manifest/
mod.rs

1//! Atomic manifest commit point for every visible persisted artifact.
2
3pub mod io;
4
5use std::collections::BTreeSet;
6use std::path::PathBuf;
7
8use crate::epoch::{
9    ComputeUnits, EmbeddingEpoch, EmbeddingRuntime, EmbeddingTower, EpochId, EpochIdentity,
10    Normalization,
11};
12use crate::format::FormatFamily;
13use crate::format::frame::{FormatError, decode_artifact, encode_artifact};
14use crate::fts::tokenizer::TokenizerEpoch;
15use crate::meta::{ColumnDefinition, ColumnId, ColumnType, Schema};
16use crate::segment::{ClusteringKeyRange, SegmentId, SegmentMeta};
17
18const CLUSTERING_RANGE_EXTENSION_MAGIC: [u8; 4] = *b"TSR1";
19const CLUSTERING_RANGE_RECORD_LEN: usize = 24;
20
21/// Interpretation-critical model/tokenizer identity for one coexistence epoch.
22#[derive(Clone, Debug, Eq, PartialEq)]
23pub struct EpochMeta {
24    /// Stable epoch identifier.
25    pub id: EpochId,
26    /// Complete authoritative embedding-model identity.
27    pub embedding: EmbeddingEpoch,
28    /// Authoritative tokenizer identity.
29    pub tokenizer: TokenizerEpoch,
30}
31
32/// The single atomic snapshot commit point.
33#[derive(Clone, Debug, Eq, PartialEq)]
34pub struct Manifest {
35    /// Monotonic committed generation.
36    pub generation: u64,
37    /// Durable WAL sequence covered by this snapshot.
38    pub log_seq: u64,
39    /// Complete reachable immutable segment set.
40    pub segments: Vec<SegmentMeta>,
41    /// Complete interpretation epoch set.
42    pub epochs: Vec<EpochMeta>,
43    /// Atomically published embedding/tokenizer identity used by queries and writes.
44    pub epoch_alias: Option<EpochIdentity>,
45    /// Typed metadata schema interpreted by every listed segment.
46    pub schema: Schema,
47}
48
49/// Typed manifest encode/load/open failure.
50#[derive(Debug)]
51pub enum ManifestError {
52    /// Filesystem operation failed for a named path.
53    Io {
54        /// Path involved in the operation.
55        path: PathBuf,
56        /// Underlying platform error.
57        source: std::io::Error,
58    },
59    /// Framing, version, length, or checksum validation failed.
60    Format(FormatError),
61    /// Hand-written payload decoding rejected malformed bytes.
62    Decode(String),
63    /// Snapshot log sequence exceeds the durable WAL end.
64    AheadOfLog {
65        /// Sequence covered by the snapshot.
66        snapshot: u64,
67        /// Sequence known durable in the WAL.
68        durable: u64,
69    },
70    /// A referenced segment header failed validation.
71    Segment(crate::segment::SegmentError),
72    /// The published alias did not name a registry entry.
73    UnknownEpochAlias {
74        /// Rejected alias.
75        alias: EpochIdentity,
76    },
77    /// A segment epoch tag did not name any registry entry.
78    UnknownSegmentEpoch {
79        /// Segment carrying the bad tag.
80        segment: SegmentId,
81        /// Rejected embedding epoch id.
82        epoch: EpochId,
83    },
84}
85
86impl ManifestError {
87    pub(crate) fn io(path: impl Into<PathBuf>, source: std::io::Error) -> Self {
88        Self::Io {
89            path: path.into(),
90            source,
91        }
92    }
93}
94
95impl std::fmt::Display for ManifestError {
96    fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
97        match self {
98            Self::Io { path, source } => {
99                write!(formatter, "manifest I/O {}: {source}", path.display())
100            }
101            Self::Format(error) => error.fmt(formatter),
102            Self::Decode(detail) => write!(formatter, "manifest decode failed: {detail}"),
103            Self::AheadOfLog { snapshot, durable } => write!(
104                formatter,
105                "manifest log sequence {snapshot} is ahead of durable WAL end {durable}"
106            ),
107            Self::Segment(error) => {
108                write!(formatter, "manifest segment validation failed: {error}")
109            }
110            Self::UnknownEpochAlias { alias } => write!(
111                formatter,
112                "manifest epoch alias ({}, {}) does not name a registry entry",
113                alias.embedding, alias.tokenizer
114            ),
115            Self::UnknownSegmentEpoch { segment, epoch } => write!(
116                formatter,
117                "manifest segment {segment} names unknown embedding epoch {epoch}"
118            ),
119        }
120    }
121}
122
123impl std::error::Error for ManifestError {
124    fn source(&self) -> Option<&(dyn std::error::Error + 'static)> {
125        match self {
126            Self::Io { source, .. } => Some(source),
127            Self::Format(error) => Some(error),
128            Self::Segment(error) => Some(error),
129            Self::Decode(_)
130            | Self::AheadOfLog { .. }
131            | Self::UnknownEpochAlias { .. }
132            | Self::UnknownSegmentEpoch { .. } => None,
133        }
134    }
135}
136
137impl From<FormatError> for ManifestError {
138    fn from(error: FormatError) -> Self {
139        Self::Format(error)
140    }
141}
142
143impl From<crate::segment::SegmentError> for ManifestError {
144    fn from(error: crate::segment::SegmentError) -> Self {
145        Self::Segment(error)
146    }
147}
148
149/// Encodes a complete manifest through a single checksummed frame.
150pub fn encode_manifest(manifest: &Manifest) -> Result<Vec<u8>, ManifestError> {
151    validate_epoch_references(manifest)?;
152    let mut payload = Vec::new();
153    payload.extend_from_slice(&manifest.generation.to_le_bytes());
154    payload.extend_from_slice(&manifest.log_seq.to_le_bytes());
155    append_u32_len(manifest.segments.len(), "segments", &mut payload)?;
156    append_u32_len(manifest.epochs.len(), "epochs", &mut payload)?;
157    append_u32_len(
158        manifest.schema.user_column_count(),
159        "schema columns",
160        &mut payload,
161    )?;
162    payload.extend_from_slice(&0_u32.to_le_bytes());
163    append_optional_identity(manifest.epoch_alias, &mut payload);
164    for segment in &manifest.segments {
165        payload.extend_from_slice(segment.id.as_bytes());
166        payload.extend_from_slice(&segment.row_count.to_le_bytes());
167        payload.extend_from_slice(&segment.scheme.to_le_bytes());
168        payload.extend_from_slice(&0_u16.to_le_bytes());
169        payload.extend_from_slice(&segment.dims.to_le_bytes());
170        payload.extend_from_slice(&segment.file_size.to_le_bytes());
171        append_optional_epoch_id(segment.epoch_id, &mut payload);
172    }
173    for epoch in &manifest.epochs {
174        payload.extend_from_slice(&epoch.id.value().to_le_bytes());
175        payload.extend_from_slice(&epoch.tokenizer.value().to_le_bytes());
176        append_embedding_epoch(&epoch.embedding, &mut payload)?;
177    }
178    for definition in manifest.schema.columns().iter().skip(1) {
179        payload.extend_from_slice(&definition.id().get().to_le_bytes());
180        payload.extend_from_slice(&column_type_id(definition.column_type()).to_le_bytes());
181        payload.extend_from_slice(&u16::from(definition.is_nullable()).to_le_bytes());
182        append_string(definition.name(), &mut payload)?;
183    }
184    append_clustering_ranges(&manifest.segments, &mut payload)?;
185    Ok(encode_artifact(FormatFamily::Manifest, 0, &payload))
186}
187
188fn append_clustering_ranges(
189    segments: &[SegmentMeta],
190    output: &mut Vec<u8>,
191) -> Result<(), ManifestError> {
192    if segments
193        .iter()
194        .all(|segment| segment.clustering_key_range == ClusteringKeyRange::Unstamped)
195    {
196        return Ok(());
197    }
198    output.extend_from_slice(&CLUSTERING_RANGE_EXTENSION_MAGIC);
199    append_u32_len(segments.len(), "clustering ranges", output)?;
200    for segment in segments {
201        let (tag, min_ts, max_ts) = match segment.clustering_key_range {
202            ClusteringKeyRange::Unstamped => (0_u8, 0_i64, 0_i64),
203            ClusteringKeyRange::Empty => (1_u8, 0_i64, 0_i64),
204            ClusteringKeyRange::Bounded { min_ts, max_ts } => {
205                if min_ts > max_ts {
206                    return Err(ManifestError::Decode(format!(
207                        "segment {} clustering range {min_ts}..={max_ts} is inverted",
208                        segment.id
209                    )));
210                }
211                (2_u8, min_ts, max_ts)
212            }
213        };
214        output.push(tag);
215        output.extend_from_slice(&[0_u8; 7]);
216        output.extend_from_slice(&min_ts.to_le_bytes());
217        output.extend_from_slice(&max_ts.to_le_bytes());
218    }
219    Ok(())
220}
221
222/// Decodes a complete manifest only after both xxh3-64 checksums validate.
223pub fn decode_manifest(artifact: &str, bytes: &[u8]) -> Result<Manifest, ManifestError> {
224    #[cfg(any(test, feature = "test-support"))]
225    let actual_family = bytes
226        .get(8..10)
227        .and_then(|family| <[u8; 2]>::try_from(family).ok())
228        .map(u16::from_le_bytes);
229    let framed = decode_artifact(artifact, FormatFamily::Manifest, bytes).map_err(|error| {
230        #[cfg(any(test, feature = "test-support"))]
231        crate::lifecycle::record_storage_manifest_format_fault(&error, actual_family);
232        ManifestError::Format(error)
233    })?;
234    let mut cursor = ManifestCursor::new(framed.payload);
235    let generation = cursor.u64()?;
236    let log_seq = cursor.u64()?;
237    let segment_count = cursor.usize_from_u32()?;
238    let epoch_count = cursor.usize_from_u32()?;
239    let schema_count = cursor.usize_from_u32()?;
240    if cursor.u32()? != 0 {
241        return Err(ManifestError::Decode(
242            "manifest reserved field is non-zero".to_owned(),
243        ));
244    }
245    let epoch_alias = cursor.optional_identity()?;
246    let mut segments = Vec::with_capacity(segment_count);
247    for _ in 0..segment_count {
248        let id_bytes: [u8; 16] = cursor
249            .take(16)?
250            .try_into()
251            .map_err(|_| ManifestError::Decode("invalid segment id".to_owned()))?;
252        let row_count = cursor.u32()?;
253        let scheme = cursor.u16()?;
254        crate::format::FormatRegistry::require_scheme(scheme)
255            .map_err(|error| ManifestError::Decode(error.to_string()))?;
256        if cursor.u16()? != 0 {
257            return Err(ManifestError::Decode(
258                "segment reserved field is non-zero".to_owned(),
259            ));
260        }
261        segments.push(SegmentMeta {
262            id: SegmentId::from_bytes(id_bytes),
263            row_count,
264            scheme,
265            dims: cursor.u32()?,
266            file_size: cursor.u64()?,
267            epoch_id: cursor.optional_epoch_id()?,
268            clustering_key_range: crate::segment::ClusteringKeyRange::Unstamped,
269        });
270    }
271    let mut epochs = Vec::with_capacity(epoch_count);
272    for _ in 0..epoch_count {
273        epochs.push(EpochMeta {
274            id: EpochId::from_value(cursor.u64()?),
275            tokenizer: TokenizerEpoch::from_value(cursor.u64()?),
276            embedding: cursor.embedding_epoch()?,
277        });
278    }
279    let mut definitions = Vec::with_capacity(schema_count);
280    for _ in 0..schema_count {
281        let id = ColumnId::new(cursor.u32()?);
282        let column_type = column_type_from_id(cursor.u16()?)?;
283        let nullable = match cursor.u16()? {
284            0 => false,
285            1 => true,
286            value => {
287                return Err(ManifestError::Decode(format!(
288                    "invalid nullable flag {value}"
289                )));
290            }
291        };
292        definitions.push(ColumnDefinition::new(
293            id,
294            cursor.string()?,
295            column_type,
296            nullable,
297        ));
298    }
299    if cursor.remaining() != 0 {
300        decode_clustering_ranges(&mut cursor, &mut segments)?;
301    }
302    cursor.finish()?;
303    let schema =
304        Schema::new(definitions).map_err(|error| ManifestError::Decode(error.to_string()))?;
305    let manifest = Manifest {
306        generation,
307        log_seq,
308        segments,
309        epochs,
310        epoch_alias,
311        schema,
312    };
313    validate_epoch_references(&manifest)?;
314    Ok(manifest)
315}
316
317fn validate_epoch_references(manifest: &Manifest) -> Result<(), ManifestError> {
318    let mut identities = BTreeSet::new();
319    for epoch in &manifest.epochs {
320        let identity = EpochIdentity::from_meta(epoch)
321            .map_err(|error| ManifestError::Decode(error.to_string()))?;
322        if !identities.insert(identity) {
323            return Err(ManifestError::Decode(format!(
324                "duplicate epoch registry identity ({}, {})",
325                identity.embedding, identity.tokenizer
326            )));
327        }
328    }
329
330    match manifest.epoch_alias {
331        Some(alias) if !identities.contains(&alias) => {
332            return Err(ManifestError::UnknownEpochAlias { alias });
333        }
334        None if !identities.is_empty() => {
335            return Err(ManifestError::Decode(
336                "non-empty epoch registry has no published alias".to_owned(),
337            ));
338        }
339        Some(_) | None => {}
340    }
341
342    for segment in &manifest.segments {
343        match segment.epoch_id {
344            Some(epoch)
345                if !identities
346                    .iter()
347                    .any(|identity| identity.embedding == epoch) =>
348            {
349                return Err(ManifestError::UnknownSegmentEpoch {
350                    segment: segment.id,
351                    epoch,
352                });
353            }
354            None if !identities.is_empty() => {
355                return Err(ManifestError::Decode(format!(
356                    "segment {} has no embedding epoch tag",
357                    segment.id
358                )));
359            }
360            Some(_) | None => {}
361        }
362    }
363    Ok(())
364}
365
366fn append_optional_identity(identity: Option<EpochIdentity>, output: &mut Vec<u8>) {
367    let (tag, embedding, tokenizer) = match identity {
368        Some(identity) => (1_u8, identity.embedding.value(), identity.tokenizer.value()),
369        None => (0, 0, 0),
370    };
371    output.push(tag);
372    output.extend_from_slice(&[0_u8; 7]);
373    output.extend_from_slice(&embedding.to_le_bytes());
374    output.extend_from_slice(&tokenizer.to_le_bytes());
375}
376
377fn append_optional_epoch_id(epoch: Option<EpochId>, output: &mut Vec<u8>) {
378    let (tag, value) = epoch.map_or((0_u8, 0_u64), |epoch| (1, epoch.value()));
379    output.push(tag);
380    output.extend_from_slice(&[0_u8; 7]);
381    output.extend_from_slice(&value.to_le_bytes());
382}
383
384fn append_embedding_epoch(
385    epoch: &EmbeddingEpoch,
386    output: &mut Vec<u8>,
387) -> Result<(), ManifestError> {
388    append_embedding_tower(&epoch.document, output)?;
389    append_embedding_tower(&epoch.query, output)?;
390    append_bytes(&epoch.alignment_digest, output)
391}
392
393fn append_embedding_tower(
394    tower: &EmbeddingTower,
395    output: &mut Vec<u8>,
396) -> Result<(), ManifestError> {
397    append_string(&tower.model_id, output)?;
398    append_string(&tower.model_version, output)?;
399    append_bytes(&tower.weights_digest, output)?;
400    output.extend_from_slice(&tower.dims.to_le_bytes());
401    output.extend_from_slice(&(tower.normalization as u16).to_le_bytes());
402    append_string(&tower.prompt_prefix, output)?;
403    output.extend_from_slice(&tower.max_tokens.to_le_bytes());
404    output.extend_from_slice(&(tower.runtime as u16).to_le_bytes());
405    output.extend_from_slice(&(tower.compute_units as u16).to_le_bytes());
406    match &tower.os_build {
407        Some(build) => {
408            output.push(1);
409            output.extend_from_slice(&[0_u8; 3]);
410            append_string(build, output)?;
411        }
412        None => {
413            output.push(0);
414            output.extend_from_slice(&[0_u8; 3]);
415        }
416    }
417    Ok(())
418}
419
420fn decode_clustering_ranges(
421    cursor: &mut ManifestCursor<'_>,
422    segments: &mut [SegmentMeta],
423) -> Result<(), ManifestError> {
424    if cursor.take(CLUSTERING_RANGE_EXTENSION_MAGIC.len())? != CLUSTERING_RANGE_EXTENSION_MAGIC {
425        return Err(ManifestError::Decode(
426            "unknown manifest extension after schema".to_owned(),
427        ));
428    }
429    let count = cursor.usize_from_u32()?;
430    if count != segments.len() {
431        return Err(ManifestError::Decode(format!(
432            "clustering range count {count} does not match segment count {}",
433            segments.len()
434        )));
435    }
436    let expected_bytes = count
437        .checked_mul(CLUSTERING_RANGE_RECORD_LEN)
438        .ok_or_else(|| ManifestError::Decode("clustering range bytes overflow".to_owned()))?;
439    if cursor.remaining() != expected_bytes {
440        return Err(ManifestError::Decode(format!(
441            "clustering range extension has {} bytes, expected {expected_bytes}",
442            cursor.remaining()
443        )));
444    }
445    for segment in segments {
446        let tag = cursor.u8()?;
447        if cursor.take(7)?.iter().any(|byte| *byte != 0) {
448            return Err(ManifestError::Decode(
449                "clustering range reserved bytes are non-zero".to_owned(),
450            ));
451        }
452        let min_ts = cursor.i64()?;
453        let max_ts = cursor.i64()?;
454        segment.clustering_key_range = match tag {
455            0 if min_ts == 0 && max_ts == 0 => ClusteringKeyRange::Unstamped,
456            1 if min_ts == 0 && max_ts == 0 => ClusteringKeyRange::Empty,
457            2 if min_ts <= max_ts => ClusteringKeyRange::Bounded { min_ts, max_ts },
458            0 | 1 => {
459                return Err(ManifestError::Decode(format!(
460                    "clustering range tag {tag} requires zero bounds"
461                )));
462            }
463            2 => {
464                return Err(ManifestError::Decode(format!(
465                    "clustering range {min_ts}..={max_ts} is inverted"
466                )));
467            }
468            unknown => {
469                return Err(ManifestError::Decode(format!(
470                    "unknown clustering range tag {unknown}"
471                )));
472            }
473        };
474    }
475    Ok(())
476}
477
478fn append_u32_len(length: usize, name: &str, output: &mut Vec<u8>) -> Result<(), ManifestError> {
479    let value = u32::try_from(length)
480        .map_err(|_| ManifestError::Decode(format!("{name} length exceeds u32")))?;
481    output.extend_from_slice(&value.to_le_bytes());
482    Ok(())
483}
484
485fn append_string(value: &str, output: &mut Vec<u8>) -> Result<(), ManifestError> {
486    append_bytes(value.as_bytes(), output)
487}
488
489fn append_bytes(value: &[u8], output: &mut Vec<u8>) -> Result<(), ManifestError> {
490    append_u32_len(value.len(), "byte field", output)?;
491    output.extend_from_slice(value);
492    Ok(())
493}
494
495fn column_type_id(column_type: ColumnType) -> u16 {
496    match column_type {
497        ColumnType::U64 => 1,
498        ColumnType::I64 => 2,
499        ColumnType::F64 => 3,
500        ColumnType::Bool => 4,
501        ColumnType::DictionaryString => 5,
502        ColumnType::RawString => 6,
503    }
504}
505
506fn column_type_from_id(id: u16) -> Result<ColumnType, ManifestError> {
507    match id {
508        1 => Ok(ColumnType::U64),
509        2 => Ok(ColumnType::I64),
510        3 => Ok(ColumnType::F64),
511        4 => Ok(ColumnType::Bool),
512        5 => Ok(ColumnType::DictionaryString),
513        6 => Ok(ColumnType::RawString),
514        _ => Err(ManifestError::Decode(format!("unknown column type {id}"))),
515    }
516}
517
518struct ManifestCursor<'a> {
519    bytes: &'a [u8],
520    position: usize,
521}
522
523impl<'a> ManifestCursor<'a> {
524    const fn new(bytes: &'a [u8]) -> Self {
525        Self { bytes, position: 0 }
526    }
527
528    fn take(&mut self, length: usize) -> Result<&'a [u8], ManifestError> {
529        let end = self
530            .position
531            .checked_add(length)
532            .ok_or_else(|| ManifestError::Decode("payload offset overflow".to_owned()))?;
533        let value = self.bytes.get(self.position..end).ok_or_else(|| {
534            ManifestError::Decode(format!(
535                "payload truncated at {}, need {length}, total {}",
536                self.position,
537                self.bytes.len()
538            ))
539        })?;
540        self.position = end;
541        Ok(value)
542    }
543
544    fn u16(&mut self) -> Result<u16, ManifestError> {
545        let raw: [u8; 2] = self
546            .take(2)?
547            .try_into()
548            .map_err(|_| ManifestError::Decode("invalid u16".to_owned()))?;
549        Ok(u16::from_le_bytes(raw))
550    }
551
552    fn u8(&mut self) -> Result<u8, ManifestError> {
553        self.take(1)?
554            .first()
555            .copied()
556            .ok_or_else(|| ManifestError::Decode("invalid u8".to_owned()))
557    }
558
559    fn u32(&mut self) -> Result<u32, ManifestError> {
560        let raw: [u8; 4] = self
561            .take(4)?
562            .try_into()
563            .map_err(|_| ManifestError::Decode("invalid u32".to_owned()))?;
564        Ok(u32::from_le_bytes(raw))
565    }
566
567    fn u64(&mut self) -> Result<u64, ManifestError> {
568        let raw: [u8; 8] = self
569            .take(8)?
570            .try_into()
571            .map_err(|_| ManifestError::Decode("invalid u64".to_owned()))?;
572        Ok(u64::from_le_bytes(raw))
573    }
574
575    fn optional_identity(&mut self) -> Result<Option<EpochIdentity>, ManifestError> {
576        let tag = self.u8()?;
577        if self.take(7)?.iter().any(|byte| *byte != 0) {
578            return Err(ManifestError::Decode(
579                "epoch alias reserved bytes are non-zero".to_owned(),
580            ));
581        }
582        let embedding = self.u64()?;
583        let tokenizer = self.u64()?;
584        match (tag, embedding, tokenizer) {
585            (0, 0, 0) => Ok(None),
586            (1, embedding, tokenizer) => Ok(Some(EpochIdentity {
587                embedding: EpochId::from_value(embedding),
588                tokenizer: TokenizerEpoch::from_value(tokenizer),
589            })),
590            (0, _, _) => Err(ManifestError::Decode(
591                "absent epoch alias carries non-zero identity bytes".to_owned(),
592            )),
593            (unknown, _, _) => Err(ManifestError::Decode(format!(
594                "unknown epoch alias presence tag {unknown}"
595            ))),
596        }
597    }
598
599    fn optional_epoch_id(&mut self) -> Result<Option<EpochId>, ManifestError> {
600        let tag = self.u8()?;
601        if self.take(7)?.iter().any(|byte| *byte != 0) {
602            return Err(ManifestError::Decode(
603                "segment epoch reserved bytes are non-zero".to_owned(),
604            ));
605        }
606        let epoch = self.u64()?;
607        match (tag, epoch) {
608            (0, 0) => Ok(None),
609            (1, epoch) => Ok(Some(EpochId::from_value(epoch))),
610            (0, _) => Err(ManifestError::Decode(
611                "absent segment epoch tag carries a non-zero id".to_owned(),
612            )),
613            (unknown, _) => Err(ManifestError::Decode(format!(
614                "unknown segment epoch presence tag {unknown}"
615            ))),
616        }
617    }
618
619    fn embedding_epoch(&mut self) -> Result<EmbeddingEpoch, ManifestError> {
620        Ok(EmbeddingEpoch {
621            document: self.embedding_tower()?,
622            query: self.embedding_tower()?,
623            alignment_digest: self.bytes()?,
624        })
625    }
626
627    fn embedding_tower(&mut self) -> Result<EmbeddingTower, ManifestError> {
628        let model_id = self.string()?;
629        let model_version = self.string()?;
630        let weights_digest = self.bytes()?;
631        let dims = self.u32()?;
632        let normalization = match self.u16()? {
633            0 => Normalization::None,
634            1 => Normalization::L2,
635            unknown => {
636                return Err(ManifestError::Decode(format!(
637                    "unknown embedding normalization {unknown}"
638                )));
639            }
640        };
641        let prompt_prefix = self.string()?;
642        let max_tokens = self.u32()?;
643        let runtime = match self.u16()? {
644            1 => EmbeddingRuntime::CoreMl,
645            2 => EmbeddingRuntime::Mlx,
646            3 => EmbeddingRuntime::CpuReference,
647            unknown => {
648                return Err(ManifestError::Decode(format!(
649                    "unknown embedding runtime {unknown}"
650                )));
651            }
652        };
653        let compute_units = match self.u16()? {
654            1 => ComputeUnits::Cpu,
655            2 => ComputeUnits::CpuAndGpu,
656            3 => ComputeUnits::CpuAndNeuralEngine,
657            4 => ComputeUnits::All,
658            unknown => {
659                return Err(ManifestError::Decode(format!(
660                    "unknown embedding compute units {unknown}"
661                )));
662            }
663        };
664        let os_tag = self.u8()?;
665        if self.take(3)?.iter().any(|byte| *byte != 0) {
666            return Err(ManifestError::Decode(
667                "embedding OS-build reserved bytes are non-zero".to_owned(),
668            ));
669        }
670        let os_build = match os_tag {
671            0 => None,
672            1 => Some(self.string()?),
673            unknown => {
674                return Err(ManifestError::Decode(format!(
675                    "unknown embedding OS-build presence tag {unknown}"
676                )));
677            }
678        };
679        Ok(EmbeddingTower {
680            model_id,
681            model_version,
682            weights_digest,
683            dims,
684            normalization,
685            prompt_prefix,
686            max_tokens,
687            runtime,
688            compute_units,
689            os_build,
690        })
691    }
692
693    fn i64(&mut self) -> Result<i64, ManifestError> {
694        let raw: [u8; 8] = self
695            .take(8)?
696            .try_into()
697            .map_err(|_| ManifestError::Decode("invalid i64".to_owned()))?;
698        Ok(i64::from_le_bytes(raw))
699    }
700
701    const fn remaining(&self) -> usize {
702        self.bytes.len().saturating_sub(self.position)
703    }
704
705    fn usize_from_u32(&mut self) -> Result<usize, ManifestError> {
706        usize::try_from(self.u32()?)
707            .map_err(|_| ManifestError::Decode("u32 length exceeds usize".to_owned()))
708    }
709
710    fn string(&mut self) -> Result<String, ManifestError> {
711        let length = self.usize_from_u32()?;
712        std::str::from_utf8(self.take(length)?)
713            .map(str::to_owned)
714            .map_err(|error| ManifestError::Decode(format!("invalid UTF-8: {error}")))
715    }
716
717    fn bytes(&mut self) -> Result<Vec<u8>, ManifestError> {
718        let length = self.usize_from_u32()?;
719        Ok(self.take(length)?.to_vec())
720    }
721
722    fn finish(&self) -> Result<(), ManifestError> {
723        if self.position == self.bytes.len() {
724            Ok(())
725        } else {
726            Err(ManifestError::Decode(format!(
727                "{} trailing payload bytes",
728                self.bytes.len().saturating_sub(self.position)
729            )))
730        }
731    }
732}
zeppelin_embed/manifest/mod.rs

zeppelin_embed/manifest/
mod.rs
