zeppelin_embed/manifest/
mod.rs

1//! Atomic manifest commit point for every visible persisted artifact.
2
3pub mod io;
4
5use std::collections::BTreeSet;
6use std::path::PathBuf;
7
8use crate::epoch::{
9    ComputeUnits, EmbeddingEpoch, EmbeddingRuntime, EmbeddingTower, EpochId, EpochIdentity,
10    Normalization,
11};
12use crate::format::FormatFamily;
13use crate::format::frame::{FormatError, decode_artifact, encode_artifact};
14use crate::fts::tokenizer::TokenizerEpoch;
15use crate::meta::{ColumnDefinition, ColumnId, ColumnType, Schema};
16use crate::segment::{ClusteringKeyRange, SegmentId, SegmentMeta};
17
18const CLUSTERING_RANGE_EXTENSION_MAGIC: [u8; 4] = *b"TSR1";
19const CLUSTERING_RANGE_RECORD_LEN: usize = 24;
20
21/// Interpretation-critical model/tokenizer identity for one coexistence epoch.
22#[derive(Clone, Debug, Eq, PartialEq)]
23pub struct EpochMeta {
24    /// Stable epoch identifier.
25    pub id: EpochId,
26    /// Complete authoritative embedding-model identity.
27    pub embedding: EmbeddingEpoch,
28    /// Authoritative tokenizer identity.
29    pub tokenizer: TokenizerEpoch,
30}
31
32/// The single atomic snapshot commit point.
33#[derive(Clone, Debug, Eq, PartialEq)]
34pub struct Manifest {
35    /// Monotonic committed generation.
36    pub generation: u64,
37    /// Durable WAL sequence covered by this snapshot.
38    pub log_seq: u64,
39    /// Complete reachable immutable segment set.
40    pub segments: Vec<SegmentMeta>,
41    /// Complete interpretation epoch set.
42    pub epochs: Vec<EpochMeta>,
43    /// Atomically published embedding/tokenizer identity used by queries and writes.
44    pub epoch_alias: Option<EpochIdentity>,
45    /// Typed metadata schema interpreted by every listed segment.
46    pub schema: Schema,
47}
48
49/// Typed manifest encode/load/open failure.
50#[derive(Debug)]
51pub enum ManifestError {
52    /// Filesystem operation failed for a named path.
53    Io {
54        /// Path involved in the operation.
55        path: PathBuf,
56        /// Underlying platform error.
57        source: std::io::Error,
58    },
59    /// Framing, version, length, or checksum validation failed.
60    Format(FormatError),
61    /// Hand-written payload decoding rejected malformed bytes.
62    Decode(String),
63    /// Snapshot log sequence exceeds the durable WAL end.
64    AheadOfLog {
65        /// Sequence covered by the snapshot.
66        snapshot: u64,
67        /// Sequence known durable in the WAL.
68        durable: u64,
69    },
70    /// A referenced segment header failed validation.
71    Segment(crate::segment::SegmentError),
72    /// The published alias did not name a registry entry.
73    UnknownEpochAlias {
74        /// Rejected alias.
75        alias: EpochIdentity,
76    },
77    /// A segment epoch tag did not name any registry entry.
78    UnknownSegmentEpoch {
79        /// Segment carrying the bad tag.
80        segment: SegmentId,
81        /// Rejected embedding epoch id.
82        epoch: EpochId,
83    },
84}
85
86impl ManifestError {
87    pub(crate) fn io(path: impl Into<PathBuf>, source: std::io::Error) -> Self {
88        Self::Io {
89            path: path.into(),
90            source,
91        }
92    }
93}
94
95impl std::fmt::Display for ManifestError {
96    fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
97        match self {
98            Self::Io { path, source } => {
99                write!(formatter, "manifest I/O {}: {source}", path.display())
100            }
101            Self::Format(error) => error.fmt(formatter),
102            Self::Decode(detail) => write!(formatter, "manifest decode failed: {detail}"),
103            Self::AheadOfLog { snapshot, durable } => write!(
104                formatter,
105                "manifest log sequence {snapshot} is ahead of durable WAL end {durable}"
106            ),
107            Self::Segment(error) => {
108                write!(formatter, "manifest segment validation failed: {error}")
109            }
110            Self::UnknownEpochAlias { alias } => write!(
111                formatter,
112                "manifest epoch alias ({}, {}) does not name a registry entry",
113                alias.embedding, alias.tokenizer
114            ),
115            Self::UnknownSegmentEpoch { segment, epoch } => write!(
116                formatter,
117                "manifest segment {segment} names unknown embedding epoch {epoch}"
118            ),
119        }
120    }
121}
122
123impl std::error::Error for ManifestError {
124    fn source(&self) -> Option<&(dyn std::error::Error + 'static)> {
125        match self {
126            Self::Io { source, .. } => Some(source),
127            Self::Format(error) => Some(error),
128            Self::Segment(error) => Some(error),
129            Self::Decode(_)
130            | Self::AheadOfLog { .. }
131            | Self::UnknownEpochAlias { .. }
132            | Self::UnknownSegmentEpoch { .. } => None,
133        }
134    }
135}
136
137impl From<FormatError> for ManifestError {
138    fn from(error: FormatError) -> Self {
139        Self::Format(error)
140    }
141}
142
143impl From<crate::segment::SegmentError> for ManifestError {
144    fn from(error: crate::segment::SegmentError) -> Self {
145        Self::Segment(error)
146    }
147}
148
149/// Encodes a complete manifest through a single checksummed frame.
150pub fn encode_manifest(manifest: &Manifest) -> Result<Vec<u8>, ManifestError> {
151    validate_epoch_references(manifest)?;
152    let mut payload = Vec::new();
153    payload.extend_from_slice(&manifest.generation.to_le_bytes());
154    payload.extend_from_slice(&manifest.log_seq.to_le_bytes());
155    append_u32_len(manifest.segments.len(), "segments", &mut payload)?;
156    append_u32_len(manifest.epochs.len(), "epochs", &mut payload)?;
157    append_u32_len(
158        manifest.schema.user_column_count(),
159        "schema columns",
160        &mut payload,
161    )?;
162    payload.extend_from_slice(&0_u32.to_le_bytes());
163    append_optional_identity(manifest.epoch_alias, &mut payload);
164    for segment in &manifest.segments {
165        payload.extend_from_slice(segment.id.as_bytes());
166        payload.extend_from_slice(&segment.row_count.to_le_bytes());
167        payload.extend_from_slice(&segment.scheme.to_le_bytes());
168        payload.extend_from_slice(&0_u16.to_le_bytes());
169        payload.extend_from_slice(&segment.dims.to_le_bytes());
170        payload.extend_from_slice(&segment.file_size.to_le_bytes());
171        append_optional_epoch_id(segment.epoch_id, &mut payload);
172    }
173    for epoch in &manifest.epochs {
174        payload.extend_from_slice(&epoch.id.value().to_le_bytes());
175        payload.extend_from_slice(&epoch.tokenizer.value().to_le_bytes());
176        append_embedding_epoch(&epoch.embedding, &mut payload)?;
177    }
178    for definition in manifest.schema.columns().iter().skip(1) {
179        payload.extend_from_slice(&definition.id().get().to_le_bytes());
180        payload.extend_from_slice(&column_type_id(definition.column_type()).to_le_bytes());
181        payload.extend_from_slice(&u16::from(definition.is_nullable()).to_le_bytes());
182        append_string(definition.name(), &mut payload)?;
183    }
184    append_clustering_ranges(&manifest.segments, &mut payload)?;
185    Ok(encode_artifact(FormatFamily::Manifest, 0, &payload))
186}
187
188fn append_clustering_ranges(
189    segments: &[SegmentMeta],
190    output: &mut Vec<u8>,
191) -> Result<(), ManifestError> {
192    if segments
193        .iter()
194        .all(|segment| segment.clustering_key_range == ClusteringKeyRange::Unstamped)
195    {
196        return Ok(());
197    }
198    output.extend_from_slice(&CLUSTERING_RANGE_EXTENSION_MAGIC);
199    append_u32_len(segments.len(), "clustering ranges", output)?;
200    for segment in segments {
201        let (tag, min_ts, max_ts) = match segment.clustering_key_range {
202            ClusteringKeyRange::Unstamped => (0_u8, 0_i64, 0_i64),
203            ClusteringKeyRange::Empty => (1_u8, 0_i64, 0_i64),
204            ClusteringKeyRange::Bounded { min_ts, max_ts } => {
205                if min_ts > max_ts {
206                    return Err(ManifestError::Decode(format!(
207                        "segment {} clustering range {min_ts}..={max_ts} is inverted",
208                        segment.id
209                    )));
210                }
211                (2_u8, min_ts, max_ts)
212            }
213        };
214        output.push(tag);
215        output.extend_from_slice(&[0_u8; 7]);
216        output.extend_from_slice(&min_ts.to_le_bytes());
217        output.extend_from_slice(&max_ts.to_le_bytes());
218    }
219    Ok(())
220}
221
222/// Decodes a complete manifest only after both xxh3-64 checksums validate.
223pub fn decode_manifest(artifact: &str, bytes: &[u8]) -> Result<Manifest, ManifestError> {
224    #[cfg(any(test, feature = "test-seams"))]
225    let actual_family = bytes
226        .get(8..10)
227        .and_then(|family| <[u8; 2]>::try_from(family).ok())
228        .map(u16::from_le_bytes);
229    let framed = decode_artifact(artifact, FormatFamily::Manifest, bytes).map_err(|error| {
230        #[cfg(any(test, feature = "test-seams"))]
231        crate::lifecycle::record_storage_manifest_format_fault(&error, actual_family);
232        ManifestError::Format(error)
233    })?;
234    let mut cursor = ManifestCursor::new(framed.payload);
235    let generation = cursor.u64()?;
236    let log_seq = cursor.u64()?;
237    let segment_count = cursor.usize_from_u32()?;
238    let epoch_count = cursor.usize_from_u32()?;
239    let schema_count = cursor.usize_from_u32()?;
240    if cursor.u32()? != 0 {
241        return Err(ManifestError::Decode(
242            "manifest reserved field is non-zero".to_owned(),
243        ));
244    }
245    let epoch_alias = cursor.optional_identity()?;
246    let mut segments = Vec::with_capacity(segment_count);
247    for _ in 0..segment_count {
248        let id_bytes: [u8; 16] = cursor
249            .take(16)?
250            .try_into()
251            .map_err(|_| ManifestError::Decode("invalid segment id".to_owned()))?;
252        let row_count = cursor.u32()?;
253        let scheme = cursor.u16()?;
254        crate::format::FormatRegistry::require_scheme(scheme)
255            .map_err(|error| ManifestError::Decode(error.to_string()))?;
256        if cursor.u16()? != 0 {
257            return Err(ManifestError::Decode(
258                "segment reserved field is non-zero".to_owned(),
259            ));
260        }
261        segments.push(SegmentMeta {
262            id: SegmentId::from_bytes(id_bytes),
263            row_count,
264            scheme,
265            dims: cursor.u32()?,
266            file_size: cursor.u64()?,
267            epoch_id: cursor.optional_epoch_id()?,
268            clustering_key_range: crate::segment::ClusteringKeyRange::Unstamped,
269        });
270    }
271    let mut epochs = Vec::with_capacity(epoch_count);
272    for _ in 0..epoch_count {
273        epochs.push(EpochMeta {
274            id: EpochId::from_value(cursor.u64()?),
275            tokenizer: TokenizerEpoch::from_value(cursor.u64()?),
276            embedding: cursor.embedding_epoch()?,
277        });
278    }
279    let mut definitions = Vec::with_capacity(schema_count);
280    for _ in 0..schema_count {
281        let id = ColumnId::new(cursor.u32()?);
282        let column_type = column_type_from_id(cursor.u16()?)?;
283        let nullable = match cursor.u16()? {
284            0 => false,
285            1 => true,
286            value => {
287                return Err(ManifestError::Decode(format!(
288                    "invalid nullable flag {value}"
289                )));
290            }
291        };
292        definitions.push(ColumnDefinition::new(
293            id,
294            cursor.string()?,
295            column_type,
296            nullable,
297        ));
298    }
299    if cursor.remaining() != 0 {
300        decode_clustering_ranges(&mut cursor, &mut segments)?;
301    }
302    cursor.finish()?;
303    let schema =
304        Schema::new(definitions).map_err(|error| ManifestError::Decode(error.to_string()))?;
305    let manifest = Manifest {
306        generation,
307        log_seq,
308        segments,
309        epochs,
310        epoch_alias,
311        schema,
312    };
313    validate_epoch_references(&manifest)?;
314    Ok(manifest)
315}
316
317fn validate_epoch_references(manifest: &Manifest) -> Result<(), ManifestError> {
318    let mut identities = BTreeSet::new();
319    for epoch in &manifest.epochs {
320        let identity = EpochIdentity::from_meta(epoch)
321            .map_err(|error| ManifestError::Decode(error.to_string()))?;
322        if !identities.insert(identity) {
323            return Err(ManifestError::Decode(format!(
324                "duplicate epoch registry identity ({}, {})",
325                identity.embedding, identity.tokenizer
326            )));
327        }
328    }
329
330    match manifest.epoch_alias {
331        Some(alias) if !identities.contains(&alias) => {
332            return Err(ManifestError::UnknownEpochAlias { alias });
333        }
334        None if !identities.is_empty() => {
335            return Err(ManifestError::Decode(
336                "non-empty epoch registry has no published alias".to_owned(),
337            ));
338        }
339        Some(_) | None => {}
340    }
341
342    for segment in &manifest.segments {
343        match segment.epoch_id {
344            Some(epoch)
345                if !identities
346                    .iter()
347                    .any(|identity| identity.embedding == epoch) =>
348            {
349                return Err(ManifestError::UnknownSegmentEpoch {
350                    segment: segment.id,
351                    epoch,
352                });
353            }
354            None if !identities.is_empty() => {
355                return Err(ManifestError::Decode(format!(
356                    "segment {} has no embedding epoch tag",
357                    segment.id
358                )));
359            }
360            Some(_) | None => {}
361        }
362    }
363    Ok(())
364}
365
366fn append_optional_identity(identity: Option<EpochIdentity>, output: &mut Vec<u8>) {
367    let (tag, embedding, tokenizer) = match identity {
368        Some(identity) => (1_u8, identity.embedding.value(), identity.tokenizer.value()),
369        None => (0, 0, 0),
370    };
371    output.push(tag);
372    output.extend_from_slice(&[0_u8; 7]);
373    output.extend_from_slice(&embedding.to_le_bytes());
374    output.extend_from_slice(&tokenizer.to_le_bytes());
375}
376
377fn append_optional_epoch_id(epoch: Option<EpochId>, output: &mut Vec<u8>) {
378    let (tag, value) = epoch.map_or((0_u8, 0_u64), |epoch| (1, epoch.value()));
379    output.push(tag);
380    output.extend_from_slice(&[0_u8; 7]);
381    output.extend_from_slice(&value.to_le_bytes());
382}
383
384fn append_embedding_epoch(
385    epoch: &EmbeddingEpoch,
386    output: &mut Vec<u8>,
387) -> Result<(), ManifestError> {
388    append_embedding_tower(&epoch.document, output)?;
389    append_embedding_tower(&epoch.query, output)?;
390    append_bytes(&epoch.alignment_digest, output)
391}
392
393fn append_embedding_tower(
394    tower: &EmbeddingTower,
395    output: &mut Vec<u8>,
396) -> Result<(), ManifestError> {
397    append_string(&tower.model_id, output)?;
398    append_string(&tower.model_version, output)?;
399    append_bytes(&tower.weights_digest, output)?;
400    output.extend_from_slice(&tower.dims.to_le_bytes());
401    output.extend_from_slice(&(tower.normalization as u16).to_le_bytes());
402    append_string(&tower.prompt_prefix, output)?;
403    output.extend_from_slice(&tower.max_tokens.to_le_bytes());
404    output.extend_from_slice(&(tower.runtime as u16).to_le_bytes());
405    output.extend_from_slice(&(tower.compute_units as u16).to_le_bytes());
406    match &tower.os_build {
407        Some(build) => {
408            output.push(1);
409            output.extend_from_slice(&[0_u8; 3]);
410            append_string(build, output)?;
411        }
412        None => {
413            output.push(0);
414            output.extend_from_slice(&[0_u8; 3]);
415        }
416    }
417    Ok(())
418}
419
420fn decode_clustering_ranges(
421    cursor: &mut ManifestCursor<'_>,
422    segments: &mut [SegmentMeta],
423) -> Result<(), ManifestError> {
424    if cursor.take(CLUSTERING_RANGE_EXTENSION_MAGIC.len())? != CLUSTERING_RANGE_EXTENSION_MAGIC {
425        return Err(ManifestError::Decode(
426            "unknown manifest extension after schema".to_owned(),
427        ));
428    }
429    let count = cursor.usize_from_u32()?;
430    if count != segments.len() {
431        return Err(ManifestError::Decode(format!(
432            "clustering range count {count} does not match segment count {}",
433            segments.len()
434        )));
435    }
436    let expected_bytes = count
437        .checked_mul(CLUSTERING_RANGE_RECORD_LEN)
438        .ok_or_else(|| ManifestError::Decode("clustering range bytes overflow".to_owned()))?;
439    if cursor.remaining() != expected_bytes {
440        return Err(ManifestError::Decode(format!(
441            "clustering range extension has {} bytes, expected {expected_bytes}",
442            cursor.remaining()
443        )));
444    }
445    for segment in segments {
446        let tag = cursor.u8()?;
447        if cursor.take(7)?.iter().any(|byte| *byte != 0) {
448            return Err(ManifestError::Decode(
449                "clustering range reserved bytes are non-zero".to_owned(),
450            ));
451        }
452        let min_ts = cursor.i64()?;
453        let max_ts = cursor.i64()?;
454        segment.clustering_key_range = match tag {
455            0 if min_ts == 0 && max_ts == 0 => ClusteringKeyRange::Unstamped,
456            1 if min_ts == 0 && max_ts == 0 => ClusteringKeyRange::Empty,
457            2 if min_ts <= max_ts => ClusteringKeyRange::Bounded { min_ts, max_ts },
458            0 | 1 => {
459                return Err(ManifestError::Decode(format!(
460                    "clustering range tag {tag} requires zero bounds"
461                )));
462            }
463            2 => {
464                return Err(ManifestError::Decode(format!(
465                    "clustering range {min_ts}..={max_ts} is inverted"
466                )));
467            }
468            unknown => {
469                return Err(ManifestError::Decode(format!(
470                    "unknown clustering range tag {unknown}"
471                )));
472            }
473        };
474    }
475    Ok(())
476}
477
478fn append_u32_len(length: usize, name: &str, output: &mut Vec<u8>) -> Result<(), ManifestError> {
479    let value = u32::try_from(length)
480        .map_err(|_| ManifestError::Decode(format!("{name} length exceeds u32")))?;
481    output.extend_from_slice(&value.to_le_bytes());
482    Ok(())
483}
484
485fn append_string(value: &str, output: &mut Vec<u8>) -> Result<(), ManifestError> {
486    append_bytes(value.as_bytes(), output)
487}
488
489fn append_bytes(value: &[u8], output: &mut Vec<u8>) -> Result<(), ManifestError> {
490    append_u32_len(value.len(), "byte field", output)?;
491    output.extend_from_slice(value);
492    Ok(())
493}
494
495fn column_type_id(column_type: ColumnType) -> u16 {
496    match column_type {
497        ColumnType::Id128 => 7,
498        ColumnType::U64 => 1,
499        ColumnType::I64 => 2,
500        ColumnType::F64 => 3,
501        ColumnType::Bool => 4,
502        ColumnType::DictionaryString => 5,
503        ColumnType::RawString => 6,
504    }
505}
506
507fn column_type_from_id(id: u16) -> Result<ColumnType, ManifestError> {
508    match id {
509        7 => Ok(ColumnType::Id128),
510        1 => Ok(ColumnType::U64),
511        2 => Ok(ColumnType::I64),
512        3 => Ok(ColumnType::F64),
513        4 => Ok(ColumnType::Bool),
514        5 => Ok(ColumnType::DictionaryString),
515        6 => Ok(ColumnType::RawString),
516        _ => Err(ManifestError::Decode(format!("unknown column type {id}"))),
517    }
518}
519
520struct ManifestCursor<'a> {
521    bytes: &'a [u8],
522    position: usize,
523}
524
525impl<'a> ManifestCursor<'a> {
526    const fn new(bytes: &'a [u8]) -> Self {
527        Self { bytes, position: 0 }
528    }
529
530    fn take(&mut self, length: usize) -> Result<&'a [u8], ManifestError> {
531        let end = self
532            .position
533            .checked_add(length)
534            .ok_or_else(|| ManifestError::Decode("payload offset overflow".to_owned()))?;
535        let value = self.bytes.get(self.position..end).ok_or_else(|| {
536            ManifestError::Decode(format!(
537                "payload truncated at {}, need {length}, total {}",
538                self.position,
539                self.bytes.len()
540            ))
541        })?;
542        self.position = end;
543        Ok(value)
544    }
545
546    fn u16(&mut self) -> Result<u16, ManifestError> {
547        let raw: [u8; 2] = self
548            .take(2)?
549            .try_into()
550            .map_err(|_| ManifestError::Decode("invalid u16".to_owned()))?;
551        Ok(u16::from_le_bytes(raw))
552    }
553
554    fn u8(&mut self) -> Result<u8, ManifestError> {
555        self.take(1)?
556            .first()
557            .copied()
558            .ok_or_else(|| ManifestError::Decode("invalid u8".to_owned()))
559    }
560
561    fn u32(&mut self) -> Result<u32, ManifestError> {
562        let raw: [u8; 4] = self
563            .take(4)?
564            .try_into()
565            .map_err(|_| ManifestError::Decode("invalid u32".to_owned()))?;
566        Ok(u32::from_le_bytes(raw))
567    }
568
569    fn u64(&mut self) -> Result<u64, ManifestError> {
570        let raw: [u8; 8] = self
571            .take(8)?
572            .try_into()
573            .map_err(|_| ManifestError::Decode("invalid u64".to_owned()))?;
574        Ok(u64::from_le_bytes(raw))
575    }
576
577    fn optional_identity(&mut self) -> Result<Option<EpochIdentity>, ManifestError> {
578        let tag = self.u8()?;
579        if self.take(7)?.iter().any(|byte| *byte != 0) {
580            return Err(ManifestError::Decode(
581                "epoch alias reserved bytes are non-zero".to_owned(),
582            ));
583        }
584        let embedding = self.u64()?;
585        let tokenizer = self.u64()?;
586        match (tag, embedding, tokenizer) {
587            (0, 0, 0) => Ok(None),
588            (1, embedding, tokenizer) => Ok(Some(EpochIdentity {
589                embedding: EpochId::from_value(embedding),
590                tokenizer: TokenizerEpoch::from_value(tokenizer),
591            })),
592            (0, _, _) => Err(ManifestError::Decode(
593                "absent epoch alias carries non-zero identity bytes".to_owned(),
594            )),
595            (unknown, _, _) => Err(ManifestError::Decode(format!(
596                "unknown epoch alias presence tag {unknown}"
597            ))),
598        }
599    }
600
601    fn optional_epoch_id(&mut self) -> Result<Option<EpochId>, ManifestError> {
602        let tag = self.u8()?;
603        if self.take(7)?.iter().any(|byte| *byte != 0) {
604            return Err(ManifestError::Decode(
605                "segment epoch reserved bytes are non-zero".to_owned(),
606            ));
607        }
608        let epoch = self.u64()?;
609        match (tag, epoch) {
610            (0, 0) => Ok(None),
611            (1, epoch) => Ok(Some(EpochId::from_value(epoch))),
612            (0, _) => Err(ManifestError::Decode(
613                "absent segment epoch tag carries a non-zero id".to_owned(),
614            )),
615            (unknown, _) => Err(ManifestError::Decode(format!(
616                "unknown segment epoch presence tag {unknown}"
617            ))),
618        }
619    }
620
621    fn embedding_epoch(&mut self) -> Result<EmbeddingEpoch, ManifestError> {
622        Ok(EmbeddingEpoch {
623            document: self.embedding_tower()?,
624            query: self.embedding_tower()?,
625            alignment_digest: self.bytes()?,
626        })
627    }
628
629    fn embedding_tower(&mut self) -> Result<EmbeddingTower, ManifestError> {
630        let model_id = self.string()?;
631        let model_version = self.string()?;
632        let weights_digest = self.bytes()?;
633        let dims = self.u32()?;
634        let normalization = match self.u16()? {
635            0 => Normalization::None,
636            1 => Normalization::L2,
637            unknown => {
638                return Err(ManifestError::Decode(format!(
639                    "unknown embedding normalization {unknown}"
640                )));
641            }
642        };
643        let prompt_prefix = self.string()?;
644        let max_tokens = self.u32()?;
645        let runtime = match self.u16()? {
646            1 => EmbeddingRuntime::CoreMl,
647            2 => EmbeddingRuntime::Mlx,
648            3 => EmbeddingRuntime::CpuReference,
649            unknown => {
650                return Err(ManifestError::Decode(format!(
651                    "unknown embedding runtime {unknown}"
652                )));
653            }
654        };
655        let compute_units = match self.u16()? {
656            1 => ComputeUnits::Cpu,
657            2 => ComputeUnits::CpuAndGpu,
658            3 => ComputeUnits::CpuAndNeuralEngine,
659            4 => ComputeUnits::All,
660            unknown => {
661                return Err(ManifestError::Decode(format!(
662                    "unknown embedding compute units {unknown}"
663                )));
664            }
665        };
666        let os_tag = self.u8()?;
667        if self.take(3)?.iter().any(|byte| *byte != 0) {
668            return Err(ManifestError::Decode(
669                "embedding OS-build reserved bytes are non-zero".to_owned(),
670            ));
671        }
672        let os_build = match os_tag {
673            0 => None,
674            1 => Some(self.string()?),
675            unknown => {
676                return Err(ManifestError::Decode(format!(
677                    "unknown embedding OS-build presence tag {unknown}"
678                )));
679            }
680        };
681        Ok(EmbeddingTower {
682            model_id,
683            model_version,
684            weights_digest,
685            dims,
686            normalization,
687            prompt_prefix,
688            max_tokens,
689            runtime,
690            compute_units,
691            os_build,
692        })
693    }
694
695    fn i64(&mut self) -> Result<i64, ManifestError> {
696        let raw: [u8; 8] = self
697            .take(8)?
698            .try_into()
699            .map_err(|_| ManifestError::Decode("invalid i64".to_owned()))?;
700        Ok(i64::from_le_bytes(raw))
701    }
702
703    const fn remaining(&self) -> usize {
704        self.bytes.len().saturating_sub(self.position)
705    }
706
707    fn usize_from_u32(&mut self) -> Result<usize, ManifestError> {
708        usize::try_from(self.u32()?)
709            .map_err(|_| ManifestError::Decode("u32 length exceeds usize".to_owned()))
710    }
711
712    fn string(&mut self) -> Result<String, ManifestError> {
713        let length = self.usize_from_u32()?;
714        std::str::from_utf8(self.take(length)?)
715            .map(str::to_owned)
716            .map_err(|error| ManifestError::Decode(format!("invalid UTF-8: {error}")))
717    }
718
719    fn bytes(&mut self) -> Result<Vec<u8>, ManifestError> {
720        let length = self.usize_from_u32()?;
721        Ok(self.take(length)?.to_vec())
722    }
723
724    fn finish(&self) -> Result<(), ManifestError> {
725        if self.position == self.bytes.len() {
726            Ok(())
727        } else {
728            Err(ManifestError::Decode(format!(
729                "{} trailing payload bytes",
730                self.bytes.len().saturating_sub(self.position)
731            )))
732        }
733    }
734}
zeppelin_embed/manifest/mod.rs

zeppelin_embed/manifest/
mod.rs
