icechunk_format/
lib.rs

1//! Icechunk data types and serialization.
2//!
3//! Defines ID types ([`SnapshotId`], [`ManifestId`], [`ChunkId`], [`NodeId`]),
4//! [`Path`] for normalized Zarr paths, and [`ByteRange`]/[`ChunkIndices`] for
5//! chunk addressing. Submodules define snapshots, manifests, and transaction logs.
6
7use core::fmt;
8use std::{
9    cmp::Ordering,
10    convert::Infallible,
11    fmt::{Debug, Display},
12    hash::Hash,
13    marker::PhantomData,
14    ops::Range,
15};
16
17use ::flatbuffers::InvalidFlatbuffer;
18use bytes::Bytes;
19use chrono::{DateTime, Utc};
20use flatbuffers::generated;
21use format_constants::FileTypeBin;
22use manifest::VirtualReferenceErrorKind;
23use rand::{RngExt as _, rng};
24use serde::{Deserialize, Serialize};
25use thiserror::Error;
26
27use icechunk_types::error::ICError;
28use icechunk_types::sealed;
29
30/// User attributes stored on arrays and groups.
31pub mod attributes;
32/// Chunk reference tables.
33pub mod manifest;
34
35/// Generated Flatbuffer types for binary serialization.
36#[path = "./flatbuffers/all_generated.rs"]
37#[allow(clippy::all, warnings)]
38pub mod flatbuffers;
39
40/// Repository metadata (version, properties).
41pub mod repo_info;
42/// Flatbuffer serialization.
43pub mod serializers;
44/// Repository state at a point in time.
45pub mod snapshot;
46/// Change records for commits.
47pub mod transaction_log;
48
49pub const CONFIG_FILE_PATH: &str = "config.yaml";
50pub const REPO_INFO_FILE_PATH: &str = "repo";
51pub const CHUNKS_FILE_PATH: &str = "chunks";
52pub const MANIFESTS_FILE_PATH: &str = "manifests";
53pub const SNAPSHOTS_FILE_PATH: &str = "snapshots";
54pub const TRANSACTION_LOGS_FILE_PATH: &str = "transactions";
55pub const OVERWRITTEN_FILES_PATH: &str = "overwritten";
56pub const V1_REFS_FILE_PATH: &str = "refs";
57
58pub use icechunk_types::{Path, PathError};
59
60/// Marker trait for object ID type tags (sealed).
61pub trait FileTypeTag: sealed::Sealed {}
62
63/// The id of a file in object store
64#[derive(Hash, Clone, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize)]
65pub struct ObjectId<const SIZE: usize, T: FileTypeTag>(
66    #[serde(with = "serde_bytes")] pub [u8; SIZE],
67    PhantomData<T>,
68);
69
70/// Type tag for [`SnapshotId`].
71#[derive(Debug, Hash, Clone, PartialEq, Eq, PartialOrd, Ord)]
72pub struct SnapshotTag;
73
74/// Type tag for [`ManifestId`].
75#[derive(Debug, Hash, Clone, PartialEq, Eq, PartialOrd, Ord)]
76pub struct ManifestTag;
77
78/// Type tag for [`ChunkId`].
79#[derive(Debug, Hash, Clone, PartialEq, Eq, PartialOrd, Ord)]
80pub struct ChunkTag;
81
82/// Type tag for [`AttributesId`].
83#[derive(Debug, Hash, Clone, PartialEq, Eq, PartialOrd, Ord)]
84pub struct AttributesTag;
85
86/// Type tag for [`NodeId`].
87#[derive(Debug, Hash, Clone, PartialEq, Eq, PartialOrd, Ord)]
88pub struct NodeTag;
89
90impl sealed::Sealed for SnapshotTag {}
91impl sealed::Sealed for ManifestTag {}
92impl sealed::Sealed for ChunkTag {}
93impl sealed::Sealed for AttributesTag {}
94impl sealed::Sealed for NodeTag {}
95impl FileTypeTag for SnapshotTag {}
96impl FileTypeTag for ManifestTag {}
97impl FileTypeTag for ChunkTag {}
98impl FileTypeTag for AttributesTag {}
99impl FileTypeTag for NodeTag {}
100
101// A 1e-9 conflict probability requires 2^33 ~ 8.5 bn chunks
102// using this site for the calculations: https://www.bdayprob.com/
103
104/// Unique identifier for a snapshot.
105pub type SnapshotId = ObjectId<12, SnapshotTag>;
106/// Unique identifier for a manifest file.
107pub type ManifestId = ObjectId<12, ManifestTag>;
108/// Unique identifier for a chunk.
109pub type ChunkId = ObjectId<12, ChunkTag>;
110/// Unique identifier for an attributes blob.
111pub type AttributesId = ObjectId<12, AttributesTag>;
112
113// A 1e-9 conflict probability requires 2^17.55 ~ 200k nodes
114/// The internal id of an array or group, unique only to a single store version
115pub type NodeId = ObjectId<8, NodeTag>;
116
117#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, Hash)]
118pub enum NodeType {
119    Group,
120    Array,
121}
122
123#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, Hash)]
124pub struct Move {
125    pub from: Path,
126    pub to: Path,
127    pub node_id: NodeId,
128    pub node_type: NodeType,
129}
130
131impl From<NodeType> for generated::NodeType {
132    fn from(value: NodeType) -> Self {
133        match value {
134            NodeType::Group => generated::NodeType::Group,
135            NodeType::Array => generated::NodeType::Array,
136        }
137    }
138}
139
140impl NodeType {
141    fn max_supported() -> u8 {
142        generated::NodeType::ENUM_MAX
143    }
144}
145
146impl TryFrom<generated::NodeType> for NodeType {
147    type Error = IcechunkFormatErrorKind;
148
149    fn try_from(value: generated::NodeType) -> Result<Self, Self::Error> {
150        match value {
151            generated::NodeType::Group => Ok(NodeType::Group),
152            generated::NodeType::Array => Ok(NodeType::Array),
153            generated::NodeType(v) => Err(IcechunkFormatErrorKind::InvalidNodeType {
154                found: v,
155                max_supported: Self::max_supported(),
156            }),
157        }
158    }
159}
160
161impl<const SIZE: usize, T: FileTypeTag> ObjectId<SIZE, T> {
162    pub fn random() -> Self {
163        let mut buf = [0u8; SIZE];
164        rng().fill(&mut buf[..]);
165        Self(buf, PhantomData)
166    }
167
168    pub const fn new(buf: [u8; SIZE]) -> Self {
169        Self(buf, PhantomData)
170    }
171
172    pub const FAKE: Self = Self([0; SIZE], PhantomData);
173}
174
175impl<const SIZE: usize, T: FileTypeTag> Debug for ObjectId<SIZE, T> {
176    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
177        write!(f, "{}", String::from(self))
178    }
179}
180
181impl<const SIZE: usize, T: FileTypeTag> TryFrom<&[u8]> for ObjectId<SIZE, T> {
182    type Error = &'static str;
183
184    fn try_from(value: &[u8]) -> Result<Self, Self::Error> {
185        let buf = value.try_into();
186        buf.map(|buf| ObjectId(buf, PhantomData))
187            .map_err(|_| "Invalid ObjectId buffer length")
188    }
189}
190
191impl<const SIZE: usize, T: FileTypeTag> TryFrom<&str> for ObjectId<SIZE, T> {
192    type Error = &'static str;
193
194    fn try_from(value: &str) -> Result<Self, Self::Error> {
195        let bytes = base32::decode(base32::Alphabet::Crockford, value);
196        let Some(bytes) = bytes else { return Err("Invalid ObjectId string") };
197        Self::try_from(bytes.as_slice())
198    }
199}
200
201impl<const SIZE: usize, T: FileTypeTag> From<&ObjectId<SIZE, T>> for String {
202    fn from(value: &ObjectId<SIZE, T>) -> Self {
203        base32::encode(base32::Alphabet::Crockford, &value.0)
204    }
205}
206
207impl<const SIZE: usize, T: FileTypeTag> From<[u8; SIZE]> for ObjectId<SIZE, T> {
208    fn from(value: [u8; SIZE]) -> Self {
209        ObjectId::new(value)
210    }
211}
212
213impl<const SIZE: usize, T: FileTypeTag> TryInto<String> for ObjectId<SIZE, T> {
214    type Error = Infallible;
215
216    fn try_into(self) -> Result<String, Self::Error> {
217        Ok(self.to_string())
218    }
219}
220
221impl<const SIZE: usize, T: FileTypeTag> TryInto<ObjectId<SIZE, T>> for String {
222    type Error = &'static str;
223
224    fn try_into(self) -> Result<ObjectId<SIZE, T>, Self::Error> {
225        self.as_str().try_into()
226    }
227}
228
229impl<const SIZE: usize, T: FileTypeTag> Display for ObjectId<SIZE, T> {
230    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
231        write!(f, "{}", String::from(self))
232    }
233}
234
235/// An ND index to an element in a chunk grid.
236#[derive(Clone, Debug, Hash, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize)]
237pub struct ChunkIndices(pub Vec<u32>);
238
239/// Byte offset within a chunk.
240pub type ChunkOffset = u64;
241/// Size of a chunk in bytes.
242pub type ChunkLength = u64;
243
244impl<'a> From<generated::ChunkIndices<'a>> for ChunkIndices {
245    fn from(value: generated::ChunkIndices<'a>) -> Self {
246        ChunkIndices(value.coords().iter().collect())
247    }
248}
249
250/// A byte range within a chunk or object.
251#[derive(Debug, Clone, PartialEq, Eq, Hash)]
252pub enum ByteRange {
253    /// The fixed length range represented by the given `Range`
254    Bounded(Range<ChunkOffset>),
255    /// All bytes from the given offset (included) to the end of the object
256    From(ChunkOffset),
257    /// The last n bytes in the object
258    Last(ChunkLength),
259    /// All bytes up to the last n bytes in the object
260    Until(ChunkOffset),
261}
262
263impl From<Range<ChunkOffset>> for ByteRange {
264    fn from(value: Range<ChunkOffset>) -> Self {
265        ByteRange::Bounded(value)
266    }
267}
268
269impl ByteRange {
270    pub fn from_offset(offset: ChunkOffset) -> Self {
271        Self::From(offset)
272    }
273
274    pub fn from_offset_with_length(offset: ChunkOffset, length: ChunkOffset) -> Self {
275        Self::Bounded(offset..offset + length)
276    }
277
278    pub fn to_offset(offset: ChunkOffset) -> Self {
279        Self::Bounded(0..offset)
280    }
281
282    pub fn bounded(start: ChunkOffset, end: ChunkOffset) -> Self {
283        (start..end).into()
284    }
285
286    pub const ALL: Self = Self::From(0);
287
288    pub fn slice(&self, bytes: &Bytes) -> Bytes {
289        match self {
290            ByteRange::Bounded(range) => {
291                bytes.slice(range.start as usize..range.end as usize)
292            }
293            ByteRange::From(from) => bytes.slice(*from as usize..),
294            ByteRange::Last(n) => bytes.slice(bytes.len() - *n as usize..),
295            ByteRange::Until(n) => bytes.slice(0usize..bytes.len() - *n as usize),
296        }
297    }
298}
299
300impl From<(Option<ChunkOffset>, Option<ChunkOffset>)> for ByteRange {
301    fn from((start, end): (Option<ChunkOffset>, Option<ChunkOffset>)) -> Self {
302        match (start, end) {
303            (Some(start), Some(end)) => Self::Bounded(start..end),
304            (Some(start), None) => Self::From(start),
305            // NOTE: This is relied upon by zarr python
306            (None, Some(end)) => Self::Until(end),
307            (None, None) => Self::ALL,
308        }
309    }
310}
311
312/// Offset within a manifest table.
313pub type TableOffset = u32;
314
315/// Format-level error types.
316#[derive(Debug, Error)]
317#[non_exhaustive]
318pub enum IcechunkFormatErrorKind {
319    #[error(transparent)]
320    VirtualReferenceError(#[from] VirtualReferenceErrorKind),
321    #[error("node not found at `{path:?}`")]
322    NodeNotFound { path: Path },
323    #[error("chunk coordinates not found `{coords:?}`")]
324    ChunkCoordinatesNotFound { coords: ChunkIndices },
325    #[error("snapshot id not found `{snapshot_id}`")]
326    SnapshotIdNotFound { snapshot_id: SnapshotId },
327    #[error("branch already exists `{branch} -> {snapshot_id}`")]
328    BranchAlreadyExists { branch: String, snapshot_id: SnapshotId },
329    #[error("branch not found `{branch}`")]
330    BranchNotFound { branch: String },
331    #[error("tag already exists `{tag}`")]
332    TagAlreadyExists { tag: String },
333    #[error("icechunk does not allow tag reuse and tag was already deleted `{tag}`")]
334    TagPreviouslyDeleted { tag: String },
335    #[error("tag not found `{tag}`")]
336    TagNotFound { tag: String },
337    #[error("snapshot id is already present in the repository: `{snapshot_id}`")]
338    DuplicateSnapshotId { snapshot_id: SnapshotId },
339    #[error("manifest information cannot be found in snapshot for id `{manifest_id}`")]
340    ManifestInfoNotFound { manifest_id: ManifestId },
341    #[error("invalid magic numbers in file")]
342    InvalidMagicNumbers, // TODO: add more info
343    #[error(
344        "invalid icechunk header size, got {found} bytes, expected at least {expected}"
345    )]
346    InvalidIcechunkHeaderSize { found: usize, expected: usize },
347    #[error("invalid node type {found}, max supported value is {max_supported}")]
348    InvalidNodeType { found: u8, max_supported: u8 },
349    #[error(
350        "this repository uses Icechunk format version {found}, but this library only supports up to version {max_supported}. Please upgrade the icechunk library"
351    )]
352    InvalidSpecVersion { found: u8, max_supported: u8 },
353    #[error("this operation is not supported for Icechunk format version {version}")]
354    UnsupportedOperationForVersion { version: u8 },
355    #[error("Icechunk cannot read this file type, expected {expected:?} got {got}")]
356    InvalidFileType { expected: FileTypeBin, got: u8 }, // TODO: add more info
357    #[error("Icechunk encountered an unknown file type code {got}")]
358    UnknownFileType { got: u8 },
359    #[error("Icechunk cannot read file, invalid compression algorithm")]
360    InvalidCompressionAlgorithm, // TODO: add more info
361    #[error("Invalid Icechunk metadata file")]
362    InvalidFlatBuffer(#[from] InvalidFlatbuffer),
363    #[error("error during metadata deserialization")]
364    DeserializationError(#[from] Box<rmp_serde::decode::Error>),
365    #[error("error during metadata serialization")]
366    SerializationError(#[from] Box<rmp_serde::encode::Error>),
367    #[error("error during metadata serialization")]
368    SerializationErrorFlexBuffers(#[from] Box<flexbuffers::SerializationError>),
369    #[error("error during metadata deserialization")]
370    DeserializationErrorFlexBuffers(#[from] Box<flexbuffers::DeserializationError>),
371    #[error("I/O error")]
372    IO(#[from] std::io::Error),
373    #[error("path error")]
374    Path(#[from] PathError),
375    #[error("invalid timestamp in file")]
376    InvalidTimestamp,
377    #[error(
378        "update timestamp is invalid, please verify if the machine clock has drifted: update time: `{new_time}`, latest update time: `{latest_time}`"
379    )]
380    InvalidUpdateTimestamp { latest_time: DateTime<Utc>, new_time: DateTime<Utc> },
381    #[error("invalid feature flag name: {name}")]
382    InvalidFeatureFlagName { name: String },
383    #[error("invalid feature flag id: {id}")]
384    InvalidFeatureFlagId { id: u16 },
385    #[error("{feature_description} is disabled by a feature flag ({feature_flag})")]
386    FeatureFlagDisabled { feature_description: String, feature_flag: String },
387    #[error(
388        "compressed chunk location present but no decompression dictionary available"
389    )]
390    MissingLocationCompressionDictionary,
391    #[error("Invalid array metadata: {0}")]
392    InvalidArrayMetadata(String),
393    #[error("Move operation missing required field '{0}'")]
394    MissingRequiredField(String),
395}
396
397pub type IcechunkFormatError = ICError<IcechunkFormatErrorKind>;
398
399impl From<Infallible> for IcechunkFormatErrorKind {
400    fn from(value: Infallible) -> Self {
401        match value {}
402    }
403}
404
405pub type IcechunkResult<T> = Result<T, IcechunkFormatError>;
406
407/// Binary format constants (file types, spec versions, compression).
408pub mod format_constants {
409    use std::sync::LazyLock;
410
411    use icechunk_types::ICResultExt as _;
412    use serde::{Deserialize, Serialize};
413
414    use super::{IcechunkFormatError, IcechunkFormatErrorKind};
415
416    /// Binary file type identifier in the file header.
417    #[repr(u8)]
418    #[derive(Debug, Clone, Copy, PartialEq, Eq)]
419    pub enum FileTypeBin {
420        Snapshot = 1u8,
421        Manifest = 2,
422        Attributes = 3,
423        TransactionLog = 4,
424        Chunk = 5,
425        RepoInfo = 6,
426    }
427
428    impl TryFrom<u8> for FileTypeBin {
429        type Error = String;
430
431        fn try_from(value: u8) -> Result<Self, Self::Error> {
432            match value {
433                n if n == FileTypeBin::Snapshot as u8 => Ok(FileTypeBin::Snapshot),
434                n if n == FileTypeBin::Manifest as u8 => Ok(FileTypeBin::Manifest),
435                n if n == FileTypeBin::Attributes as u8 => Ok(FileTypeBin::Attributes),
436                n if n == FileTypeBin::RepoInfo as u8 => Ok(FileTypeBin::RepoInfo),
437                n if n == FileTypeBin::TransactionLog as u8 => {
438                    Ok(FileTypeBin::TransactionLog)
439                }
440                n if n == FileTypeBin::Chunk as u8 => Ok(FileTypeBin::Chunk),
441                n => Err(format!("Bad file type code: {n}")),
442            }
443        }
444    }
445
446    /// Icechunk format specification version.
447    #[repr(u8)]
448    #[derive(
449        Debug,
450        Clone,
451        Copy,
452        PartialEq,
453        Eq,
454        Serialize,
455        Deserialize,
456        Default,
457        PartialOrd,
458        Ord,
459    )]
460    pub enum SpecVersionBin {
461        V1 = 1u8,
462        #[default]
463        V2 = 2u8,
464        // When adding new versions here, don't forget to update the
465        // PySpecVersion enum in icechunk-python/src/repository.rs too!
466    }
467
468    impl TryFrom<u8> for SpecVersionBin {
469        type Error = IcechunkFormatErrorKind;
470
471        fn try_from(value: u8) -> Result<Self, Self::Error> {
472            match value {
473                n if n == SpecVersionBin::V1 as u8 => Ok(SpecVersionBin::V1),
474                n if n == SpecVersionBin::V2 as u8 => Ok(SpecVersionBin::V2),
475                n => Err(IcechunkFormatErrorKind::InvalidSpecVersion {
476                    found: n,
477                    max_supported: Self::current() as u8,
478                }),
479            }
480        }
481    }
482
483    impl SpecVersionBin {
484        pub fn current() -> Self {
485            Default::default()
486        }
487    }
488
489    impl std::fmt::Display for SpecVersionBin {
490        fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
491            let s = match self {
492                SpecVersionBin::V1 => "1.0",
493                SpecVersionBin::V2 => "2.0",
494            };
495            write!(f, "{s}")
496        }
497    }
498
499    /// Compression algorithm used for metadata files.
500    #[repr(u8)]
501    #[derive(Debug, Clone, Copy, PartialEq, Eq)]
502    pub enum CompressionAlgorithmBin {
503        None = 0u8,
504        Zstd = 1u8,
505    }
506
507    impl TryFrom<u8> for CompressionAlgorithmBin {
508        type Error = String;
509
510        fn try_from(value: u8) -> Result<Self, Self::Error> {
511            match value {
512                n if n == CompressionAlgorithmBin::None as u8 => {
513                    Ok(CompressionAlgorithmBin::None)
514                }
515                n if n == CompressionAlgorithmBin::Zstd as u8 => {
516                    Ok(CompressionAlgorithmBin::Zstd)
517                }
518                n => Err(format!("Bad cmpression algorithm code: {n}")),
519            }
520        }
521    }
522
523    pub const ICECHUNK_FORMAT_MAGIC_BYTES: &[u8] = "ICE🧊CHUNK".as_bytes();
524    // offsets assume a 12-byte magic
525    const _: () = assert!(ICECHUNK_FORMAT_MAGIC_BYTES.len() == 12);
526
527    // Binary file header layout: magic | impl name | spec version | file type | compression.
528    // Reader (check_header) and writer (binary_file_header) both derive offsets from these.
529    pub const ICECHUNK_IMPL_NAME_LEN: usize = 24;
530    pub const ICECHUNK_SPEC_VERSION_OFFSET: usize =
531        ICECHUNK_FORMAT_MAGIC_BYTES.len() + ICECHUNK_IMPL_NAME_LEN;
532    pub const ICECHUNK_FILE_TYPE_OFFSET: usize = ICECHUNK_SPEC_VERSION_OFFSET + 1;
533    pub const ICECHUNK_COMPRESSION_OFFSET: usize = ICECHUNK_FILE_TYPE_OFFSET + 1;
534    pub const ICECHUNK_FILE_HEADER_LEN: usize = ICECHUNK_COMPRESSION_OFFSET + 1;
535
536    // Object-metadata keys use letters only. Azure needs a valid C# identifier, which
537    // bars `-`, and nginx-fronted S3 gateways drop header names that contain `_`.
538    // The `_DEPRECATED` keys name what repositories written before that change carry.
539    pub const LATEST_ICECHUNK_FORMAT_VERSION_METADATA_KEY: &str = "icspecver";
540    pub const LATEST_ICECHUNK_FORMAT_VERSION_METADATA_KEY_DEPRECATED: &str =
541        "ic_spec_ver";
542
543    pub const ICECHUNK_LIB_VERSION: &str = env!("CARGO_PKG_VERSION");
544
545    pub static ICECHUNK_CLIENT_NAME: LazyLock<String> =
546        LazyLock::new(|| "ic-".to_string() + ICECHUNK_LIB_VERSION);
547    pub const ICECHUNK_CLIENT_NAME_METADATA_KEY: &str = "icclient";
548    pub const ICECHUNK_CLIENT_NAME_METADATA_KEY_DEPRECATED: &str = "ic_client";
549
550    pub const ICECHUNK_FILE_TYPE_SNAPSHOT: &str = "snapshot";
551    pub const ICECHUNK_FILE_TYPE_MANIFEST: &str = "manifest";
552    pub const ICECHUNK_FILE_TYPE_TRANSACTION_LOG: &str = "transaction-log";
553    pub const ICECHUNK_FILE_TYPE_REPO_INFO: &str = "repo-info";
554    pub const ICECHUNK_FILE_TYPE_METADATA_KEY: &str = "icfiletype";
555    pub const ICECHUNK_FILE_TYPE_METADATA_KEY_DEPRECATED: &str = "ic_file_type";
556
557    pub const ICECHUNK_COMPRESSION_METADATA_KEY: &str = "iccompalg";
558    pub const ICECHUNK_COMPRESSION_METADATA_KEY_DEPRECATED: &str = "ic_comp_alg";
559    pub const ICECHUNK_COMPRESSION_ZSTD: &str = "zstd";
560
561    /// Decoded contents of a metadata file's binary header.
562    ///
563    /// Parsing of `impl_name` and `app_name` is best effort only.
564    #[derive(Debug, Clone, PartialEq, Eq)]
565    pub struct FileHeader {
566        /// Raw, trimmed implementation/client string, e.g. `"ic-2.1.0"`.
567        pub impl_name: String,
568        /// Best-effort app name: `impl_name` before the first `'-'` (e.g. `"ic"`).
569        pub app_name: String,
570        /// Best-effort version: `impl_name` after the first `'-'` (e.g. `"2.1.0"`);
571        /// `None` if there is no `'-'`.
572        pub app_version: Option<String>,
573        pub spec_version: SpecVersionBin,
574        pub file_type: FileTypeBin,
575        pub compression: CompressionAlgorithmBin,
576    }
577
578    /// Parse a file header from the first [`ICECHUNK_FILE_HEADER_LEN`] bytes of any
579    /// metadata file.
580    ///
581    /// Discovers (does not assert) the file type, so it works for every metadata
582    /// file kind. Tolerant about the impl-name string content.
583    pub fn parse_file_header(buf: &[u8]) -> Result<FileHeader, IcechunkFormatError> {
584        use IcechunkFormatErrorKind as K;
585
586        if buf.len() < ICECHUNK_FILE_HEADER_LEN {
587            return Err(K::InvalidIcechunkHeaderSize {
588                found: buf.len(),
589                expected: ICECHUNK_FILE_HEADER_LEN,
590            })
591            .capture();
592        }
593        if !buf.starts_with(ICECHUNK_FORMAT_MAGIC_BYTES) {
594            return Err(K::InvalidMagicNumbers).capture();
595        }
596
597        let impl_bytes =
598            &buf[ICECHUNK_FORMAT_MAGIC_BYTES.len()..ICECHUNK_SPEC_VERSION_OFFSET];
599        let impl_name =
600            String::from_utf8_lossy(impl_bytes).trim_end_matches([' ', '\0']).to_string();
601        let (app_name, app_version) = match impl_name.split_once('-') {
602            Some((name, version)) => (name.to_string(), Some(version.to_string())),
603            None => (impl_name.clone(), None),
604        };
605
606        let spec_version =
607            SpecVersionBin::try_from(buf[ICECHUNK_SPEC_VERSION_OFFSET]).capture()?;
608
609        let file_type_byte = buf[ICECHUNK_FILE_TYPE_OFFSET];
610        let file_type = FileTypeBin::try_from(file_type_byte)
611            .map_err(|_| K::UnknownFileType { got: file_type_byte })
612            .capture()?;
613
614        let compression =
615            CompressionAlgorithmBin::try_from(buf[ICECHUNK_COMPRESSION_OFFSET])
616                .map_err(|_| K::InvalidCompressionAlgorithm)
617                .capture()?;
618
619        Ok(FileHeader {
620            impl_name,
621            app_name,
622            app_version,
623            spec_version,
624            file_type,
625            compression,
626        })
627    }
628}
629
630#[inline(always)]
631#[expect(clippy::needless_pass_by_value)]
632pub fn lookup_index_by_key<'a, T: ::flatbuffers::Follow<'a> + 'a, K: Ord>(
633    v: ::flatbuffers::Vector<'a, T>,
634    key: K,
635    f: fn(&<T as ::flatbuffers::Follow<'a>>::Inner, &K) -> Ordering,
636) -> Option<usize> {
637    if v.is_empty() {
638        return None;
639    }
640
641    let mut left: usize = 0;
642    let mut right = v.len() - 1;
643
644    while left <= right {
645        let mid = (left + right) / 2;
646        let value = v.get(mid);
647        match f(&value, &key) {
648            Ordering::Equal => return Some(mid),
649            Ordering::Less => left = mid + 1,
650            Ordering::Greater => {
651                if mid == 0 {
652                    return None;
653                }
654                right = mid - 1;
655            }
656        }
657    }
658
659    None
660}
661
662// This macro is used for creating property tests
663// which check that serializing and deserializing
664// an instance of a type T is equivalent to the
665// identity function
666// Given pairs of test names and arbitraries to be used
667// for the tests, e.g., (n1, a1), (n2, a2),... (nx, ax)
668// the tests can be created by doing
669// roundtrip_serialization_tests!(n1 - a1, n2 - a2, .... nx - ax)
670#[macro_export]
671macro_rules! roundtrip_serialization_tests {
672    ($($test_name: ident - $generator: ident), +) => {
673        $(
674            proptest!{
675                #[icechunk_macros::test]
676                fn $test_name(elem in $generator()) {
677                    let bytes = rmp_serde::to_vec(&elem).unwrap();
678                    let roundtrip = rmp_serde::from_slice(&bytes).unwrap();
679                    assert_eq!(elem, roundtrip);
680                }
681            }
682        )*
683    }
684}
685
686#[cfg(test)]
687pub mod strategies;
688
689#[cfg(test)]
690mod tests {
691    use super::*;
692    use crate::strategies::{attributes_id, spec_version};
693    use pretty_assertions::assert_eq;
694    use proptest::prelude::*;
695
696    roundtrip_serialization_tests!(
697        serialize_and_deserialize_attribute_ids - attributes_id,
698        serialize_and_deserialize_spec_version_bin - spec_version
699    );
700
701    #[icechunk_macros::test]
702    fn test_object_id_serialization() {
703        let sid = SnapshotId::new([0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11]);
704        assert_eq!(
705            serde_json::to_string(&sid).unwrap(),
706            r#"[[0,1,2,3,4,5,6,7,8,9,10,11],null]"#
707        );
708        assert_eq!(String::from(&sid), "000G40R40M30E209185G");
709        assert_eq!(sid, SnapshotId::try_from("000G40R40M30E209185G").unwrap());
710        let sid = SnapshotId::random();
711        assert_eq!(
712            serde_json::from_slice::<SnapshotId>(
713                serde_json::to_vec(&sid).unwrap().as_slice()
714            )
715            .unwrap(),
716            sid,
717        );
718    }
719
720    #[icechunk_macros::test]
721    fn test_unknown_spec_version_gives_nice_error() {
722        use format_constants::SpecVersionBin;
723
724        let future_version: u8 = 3;
725        let result = SpecVersionBin::try_from(future_version);
726        assert!(result.is_err());
727        let err = result.unwrap_err();
728
729        let msg = err.to_string();
730        assert!(
731            msg.contains("format version 3"),
732            "Error should mention the found version: {msg}"
733        );
734        assert!(
735            msg.contains("upgrade the icechunk library"),
736            "Error should suggest upgrading: {msg}"
737        );
738    }
739
740    /// Build a binary file header with an arbitrary impl string (right-padded to
741    /// `ICECHUNK_IMPL_NAME_LEN`, mirroring `binary_file_header` in the `icechunk`
742    /// crate), so tests can exercise `parse_file_header` directly.
743    fn make_header(impl_name: &str, spec: u8, file_type: u8, compression: u8) -> Vec<u8> {
744        use format_constants::*;
745        let mut buf = Vec::with_capacity(ICECHUNK_FILE_HEADER_LEN);
746        buf.extend_from_slice(ICECHUNK_FORMAT_MAGIC_BYTES);
747        let padded = format!("{impl_name:<ICECHUNK_IMPL_NAME_LEN$}");
748        buf.extend_from_slice(&padded.as_bytes()[..ICECHUNK_IMPL_NAME_LEN]);
749        buf.push(spec);
750        buf.push(file_type);
751        buf.push(compression);
752        buf
753    }
754
755    #[icechunk_macros::test]
756    fn test_parse_file_header_roundtrip() {
757        use format_constants::*;
758        let mut buf = make_header(
759            ICECHUNK_CLIENT_NAME.as_str(),
760            SpecVersionBin::V2 as u8,
761            FileTypeBin::Snapshot as u8,
762            CompressionAlgorithmBin::Zstd as u8,
763        );
764        // body bytes after the header must be ignored
765        buf.extend_from_slice(b"a compressed body would go here");
766
767        let header = parse_file_header(&buf).unwrap();
768        assert_eq!(header.impl_name, *ICECHUNK_CLIENT_NAME);
769        assert_eq!(header.app_name, "ic");
770        assert_eq!(header.app_version.as_deref(), Some(ICECHUNK_LIB_VERSION));
771        assert_eq!(header.spec_version, SpecVersionBin::V2);
772        assert_eq!(header.file_type, FileTypeBin::Snapshot);
773        assert_eq!(header.compression, CompressionAlgorithmBin::Zstd);
774    }
775
776    #[icechunk_macros::test]
777    fn test_parse_file_header_trims_padding() {
778        use format_constants::*;
779        // exactly the header, no body: the 24-byte impl field is space-padded
780        let buf = make_header(
781            "ic-9.9.9",
782            SpecVersionBin::V1 as u8,
783            FileTypeBin::Manifest as u8,
784            CompressionAlgorithmBin::None as u8,
785        );
786        let header = parse_file_header(&buf).unwrap();
787        assert_eq!(header.impl_name, "ic-9.9.9");
788        assert_eq!(header.app_name, "ic");
789        assert_eq!(header.app_version.as_deref(), Some("9.9.9"));
790        assert_eq!(header.spec_version, SpecVersionBin::V1);
791        assert_eq!(header.file_type, FileTypeBin::Manifest);
792        assert_eq!(header.compression, CompressionAlgorithmBin::None);
793    }
794
795    #[icechunk_macros::test]
796    fn test_parse_file_header_splits_on_first_dash() {
797        use format_constants::*;
798        let buf = make_header(
799            "ic-2.0.0-alpha.4",
800            SpecVersionBin::V2 as u8,
801            FileTypeBin::RepoInfo as u8,
802            CompressionAlgorithmBin::Zstd as u8,
803        );
804        let header = parse_file_header(&buf).unwrap();
805        assert_eq!(header.impl_name, "ic-2.0.0-alpha.4");
806        assert_eq!(header.app_name, "ic");
807        assert_eq!(header.app_version.as_deref(), Some("2.0.0-alpha.4"));
808    }
809
810    #[icechunk_macros::test]
811    fn test_parse_file_header_no_dash_fallback() {
812        use format_constants::*;
813        let buf = make_header(
814            "icjs",
815            SpecVersionBin::V2 as u8,
816            FileTypeBin::TransactionLog as u8,
817            CompressionAlgorithmBin::Zstd as u8,
818        );
819        let header = parse_file_header(&buf).unwrap();
820        assert_eq!(header.impl_name, "icjs");
821        assert_eq!(header.app_name, "icjs");
822        assert_eq!(header.app_version, None);
823    }
824
825    #[icechunk_macros::test]
826    fn test_parse_file_header_bad_magic() {
827        use format_constants::*;
828        let mut buf = make_header(
829            "ic-1.0.0",
830            SpecVersionBin::V2 as u8,
831            FileTypeBin::Snapshot as u8,
832            CompressionAlgorithmBin::Zstd as u8,
833        );
834        buf[0] = b'X';
835        let err = parse_file_header(&buf).unwrap_err();
836        assert!(matches!(err.kind(), IcechunkFormatErrorKind::InvalidMagicNumbers));
837    }
838
839    #[icechunk_macros::test]
840    fn test_parse_file_header_too_short() {
841        use format_constants::*;
842        let buf = vec![0u8; ICECHUNK_FILE_HEADER_LEN - 1];
843        let err = parse_file_header(&buf).unwrap_err();
844        assert!(matches!(
845            err.kind(),
846            IcechunkFormatErrorKind::InvalidIcechunkHeaderSize { .. }
847        ));
848    }
849
850    #[icechunk_macros::test]
851    fn test_parse_file_header_unknown_file_type() {
852        use format_constants::*;
853        let buf = make_header(
854            "ic-1.0.0",
855            SpecVersionBin::V2 as u8,
856            99,
857            CompressionAlgorithmBin::Zstd as u8,
858        );
859        let err = parse_file_header(&buf).unwrap_err();
860        assert!(matches!(
861            err.kind(),
862            IcechunkFormatErrorKind::UnknownFileType { got: 99 }
863        ));
864    }
865
866    #[icechunk_macros::test]
867    fn test_parse_file_header_bad_spec_version() {
868        use format_constants::*;
869        let buf = make_header(
870            "ic-1.0.0",
871            99,
872            FileTypeBin::Snapshot as u8,
873            CompressionAlgorithmBin::Zstd as u8,
874        );
875        let err = parse_file_header(&buf).unwrap_err();
876        assert!(matches!(
877            err.kind(),
878            IcechunkFormatErrorKind::InvalidSpecVersion { found: 99, .. }
879        ));
880    }
881
882    #[icechunk_macros::test]
883    fn test_object_metadata_keys_are_portable() {
884        use format_constants::*;
885        for key in [
886            LATEST_ICECHUNK_FORMAT_VERSION_METADATA_KEY,
887            ICECHUNK_CLIENT_NAME_METADATA_KEY,
888            ICECHUNK_FILE_TYPE_METADATA_KEY,
889            ICECHUNK_COMPRESSION_METADATA_KEY,
890        ] {
891            assert!(
892                !key.is_empty()
893                    && key.starts_with(|c: char| c.is_ascii_alphabetic())
894                    && key.chars().all(|c| c.is_ascii_alphanumeric()),
895                "{key} must be ASCII alphanumeric and start with a letter: \
896                 Azure rejects `-` and nginx-fronted S3 gateways drop `_`"
897            );
898        }
899    }
900}
icechunk_format/lib.rs

icechunk_format/
lib.rs
