icechunk_format/
lib.rs

1//! Icechunk data types and serialization.
2//!
3//! Defines ID types ([`SnapshotId`], [`ManifestId`], [`ChunkId`], [`NodeId`]),
4//! [`Path`] for normalized Zarr paths, and [`ByteRange`]/[`ChunkIndices`] for
5//! chunk addressing. Submodules define snapshots, manifests, and transaction logs.
6
7use core::fmt;
8use std::{
9    cmp::Ordering,
10    collections::HashSet,
11    convert::Infallible,
12    fmt::{Debug, Display},
13    hash::Hash,
14    marker::PhantomData,
15    ops::Range,
16    sync::LazyLock,
17};
18
19use ::flatbuffers::InvalidFlatbuffer;
20use bytes::Bytes;
21use chrono::{DateTime, Utc};
22use flatbuffers::generated;
23use format_constants::FileTypeBin;
24use manifest::VirtualReferenceErrorKind;
25use rand::{RngExt as _, rng};
26use serde::{Deserialize, Serialize};
27use thiserror::Error;
28
29use icechunk_types::error::ICError;
30use icechunk_types::sealed;
31
32/// User attributes stored on arrays and groups.
33pub mod attributes;
34/// Chunk reference tables.
35pub mod manifest;
36
37/// Generated Flatbuffer types for binary serialization.
38#[path = "./flatbuffers/all_generated.rs"]
39#[allow(clippy::all, warnings)]
40pub mod flatbuffers;
41
42/// Repository metadata (version, properties).
43pub mod repo_info;
44/// Flatbuffer serialization.
45pub mod serializers;
46/// Repository state at a point in time.
47pub mod snapshot;
48/// Change records for commits.
49pub mod transaction_log;
50
51pub const CONFIG_FILE_PATH: &str = "config.yaml";
52pub const REPO_INFO_FILE_PATH: &str = "repo";
53pub const CHUNKS_FILE_PATH: &str = "chunks";
54pub const MANIFESTS_FILE_PATH: &str = "manifests";
55pub const SNAPSHOTS_FILE_PATH: &str = "snapshots";
56pub const TRANSACTION_LOGS_FILE_PATH: &str = "transactions";
57pub const OVERWRITTEN_FILES_PATH: &str = "overwritten";
58pub const V1_REFS_FILE_PATH: &str = "refs";
59
60pub use icechunk_types::{Path, PathError};
61
62/// Marker trait for object ID type tags (sealed).
63pub trait FileTypeTag: sealed::Sealed {}
64
65/// The id of a file in object store
66#[derive(Hash, Clone, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize)]
67pub struct ObjectId<const SIZE: usize, T: FileTypeTag>(
68    #[serde(with = "serde_bytes")] pub [u8; SIZE],
69    PhantomData<T>,
70);
71
72/// Type tag for [`SnapshotId`].
73#[derive(Debug, Hash, Clone, PartialEq, Eq, PartialOrd, Ord)]
74pub struct SnapshotTag;
75
76/// Type tag for [`ManifestId`].
77#[derive(Debug, Hash, Clone, PartialEq, Eq, PartialOrd, Ord)]
78pub struct ManifestTag;
79
80/// Type tag for [`ChunkId`].
81#[derive(Debug, Hash, Clone, PartialEq, Eq, PartialOrd, Ord)]
82pub struct ChunkTag;
83
84/// Type tag for [`AttributesId`].
85#[derive(Debug, Hash, Clone, PartialEq, Eq, PartialOrd, Ord)]
86pub struct AttributesTag;
87
88/// Type tag for [`NodeId`].
89#[derive(Debug, Hash, Clone, PartialEq, Eq, PartialOrd, Ord)]
90pub struct NodeTag;
91
92impl sealed::Sealed for SnapshotTag {}
93impl sealed::Sealed for ManifestTag {}
94impl sealed::Sealed for ChunkTag {}
95impl sealed::Sealed for AttributesTag {}
96impl sealed::Sealed for NodeTag {}
97impl FileTypeTag for SnapshotTag {}
98impl FileTypeTag for ManifestTag {}
99impl FileTypeTag for ChunkTag {}
100impl FileTypeTag for AttributesTag {}
101impl FileTypeTag for NodeTag {}
102
103// A 1e-9 conflict probability requires 2^33 ~ 8.5 bn chunks
104// using this site for the calculations: https://www.bdayprob.com/
105
106/// Unique identifier for a snapshot.
107pub type SnapshotId = ObjectId<12, SnapshotTag>;
108/// Unique identifier for a manifest file.
109pub type ManifestId = ObjectId<12, ManifestTag>;
110/// Unique identifier for a chunk.
111pub type ChunkId = ObjectId<12, ChunkTag>;
112/// Unique identifier for an attributes blob.
113pub type AttributesId = ObjectId<12, AttributesTag>;
114
115// A 1e-9 conflict probability requires 2^17.55 ~ 200k nodes
116/// The internal id of an array or group, unique only to a single store version
117pub type NodeId = ObjectId<8, NodeTag>;
118
119#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, Hash)]
120pub enum NodeType {
121    Group,
122    Array,
123}
124
125#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, Hash)]
126pub struct Move {
127    pub from: Path,
128    pub to: Path,
129    pub node_id: NodeId,
130    pub node_type: NodeType,
131}
132
133impl From<NodeType> for generated::NodeType {
134    fn from(value: NodeType) -> Self {
135        match value {
136            NodeType::Group => generated::NodeType::Group,
137            NodeType::Array => generated::NodeType::Array,
138        }
139    }
140}
141
142impl NodeType {
143    fn max_supported() -> u8 {
144        generated::NodeType::ENUM_MAX
145    }
146}
147
148impl TryFrom<generated::NodeType> for NodeType {
149    type Error = IcechunkFormatErrorKind;
150
151    fn try_from(value: generated::NodeType) -> Result<Self, Self::Error> {
152        match value {
153            generated::NodeType::Group => Ok(NodeType::Group),
154            generated::NodeType::Array => Ok(NodeType::Array),
155            generated::NodeType(v) => Err(IcechunkFormatErrorKind::InvalidNodeType {
156                found: v,
157                max_supported: Self::max_supported(),
158            }),
159        }
160    }
161}
162
163impl<const SIZE: usize, T: FileTypeTag> ObjectId<SIZE, T> {
164    pub fn random() -> Self {
165        let mut buf = [0u8; SIZE];
166        rng().fill(&mut buf[..]);
167        Self(buf, PhantomData)
168    }
169
170    pub const fn new(buf: [u8; SIZE]) -> Self {
171        Self(buf, PhantomData)
172    }
173
174    pub const FAKE: Self = Self([0; SIZE], PhantomData);
175}
176
177impl<const SIZE: usize, T: FileTypeTag> Debug for ObjectId<SIZE, T> {
178    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
179        write!(f, "{}", String::from(self))
180    }
181}
182
183impl<const SIZE: usize, T: FileTypeTag> TryFrom<&[u8]> for ObjectId<SIZE, T> {
184    type Error = &'static str;
185
186    fn try_from(value: &[u8]) -> Result<Self, Self::Error> {
187        let buf = value.try_into();
188        buf.map(|buf| ObjectId(buf, PhantomData))
189            .map_err(|_| "Invalid ObjectId buffer length")
190    }
191}
192
193impl<const SIZE: usize, T: FileTypeTag> TryFrom<&str> for ObjectId<SIZE, T> {
194    type Error = &'static str;
195
196    fn try_from(value: &str) -> Result<Self, Self::Error> {
197        let bytes = base32::decode(base32::Alphabet::Crockford, value);
198        let Some(bytes) = bytes else { return Err("Invalid ObjectId string") };
199        Self::try_from(bytes.as_slice())
200    }
201}
202
203impl<const SIZE: usize, T: FileTypeTag> From<&ObjectId<SIZE, T>> for String {
204    fn from(value: &ObjectId<SIZE, T>) -> Self {
205        base32::encode(base32::Alphabet::Crockford, &value.0)
206    }
207}
208
209/// The Crockford base32 alphabet, which `base32` keeps private. Every [`ObjectId`]
210/// starts with one of these, so a listing split on them covers every id.
211pub static OBJECT_ID_FIRST_CHARS: LazyLock<HashSet<char>> =
212    LazyLock::new(|| "0123456789ABCDEFGHJKMNPQRSTVWXYZ".chars().collect());
213
214/// The 32 one-character id prefixes GC and stats list concurrently. Sorted, so
215/// the order a listing fans out in does not depend on the hash set's iteration.
216pub static OBJECT_ID_ONE_CHAR_PREFIXES: LazyLock<Vec<String>> = LazyLock::new(|| {
217    let mut prefixes: Vec<String> =
218        OBJECT_ID_FIRST_CHARS.iter().map(|c| c.to_string()).collect();
219    prefixes.sort_unstable();
220    prefixes
221});
222
223/// The 32 × 32 two-character id prefixes, sorted. Used to list large object
224/// sets with more concurrency than the 32 one-character prefixes allow.
225pub static OBJECT_ID_TWO_CHAR_PREFIXES: LazyLock<Vec<String>> = LazyLock::new(|| {
226    let mut prefixes: Vec<String> = OBJECT_ID_ONE_CHAR_PREFIXES
227        .iter()
228        .flat_map(|a| OBJECT_ID_ONE_CHAR_PREFIXES.iter().map(move |b| format!("{a}{b}")))
229        .collect();
230    prefixes.sort_unstable();
231    prefixes
232});
233
234impl<const SIZE: usize, T: FileTypeTag> From<[u8; SIZE]> for ObjectId<SIZE, T> {
235    fn from(value: [u8; SIZE]) -> Self {
236        ObjectId::new(value)
237    }
238}
239
240impl<const SIZE: usize, T: FileTypeTag> TryInto<String> for ObjectId<SIZE, T> {
241    type Error = Infallible;
242
243    fn try_into(self) -> Result<String, Self::Error> {
244        Ok(self.to_string())
245    }
246}
247
248impl<const SIZE: usize, T: FileTypeTag> TryInto<ObjectId<SIZE, T>> for String {
249    type Error = &'static str;
250
251    fn try_into(self) -> Result<ObjectId<SIZE, T>, Self::Error> {
252        self.as_str().try_into()
253    }
254}
255
256impl<const SIZE: usize, T: FileTypeTag> Display for ObjectId<SIZE, T> {
257    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
258        write!(f, "{}", String::from(self))
259    }
260}
261
262/// An ND index to an element in a chunk grid.
263#[derive(Clone, Debug, Hash, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize)]
264pub struct ChunkIndices(pub Vec<u32>);
265
266/// Byte offset within a chunk.
267pub type ChunkOffset = u64;
268/// Size of a chunk in bytes.
269pub type ChunkLength = u64;
270
271impl<'a> From<generated::ChunkIndices<'a>> for ChunkIndices {
272    fn from(value: generated::ChunkIndices<'a>) -> Self {
273        ChunkIndices(value.coords().iter().collect())
274    }
275}
276
277/// A byte range within a chunk or object.
278#[derive(Debug, Clone, PartialEq, Eq, Hash)]
279pub enum ByteRange {
280    /// The fixed length range represented by the given `Range`
281    Bounded(Range<ChunkOffset>),
282    /// All bytes from the given offset (included) to the end of the object
283    From(ChunkOffset),
284    /// The last n bytes in the object
285    Last(ChunkLength),
286    /// All bytes up to the last n bytes in the object
287    Until(ChunkOffset),
288}
289
290impl From<Range<ChunkOffset>> for ByteRange {
291    fn from(value: Range<ChunkOffset>) -> Self {
292        ByteRange::Bounded(value)
293    }
294}
295
296impl ByteRange {
297    pub fn from_offset(offset: ChunkOffset) -> Self {
298        Self::From(offset)
299    }
300
301    pub fn from_offset_with_length(offset: ChunkOffset, length: ChunkOffset) -> Self {
302        Self::Bounded(offset..offset + length)
303    }
304
305    pub fn to_offset(offset: ChunkOffset) -> Self {
306        Self::Bounded(0..offset)
307    }
308
309    pub fn bounded(start: ChunkOffset, end: ChunkOffset) -> Self {
310        (start..end).into()
311    }
312
313    pub const ALL: Self = Self::From(0);
314
315    pub fn slice(&self, bytes: &Bytes) -> Bytes {
316        match self {
317            ByteRange::Bounded(range) => {
318                bytes.slice(range.start as usize..range.end as usize)
319            }
320            ByteRange::From(from) => bytes.slice(*from as usize..),
321            ByteRange::Last(n) => bytes.slice(bytes.len() - *n as usize..),
322            ByteRange::Until(n) => bytes.slice(0usize..bytes.len() - *n as usize),
323        }
324    }
325}
326
327impl From<(Option<ChunkOffset>, Option<ChunkOffset>)> for ByteRange {
328    fn from((start, end): (Option<ChunkOffset>, Option<ChunkOffset>)) -> Self {
329        match (start, end) {
330            (Some(start), Some(end)) => Self::Bounded(start..end),
331            (Some(start), None) => Self::From(start),
332            // NOTE: This is relied upon by zarr python
333            (None, Some(end)) => Self::Until(end),
334            (None, None) => Self::ALL,
335        }
336    }
337}
338
339/// Offset within a manifest table.
340pub type TableOffset = u32;
341
342/// Format-level error types.
343#[derive(Debug, Error)]
344#[non_exhaustive]
345pub enum IcechunkFormatErrorKind {
346    #[error(transparent)]
347    VirtualReferenceError(#[from] VirtualReferenceErrorKind),
348    #[error("node not found at `{path:?}`")]
349    NodeNotFound { path: Path },
350    #[error("chunk coordinates not found `{coords:?}`")]
351    ChunkCoordinatesNotFound { coords: ChunkIndices },
352    #[error("snapshot id not found `{snapshot_id}`")]
353    SnapshotIdNotFound { snapshot_id: SnapshotId },
354    #[error("branch already exists `{branch} -> {snapshot_id}`")]
355    BranchAlreadyExists { branch: String, snapshot_id: SnapshotId },
356    #[error("branch not found `{branch}`")]
357    BranchNotFound { branch: String },
358    #[error("tag already exists `{tag}`")]
359    TagAlreadyExists { tag: String },
360    #[error("icechunk does not allow tag reuse and tag was already deleted `{tag}`")]
361    TagPreviouslyDeleted { tag: String },
362    #[error("tag not found `{tag}`")]
363    TagNotFound { tag: String },
364    #[error("snapshot id is already present in the repository: `{snapshot_id}`")]
365    DuplicateSnapshotId { snapshot_id: SnapshotId },
366    #[error("manifest information cannot be found in snapshot for id `{manifest_id}`")]
367    ManifestInfoNotFound { manifest_id: ManifestId },
368    #[error("invalid magic numbers in file")]
369    InvalidMagicNumbers, // TODO: add more info
370    #[error(
371        "invalid icechunk header size, got {found} bytes, expected at least {expected}"
372    )]
373    InvalidIcechunkHeaderSize { found: usize, expected: usize },
374    #[error("invalid node type {found}, max supported value is {max_supported}")]
375    InvalidNodeType { found: u8, max_supported: u8 },
376    #[error(
377        "this repository uses Icechunk format version {found}, but this library only supports up to version {max_supported}. Please upgrade the icechunk library"
378    )]
379    InvalidSpecVersion { found: u8, max_supported: u8 },
380    #[error("this operation is not supported for Icechunk format version {version}")]
381    UnsupportedOperationForVersion { version: u8 },
382    #[error("Icechunk cannot read this file type, expected {expected:?} got {got}")]
383    InvalidFileType { expected: FileTypeBin, got: u8 }, // TODO: add more info
384    #[error("Icechunk encountered an unknown file type code {got}")]
385    UnknownFileType { got: u8 },
386    #[error("Icechunk cannot read file, invalid compression algorithm")]
387    InvalidCompressionAlgorithm, // TODO: add more info
388    #[error("Invalid Icechunk metadata file")]
389    InvalidFlatBuffer(#[from] InvalidFlatbuffer),
390    #[error("error during metadata deserialization")]
391    DeserializationError(#[from] Box<rmp_serde::decode::Error>),
392    #[error("error during metadata serialization")]
393    SerializationError(#[from] Box<rmp_serde::encode::Error>),
394    #[error("error during metadata serialization")]
395    SerializationErrorFlexBuffers(#[from] Box<flexbuffers::SerializationError>),
396    #[error("error during metadata deserialization")]
397    DeserializationErrorFlexBuffers(#[from] Box<flexbuffers::DeserializationError>),
398    #[error("I/O error")]
399    IO(#[from] std::io::Error),
400    #[error("path error")]
401    Path(#[from] PathError),
402    #[error("invalid timestamp in file")]
403    InvalidTimestamp,
404    #[error(
405        "update timestamp is invalid, please verify if the machine clock has drifted: update time: `{new_time}`, latest update time: `{latest_time}`"
406    )]
407    InvalidUpdateTimestamp { latest_time: DateTime<Utc>, new_time: DateTime<Utc> },
408    #[error("invalid feature flag name: {name}")]
409    InvalidFeatureFlagName { name: String },
410    #[error("invalid feature flag id: {id}")]
411    InvalidFeatureFlagId { id: u16 },
412    #[error("{feature_description} is disabled by a feature flag ({feature_flag})")]
413    FeatureFlagDisabled { feature_description: String, feature_flag: String },
414    #[error(
415        "compressed chunk location present but no decompression dictionary available"
416    )]
417    MissingLocationCompressionDictionary,
418    #[error("Invalid array metadata: {0}")]
419    InvalidArrayMetadata(String),
420    #[error("Move operation missing required field '{0}'")]
421    MissingRequiredField(String),
422}
423
424pub type IcechunkFormatError = ICError<IcechunkFormatErrorKind>;
425
426impl From<Infallible> for IcechunkFormatErrorKind {
427    fn from(value: Infallible) -> Self {
428        match value {}
429    }
430}
431
432pub type IcechunkResult<T> = Result<T, IcechunkFormatError>;
433
434/// Binary format constants (file types, spec versions, compression).
435pub mod format_constants {
436    use std::sync::LazyLock;
437
438    use icechunk_types::ICResultExt as _;
439    use serde::{Deserialize, Serialize};
440
441    use super::{IcechunkFormatError, IcechunkFormatErrorKind};
442
443    /// Binary file type identifier in the file header.
444    #[repr(u8)]
445    #[derive(Debug, Clone, Copy, PartialEq, Eq)]
446    pub enum FileTypeBin {
447        Snapshot = 1u8,
448        Manifest = 2,
449        Attributes = 3,
450        TransactionLog = 4,
451        Chunk = 5,
452        RepoInfo = 6,
453    }
454
455    impl TryFrom<u8> for FileTypeBin {
456        type Error = String;
457
458        fn try_from(value: u8) -> Result<Self, Self::Error> {
459            match value {
460                n if n == FileTypeBin::Snapshot as u8 => Ok(FileTypeBin::Snapshot),
461                n if n == FileTypeBin::Manifest as u8 => Ok(FileTypeBin::Manifest),
462                n if n == FileTypeBin::Attributes as u8 => Ok(FileTypeBin::Attributes),
463                n if n == FileTypeBin::RepoInfo as u8 => Ok(FileTypeBin::RepoInfo),
464                n if n == FileTypeBin::TransactionLog as u8 => {
465                    Ok(FileTypeBin::TransactionLog)
466                }
467                n if n == FileTypeBin::Chunk as u8 => Ok(FileTypeBin::Chunk),
468                n => Err(format!("Bad file type code: {n}")),
469            }
470        }
471    }
472
473    /// Icechunk format specification version.
474    #[repr(u8)]
475    #[derive(
476        Debug,
477        Clone,
478        Copy,
479        PartialEq,
480        Eq,
481        Serialize,
482        Deserialize,
483        Default,
484        PartialOrd,
485        Ord,
486    )]
487    pub enum SpecVersionBin {
488        V1 = 1u8,
489        #[default]
490        V2 = 2u8,
491        // When adding new versions here, don't forget to update the
492        // PySpecVersion enum in icechunk-python/src/repository.rs too!
493    }
494
495    impl TryFrom<u8> for SpecVersionBin {
496        type Error = IcechunkFormatErrorKind;
497
498        fn try_from(value: u8) -> Result<Self, Self::Error> {
499            match value {
500                n if n == SpecVersionBin::V1 as u8 => Ok(SpecVersionBin::V1),
501                n if n == SpecVersionBin::V2 as u8 => Ok(SpecVersionBin::V2),
502                n => Err(IcechunkFormatErrorKind::InvalidSpecVersion {
503                    found: n,
504                    max_supported: Self::current() as u8,
505                }),
506            }
507        }
508    }
509
510    impl SpecVersionBin {
511        pub fn current() -> Self {
512            Default::default()
513        }
514    }
515
516    impl std::fmt::Display for SpecVersionBin {
517        fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
518            let s = match self {
519                SpecVersionBin::V1 => "1.0",
520                SpecVersionBin::V2 => "2.0",
521            };
522            write!(f, "{s}")
523        }
524    }
525
526    /// Compression algorithm used for metadata files.
527    #[repr(u8)]
528    #[derive(Debug, Clone, Copy, PartialEq, Eq)]
529    pub enum CompressionAlgorithmBin {
530        None = 0u8,
531        Zstd = 1u8,
532    }
533
534    impl TryFrom<u8> for CompressionAlgorithmBin {
535        type Error = String;
536
537        fn try_from(value: u8) -> Result<Self, Self::Error> {
538            match value {
539                n if n == CompressionAlgorithmBin::None as u8 => {
540                    Ok(CompressionAlgorithmBin::None)
541                }
542                n if n == CompressionAlgorithmBin::Zstd as u8 => {
543                    Ok(CompressionAlgorithmBin::Zstd)
544                }
545                n => Err(format!("Bad cmpression algorithm code: {n}")),
546            }
547        }
548    }
549
550    pub const ICECHUNK_FORMAT_MAGIC_BYTES: &[u8] = "ICE🧊CHUNK".as_bytes();
551    // offsets assume a 12-byte magic
552    const _: () = assert!(ICECHUNK_FORMAT_MAGIC_BYTES.len() == 12);
553
554    // Binary file header layout: magic | impl name | spec version | file type | compression.
555    // Reader (check_header) and writer (binary_file_header) both derive offsets from these.
556    pub const ICECHUNK_IMPL_NAME_LEN: usize = 24;
557    pub const ICECHUNK_SPEC_VERSION_OFFSET: usize =
558        ICECHUNK_FORMAT_MAGIC_BYTES.len() + ICECHUNK_IMPL_NAME_LEN;
559    pub const ICECHUNK_FILE_TYPE_OFFSET: usize = ICECHUNK_SPEC_VERSION_OFFSET + 1;
560    pub const ICECHUNK_COMPRESSION_OFFSET: usize = ICECHUNK_FILE_TYPE_OFFSET + 1;
561    pub const ICECHUNK_FILE_HEADER_LEN: usize = ICECHUNK_COMPRESSION_OFFSET + 1;
562
563    // Object-metadata keys use letters only. Azure needs a valid C# identifier, which
564    // bars `-`, and nginx-fronted S3 gateways drop header names that contain `_`.
565    // The `_DEPRECATED` keys name what repositories written before that change carry.
566    pub const LATEST_ICECHUNK_FORMAT_VERSION_METADATA_KEY: &str = "icspecver";
567    pub const LATEST_ICECHUNK_FORMAT_VERSION_METADATA_KEY_DEPRECATED: &str =
568        "ic_spec_ver";
569
570    // The Cargo spelling, so a prerelease stamps `ic-2.3.0-alpha.1`. The PyPI version
571    // of that same build is `2.3.0a1`, so compare the two parsed, never as strings.
572    pub const ICECHUNK_LIB_VERSION: &str = env!("CARGO_PKG_VERSION");
573
574    pub static ICECHUNK_CLIENT_NAME: LazyLock<String> =
575        LazyLock::new(|| "ic-".to_string() + ICECHUNK_LIB_VERSION);
576    pub const ICECHUNK_CLIENT_NAME_METADATA_KEY: &str = "icclient";
577    pub const ICECHUNK_CLIENT_NAME_METADATA_KEY_DEPRECATED: &str = "ic_client";
578
579    pub const ICECHUNK_FILE_TYPE_SNAPSHOT: &str = "snapshot";
580    pub const ICECHUNK_FILE_TYPE_MANIFEST: &str = "manifest";
581    pub const ICECHUNK_FILE_TYPE_TRANSACTION_LOG: &str = "transaction-log";
582    pub const ICECHUNK_FILE_TYPE_REPO_INFO: &str = "repo-info";
583    pub const ICECHUNK_FILE_TYPE_METADATA_KEY: &str = "icfiletype";
584    pub const ICECHUNK_FILE_TYPE_METADATA_KEY_DEPRECATED: &str = "ic_file_type";
585
586    pub const ICECHUNK_COMPRESSION_METADATA_KEY: &str = "iccompalg";
587    pub const ICECHUNK_COMPRESSION_METADATA_KEY_DEPRECATED: &str = "ic_comp_alg";
588    pub const ICECHUNK_COMPRESSION_ZSTD: &str = "zstd";
589
590    /// Decoded contents of a metadata file's binary header.
591    ///
592    /// Parsing of `impl_name` and `app_name` is best effort only.
593    #[derive(Debug, Clone, PartialEq, Eq)]
594    pub struct FileHeader {
595        /// Raw, trimmed implementation/client string, e.g. `"ic-2.1.0"`.
596        pub impl_name: String,
597        /// Best-effort app name: `impl_name` before the first `'-'` (e.g. `"ic"`).
598        pub app_name: String,
599        /// Best-effort version: `impl_name` after the first `'-'` (e.g. `"2.1.0"`);
600        /// `None` if there is no `'-'`.
601        pub app_version: Option<String>,
602        pub spec_version: SpecVersionBin,
603        pub file_type: FileTypeBin,
604        pub compression: CompressionAlgorithmBin,
605    }
606
607    /// Parse a file header from the first [`ICECHUNK_FILE_HEADER_LEN`] bytes of any
608    /// metadata file.
609    ///
610    /// Discovers (does not assert) the file type, so it works for every metadata
611    /// file kind. Tolerant about the impl-name string content.
612    pub fn parse_file_header(buf: &[u8]) -> Result<FileHeader, IcechunkFormatError> {
613        use IcechunkFormatErrorKind as K;
614
615        if buf.len() < ICECHUNK_FILE_HEADER_LEN {
616            return Err(K::InvalidIcechunkHeaderSize {
617                found: buf.len(),
618                expected: ICECHUNK_FILE_HEADER_LEN,
619            })
620            .capture();
621        }
622        if !buf.starts_with(ICECHUNK_FORMAT_MAGIC_BYTES) {
623            return Err(K::InvalidMagicNumbers).capture();
624        }
625
626        let impl_bytes =
627            &buf[ICECHUNK_FORMAT_MAGIC_BYTES.len()..ICECHUNK_SPEC_VERSION_OFFSET];
628        let impl_name =
629            String::from_utf8_lossy(impl_bytes).trim_end_matches([' ', '\0']).to_string();
630        let (app_name, app_version) = match impl_name.split_once('-') {
631            Some((name, version)) => (name.to_string(), Some(version.to_string())),
632            None => (impl_name.clone(), None),
633        };
634
635        let spec_version =
636            SpecVersionBin::try_from(buf[ICECHUNK_SPEC_VERSION_OFFSET]).capture()?;
637
638        let file_type_byte = buf[ICECHUNK_FILE_TYPE_OFFSET];
639        let file_type = FileTypeBin::try_from(file_type_byte)
640            .map_err(|_| K::UnknownFileType { got: file_type_byte })
641            .capture()?;
642
643        let compression =
644            CompressionAlgorithmBin::try_from(buf[ICECHUNK_COMPRESSION_OFFSET])
645                .map_err(|_| K::InvalidCompressionAlgorithm)
646                .capture()?;
647
648        Ok(FileHeader {
649            impl_name,
650            app_name,
651            app_version,
652            spec_version,
653            file_type,
654            compression,
655        })
656    }
657}
658
659#[inline(always)]
660#[expect(clippy::needless_pass_by_value)]
661pub fn lookup_index_by_key<'a, T: ::flatbuffers::Follow<'a> + 'a, K: Ord>(
662    v: ::flatbuffers::Vector<'a, T>,
663    key: K,
664    f: fn(&<T as ::flatbuffers::Follow<'a>>::Inner, &K) -> Ordering,
665) -> Option<usize> {
666    if v.is_empty() {
667        return None;
668    }
669
670    let mut left: usize = 0;
671    let mut right = v.len() - 1;
672
673    while left <= right {
674        let mid = (left + right) / 2;
675        let value = v.get(mid);
676        match f(&value, &key) {
677            Ordering::Equal => return Some(mid),
678            Ordering::Less => left = mid + 1,
679            Ordering::Greater => {
680                if mid == 0 {
681                    return None;
682                }
683                right = mid - 1;
684            }
685        }
686    }
687
688    None
689}
690
691// This macro is used for creating property tests
692// which check that serializing and deserializing
693// an instance of a type T is equivalent to the
694// identity function
695// Given pairs of test names and arbitraries to be used
696// for the tests, e.g., (n1, a1), (n2, a2),... (nx, ax)
697// the tests can be created by doing
698// roundtrip_serialization_tests!(n1 - a1, n2 - a2, .... nx - ax)
699#[macro_export]
700macro_rules! roundtrip_serialization_tests {
701    ($($test_name: ident - $generator: ident), +) => {
702        $(
703            proptest!{
704                #[icechunk_macros::test]
705                fn $test_name(elem in $generator()) {
706                    let bytes = rmp_serde::to_vec(&elem).unwrap();
707                    let roundtrip = rmp_serde::from_slice(&bytes).unwrap();
708                    assert_eq!(elem, roundtrip);
709                }
710            }
711        )*
712    }
713}
714
715#[cfg(test)]
716pub mod strategies;
717
718#[cfg(test)]
719mod tests {
720    use super::*;
721    use crate::strategies::{attributes_id, spec_version};
722    use pretty_assertions::assert_eq;
723    use proptest::prelude::*;
724
725    roundtrip_serialization_tests!(
726        serialize_and_deserialize_attribute_ids - attributes_id,
727        serialize_and_deserialize_spec_version_bin - spec_version
728    );
729
730    #[icechunk_macros::test]
731    fn test_object_id_serialization() {
732        let sid = SnapshotId::new([0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11]);
733        assert_eq!(
734            serde_json::to_string(&sid).unwrap(),
735            r#"[[0,1,2,3,4,5,6,7,8,9,10,11],null]"#
736        );
737        assert_eq!(String::from(&sid), "000G40R40M30E209185G");
738        assert_eq!(sid, SnapshotId::try_from("000G40R40M30E209185G").unwrap());
739        let sid = SnapshotId::random();
740        assert_eq!(
741            serde_json::from_slice::<SnapshotId>(
742                serde_json::to_vec(&sid).unwrap().as_slice()
743            )
744            .unwrap(),
745            sid,
746        );
747    }
748
749    #[icechunk_macros::test]
750    fn test_object_id_first_chars_are_every_possible_first_char() {
751        // The first character encodes the top 5 bits of the first byte.
752        let first_chars: HashSet<char> = (0..=u8::MAX)
753            .filter_map(|first_byte| {
754                let mut buf = [0u8; 12];
755                buf[0] = first_byte;
756                String::from(&SnapshotId::new(buf)).chars().next()
757            })
758            .collect();
759        assert_eq!(first_chars, *OBJECT_ID_FIRST_CHARS);
760    }
761
762    #[icechunk_macros::test]
763    fn two_char_prefixes_cover_the_alphabet() {
764        let p = &*OBJECT_ID_TWO_CHAR_PREFIXES;
765        assert_eq!(p.len(), 1024);
766        assert!(p.windows(2).all(|w| w[0] < w[1]), "sorted and distinct");
767        assert!(p.iter().all(
768            |s| s.len() == 2 && s.chars().all(|c| OBJECT_ID_FIRST_CHARS.contains(&c))
769        ));
770        // every random id starts with exactly one of them
771        for _ in 0..1000 {
772            let id = ChunkId::random().to_string();
773            assert_eq!(p.iter().filter(|s| id.starts_with(s.as_str())).count(), 1);
774        }
775    }
776
777    #[icechunk_macros::test]
778    fn test_unknown_spec_version_gives_nice_error() {
779        use format_constants::SpecVersionBin;
780
781        let future_version: u8 = 3;
782        let result = SpecVersionBin::try_from(future_version);
783        assert!(result.is_err());
784        let err = result.unwrap_err();
785
786        let msg = err.to_string();
787        assert!(
788            msg.contains("format version 3"),
789            "Error should mention the found version: {msg}"
790        );
791        assert!(
792            msg.contains("upgrade the icechunk library"),
793            "Error should suggest upgrading: {msg}"
794        );
795    }
796
797    /// Build a binary file header with an arbitrary impl string (right-padded to
798    /// `ICECHUNK_IMPL_NAME_LEN`, mirroring `binary_file_header` in the `icechunk`
799    /// crate), so tests can exercise `parse_file_header` directly.
800    fn make_header(impl_name: &str, spec: u8, file_type: u8, compression: u8) -> Vec<u8> {
801        use format_constants::*;
802        let mut buf = Vec::with_capacity(ICECHUNK_FILE_HEADER_LEN);
803        buf.extend_from_slice(ICECHUNK_FORMAT_MAGIC_BYTES);
804        let padded = format!("{impl_name:<ICECHUNK_IMPL_NAME_LEN$}");
805        buf.extend_from_slice(&padded.as_bytes()[..ICECHUNK_IMPL_NAME_LEN]);
806        buf.push(spec);
807        buf.push(file_type);
808        buf.push(compression);
809        buf
810    }
811
812    #[icechunk_macros::test]
813    fn test_parse_file_header_roundtrip() {
814        use format_constants::*;
815        let mut buf = make_header(
816            ICECHUNK_CLIENT_NAME.as_str(),
817            SpecVersionBin::V2 as u8,
818            FileTypeBin::Snapshot as u8,
819            CompressionAlgorithmBin::Zstd as u8,
820        );
821        // body bytes after the header must be ignored
822        buf.extend_from_slice(b"a compressed body would go here");
823
824        let header = parse_file_header(&buf).unwrap();
825        assert_eq!(header.impl_name, *ICECHUNK_CLIENT_NAME);
826        assert_eq!(header.app_name, "ic");
827        assert_eq!(header.app_version.as_deref(), Some(ICECHUNK_LIB_VERSION));
828        assert_eq!(header.spec_version, SpecVersionBin::V2);
829        assert_eq!(header.file_type, FileTypeBin::Snapshot);
830        assert_eq!(header.compression, CompressionAlgorithmBin::Zstd);
831    }
832
833    #[icechunk_macros::test]
834    fn test_parse_file_header_trims_padding() {
835        use format_constants::*;
836        // exactly the header, no body: the 24-byte impl field is space-padded
837        let buf = make_header(
838            "ic-9.9.9",
839            SpecVersionBin::V1 as u8,
840            FileTypeBin::Manifest as u8,
841            CompressionAlgorithmBin::None as u8,
842        );
843        let header = parse_file_header(&buf).unwrap();
844        assert_eq!(header.impl_name, "ic-9.9.9");
845        assert_eq!(header.app_name, "ic");
846        assert_eq!(header.app_version.as_deref(), Some("9.9.9"));
847        assert_eq!(header.spec_version, SpecVersionBin::V1);
848        assert_eq!(header.file_type, FileTypeBin::Manifest);
849        assert_eq!(header.compression, CompressionAlgorithmBin::None);
850    }
851
852    #[icechunk_macros::test]
853    fn test_parse_file_header_splits_on_first_dash() {
854        use format_constants::*;
855        let buf = make_header(
856            "ic-2.0.0-alpha.4",
857            SpecVersionBin::V2 as u8,
858            FileTypeBin::RepoInfo as u8,
859            CompressionAlgorithmBin::Zstd as u8,
860        );
861        let header = parse_file_header(&buf).unwrap();
862        assert_eq!(header.impl_name, "ic-2.0.0-alpha.4");
863        assert_eq!(header.app_name, "ic");
864        assert_eq!(header.app_version.as_deref(), Some("2.0.0-alpha.4"));
865    }
866
867    #[icechunk_macros::test]
868    fn test_parse_file_header_no_dash_fallback() {
869        use format_constants::*;
870        let buf = make_header(
871            "icjs",
872            SpecVersionBin::V2 as u8,
873            FileTypeBin::TransactionLog as u8,
874            CompressionAlgorithmBin::Zstd as u8,
875        );
876        let header = parse_file_header(&buf).unwrap();
877        assert_eq!(header.impl_name, "icjs");
878        assert_eq!(header.app_name, "icjs");
879        assert_eq!(header.app_version, None);
880    }
881
882    #[icechunk_macros::test]
883    fn test_parse_file_header_bad_magic() {
884        use format_constants::*;
885        let mut buf = make_header(
886            "ic-1.0.0",
887            SpecVersionBin::V2 as u8,
888            FileTypeBin::Snapshot as u8,
889            CompressionAlgorithmBin::Zstd as u8,
890        );
891        buf[0] = b'X';
892        let err = parse_file_header(&buf).unwrap_err();
893        assert!(matches!(err.kind(), IcechunkFormatErrorKind::InvalidMagicNumbers));
894    }
895
896    #[icechunk_macros::test]
897    fn test_parse_file_header_too_short() {
898        use format_constants::*;
899        let buf = vec![0u8; ICECHUNK_FILE_HEADER_LEN - 1];
900        let err = parse_file_header(&buf).unwrap_err();
901        assert!(matches!(
902            err.kind(),
903            IcechunkFormatErrorKind::InvalidIcechunkHeaderSize { .. }
904        ));
905    }
906
907    #[icechunk_macros::test]
908    fn test_parse_file_header_unknown_file_type() {
909        use format_constants::*;
910        let buf = make_header(
911            "ic-1.0.0",
912            SpecVersionBin::V2 as u8,
913            99,
914            CompressionAlgorithmBin::Zstd as u8,
915        );
916        let err = parse_file_header(&buf).unwrap_err();
917        assert!(matches!(
918            err.kind(),
919            IcechunkFormatErrorKind::UnknownFileType { got: 99 }
920        ));
921    }
922
923    #[icechunk_macros::test]
924    fn test_parse_file_header_bad_spec_version() {
925        use format_constants::*;
926        let buf = make_header(
927            "ic-1.0.0",
928            99,
929            FileTypeBin::Snapshot as u8,
930            CompressionAlgorithmBin::Zstd as u8,
931        );
932        let err = parse_file_header(&buf).unwrap_err();
933        assert!(matches!(
934            err.kind(),
935            IcechunkFormatErrorKind::InvalidSpecVersion { found: 99, .. }
936        ));
937    }
938
939    #[icechunk_macros::test]
940    fn test_object_metadata_keys_are_portable() {
941        use format_constants::*;
942        for key in [
943            LATEST_ICECHUNK_FORMAT_VERSION_METADATA_KEY,
944            ICECHUNK_CLIENT_NAME_METADATA_KEY,
945            ICECHUNK_FILE_TYPE_METADATA_KEY,
946            ICECHUNK_COMPRESSION_METADATA_KEY,
947        ] {
948            assert!(
949                !key.is_empty()
950                    && key.starts_with(|c: char| c.is_ascii_alphabetic())
951                    && key.chars().all(|c| c.is_ascii_alphanumeric()),
952                "{key} must be ASCII alphanumeric and start with a letter: \
953                 Azure rejects `-` and nginx-fronted S3 gateways drop `_`"
954            );
955        }
956    }
957}
icechunk_format/lib.rs

icechunk_format/
lib.rs
