icechunk_format/
lib.rs

1//! Icechunk data types and serialization.
2//!
3//! Defines ID types ([`SnapshotId`], [`ManifestId`], [`ChunkId`], [`NodeId`]),
4//! [`Path`] for normalized Zarr paths, and [`ByteRange`]/[`ChunkIndices`] for
5//! chunk addressing. Submodules define snapshots, manifests, and transaction logs.
6
7use core::fmt;
8use std::{
9    cmp::Ordering,
10    collections::HashSet,
11    convert::Infallible,
12    fmt::{Debug, Display},
13    hash::Hash,
14    marker::PhantomData,
15    ops::Range,
16    sync::LazyLock,
17};
18
19use ::flatbuffers::InvalidFlatbuffer;
20use bytes::Bytes;
21use chrono::{DateTime, Utc};
22use flatbuffers::generated;
23use format_constants::FileTypeBin;
24use manifest::VirtualReferenceErrorKind;
25use rand::{RngExt as _, rng};
26use serde::{Deserialize, Serialize};
27use thiserror::Error;
28
29use icechunk_types::error::ICError;
30use icechunk_types::sealed;
31
32/// User attributes stored on arrays and groups.
33pub mod attributes;
34/// Chunk reference tables.
35pub mod manifest;
36
37/// Generated Flatbuffer types for binary serialization.
38#[path = "./flatbuffers/all_generated.rs"]
39#[allow(clippy::all, warnings)]
40pub mod flatbuffers;
41
42/// Repository metadata (version, properties).
43pub mod repo_info;
44/// Flatbuffer serialization.
45pub mod serializers;
46/// Repository state at a point in time.
47pub mod snapshot;
48/// Change records for commits.
49pub mod transaction_log;
50
51pub const CONFIG_FILE_PATH: &str = "config.yaml";
52pub const REPO_INFO_FILE_PATH: &str = "repo";
53pub const CHUNKS_FILE_PATH: &str = "chunks";
54pub const MANIFESTS_FILE_PATH: &str = "manifests";
55pub const SNAPSHOTS_FILE_PATH: &str = "snapshots";
56pub const TRANSACTION_LOGS_FILE_PATH: &str = "transactions";
57pub const OVERWRITTEN_FILES_PATH: &str = "overwritten";
58pub const V1_REFS_FILE_PATH: &str = "refs";
59
60pub use icechunk_types::{Path, PathError};
61
62/// Marker trait for object ID type tags (sealed).
63pub trait FileTypeTag: sealed::Sealed {}
64
65/// The id of a file in object store
66#[derive(Hash, Clone, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize)]
67pub struct ObjectId<const SIZE: usize, T: FileTypeTag>(
68    #[serde(with = "serde_bytes")] pub [u8; SIZE],
69    PhantomData<T>,
70);
71
72/// Type tag for [`SnapshotId`].
73#[derive(Debug, Hash, Clone, PartialEq, Eq, PartialOrd, Ord)]
74pub struct SnapshotTag;
75
76/// Type tag for [`ManifestId`].
77#[derive(Debug, Hash, Clone, PartialEq, Eq, PartialOrd, Ord)]
78pub struct ManifestTag;
79
80/// Type tag for [`ChunkId`].
81#[derive(Debug, Hash, Clone, PartialEq, Eq, PartialOrd, Ord)]
82pub struct ChunkTag;
83
84/// Type tag for [`AttributesId`].
85#[derive(Debug, Hash, Clone, PartialEq, Eq, PartialOrd, Ord)]
86pub struct AttributesTag;
87
88/// Type tag for [`NodeId`].
89#[derive(Debug, Hash, Clone, PartialEq, Eq, PartialOrd, Ord)]
90pub struct NodeTag;
91
92impl sealed::Sealed for SnapshotTag {}
93impl sealed::Sealed for ManifestTag {}
94impl sealed::Sealed for ChunkTag {}
95impl sealed::Sealed for AttributesTag {}
96impl sealed::Sealed for NodeTag {}
97impl FileTypeTag for SnapshotTag {}
98impl FileTypeTag for ManifestTag {}
99impl FileTypeTag for ChunkTag {}
100impl FileTypeTag for AttributesTag {}
101impl FileTypeTag for NodeTag {}
102
103// A 1e-9 conflict probability requires 2^33 ~ 8.5 bn chunks
104// using this site for the calculations: https://www.bdayprob.com/
105
106/// Unique identifier for a snapshot.
107pub type SnapshotId = ObjectId<12, SnapshotTag>;
108/// Unique identifier for a manifest file.
109pub type ManifestId = ObjectId<12, ManifestTag>;
110/// Unique identifier for a chunk.
111pub type ChunkId = ObjectId<12, ChunkTag>;
112/// Unique identifier for an attributes blob.
113pub type AttributesId = ObjectId<12, AttributesTag>;
114
115// A 1e-9 conflict probability requires 2^17.55 ~ 200k nodes
116/// The internal id of an array or group, unique only to a single store version
117pub type NodeId = ObjectId<8, NodeTag>;
118
119#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, Hash)]
120pub enum NodeType {
121    Group,
122    Array,
123}
124
125#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, Hash)]
126pub struct Move {
127    pub from: Path,
128    pub to: Path,
129    pub node_id: NodeId,
130    pub node_type: NodeType,
131}
132
133impl From<NodeType> for generated::NodeType {
134    fn from(value: NodeType) -> Self {
135        match value {
136            NodeType::Group => generated::NodeType::Group,
137            NodeType::Array => generated::NodeType::Array,
138        }
139    }
140}
141
142impl NodeType {
143    fn max_supported() -> u8 {
144        generated::NodeType::ENUM_MAX
145    }
146}
147
148impl TryFrom<generated::NodeType> for NodeType {
149    type Error = IcechunkFormatErrorKind;
150
151    fn try_from(value: generated::NodeType) -> Result<Self, Self::Error> {
152        match value {
153            generated::NodeType::Group => Ok(NodeType::Group),
154            generated::NodeType::Array => Ok(NodeType::Array),
155            generated::NodeType(v) => Err(IcechunkFormatErrorKind::InvalidNodeType {
156                found: v,
157                max_supported: Self::max_supported(),
158            }),
159        }
160    }
161}
162
163impl<const SIZE: usize, T: FileTypeTag> ObjectId<SIZE, T> {
164    pub fn random() -> Self {
165        let mut buf = [0u8; SIZE];
166        rng().fill(&mut buf[..]);
167        Self(buf, PhantomData)
168    }
169
170    pub const fn new(buf: [u8; SIZE]) -> Self {
171        Self(buf, PhantomData)
172    }
173
174    pub const FAKE: Self = Self([0; SIZE], PhantomData);
175}
176
177impl<const SIZE: usize, T: FileTypeTag> Debug for ObjectId<SIZE, T> {
178    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
179        write!(f, "{}", String::from(self))
180    }
181}
182
183impl<const SIZE: usize, T: FileTypeTag> TryFrom<&[u8]> for ObjectId<SIZE, T> {
184    type Error = &'static str;
185
186    fn try_from(value: &[u8]) -> Result<Self, Self::Error> {
187        let buf = value.try_into();
188        buf.map(|buf| ObjectId(buf, PhantomData))
189            .map_err(|_| "Invalid ObjectId buffer length")
190    }
191}
192
193impl<const SIZE: usize, T: FileTypeTag> TryFrom<&str> for ObjectId<SIZE, T> {
194    type Error = &'static str;
195
196    fn try_from(value: &str) -> Result<Self, Self::Error> {
197        let bytes = base32::decode(base32::Alphabet::Crockford, value);
198        let Some(bytes) = bytes else { return Err("Invalid ObjectId string") };
199        Self::try_from(bytes.as_slice())
200    }
201}
202
203impl<const SIZE: usize, T: FileTypeTag> From<&ObjectId<SIZE, T>> for String {
204    fn from(value: &ObjectId<SIZE, T>) -> Self {
205        base32::encode(base32::Alphabet::Crockford, &value.0)
206    }
207}
208
209/// The Crockford base32 alphabet, which `base32` keeps private. Every [`ObjectId`]
210/// starts with one of these, so a listing split on them covers every id.
211pub static OBJECT_ID_FIRST_CHARS: LazyLock<HashSet<char>> =
212    LazyLock::new(|| "0123456789ABCDEFGHJKMNPQRSTVWXYZ".chars().collect());
213
214impl<const SIZE: usize, T: FileTypeTag> From<[u8; SIZE]> for ObjectId<SIZE, T> {
215    fn from(value: [u8; SIZE]) -> Self {
216        ObjectId::new(value)
217    }
218}
219
220impl<const SIZE: usize, T: FileTypeTag> TryInto<String> for ObjectId<SIZE, T> {
221    type Error = Infallible;
222
223    fn try_into(self) -> Result<String, Self::Error> {
224        Ok(self.to_string())
225    }
226}
227
228impl<const SIZE: usize, T: FileTypeTag> TryInto<ObjectId<SIZE, T>> for String {
229    type Error = &'static str;
230
231    fn try_into(self) -> Result<ObjectId<SIZE, T>, Self::Error> {
232        self.as_str().try_into()
233    }
234}
235
236impl<const SIZE: usize, T: FileTypeTag> Display for ObjectId<SIZE, T> {
237    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
238        write!(f, "{}", String::from(self))
239    }
240}
241
242/// An ND index to an element in a chunk grid.
243#[derive(Clone, Debug, Hash, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize)]
244pub struct ChunkIndices(pub Vec<u32>);
245
246/// Byte offset within a chunk.
247pub type ChunkOffset = u64;
248/// Size of a chunk in bytes.
249pub type ChunkLength = u64;
250
251impl<'a> From<generated::ChunkIndices<'a>> for ChunkIndices {
252    fn from(value: generated::ChunkIndices<'a>) -> Self {
253        ChunkIndices(value.coords().iter().collect())
254    }
255}
256
257/// A byte range within a chunk or object.
258#[derive(Debug, Clone, PartialEq, Eq, Hash)]
259pub enum ByteRange {
260    /// The fixed length range represented by the given `Range`
261    Bounded(Range<ChunkOffset>),
262    /// All bytes from the given offset (included) to the end of the object
263    From(ChunkOffset),
264    /// The last n bytes in the object
265    Last(ChunkLength),
266    /// All bytes up to the last n bytes in the object
267    Until(ChunkOffset),
268}
269
270impl From<Range<ChunkOffset>> for ByteRange {
271    fn from(value: Range<ChunkOffset>) -> Self {
272        ByteRange::Bounded(value)
273    }
274}
275
276impl ByteRange {
277    pub fn from_offset(offset: ChunkOffset) -> Self {
278        Self::From(offset)
279    }
280
281    pub fn from_offset_with_length(offset: ChunkOffset, length: ChunkOffset) -> Self {
282        Self::Bounded(offset..offset + length)
283    }
284
285    pub fn to_offset(offset: ChunkOffset) -> Self {
286        Self::Bounded(0..offset)
287    }
288
289    pub fn bounded(start: ChunkOffset, end: ChunkOffset) -> Self {
290        (start..end).into()
291    }
292
293    pub const ALL: Self = Self::From(0);
294
295    pub fn slice(&self, bytes: &Bytes) -> Bytes {
296        match self {
297            ByteRange::Bounded(range) => {
298                bytes.slice(range.start as usize..range.end as usize)
299            }
300            ByteRange::From(from) => bytes.slice(*from as usize..),
301            ByteRange::Last(n) => bytes.slice(bytes.len() - *n as usize..),
302            ByteRange::Until(n) => bytes.slice(0usize..bytes.len() - *n as usize),
303        }
304    }
305}
306
307impl From<(Option<ChunkOffset>, Option<ChunkOffset>)> for ByteRange {
308    fn from((start, end): (Option<ChunkOffset>, Option<ChunkOffset>)) -> Self {
309        match (start, end) {
310            (Some(start), Some(end)) => Self::Bounded(start..end),
311            (Some(start), None) => Self::From(start),
312            // NOTE: This is relied upon by zarr python
313            (None, Some(end)) => Self::Until(end),
314            (None, None) => Self::ALL,
315        }
316    }
317}
318
319/// Offset within a manifest table.
320pub type TableOffset = u32;
321
322/// Format-level error types.
323#[derive(Debug, Error)]
324#[non_exhaustive]
325pub enum IcechunkFormatErrorKind {
326    #[error(transparent)]
327    VirtualReferenceError(#[from] VirtualReferenceErrorKind),
328    #[error("node not found at `{path:?}`")]
329    NodeNotFound { path: Path },
330    #[error("chunk coordinates not found `{coords:?}`")]
331    ChunkCoordinatesNotFound { coords: ChunkIndices },
332    #[error("snapshot id not found `{snapshot_id}`")]
333    SnapshotIdNotFound { snapshot_id: SnapshotId },
334    #[error("branch already exists `{branch} -> {snapshot_id}`")]
335    BranchAlreadyExists { branch: String, snapshot_id: SnapshotId },
336    #[error("branch not found `{branch}`")]
337    BranchNotFound { branch: String },
338    #[error("tag already exists `{tag}`")]
339    TagAlreadyExists { tag: String },
340    #[error("icechunk does not allow tag reuse and tag was already deleted `{tag}`")]
341    TagPreviouslyDeleted { tag: String },
342    #[error("tag not found `{tag}`")]
343    TagNotFound { tag: String },
344    #[error("snapshot id is already present in the repository: `{snapshot_id}`")]
345    DuplicateSnapshotId { snapshot_id: SnapshotId },
346    #[error("manifest information cannot be found in snapshot for id `{manifest_id}`")]
347    ManifestInfoNotFound { manifest_id: ManifestId },
348    #[error("invalid magic numbers in file")]
349    InvalidMagicNumbers, // TODO: add more info
350    #[error(
351        "invalid icechunk header size, got {found} bytes, expected at least {expected}"
352    )]
353    InvalidIcechunkHeaderSize { found: usize, expected: usize },
354    #[error("invalid node type {found}, max supported value is {max_supported}")]
355    InvalidNodeType { found: u8, max_supported: u8 },
356    #[error(
357        "this repository uses Icechunk format version {found}, but this library only supports up to version {max_supported}. Please upgrade the icechunk library"
358    )]
359    InvalidSpecVersion { found: u8, max_supported: u8 },
360    #[error("this operation is not supported for Icechunk format version {version}")]
361    UnsupportedOperationForVersion { version: u8 },
362    #[error("Icechunk cannot read this file type, expected {expected:?} got {got}")]
363    InvalidFileType { expected: FileTypeBin, got: u8 }, // TODO: add more info
364    #[error("Icechunk encountered an unknown file type code {got}")]
365    UnknownFileType { got: u8 },
366    #[error("Icechunk cannot read file, invalid compression algorithm")]
367    InvalidCompressionAlgorithm, // TODO: add more info
368    #[error("Invalid Icechunk metadata file")]
369    InvalidFlatBuffer(#[from] InvalidFlatbuffer),
370    #[error("error during metadata deserialization")]
371    DeserializationError(#[from] Box<rmp_serde::decode::Error>),
372    #[error("error during metadata serialization")]
373    SerializationError(#[from] Box<rmp_serde::encode::Error>),
374    #[error("error during metadata serialization")]
375    SerializationErrorFlexBuffers(#[from] Box<flexbuffers::SerializationError>),
376    #[error("error during metadata deserialization")]
377    DeserializationErrorFlexBuffers(#[from] Box<flexbuffers::DeserializationError>),
378    #[error("I/O error")]
379    IO(#[from] std::io::Error),
380    #[error("path error")]
381    Path(#[from] PathError),
382    #[error("invalid timestamp in file")]
383    InvalidTimestamp,
384    #[error(
385        "update timestamp is invalid, please verify if the machine clock has drifted: update time: `{new_time}`, latest update time: `{latest_time}`"
386    )]
387    InvalidUpdateTimestamp { latest_time: DateTime<Utc>, new_time: DateTime<Utc> },
388    #[error("invalid feature flag name: {name}")]
389    InvalidFeatureFlagName { name: String },
390    #[error("invalid feature flag id: {id}")]
391    InvalidFeatureFlagId { id: u16 },
392    #[error("{feature_description} is disabled by a feature flag ({feature_flag})")]
393    FeatureFlagDisabled { feature_description: String, feature_flag: String },
394    #[error(
395        "compressed chunk location present but no decompression dictionary available"
396    )]
397    MissingLocationCompressionDictionary,
398    #[error("Invalid array metadata: {0}")]
399    InvalidArrayMetadata(String),
400    #[error("Move operation missing required field '{0}'")]
401    MissingRequiredField(String),
402}
403
404pub type IcechunkFormatError = ICError<IcechunkFormatErrorKind>;
405
406impl From<Infallible> for IcechunkFormatErrorKind {
407    fn from(value: Infallible) -> Self {
408        match value {}
409    }
410}
411
412pub type IcechunkResult<T> = Result<T, IcechunkFormatError>;
413
414/// Binary format constants (file types, spec versions, compression).
415pub mod format_constants {
416    use std::sync::LazyLock;
417
418    use icechunk_types::ICResultExt as _;
419    use serde::{Deserialize, Serialize};
420
421    use super::{IcechunkFormatError, IcechunkFormatErrorKind};
422
423    /// Binary file type identifier in the file header.
424    #[repr(u8)]
425    #[derive(Debug, Clone, Copy, PartialEq, Eq)]
426    pub enum FileTypeBin {
427        Snapshot = 1u8,
428        Manifest = 2,
429        Attributes = 3,
430        TransactionLog = 4,
431        Chunk = 5,
432        RepoInfo = 6,
433    }
434
435    impl TryFrom<u8> for FileTypeBin {
436        type Error = String;
437
438        fn try_from(value: u8) -> Result<Self, Self::Error> {
439            match value {
440                n if n == FileTypeBin::Snapshot as u8 => Ok(FileTypeBin::Snapshot),
441                n if n == FileTypeBin::Manifest as u8 => Ok(FileTypeBin::Manifest),
442                n if n == FileTypeBin::Attributes as u8 => Ok(FileTypeBin::Attributes),
443                n if n == FileTypeBin::RepoInfo as u8 => Ok(FileTypeBin::RepoInfo),
444                n if n == FileTypeBin::TransactionLog as u8 => {
445                    Ok(FileTypeBin::TransactionLog)
446                }
447                n if n == FileTypeBin::Chunk as u8 => Ok(FileTypeBin::Chunk),
448                n => Err(format!("Bad file type code: {n}")),
449            }
450        }
451    }
452
453    /// Icechunk format specification version.
454    #[repr(u8)]
455    #[derive(
456        Debug,
457        Clone,
458        Copy,
459        PartialEq,
460        Eq,
461        Serialize,
462        Deserialize,
463        Default,
464        PartialOrd,
465        Ord,
466    )]
467    pub enum SpecVersionBin {
468        V1 = 1u8,
469        #[default]
470        V2 = 2u8,
471        // When adding new versions here, don't forget to update the
472        // PySpecVersion enum in icechunk-python/src/repository.rs too!
473    }
474
475    impl TryFrom<u8> for SpecVersionBin {
476        type Error = IcechunkFormatErrorKind;
477
478        fn try_from(value: u8) -> Result<Self, Self::Error> {
479            match value {
480                n if n == SpecVersionBin::V1 as u8 => Ok(SpecVersionBin::V1),
481                n if n == SpecVersionBin::V2 as u8 => Ok(SpecVersionBin::V2),
482                n => Err(IcechunkFormatErrorKind::InvalidSpecVersion {
483                    found: n,
484                    max_supported: Self::current() as u8,
485                }),
486            }
487        }
488    }
489
490    impl SpecVersionBin {
491        pub fn current() -> Self {
492            Default::default()
493        }
494    }
495
496    impl std::fmt::Display for SpecVersionBin {
497        fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
498            let s = match self {
499                SpecVersionBin::V1 => "1.0",
500                SpecVersionBin::V2 => "2.0",
501            };
502            write!(f, "{s}")
503        }
504    }
505
506    /// Compression algorithm used for metadata files.
507    #[repr(u8)]
508    #[derive(Debug, Clone, Copy, PartialEq, Eq)]
509    pub enum CompressionAlgorithmBin {
510        None = 0u8,
511        Zstd = 1u8,
512    }
513
514    impl TryFrom<u8> for CompressionAlgorithmBin {
515        type Error = String;
516
517        fn try_from(value: u8) -> Result<Self, Self::Error> {
518            match value {
519                n if n == CompressionAlgorithmBin::None as u8 => {
520                    Ok(CompressionAlgorithmBin::None)
521                }
522                n if n == CompressionAlgorithmBin::Zstd as u8 => {
523                    Ok(CompressionAlgorithmBin::Zstd)
524                }
525                n => Err(format!("Bad cmpression algorithm code: {n}")),
526            }
527        }
528    }
529
530    pub const ICECHUNK_FORMAT_MAGIC_BYTES: &[u8] = "ICE🧊CHUNK".as_bytes();
531    // offsets assume a 12-byte magic
532    const _: () = assert!(ICECHUNK_FORMAT_MAGIC_BYTES.len() == 12);
533
534    // Binary file header layout: magic | impl name | spec version | file type | compression.
535    // Reader (check_header) and writer (binary_file_header) both derive offsets from these.
536    pub const ICECHUNK_IMPL_NAME_LEN: usize = 24;
537    pub const ICECHUNK_SPEC_VERSION_OFFSET: usize =
538        ICECHUNK_FORMAT_MAGIC_BYTES.len() + ICECHUNK_IMPL_NAME_LEN;
539    pub const ICECHUNK_FILE_TYPE_OFFSET: usize = ICECHUNK_SPEC_VERSION_OFFSET + 1;
540    pub const ICECHUNK_COMPRESSION_OFFSET: usize = ICECHUNK_FILE_TYPE_OFFSET + 1;
541    pub const ICECHUNK_FILE_HEADER_LEN: usize = ICECHUNK_COMPRESSION_OFFSET + 1;
542
543    // Object-metadata keys use letters only. Azure needs a valid C# identifier, which
544    // bars `-`, and nginx-fronted S3 gateways drop header names that contain `_`.
545    // The `_DEPRECATED` keys name what repositories written before that change carry.
546    pub const LATEST_ICECHUNK_FORMAT_VERSION_METADATA_KEY: &str = "icspecver";
547    pub const LATEST_ICECHUNK_FORMAT_VERSION_METADATA_KEY_DEPRECATED: &str =
548        "ic_spec_ver";
549
550    pub const ICECHUNK_LIB_VERSION: &str = env!("CARGO_PKG_VERSION");
551
552    pub static ICECHUNK_CLIENT_NAME: LazyLock<String> =
553        LazyLock::new(|| "ic-".to_string() + ICECHUNK_LIB_VERSION);
554    pub const ICECHUNK_CLIENT_NAME_METADATA_KEY: &str = "icclient";
555    pub const ICECHUNK_CLIENT_NAME_METADATA_KEY_DEPRECATED: &str = "ic_client";
556
557    pub const ICECHUNK_FILE_TYPE_SNAPSHOT: &str = "snapshot";
558    pub const ICECHUNK_FILE_TYPE_MANIFEST: &str = "manifest";
559    pub const ICECHUNK_FILE_TYPE_TRANSACTION_LOG: &str = "transaction-log";
560    pub const ICECHUNK_FILE_TYPE_REPO_INFO: &str = "repo-info";
561    pub const ICECHUNK_FILE_TYPE_METADATA_KEY: &str = "icfiletype";
562    pub const ICECHUNK_FILE_TYPE_METADATA_KEY_DEPRECATED: &str = "ic_file_type";
563
564    pub const ICECHUNK_COMPRESSION_METADATA_KEY: &str = "iccompalg";
565    pub const ICECHUNK_COMPRESSION_METADATA_KEY_DEPRECATED: &str = "ic_comp_alg";
566    pub const ICECHUNK_COMPRESSION_ZSTD: &str = "zstd";
567
568    /// Decoded contents of a metadata file's binary header.
569    ///
570    /// Parsing of `impl_name` and `app_name` is best effort only.
571    #[derive(Debug, Clone, PartialEq, Eq)]
572    pub struct FileHeader {
573        /// Raw, trimmed implementation/client string, e.g. `"ic-2.1.0"`.
574        pub impl_name: String,
575        /// Best-effort app name: `impl_name` before the first `'-'` (e.g. `"ic"`).
576        pub app_name: String,
577        /// Best-effort version: `impl_name` after the first `'-'` (e.g. `"2.1.0"`);
578        /// `None` if there is no `'-'`.
579        pub app_version: Option<String>,
580        pub spec_version: SpecVersionBin,
581        pub file_type: FileTypeBin,
582        pub compression: CompressionAlgorithmBin,
583    }
584
585    /// Parse a file header from the first [`ICECHUNK_FILE_HEADER_LEN`] bytes of any
586    /// metadata file.
587    ///
588    /// Discovers (does not assert) the file type, so it works for every metadata
589    /// file kind. Tolerant about the impl-name string content.
590    pub fn parse_file_header(buf: &[u8]) -> Result<FileHeader, IcechunkFormatError> {
591        use IcechunkFormatErrorKind as K;
592
593        if buf.len() < ICECHUNK_FILE_HEADER_LEN {
594            return Err(K::InvalidIcechunkHeaderSize {
595                found: buf.len(),
596                expected: ICECHUNK_FILE_HEADER_LEN,
597            })
598            .capture();
599        }
600        if !buf.starts_with(ICECHUNK_FORMAT_MAGIC_BYTES) {
601            return Err(K::InvalidMagicNumbers).capture();
602        }
603
604        let impl_bytes =
605            &buf[ICECHUNK_FORMAT_MAGIC_BYTES.len()..ICECHUNK_SPEC_VERSION_OFFSET];
606        let impl_name =
607            String::from_utf8_lossy(impl_bytes).trim_end_matches([' ', '\0']).to_string();
608        let (app_name, app_version) = match impl_name.split_once('-') {
609            Some((name, version)) => (name.to_string(), Some(version.to_string())),
610            None => (impl_name.clone(), None),
611        };
612
613        let spec_version =
614            SpecVersionBin::try_from(buf[ICECHUNK_SPEC_VERSION_OFFSET]).capture()?;
615
616        let file_type_byte = buf[ICECHUNK_FILE_TYPE_OFFSET];
617        let file_type = FileTypeBin::try_from(file_type_byte)
618            .map_err(|_| K::UnknownFileType { got: file_type_byte })
619            .capture()?;
620
621        let compression =
622            CompressionAlgorithmBin::try_from(buf[ICECHUNK_COMPRESSION_OFFSET])
623                .map_err(|_| K::InvalidCompressionAlgorithm)
624                .capture()?;
625
626        Ok(FileHeader {
627            impl_name,
628            app_name,
629            app_version,
630            spec_version,
631            file_type,
632            compression,
633        })
634    }
635}
636
637#[inline(always)]
638#[expect(clippy::needless_pass_by_value)]
639pub fn lookup_index_by_key<'a, T: ::flatbuffers::Follow<'a> + 'a, K: Ord>(
640    v: ::flatbuffers::Vector<'a, T>,
641    key: K,
642    f: fn(&<T as ::flatbuffers::Follow<'a>>::Inner, &K) -> Ordering,
643) -> Option<usize> {
644    if v.is_empty() {
645        return None;
646    }
647
648    let mut left: usize = 0;
649    let mut right = v.len() - 1;
650
651    while left <= right {
652        let mid = (left + right) / 2;
653        let value = v.get(mid);
654        match f(&value, &key) {
655            Ordering::Equal => return Some(mid),
656            Ordering::Less => left = mid + 1,
657            Ordering::Greater => {
658                if mid == 0 {
659                    return None;
660                }
661                right = mid - 1;
662            }
663        }
664    }
665
666    None
667}
668
669// This macro is used for creating property tests
670// which check that serializing and deserializing
671// an instance of a type T is equivalent to the
672// identity function
673// Given pairs of test names and arbitraries to be used
674// for the tests, e.g., (n1, a1), (n2, a2),... (nx, ax)
675// the tests can be created by doing
676// roundtrip_serialization_tests!(n1 - a1, n2 - a2, .... nx - ax)
677#[macro_export]
678macro_rules! roundtrip_serialization_tests {
679    ($($test_name: ident - $generator: ident), +) => {
680        $(
681            proptest!{
682                #[icechunk_macros::test]
683                fn $test_name(elem in $generator()) {
684                    let bytes = rmp_serde::to_vec(&elem).unwrap();
685                    let roundtrip = rmp_serde::from_slice(&bytes).unwrap();
686                    assert_eq!(elem, roundtrip);
687                }
688            }
689        )*
690    }
691}
692
693#[cfg(test)]
694pub mod strategies;
695
696#[cfg(test)]
697mod tests {
698    use super::*;
699    use crate::strategies::{attributes_id, spec_version};
700    use pretty_assertions::assert_eq;
701    use proptest::prelude::*;
702
703    roundtrip_serialization_tests!(
704        serialize_and_deserialize_attribute_ids - attributes_id,
705        serialize_and_deserialize_spec_version_bin - spec_version
706    );
707
708    #[icechunk_macros::test]
709    fn test_object_id_serialization() {
710        let sid = SnapshotId::new([0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11]);
711        assert_eq!(
712            serde_json::to_string(&sid).unwrap(),
713            r#"[[0,1,2,3,4,5,6,7,8,9,10,11],null]"#
714        );
715        assert_eq!(String::from(&sid), "000G40R40M30E209185G");
716        assert_eq!(sid, SnapshotId::try_from("000G40R40M30E209185G").unwrap());
717        let sid = SnapshotId::random();
718        assert_eq!(
719            serde_json::from_slice::<SnapshotId>(
720                serde_json::to_vec(&sid).unwrap().as_slice()
721            )
722            .unwrap(),
723            sid,
724        );
725    }
726
727    #[icechunk_macros::test]
728    fn test_object_id_first_chars_are_every_possible_first_char() {
729        // The first character encodes the top 5 bits of the first byte.
730        let first_chars: HashSet<char> = (0..=u8::MAX)
731            .filter_map(|first_byte| {
732                let mut buf = [0u8; 12];
733                buf[0] = first_byte;
734                String::from(&SnapshotId::new(buf)).chars().next()
735            })
736            .collect();
737        assert_eq!(first_chars, *OBJECT_ID_FIRST_CHARS);
738    }
739
740    #[icechunk_macros::test]
741    fn test_unknown_spec_version_gives_nice_error() {
742        use format_constants::SpecVersionBin;
743
744        let future_version: u8 = 3;
745        let result = SpecVersionBin::try_from(future_version);
746        assert!(result.is_err());
747        let err = result.unwrap_err();
748
749        let msg = err.to_string();
750        assert!(
751            msg.contains("format version 3"),
752            "Error should mention the found version: {msg}"
753        );
754        assert!(
755            msg.contains("upgrade the icechunk library"),
756            "Error should suggest upgrading: {msg}"
757        );
758    }
759
760    /// Build a binary file header with an arbitrary impl string (right-padded to
761    /// `ICECHUNK_IMPL_NAME_LEN`, mirroring `binary_file_header` in the `icechunk`
762    /// crate), so tests can exercise `parse_file_header` directly.
763    fn make_header(impl_name: &str, spec: u8, file_type: u8, compression: u8) -> Vec<u8> {
764        use format_constants::*;
765        let mut buf = Vec::with_capacity(ICECHUNK_FILE_HEADER_LEN);
766        buf.extend_from_slice(ICECHUNK_FORMAT_MAGIC_BYTES);
767        let padded = format!("{impl_name:<ICECHUNK_IMPL_NAME_LEN$}");
768        buf.extend_from_slice(&padded.as_bytes()[..ICECHUNK_IMPL_NAME_LEN]);
769        buf.push(spec);
770        buf.push(file_type);
771        buf.push(compression);
772        buf
773    }
774
775    #[icechunk_macros::test]
776    fn test_parse_file_header_roundtrip() {
777        use format_constants::*;
778        let mut buf = make_header(
779            ICECHUNK_CLIENT_NAME.as_str(),
780            SpecVersionBin::V2 as u8,
781            FileTypeBin::Snapshot as u8,
782            CompressionAlgorithmBin::Zstd as u8,
783        );
784        // body bytes after the header must be ignored
785        buf.extend_from_slice(b"a compressed body would go here");
786
787        let header = parse_file_header(&buf).unwrap();
788        assert_eq!(header.impl_name, *ICECHUNK_CLIENT_NAME);
789        assert_eq!(header.app_name, "ic");
790        assert_eq!(header.app_version.as_deref(), Some(ICECHUNK_LIB_VERSION));
791        assert_eq!(header.spec_version, SpecVersionBin::V2);
792        assert_eq!(header.file_type, FileTypeBin::Snapshot);
793        assert_eq!(header.compression, CompressionAlgorithmBin::Zstd);
794    }
795
796    #[icechunk_macros::test]
797    fn test_parse_file_header_trims_padding() {
798        use format_constants::*;
799        // exactly the header, no body: the 24-byte impl field is space-padded
800        let buf = make_header(
801            "ic-9.9.9",
802            SpecVersionBin::V1 as u8,
803            FileTypeBin::Manifest as u8,
804            CompressionAlgorithmBin::None as u8,
805        );
806        let header = parse_file_header(&buf).unwrap();
807        assert_eq!(header.impl_name, "ic-9.9.9");
808        assert_eq!(header.app_name, "ic");
809        assert_eq!(header.app_version.as_deref(), Some("9.9.9"));
810        assert_eq!(header.spec_version, SpecVersionBin::V1);
811        assert_eq!(header.file_type, FileTypeBin::Manifest);
812        assert_eq!(header.compression, CompressionAlgorithmBin::None);
813    }
814
815    #[icechunk_macros::test]
816    fn test_parse_file_header_splits_on_first_dash() {
817        use format_constants::*;
818        let buf = make_header(
819            "ic-2.0.0-alpha.4",
820            SpecVersionBin::V2 as u8,
821            FileTypeBin::RepoInfo as u8,
822            CompressionAlgorithmBin::Zstd as u8,
823        );
824        let header = parse_file_header(&buf).unwrap();
825        assert_eq!(header.impl_name, "ic-2.0.0-alpha.4");
826        assert_eq!(header.app_name, "ic");
827        assert_eq!(header.app_version.as_deref(), Some("2.0.0-alpha.4"));
828    }
829
830    #[icechunk_macros::test]
831    fn test_parse_file_header_no_dash_fallback() {
832        use format_constants::*;
833        let buf = make_header(
834            "icjs",
835            SpecVersionBin::V2 as u8,
836            FileTypeBin::TransactionLog as u8,
837            CompressionAlgorithmBin::Zstd as u8,
838        );
839        let header = parse_file_header(&buf).unwrap();
840        assert_eq!(header.impl_name, "icjs");
841        assert_eq!(header.app_name, "icjs");
842        assert_eq!(header.app_version, None);
843    }
844
845    #[icechunk_macros::test]
846    fn test_parse_file_header_bad_magic() {
847        use format_constants::*;
848        let mut buf = make_header(
849            "ic-1.0.0",
850            SpecVersionBin::V2 as u8,
851            FileTypeBin::Snapshot as u8,
852            CompressionAlgorithmBin::Zstd as u8,
853        );
854        buf[0] = b'X';
855        let err = parse_file_header(&buf).unwrap_err();
856        assert!(matches!(err.kind(), IcechunkFormatErrorKind::InvalidMagicNumbers));
857    }
858
859    #[icechunk_macros::test]
860    fn test_parse_file_header_too_short() {
861        use format_constants::*;
862        let buf = vec![0u8; ICECHUNK_FILE_HEADER_LEN - 1];
863        let err = parse_file_header(&buf).unwrap_err();
864        assert!(matches!(
865            err.kind(),
866            IcechunkFormatErrorKind::InvalidIcechunkHeaderSize { .. }
867        ));
868    }
869
870    #[icechunk_macros::test]
871    fn test_parse_file_header_unknown_file_type() {
872        use format_constants::*;
873        let buf = make_header(
874            "ic-1.0.0",
875            SpecVersionBin::V2 as u8,
876            99,
877            CompressionAlgorithmBin::Zstd as u8,
878        );
879        let err = parse_file_header(&buf).unwrap_err();
880        assert!(matches!(
881            err.kind(),
882            IcechunkFormatErrorKind::UnknownFileType { got: 99 }
883        ));
884    }
885
886    #[icechunk_macros::test]
887    fn test_parse_file_header_bad_spec_version() {
888        use format_constants::*;
889        let buf = make_header(
890            "ic-1.0.0",
891            99,
892            FileTypeBin::Snapshot as u8,
893            CompressionAlgorithmBin::Zstd as u8,
894        );
895        let err = parse_file_header(&buf).unwrap_err();
896        assert!(matches!(
897            err.kind(),
898            IcechunkFormatErrorKind::InvalidSpecVersion { found: 99, .. }
899        ));
900    }
901
902    #[icechunk_macros::test]
903    fn test_object_metadata_keys_are_portable() {
904        use format_constants::*;
905        for key in [
906            LATEST_ICECHUNK_FORMAT_VERSION_METADATA_KEY,
907            ICECHUNK_CLIENT_NAME_METADATA_KEY,
908            ICECHUNK_FILE_TYPE_METADATA_KEY,
909            ICECHUNK_COMPRESSION_METADATA_KEY,
910        ] {
911            assert!(
912                !key.is_empty()
913                    && key.starts_with(|c: char| c.is_ascii_alphabetic())
914                    && key.chars().all(|c| c.is_ascii_alphanumeric()),
915                "{key} must be ASCII alphanumeric and start with a letter: \
916                 Azure rejects `-` and nginx-fronted S3 gateways drop `_`"
917            );
918        }
919    }
920}
icechunk_format/lib.rs

icechunk_format/
lib.rs
