icechunk_format/serializers/
mod.rs

1//! Flatbuffer serialization for Icechunk metadata.
2//!
3//! How serializers work:
4//!
5//! - Main goal is to make sure newer version of Icechunk can read metadata files created using
6//!   older versions. In this way, a repository can evolve during its life. As users upgrade their
7//!   Icechunk versions they don't need to migrate their data.
8//! - Of course we may choose to limit backwards compatibility after certain number of versions or
9//!   a time limit.
10//! - Performance is critical, so we cannot copy much data around during the process of
11//!   serialization/deserialization
12//! - For serialization:
13//!     - We define a new `XSerializer` for each metadata file type `X`. Example: `SnapshotSerializer`.
14//!     - This type implement [`serde::Serialize`]
15//!     - This type holds only references to the same fields as `X`
16//!     - This type implements `From<&X>` (notice by reference) Example:
17//!       ```ignore
18//!       impl<'a> From<&'a Snapshot> for SnapshotSerializer<'a> {
19//!       ...
20//!       }
21//!       ```
22//!     - Because the serializer only holds references it's essentially free to call
23//!       `snapshot.into()` to get one.
24//!     - Then this object is serialized using serde.
25//! - For deserialization:
26//!     - We define a new `XDeserializer` for each metadata file type `X`. Example: `SnapshotDeserializer`.
27//!     - This type implement [`serde::Deserialize`]
28//!     - This type holds the same fields as `X` by value
29//!     - `X` implements `From<XDeserializer>` (notice by value). Example:
30//!       ```ignore
31//!        impl From<SnapshotDeserializer> for Snapshot {
32//!        ...
33//!        }
34//!       ```
35//!     - Because the deserializer can be destructed and `X` implements `From`,  it's essentially free to call
36//!       obtain the original type `X`
37//!     - Then this new type `XDeserializer` is deserialized using serde and converted with `into`.
38//!
39//! - `serializers.current.rs` holds all the serializers and deserializers for the current version
40//!   of the spec
41//! - `serializers.version_foo.rs` holds all the serializers and deserializers for version foo of
42//!   the spec
43//! - The `serializers` module root has functions `serialize_X` and `deserialize_X` that take a
44//!   spec version number and use the right (de)-serializer to do the job.
45use std::io::Write;
46
47use icechunk_types::ICResultExt as _;
48
49use crate::{
50    IcechunkFormatError, IcechunkFormatErrorKind, format_constants::SpecVersionBin,
51    manifest::Manifest, repo_info::RepoInfo, snapshot::Snapshot,
52    transaction_log::TransactionLog,
53};
54
55pub fn serialize_snapshot(
56    snapshot: &Snapshot,
57    version: SpecVersionBin,
58    write: &mut impl Write,
59) -> Result<(), std::io::Error> {
60    match version {
61        SpecVersionBin::V1 | SpecVersionBin::V2 => write.write_all(snapshot.bytes()),
62    }
63}
64
65pub fn serialize_manifest(
66    manifest: &Manifest,
67    version: SpecVersionBin,
68    write: &mut impl Write,
69) -> Result<(), std::io::Error> {
70    match version {
71        SpecVersionBin::V1 | SpecVersionBin::V2 => write.write_all(manifest.bytes()),
72    }
73}
74
75pub fn serialize_transaction_log(
76    transaction_log: &TransactionLog,
77    version: SpecVersionBin,
78    write: &mut impl Write,
79) -> Result<(), std::io::Error> {
80    match version {
81        SpecVersionBin::V1 | SpecVersionBin::V2 => {
82            write.write_all(transaction_log.bytes())
83        }
84    }
85}
86
87pub fn serialize_repo_info(
88    info: &RepoInfo,
89    version: SpecVersionBin,
90    write: &mut impl Write,
91) -> Result<(), std::io::Error> {
92    match version {
93        SpecVersionBin::V2 => write.write_all(info.bytes()),
94        SpecVersionBin::V1 => Err(std::io::Error::new(
95            std::io::ErrorKind::Unsupported,
96            "Trying to write to an old Icechunk format version. Aborting.",
97        )),
98    }
99}
100
101pub fn deserialize_snapshot(
102    version: SpecVersionBin,
103    buffer: Vec<u8>,
104) -> Result<Snapshot, IcechunkFormatError> {
105    match version {
106        SpecVersionBin::V1 | SpecVersionBin::V2 => Snapshot::from_buffer(version, buffer),
107    }
108}
109
110pub fn deserialize_manifest(
111    version: SpecVersionBin,
112    buffer: Vec<u8>,
113) -> Result<Manifest, IcechunkFormatError> {
114    match version {
115        SpecVersionBin::V1 | SpecVersionBin::V2 => Manifest::from_buffer(buffer),
116    }
117}
118
119pub fn deserialize_transaction_log(
120    version: SpecVersionBin,
121    buffer: Vec<u8>,
122) -> Result<TransactionLog, IcechunkFormatError> {
123    match version {
124        SpecVersionBin::V1 | SpecVersionBin::V2 => TransactionLog::from_buffer(buffer),
125    }
126}
127
128pub fn deserialize_repo_info(
129    version: SpecVersionBin,
130    buffer: Vec<u8>,
131) -> Result<RepoInfo, IcechunkFormatError> {
132    match version {
133        SpecVersionBin::V2 => RepoInfo::from_buffer(buffer),
134        SpecVersionBin::V1 => {
135            Err(IcechunkFormatErrorKind::UnsupportedOperationForVersion {
136                version: SpecVersionBin::V1 as u8,
137            })
138            .capture()
139        }
140    }
141}
icechunk_format/serializers/mod.rs

icechunk_format/serializers/
mod.rs
