iris_format/layout.rs
1//! Where everything sits in the file.
2//!
3//! ```text
4//! +--------------------------------------------------+ 0
5//! | header, 16 bytes |
6//! +--------------------------------------------------+ 16
7//! | sections, back to back, each 8 byte aligned |
8//! +--------------------------------------------------+ footer_offset
9//! | footer, a run of iris-abi records |
10//! +--------------------------------------------------+ len - 56
11//! | trailer, 56 bytes |
12//! +--------------------------------------------------+ len
13//! ```
14//!
15//! The directory lives at the end rather than the front, because a writer that streams a large
16//! dataset does not know how long a section is until it has finished writing it, and the
17//! alternative is either buffering the whole thing or seeking back over it. The magic appears at
18//! both ends so that a file truncated in the middle is distinguishable from a file that was never
19//! an iris container in the first place.
20
21use iris_abi::Tag;
22
23/// The first eight bytes of every container, and the last eight.
24///
25/// The carriage return and line feed catch a transport that helpfully converts line endings, and
26/// the `0x1a` stops a Windows `type` from printing the rest of the file. Both of those are older
27/// than most of the people who will read this and both still happen.
28pub const MAGIC: [u8; 8] = *b"IRIS\r\n\x1a\n";
29
30/// The format major version this build writes and reads.
31///
32/// A major version goes up when a reader that does not know about the change would get the wrong
33/// answer rather than an error.
34pub const FORMAT_MAJOR: u16 = 0;
35
36/// The format minor version this build writes.
37///
38/// A minor version goes up when something is added that an older reader can safely ignore, which in
39/// practice means a new footer record or a new field at the end of an existing one.
40pub const FORMAT_MINOR: u16 = 1;
41
42/// How wide the header is, in bytes.
43pub const HEADER_SIZE: usize = 16;
44
45/// How wide the trailer is, in bytes.
46pub const TRAILER_SIZE: usize = 56;
47
48/// The smallest a container can possibly be, which is a header, an empty footer and a trailer.
49pub const MIN_SIZE: usize = HEADER_SIZE + TRAILER_SIZE;
50
51/// How wide a digest is, in bytes.
52pub const DIGEST_SIZE: usize = 32;
53
54/// The footer record tags.
55///
56/// These share the `Tag` newtype with the call protocol in `iris-abi` but not its number space. A
57/// footer record and a call record never appear in the same byte stream, so there is nothing to
58/// collide, and reusing the framing means the skip an unknown record rule is the one that is
59/// already written down and already tested.
60pub mod tag {
61 use super::Tag;
62
63 /// How many rows the dataset has, and what it is called.
64 pub const DATASET: Tag = Tag(0x0100);
65 /// The Arrow schema, as opaque bytes plus an encoding.
66 pub const SCHEMA: Tag = Tag(0x0101);
67 /// Which decoder reads this dataset, and what it needs from the host.
68 pub const DECODER: Tag = Tag(0x0102);
69 /// One run of bytes in the payload area, with its digest.
70 pub const SECTION: Tag = Tag(0x0103);
71}
72
73/// How the schema bytes are encoded.
74#[derive(Clone, Copy, PartialEq, Eq, Debug, serde::Serialize, serde::Deserialize)]
75#[serde(rename_all = "kebab-case")]
76#[non_exhaustive]
77pub enum SchemaEncoding {
78 /// An Arrow IPC schema message, exactly as `arrow-ipc` writes one.
79 ///
80 /// This crate does not depend on Arrow and does not look inside these bytes. Carrying the
81 /// encoding rather than assuming it means a later version can add a second one without a new
82 /// major version, and it means this crate stays small enough to fuzz properly.
83 ArrowIpc,
84 /// Something this build does not know about, carried through so that a tool which only wants
85 /// the section table is not stopped by a schema it cannot read.
86 Unknown(u32),
87}
88
89impl SchemaEncoding {
90 /// The number this encoding is written as.
91 #[must_use]
92 pub const fn code(self) -> u32 {
93 match self {
94 Self::ArrowIpc => 1,
95 Self::Unknown(code) => code,
96 }
97 }
98
99 /// The encoding a code stands for.
100 #[must_use]
101 pub const fn from_code(code: u32) -> Self {
102 match code {
103 1 => Self::ArrowIpc,
104 other => Self::Unknown(other),
105 }
106 }
107}
108
109/// What a section holds.
110#[derive(Clone, Copy, PartialEq, Eq, Debug, serde::Serialize, serde::Deserialize)]
111#[serde(rename_all = "kebab-case")]
112#[non_exhaustive]
113pub enum SectionKind {
114 /// Encoded column data, in whatever form the decoder expects.
115 Data,
116 /// A decoder module, embedded in the container.
117 Decoder,
118 /// An index, a dictionary, a footer of some inner format, anything the decoder wants to find
119 /// quickly without reading the data.
120 Sidecar,
121 /// A kind this build does not know about. A section is a run of bytes with a digest, so an
122 /// unknown kind is still checkable and still skippable.
123 Unknown(u32),
124}
125
126impl SectionKind {
127 /// The number this kind is written as.
128 #[must_use]
129 pub const fn code(self) -> u32 {
130 match self {
131 Self::Data => 1,
132 Self::Decoder => 2,
133 Self::Sidecar => 3,
134 Self::Unknown(code) => code,
135 }
136 }
137
138 /// The kind a code stands for.
139 #[must_use]
140 pub const fn from_code(code: u32) -> Self {
141 match code {
142 1 => Self::Data,
143 2 => Self::Decoder,
144 3 => Self::Sidecar,
145 other => Self::Unknown(other),
146 }
147 }
148}
149
150/// Where the decoder module for a dataset lives.
151#[derive(Clone, Copy, PartialEq, Eq, Debug, serde::Serialize, serde::Deserialize)]
152#[serde(rename_all = "kebab-case")]
153#[non_exhaustive]
154pub enum DecoderLocation {
155 /// In a section of this container, named by its section id.
156 ///
157 /// This is the self-decoding case and the one the project is about. The dataset carries the
158 /// code that reads it, so a host that has never seen the encoding can still read the data.
159 Embedded {
160 /// Which section holds the module.
161 section: u32,
162 },
163 /// Somewhere else, to be resolved by digest.
164 ///
165 /// Useful when many datasets share one decoder and nobody wants a copy of it in every file.
166 /// The digest is what makes this safe: the host either finds bytes that hash to it or it does
167 /// not run anything.
168 External,
169}