mlt_core/decoder/stream/
model.rs

1use derive_debug::Dbg;
2use num_enum::TryFromPrimitive;
3
4use crate::utils::formatter::{bytes_dbg, compact_dbg};
5use crate::{MltError, MltResult};
6
7/// Logical encoding technique used for a column, as stored in the tile
8///
9/// Variants are already shifted into the primary logical field of the v1 encoding byte (bits 7-5),
10/// so that field is matched with a mask rather than shifted down first.
11/// The secondary field (bits 4-2) holds the same patterns three bits lower.
12#[derive(Debug, Clone, Copy, PartialEq, TryFromPrimitive)]
13#[repr(u8)]
14pub enum LogicalTechnique {
15    None = 0b0000_0000,
16    Delta = 0b0010_0000,
17    ComponentwiseDelta = 0b0100_0000,
18    Rle = 0b0110_0000,
19    Morton = 0b1000_0000,
20}
21
22/// The combinations of the two [`LogicalTechnique`] fields that are legal on the wire
23///
24/// Each variant is the whole logical part of the v1 encoding byte, i.e. the primary field
25/// (bits 7-5) or-ed with the secondary field (bits 4-2).
26/// Any other pairing of the two fields is rejected while parsing.
27#[derive(Debug, Clone, Copy, PartialEq, TryFromPrimitive)]
28#[repr(u8)]
29pub enum LogicalCombination {
30    None = 0b0000_0000,
31    Delta = 0b0010_0000,
32    DeltaRle = 0b0010_1100,
33    ComponentwiseDelta = 0b0100_0000,
34    Rle = 0b0110_0000,
35    Morton = 0b1000_0000,
36    MortonDelta = 0b1000_0100,
37    MortonRle = 0b1000_1100,
38}
39
40/// Which RLE stream layout the encoder should produce.
41///
42/// A data-less selector chosen up front by the wire format (see
43/// [`WireVersion::rle_layout`](crate::encoder::WireVersion)); the realized
44/// per-stream metadata is [`RleMeta`], whose variants mirror these.
45#[derive(Debug, Clone, Copy, PartialEq, Eq)]
46pub enum RleLayout {
47    /// Tag `0x01`: all run lengths first, then all values.
48    Split,
49    /// Tag `0x02`: `(run_length, value)` pairs. Requires the `unstable-v2` feature.
50    #[cfg(feature = "unstable-v2")]
51    Interleaved,
52}
53
54/// Metadata for RLE decoding, one variant per [`RleLayout`].
55#[derive(Debug, Clone, Copy, PartialEq)]
56pub enum RleMeta {
57    /// Tag `0x01`: physically-decoded words are `[run_len × runs][value × runs]`.
58    /// `runs` is the split point; `num_rle_values` is the expanded element count.
59    Split { runs: u32, num_rle_values: u32 },
60    /// Tag `0x02`: physically-decoded words are `(run_len, value)` pairs. The run
61    /// count is derived from the data length, so only the expanded element count
62    /// (`num_rle_values`, from the stream's count context) is carried.
63    /// Requires the `unstable-v2` feature.
64    #[cfg(feature = "unstable-v2")]
65    Interleaved { num_rle_values: u32 },
66}
67
68impl RleMeta {
69    /// The total expanded element count, common to both layouts.
70    #[cfg(feature = "unstable-v2")]
71    #[must_use]
72    pub(crate) fn num_rle_values(self) -> u32 {
73        match self {
74            Self::Split { num_rle_values, .. } => num_rle_values,
75            #[cfg(feature = "unstable-v2")]
76            Self::Interleaved { num_rle_values } => num_rle_values,
77        }
78    }
79}
80
81/// Metadata for Morton decoding
82#[derive(Debug, Clone, Copy, PartialEq)]
83pub struct Morton {
84    /// Number of bits used
85    pub(crate) bits: u32,
86    /// Coordinate shift
87    pub(crate) shift: u32,
88}
89
90impl Morton {
91    pub fn new(bits: u32, shift: u32) -> MltResult<Self> {
92        if bits <= 16 {
93            Ok(Self { bits, shift })
94        } else {
95            Err(MltError::InvalidMortonBits(bits))
96        }
97    }
98}
99
100/// The decimal scaling an ALP column uses: `i = round(v * 10^e / 10^f)`.
101/// Chosen before any value is seen, so it carries no frame of reference.
102#[derive(Debug, Clone, Copy, PartialEq, Eq)]
103pub struct AlpScale {
104    /// Decimal exponent the values were scaled by.
105    pub(crate) e: u8,
106    /// Factor dividing out the trailing zeros `e` introduced, never exceeding `e`.
107    pub(crate) f: u8,
108}
109
110/// ALP parameters: `v = (base + offset) * 10^f / 10^e`.
111/// Written as three header varints, the stream itself holding the unsigned offsets.
112#[derive(Clone, Copy, PartialEq, Eq)]
113pub struct Alp {
114    pub(crate) scale: AlpScale,
115    /// Frame of reference the offsets are measured from, the smallest scaled integer in the column.
116    pub(crate) base: i64,
117}
118
119#[cfg(feature = "unstable-v2")]
120impl AlpScale {
121    /// Largest exponent the codes can carry, past which `v * 10^e` leaves the `i64` range.
122    pub(crate) const MAX_EXPONENT: u8 = 18;
123
124    /// Net power of ten the codes carry, which fixes their magnitude and so their stored size.
125    /// The order [`candidates`](crate::codecs::alp::candidates) walks, and so what lets the
126    /// encoder stop at the first scale that fits; only the tests holding that invariant name it.
127    #[cfg(test)]
128    pub(crate) fn net(self) -> u8 {
129        self.e - self.f
130    }
131}
132
133/// Flattened, since the nesting is an encoder concern and these appear in stream labels.
134impl std::fmt::Debug for Alp {
135    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
136        write!(
137            f,
138            "Alp {{ e: {}, f: {}, base: {} }}",
139            self.scale.e, self.scale.f, self.base
140        )
141    }
142}
143
144#[cfg(feature = "unstable-v2")]
145impl Alp {
146    pub(crate) fn new(e: u8, f: u8, base: i64) -> MltResult<Self> {
147        if e <= AlpScale::MAX_EXPONENT && f <= e {
148            Ok(Self {
149                scale: AlpScale { e, f },
150                base,
151            })
152        } else {
153            Err(MltError::InvalidAlpParams(e, f))
154        }
155    }
156
157    /// Measure a scaled integer from the frame of reference, giving the offset the stream stores.
158    /// Wrapping, so a column spanning the whole `i64` range still subtracts exactly.
159    #[expect(
160        clippy::cast_sign_loss,
161        reason = "the bit pattern is the point; `code_at` casts it back"
162    )]
163    pub(crate) fn offset_of(self, code: i64) -> u64 {
164        (code as u64).wrapping_sub(self.base as u64)
165    }
166
167    /// Put a stored offset back on the frame of reference, inverting [`Self::offset_of`].
168    #[expect(
169        clippy::cast_possible_wrap,
170        clippy::cast_sign_loss,
171        reason = "the bit pattern is the point; inverts `offset_of` exactly"
172    )]
173    pub(crate) fn code_at(self, offset: u64) -> i64 {
174        (self.base as u64).wrapping_add(offset) as i64
175    }
176}
177
178/// What kind of values a stream holds, which fixes the encodings it can name.
179/// Neither wire format stores it: v1 reads it from the column type, v2 from the stream's context.
180#[derive(Debug, Clone, Copy, PartialEq, Eq)]
181pub enum ValueKind {
182    Int,
183    Bool,
184    Float,
185    Vertex,
186}
187
188/// Logical encoding of a stream of integer values.
189/// Covers the id columns, the integer property columns, and the geometry length and offset streams.
190#[derive(Debug, Clone, Copy, PartialEq)]
191pub enum IntLogical {
192    None,
193    Delta,
194    Rle(RleMeta),
195    DeltaRle(RleMeta),
196}
197
198/// Logical encoding of a bool column's data stream or a presence bitfield.
199#[derive(Debug, Clone, Copy, PartialEq)]
200pub enum BoolLogical {
201    /// A raw packed bitmap, one bit per value.
202    None,
203    /// A byte-RLE compressed bitmap.
204    /// Its run parameters come from the stream's context rather than from its header.
205    ByteRle(RleMeta),
206}
207
208/// Logical encoding of a float column's data stream.
209#[derive(Debug, Clone, Copy, PartialEq)]
210pub enum FloatLogical {
211    /// Fixed-width little-endian values, one per element.
212    None,
213    /// A decimal split across two integer streams, which v1 can name but no decoder here implements.
214    /// One code per element, with the distinct values following as a second stream.
215    /// Only the tag `0x02` codec reads or writes it.
216    Dict,
217    /// Integers scaled by these parameters.
218    /// Only the tag `0x02` codec reads or writes it.
219    Alp(Alp),
220}
221
222/// Logical encoding of a geometry vertex stream, whose values are interleaved coordinate pairs.
223#[derive(Debug, Clone, Copy, PartialEq)]
224pub enum VertexLogical {
225    None,
226    Delta,
227    ComponentwiseDelta,
228    Morton(Morton),
229    MortonDelta(Morton),
230    MortonRle(Morton),
231}
232
233/// How should the stream be interpreted at the logical level (second pass of decoding)
234///
235/// Split per [`ValueKind`] so a stream can name only the encodings its values can have.
236#[derive(Debug, Clone, Copy, PartialEq)]
237pub enum LogicalEncoding {
238    Int(IntLogical),
239    Bool(BoolLogical),
240    Float(FloatLogical),
241    Vertex(VertexLogical),
242}
243
244impl LogicalEncoding {
245    /// The kind of values this encoding belongs to.
246    #[must_use]
247    pub fn kind(self) -> ValueKind {
248        match self {
249            Self::Int(_) => ValueKind::Int,
250            Self::Bool(_) => ValueKind::Bool,
251            Self::Float(_) => ValueKind::Float,
252            Self::Vertex(_) => ValueKind::Vertex,
253        }
254    }
255
256    /// The identity encoding for `kind`, i.e. values stored as they are.
257    #[must_use]
258    pub(crate) fn none(kind: ValueKind) -> Self {
259        match kind {
260            ValueKind::Int => Self::Int(IntLogical::None),
261            ValueKind::Bool => Self::Bool(BoolLogical::None),
262            ValueKind::Float => Self::Float(FloatLogical::None),
263            ValueKind::Vertex => Self::Vertex(VertexLogical::None),
264        }
265    }
266
267    /// Whether the stream's own logical pass is a no-op, so the physical words are already the output.
268    /// True for a float dictionary's codes, which the column turns back into floats.
269    /// Not true for ALP, whose offsets still need the frame of reference added back.
270    #[must_use]
271    pub(crate) fn is_identity(self) -> bool {
272        matches!(
273            self,
274            Self::Int(IntLogical::None)
275                | Self::Bool(BoolLogical::None)
276                | Self::Float(FloatLogical::None | FloatLogical::Dict)
277                | Self::Vertex(VertexLogical::None)
278        )
279    }
280}
281
282/// Carries the stream metadata needed to perform the logical decode pass.
283///
284/// Construct with [`LogicalValue::new`] after the physical decode pass fills a
285/// `&[u32]` or `&[u64]` buffer, then call the appropriate `decode_*` method,
286/// passing that slice as `data`.
287#[derive(Debug, PartialEq)]
288pub struct LogicalValue {
289    pub(crate) meta: StreamMeta,
290}
291
292// Physical encoding types
293
294/// Dictionary type used for a column, as stored in the tile
295#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, PartialOrd, Ord, TryFromPrimitive)]
296#[repr(u8)]
297pub enum DictionaryType {
298    None = 0b0000_0000,
299    Single = 0b0000_0001,
300    Shared = 0b0000_0010,
301    Vertex = 0b0000_0011,
302    Morton = 0b0000_0100,
303    Fsst = 0b0000_0101,
304}
305
306/// Offset type used for a column, as stored in the tile
307#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, PartialOrd, Ord, TryFromPrimitive)]
308#[repr(u8)]
309pub enum OffsetType {
310    Vertex = 0b0000_0000,
311    Index = 0b0000_0001,
312    String = 0b0000_0010,
313    Key = 0b0000_0011,
314}
315
316/// Length type used for a column, as stored in the tile
317#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, PartialOrd, Ord, TryFromPrimitive)]
318#[repr(u8)]
319pub enum LengthType {
320    VarBinary = 0b0000_0000,
321    Geometries = 0b0000_0001,
322    Parts = 0b0000_0010,
323    Rings = 0b0000_0011,
324    Triangles = 0b0000_0100,
325    Symbol = 0b0000_0101,
326    Dictionary = 0b0000_0110,
327    /// A nested list or map node's lengths, one per present value of the node.
328    #[cfg(feature = "unstable-v2")]
329    Nested = 0b0000_0111,
330}
331
332/// How should the stream be interpreted at the physical level (first pass of decoding)
333#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, PartialOrd, Ord)]
334pub enum StreamType {
335    Present,
336    Data(DictionaryType),
337    Offset(OffsetType),
338    Length(LengthType),
339}
340
341/// Physical encoding used for a column, as stored in the tile
342#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, PartialOrd, Ord)]
343pub enum PhysicalEncoding {
344    None,
345    /// Preferred, tends to produce the best compression ratio and decoding performance.
346    /// But currently limited to 32-bit integer.
347    FastPFor(FastPForKind),
348    /// Can produce better results in combination with a heavyweight compression scheme like `Gzip`.
349    /// Simple compression scheme where the encoding is easier to implement compared to `FastPfor`.
350    VarInt,
351    /// Every value in the same number of bits, which a leading payload byte names.
352    /// Beats `VarInt` on the short, small-alphabet streams a dictionary's codes form,
353    /// where `FastPFor`'s 128-value blocks are too coarse to pay for themselves.
354    #[cfg(feature = "unstable-v2")]
355    BitPacked,
356}
357
358/// The `FastPFor` block size and word order a wire version codes its integer streams with.
359#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, PartialOrd, Ord)]
360pub enum FastPForKind {
361    /// 256-value blocks over big-endian words
362    Block256Be,
363    /// 128-value blocks over little-endian words
364    #[cfg(feature = "unstable-v2")]
365    Block128Le,
366}
367
368// RawStream types
369
370#[derive(Debug, Clone, Copy, PartialEq)]
371pub struct IntEncoding {
372    pub logical: LogicalEncoding,
373    pub physical: PhysicalEncoding,
374}
375
376impl IntEncoding {
377    #[must_use]
378    pub(crate) const fn new(logical: LogicalEncoding, physical: PhysicalEncoding) -> Self {
379        Self { logical, physical }
380    }
381
382    #[must_use]
383    pub(crate) fn none(kind: ValueKind) -> Self {
384        Self::new(LogicalEncoding::none(kind), PhysicalEncoding::None)
385    }
386}
387
388/// Metadata about an encoded stream
389#[derive(Clone, Copy, Dbg, PartialEq)]
390pub struct StreamMeta {
391    #[dbg(formatter = "compact_dbg")]
392    pub stream_type: StreamType,
393    #[dbg(formatter = "compact_dbg")]
394    pub encoding: IntEncoding,
395    pub(crate) num_values: u32,
396}
397
398impl StreamMeta {
399    #[inline]
400    pub(crate) fn new(stream_type: StreamType, encoding: IntEncoding, num_values: u32) -> Self {
401        Self {
402            stream_type,
403            encoding,
404            num_values,
405        }
406    }
407
408    #[inline]
409    pub(crate) fn new2(
410        stream_type: StreamType,
411        logical: LogicalEncoding,
412        physical: PhysicalEncoding,
413        num_values: usize,
414    ) -> MltResult<Self> {
415        let enc = IntEncoding::new(logical, physical);
416        Ok(Self::new(stream_type, enc, u32::try_from(num_values)?))
417    }
418
419    #[inline]
420    pub(crate) fn new_none(
421        stream_type: StreamType,
422        kind: ValueKind,
423        num_values: usize,
424    ) -> MltResult<Self> {
425        let enc = IntEncoding::none(kind);
426        Ok(Self::new(stream_type, enc, u32::try_from(num_values)?))
427    }
428}
429
430/// Representation of an encoded stream
431#[derive(Clone, Dbg, PartialEq)]
432pub struct RawStream<'a> {
433    pub meta: StreamMeta,
434    #[dbg(formatter = "bytes_dbg")]
435    pub(crate) data: &'a [u8],
436}
437
438impl<'a> RawStream<'a> {
439    #[must_use]
440    pub(crate) fn new(meta: StreamMeta, data: &'a [u8]) -> Self {
441        Self { meta, data }
442    }
443}
mlt_core/decoder/stream/model.rs

mlt_core/decoder/stream/
model.rs
