mlt_core/encoder/
model.rs

1use std::borrow::Cow;
2use std::collections::HashSet;
3
4use derive_debug::Dbg;
5
6#[cfg(feature = "unstable-v2")]
7use crate::decoder::RleLayout;
8use crate::decoder::{DictionaryType, FastPForKind, GeometryValues, PhysicalEncoding, StreamType};
9use crate::encoder::geometry::VertexBufferType;
10use crate::encoder::{IntEncoder, StagedId, StagedProperty};
11use crate::tile::Extent;
12use crate::{MltError, MltResult};
13
14/// Owned variant of `Unknown`.
15#[derive(Debug, Clone, Default, PartialEq)]
16pub struct EncodedUnknown {
17    pub(crate) tag: u8,
18    pub(crate) value: Vec<u8>,
19}
20
21impl EncodedUnknown {
22    pub fn new(tag: u8, value: Vec<u8>) -> MltResult<Self> {
23        if tag == 1 {
24            return Err(MltError::ParsingColumnType(tag));
25        }
26        Ok(Self { tag, value })
27    }
28
29    #[must_use]
30    pub fn tag(&self) -> u32 {
31        u32::from(self.tag)
32    }
33
34    #[must_use]
35    pub fn data(&self) -> &[u8] {
36        &self.value
37    }
38}
39
40/// Parameters derived from the vertex set of a feature collection, used to
41/// normalize coordinates before space-filling-curve key computation.
42#[derive(Debug, Clone, Copy, PartialEq)]
43pub struct CurveParams {
44    pub shift: u32,
45    pub bits: u32,
46}
47
48impl Default for CurveParams {
49    fn default() -> Self {
50        Self { shift: 0, bits: 1 }
51    }
52}
53
54impl CurveParams {
55    /// Compute params from a flat `[x0, y0, x1, y1, …]` vertex slice.
56    #[must_use]
57    pub fn from_vertices(vertices: &[i32]) -> Self {
58        if vertices.is_empty() {
59            return Self::default();
60        }
61        let (min, max) = vertices
62            .iter()
63            .fold((i32::MAX, i32::MIN), |(mn, mx), &v| (mn.min(v), mx.max(v)));
64        crate::codecs::hilbert::hilbert_curve_params_from_bounds(min, max)
65    }
66}
67
68/// Columnar layer data being prepared for encoding (stage 2 of the encoding pipeline).
69///
70/// Holds fully-owned columnar data. Constructed directly (synthetics, benches) or
71/// converted from [`TileLayer`](crate::TileLayer).
72/// Consumed by encoding via [`StagedLayer::encode_into`] or `StagedLayer::encode_explicit`
73/// (with explicit encoding mode enabled).
74#[derive(Debug, PartialEq, Clone)]
75pub struct StagedLayer {
76    pub(crate) name: String,
77    pub(crate) extent: Extent,
78    pub(crate) id: StagedId,
79    pub(crate) geometry: GeometryValues,
80    pub(crate) properties: Vec<StagedProperty>,
81}
82
83#[cfg_attr(not(feature = "__private"), allow(dead_code))]
84impl StagedLayer {
85    pub fn new(
86        name: impl Into<String>,
87        extent: u32,
88        id: StagedId,
89        geometry: GeometryValues,
90        properties: Vec<StagedProperty>,
91    ) -> MltResult<Self> {
92        let name = name.into();
93        if name.is_empty() {
94            return Err(MltError::MissingLayerName);
95        }
96        let extent = Extent::new(extent)?;
97        let feature_count = geometry.feature_count();
98        if let Some(actual) = id.feature_count()
99            && actual != feature_count
100        {
101            return Err(MltError::StagedFeatureCountMismatch {
102                column: "id".into(),
103                expected: feature_count,
104                actual,
105            });
106        }
107        // Column names must be unique within a layer. A shared dictionary's `name()` is
108        // only its prefix (which may repeat); its real columns are `{prefix}{suffix}`.
109        // Scoped so `seen` releases its borrow of `properties` before the move below.
110        {
111            let mut seen: HashSet<Cow<str>> = HashSet::new();
112            for property in &properties {
113                let actual = property.feature_count();
114                if actual != feature_count {
115                    return Err(MltError::StagedFeatureCountMismatch {
116                        column: property.name().to_string(),
117                        expected: feature_count,
118                        actual,
119                    });
120                }
121                if let StagedProperty::SharedDict(sd) = property {
122                    for item in &sd.items {
123                        if !seen.insert(Cow::Owned(format!("{}{}", sd.prefix, item.suffix))) {
124                            return Err(MltError::DuplicatePropertyName(format!(
125                                "{}{}",
126                                sd.prefix, item.suffix
127                            )));
128                        }
129                    }
130                } else if !seen.insert(Cow::Borrowed(property.name())) {
131                    return Err(MltError::DuplicatePropertyName(property.name().to_string()));
132                }
133            }
134        }
135        Ok(Self {
136            name,
137            extent,
138            id,
139            geometry,
140            properties,
141        })
142    }
143
144    #[must_use]
145    pub fn name(&self) -> &str {
146        &self.name
147    }
148
149    #[must_use]
150    pub fn extent(&self) -> Extent {
151        self.extent
152    }
153
154    #[must_use]
155    pub fn id(&self) -> &StagedId {
156        &self.id
157    }
158
159    #[must_use]
160    pub fn geometry(&self) -> &GeometryValues {
161        &self.geometry
162    }
163
164    #[must_use]
165    pub fn properties(&self) -> &[StagedProperty] {
166        &self.properties
167    }
168}
169
170/// Which wire format layers are encoded to.
171#[cfg(feature = "unstable-v2")]
172#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Default)]
173#[cfg_attr(feature = "arbitrary", derive(arbitrary::Arbitrary))]
174pub enum WireVersion {
175    /// Tag `0x01` - the stable v1 format.
176    #[default]
177    V01,
178    /// Tag `0x02` - the experimental v2 format (see `docs/migrating-to-v2.md`).
179    ///
180    /// Currently limited to ID, scalar, string, shared-dictionary and geometry columns.
181    V02,
182}
183
184#[cfg(feature = "unstable-v2")]
185impl WireVersion {
186    /// The layer tag byte identifying this format on the wire.
187    #[must_use]
188    pub(crate) fn tag(self) -> u8 {
189        match self {
190            Self::V01 => 1,
191            Self::V02 => 2,
192        }
193    }
194
195    /// The `FastPFor` block size and word order used by this format.
196    #[must_use]
197    pub(crate) fn fastpfor_kind(self) -> FastPForKind {
198        match self {
199            Self::V01 => FastPForKind::Block256Be,
200            Self::V02 => FastPForKind::Block128Le,
201        }
202    }
203
204    /// The RLE stream data layout used by this format.
205    #[must_use]
206    pub(crate) fn rle_layout(self) -> RleLayout {
207        match self {
208            Self::V01 => RleLayout::Split,
209            Self::V02 => RleLayout::Interleaved,
210        }
211    }
212}
213
214/// Global encoder settings controlling which optimization strategies are attempted.
215#[derive(Debug, Clone, Copy, PartialEq, Hash)]
216#[expect(
217    clippy::struct_excessive_bools,
218    reason = "enums would not model this better, not a state machine"
219)]
220pub struct EncoderConfig {
221    /// The wire format to encode layers to.
222    #[cfg(feature = "unstable-v2")]
223    wire_version: WireVersion,
224    /// Generate tessellation data for polygons and multi-polygons.
225    tessellate: bool,
226    /// Try sorting features by the Z-order (Morton) curve index of their first vertex.
227    attempt_spatial_morton_sort: bool,
228    /// Try sorting features by the Hilbert curve index of their first vertex.
229    attempt_spatial_hilbert_sort: bool,
230    /// Try sorting features by their feature ID in ascending order.
231    attempt_id_sort: bool,
232    /// Allow `FSST` string compression
233    allow_fsst: bool,
234    /// Allow `FastPFOR` integer compression
235    allow_fastpfor: bool,
236    /// Allow string grouping into shared dictionaries
237    allow_shared_dict: bool,
238    /// Allow the v2-only float dictionary encoding
239    #[cfg(feature = "unstable-v2")]
240    allow_float_dict: bool,
241    /// Allow the v2-only ALP float encoding
242    #[cfg(feature = "unstable-v2")]
243    allow_float_alp: bool,
244}
245impl Default for EncoderConfig {
246    fn default() -> Self {
247        Self {
248            #[cfg(feature = "unstable-v2")]
249            wire_version: WireVersion::V01,
250            tessellate: false,
251            attempt_spatial_morton_sort: true,
252            attempt_spatial_hilbert_sort: true,
253            attempt_id_sort: true,
254            allow_fsst: true,
255            allow_fastpfor: true,
256            allow_shared_dict: true,
257            // Off by default while the encoding is still being measured.
258            #[cfg(feature = "unstable-v2")]
259            allow_float_dict: false,
260            #[cfg(feature = "unstable-v2")]
261            allow_float_alp: false,
262        }
263    }
264}
265
266impl EncoderConfig {
267    #[cfg(feature = "unstable-v2")]
268    #[must_use]
269    pub fn wire_version(self) -> WireVersion {
270        self.wire_version
271    }
272
273    #[must_use]
274    pub fn tessellate(self) -> bool {
275        self.tessellate
276    }
277
278    #[must_use]
279    pub fn attempt_spatial_morton_sort(self) -> bool {
280        self.attempt_spatial_morton_sort
281    }
282
283    #[must_use]
284    pub fn attempt_spatial_hilbert_sort(self) -> bool {
285        self.attempt_spatial_hilbert_sort
286    }
287
288    #[must_use]
289    pub fn attempt_id_sort(self) -> bool {
290        self.attempt_id_sort
291    }
292
293    #[must_use]
294    pub fn allow_fsst(self) -> bool {
295        self.allow_fsst
296    }
297
298    #[must_use]
299    pub fn allow_fastpfor(self) -> bool {
300        self.allow_fastpfor
301    }
302
303    /// The `FastPFor` encoding to race, or `None` when `FastPFor` is switched off.
304    #[must_use]
305    pub(crate) fn fastpfor(self) -> Option<PhysicalEncoding> {
306        #[cfg(feature = "unstable-v2")]
307        let kind = self.wire_version.fastpfor_kind();
308        #[cfg(not(feature = "unstable-v2"))]
309        let kind = FastPForKind::Block256Be;
310        self.allow_fastpfor
311            .then_some(PhysicalEncoding::FastPFor(kind))
312    }
313
314    #[must_use]
315    pub fn allow_shared_dict(self) -> bool {
316        self.allow_shared_dict
317    }
318
319    /// Whether float columns may use a dictionary, which only v2 can express.
320    #[cfg(feature = "unstable-v2")]
321    #[must_use]
322    pub fn allow_float_dict(self) -> bool {
323        self.allow_float_dict && self.wire_version != WireVersion::V01
324    }
325
326    /// Whether float columns may use ALP, which only v2 can express.
327    #[cfg(feature = "unstable-v2")]
328    #[must_use]
329    pub fn allow_float_alp(self) -> bool {
330        self.allow_float_alp && self.wire_version != WireVersion::V01
331    }
332
333    #[cfg(feature = "unstable-v2")]
334    #[must_use]
335    pub fn with_wire_version(mut self, version: WireVersion) -> Self {
336        self.wire_version = version;
337        self
338    }
339
340    #[must_use]
341    pub fn with_tessellation(mut self, enabled: bool) -> Self {
342        self.tessellate = enabled;
343        self
344    }
345
346    #[must_use]
347    pub fn with_spatial_morton_sort(mut self, enabled: bool) -> Self {
348        self.attempt_spatial_morton_sort = enabled;
349        self
350    }
351
352    #[must_use]
353    pub fn with_spatial_hilbert_sort(mut self, enabled: bool) -> Self {
354        self.attempt_spatial_hilbert_sort = enabled;
355        self
356    }
357
358    #[must_use]
359    pub fn with_id_sort(mut self, enabled: bool) -> Self {
360        self.attempt_id_sort = enabled;
361        self
362    }
363
364    #[must_use]
365    pub fn with_fsst(mut self, enabled: bool) -> Self {
366        self.allow_fsst = enabled;
367        self
368    }
369
370    #[must_use]
371    pub fn with_fastpfor(mut self, enabled: bool) -> Self {
372        self.allow_fastpfor = enabled;
373        self
374    }
375
376    #[must_use]
377    pub fn with_shared_dict(mut self, enabled: bool) -> Self {
378        self.allow_shared_dict = enabled;
379        self
380    }
381
382    /// Allow float columns to store one code per value into a dictionary of the distinct ones.
383    /// Off by default, and only v2 can express it.
384    /// A column takes it only when it comes out strictly smaller in stored bytes.
385    #[cfg(feature = "unstable-v2")]
386    #[must_use]
387    pub fn with_float_dict(mut self, enabled: bool) -> Self {
388        self.allow_float_dict = enabled;
389        self
390    }
391
392    /// Allow float columns to store each value as a decimal-scaled integer.
393    #[cfg(feature = "unstable-v2")]
394    #[must_use]
395    pub fn with_float_alp(mut self, enabled: bool) -> Self {
396        self.allow_float_alp = enabled;
397        self
398    }
399}
400
401/// How to encode a string column.
402#[derive(Debug, Clone, Copy, PartialEq, Eq)]
403pub enum StrEncoding {
404    Plain,
405    Dict,
406    Fsst,
407    FsstDict,
408    FrontDict,
409    FsstFrontDict,
410}
411
412/// How to encode a float column, pinned rather than costed against the alternatives.
413#[derive(Debug, Clone, Copy, PartialEq, Eq)]
414pub enum FloatEncoding {
415    None,
416    #[cfg(feature = "unstable-v2")]
417    Dict,
418    #[cfg(feature = "unstable-v2")]
419    Alp,
420}
421
422#[derive(Debug, Clone, Copy, PartialEq)]
423pub enum ColumnKind {
424    Id,
425    Geometry,
426    Property,
427}
428
429/// Context for per-stream encoding decisions in [`ExplicitEncoder`] callbacks.
430#[derive(Clone, Copy, Debug, PartialEq)]
431pub struct StreamCtx<'a> {
432    pub kind: ColumnKind,
433    pub stream_type: StreamType,
434    pub name: &'a str,
435    pub subname: &'a str,
436}
437impl<'a> StreamCtx<'a> {
438    /// Stream with a logical sub-part (e.g. string column `"lengths"` / `"offsets"`, shared-dict child suffix).
439    #[inline]
440    #[must_use]
441    pub const fn new(
442        kind: ColumnKind,
443        stream_type: StreamType,
444        name: &'a str,
445        subname: &'a str,
446    ) -> Self {
447        Self {
448            kind,
449            stream_type,
450            name,
451            subname,
452        }
453    }
454
455    #[inline]
456    #[must_use]
457    pub const fn id(stream_type: StreamType) -> Self {
458        Self::new(ColumnKind::Id, stream_type, "", "")
459    }
460
461    #[inline]
462    #[must_use]
463    pub const fn geom(stream_type: StreamType, name: &'a str) -> Self {
464        Self::new(ColumnKind::Geometry, stream_type, name, "")
465    }
466
467    #[inline]
468    #[must_use]
469    pub const fn prop(stream_type: StreamType, name: &'a str) -> Self {
470        Self::new(ColumnKind::Property, stream_type, name, "")
471    }
472
473    #[inline]
474    #[must_use]
475    pub const fn prop_data(name: &'a str) -> Self {
476        let stream_type = StreamType::Data(DictionaryType::None);
477        Self::new(ColumnKind::Property, stream_type, name, "")
478    }
479
480    #[inline]
481    #[must_use]
482    pub const fn prop2(stream_type: StreamType, prefix: &'a str, suffix: &'a str) -> Self {
483        Self::new(ColumnKind::Property, stream_type, prefix, suffix)
484    }
485}
486
487/// Explicit, deterministic encoding configuration for synthetics and tests.
488///
489/// All encoding choices are caller-specified via callbacks so one struct can cover any combination without per-stream boilerplate.
490#[derive(Dbg)]
491pub struct ExplicitEncoder {
492    /// Vertex buffer layout for geometry streams.
493    pub vertex_buffer_type: VertexBufferType,
494    /// Per-stream override for the skip-empty-stream rule used by `write_geo_u32_stream`.
495    #[dbg(skip)]
496    pub force_stream: Box<dyn for<'a> Fn(&'a StreamCtx<'a>) -> bool>,
497    /// Return the [`IntEncoder`] for a stream identified by [`StreamCtx`].
498    #[dbg(skip)]
499    pub get_int_encoder: Box<dyn for<'a> Fn(&'a StreamCtx<'a>) -> IntEncoder>,
500    /// Return the string encoding strategy for a string property column.
501    #[dbg(skip)]
502    pub get_str_encoding: Box<dyn Fn(&str) -> StrEncoding>,
503    /// Return the logical encoding for a float property column.
504    #[dbg(skip)]
505    pub get_float_encoding: Box<dyn Fn(&str) -> FloatEncoding>,
506}
mlt_core/encoder/model.rs

mlt_core/encoder/
model.rs
