mlt_core/encoder/
model.rs

1use std::borrow::Cow;
2#[cfg(feature = "unstable-v2")]
3use std::collections::HashSet;
4
5use derive_debug::Dbg;
6
7#[cfg(feature = "unstable-v2")]
8use crate::decoder::RleLayout;
9use crate::decoder::{DictionaryType, FastPForKind, GeometryValues, PhysicalEncoding, StreamType};
10use crate::encoder::geometry::VertexBufferType;
11use crate::encoder::{IntEncoder, StagedId, StagedProperty};
12#[cfg(feature = "unstable-v2")]
13use crate::encoder::{StagedInterior, StagedMValue, StagedNested, StagedNode};
14#[cfg(feature = "unstable-v2")]
15use crate::tile::MAX_NESTED_DEPTH;
16use crate::tile::{ColumnRole, Extent, reject_taken_name};
17use crate::{MltError, MltResult};
18
19/// Owned variant of `Unknown`.
20#[derive(Debug, Clone, Default, PartialEq)]
21pub struct EncodedUnknown {
22    pub(crate) tag: u8,
23    pub(crate) value: Vec<u8>,
24}
25
26impl EncodedUnknown {
27    pub fn new(tag: u8, value: Vec<u8>) -> MltResult<Self> {
28        if tag == 1 {
29            return Err(MltError::ParsingColumnType(tag));
30        }
31        Ok(Self { tag, value })
32    }
33
34    #[must_use]
35    pub fn tag(&self) -> u32 {
36        u32::from(self.tag)
37    }
38
39    #[must_use]
40    pub fn data(&self) -> &[u8] {
41        &self.value
42    }
43}
44
45/// Parameters derived from the vertex set of a feature collection, used to
46/// normalize coordinates before space-filling-curve key computation.
47#[derive(Debug, Clone, Copy, PartialEq)]
48pub struct CurveParams {
49    pub shift: u32,
50    pub bits: u32,
51}
52
53impl Default for CurveParams {
54    fn default() -> Self {
55        Self { shift: 0, bits: 1 }
56    }
57}
58
59impl CurveParams {
60    /// Compute params from a flat `[x0, y0, x1, y1, …]` vertex slice.
61    #[must_use]
62    pub fn from_vertices(vertices: &[i32]) -> Self {
63        if vertices.is_empty() {
64            return Self::default();
65        }
66        let (min, max) = vertices
67            .iter()
68            .fold((i32::MAX, i32::MIN), |(mn, mx), &v| (mn.min(v), mx.max(v)));
69        crate::codecs::hilbert::hilbert_curve_params_from_bounds(min, max)
70    }
71}
72
73/// Columnar layer data being prepared for encoding (stage 2 of the encoding pipeline).
74///
75/// Holds fully-owned columnar data. Constructed directly (synthetics, benches) or
76/// converted from [`TileLayer`](crate::TileLayer).
77/// Consumed by encoding via [`StagedLayer::encode_into`] or `StagedLayer::encode_explicit`
78/// (with explicit encoding mode enabled).
79#[derive(Debug, PartialEq, Clone)]
80pub struct StagedLayer {
81    pub(crate) name: String,
82    pub(crate) extent: Extent,
83    pub(crate) id: StagedId,
84    pub(crate) geometry: GeometryValues,
85    pub(crate) properties: Vec<StagedProperty>,
86    /// Vertex-scoped columns, which only a v2 layer can be written with.
87    #[cfg(feature = "unstable-v2")]
88    pub(crate) m_values: Vec<StagedMValue>,
89    /// Nested columns, which only a v2 layer can be written with.
90    #[cfg(feature = "unstable-v2")]
91    pub(crate) nested: Vec<StagedNested>,
92}
93
94#[cfg_attr(not(feature = "__private"), allow(dead_code))]
95impl StagedLayer {
96    pub fn new(
97        name: impl Into<String>,
98        extent: u32,
99        id: StagedId,
100        geometry: GeometryValues,
101        properties: Vec<StagedProperty>,
102    ) -> MltResult<Self> {
103        let (name, extent, _) = validate_staged(name, extent, &id, &geometry, &properties)?;
104        Ok(Self {
105            name,
106            extent,
107            id,
108            geometry,
109            properties,
110            #[cfg(feature = "unstable-v2")]
111            m_values: Vec::new(),
112            #[cfg(feature = "unstable-v2")]
113            nested: Vec::new(),
114        })
115    }
116
117    /// As [`Self::new`], with the layer's vertex-scoped columns.
118    #[cfg(feature = "unstable-v2")]
119    pub fn with_m_values(
120        name: impl Into<String>,
121        extent: u32,
122        id: StagedId,
123        geometry: GeometryValues,
124        properties: Vec<StagedProperty>,
125        m_values: Vec<StagedMValue>,
126    ) -> MltResult<Self> {
127        let (name, extent, columns) = validate_staged(name, extent, &id, &geometry, &properties)?;
128        validate_m_values(&geometry, &columns, &m_values)?;
129        Ok(Self {
130            name,
131            extent,
132            id,
133            geometry,
134            properties,
135            m_values,
136            nested: Vec::new(),
137        })
138    }
139
140    /// As [`Self::new`], with both the layer's vertex-scoped and its nested columns.
141    #[cfg(feature = "unstable-v2")]
142    pub fn with_nested(
143        name: impl Into<String>,
144        extent: u32,
145        id: StagedId,
146        geometry: GeometryValues,
147        properties: Vec<StagedProperty>,
148        m_values: Vec<StagedMValue>,
149        nested: Vec<StagedNested>,
150    ) -> MltResult<Self> {
151        let (name, extent, columns) = validate_staged(name, extent, &id, &geometry, &properties)?;
152        validate_m_values(&geometry, &columns, &m_values)?;
153        validate_nested(&geometry, &columns, &m_values, &nested)?;
154        Ok(Self {
155            name,
156            extent,
157            id,
158            geometry,
159            properties,
160            m_values,
161            nested,
162        })
163    }
164
165    #[must_use]
166    pub fn name(&self) -> &str {
167        &self.name
168    }
169
170    #[must_use]
171    pub fn extent(&self) -> Extent {
172        self.extent
173    }
174
175    #[must_use]
176    pub fn id(&self) -> &StagedId {
177        &self.id
178    }
179
180    #[must_use]
181    pub fn geometry(&self) -> &GeometryValues {
182        &self.geometry
183    }
184
185    #[must_use]
186    pub fn properties(&self) -> &[StagedProperty] {
187        &self.properties
188    }
189
190    #[cfg(feature = "unstable-v2")]
191    #[must_use]
192    pub fn m_values(&self) -> &[StagedMValue] {
193        &self.m_values
194    }
195
196    #[cfg(feature = "unstable-v2")]
197    #[must_use]
198    pub fn nested(&self) -> &[StagedNested] {
199        &self.nested
200    }
201}
202
203/// Check a staged layer's name, extent and columns, returning the validated fields and the column names.
204fn validate_staged<'p>(
205    name: impl Into<String>,
206    extent: u32,
207    id: &StagedId,
208    geometry: &GeometryValues,
209    properties: &'p [StagedProperty],
210) -> MltResult<(String, Extent, Vec<Cow<'p, str>>)> {
211    let name = name.into();
212    if name.is_empty() {
213        return Err(MltError::MissingLayerName);
214    }
215    let extent = Extent::new(extent)?;
216    let feature_count = geometry.feature_count();
217    if let Some(actual) = id.feature_count()
218        && actual != feature_count
219    {
220        return Err(MltError::StagedFeatureCountMismatch {
221            column: "id".into(),
222            expected: feature_count,
223            actual,
224        });
225    }
226    // A shared dictionary's `name()` is only its prefix, which may repeat.
227    // Its real columns are `{prefix}{suffix}`, so those are the names the layer's namespace holds.
228    let mut columns: Vec<Cow<'p, str>> = Vec::with_capacity(properties.len());
229    for property in properties {
230        let actual = property.feature_count();
231        if actual != feature_count {
232            return Err(MltError::StagedFeatureCountMismatch {
233                column: property.name().to_string(),
234                expected: feature_count,
235                actual,
236            });
237        }
238        if let StagedProperty::SharedDict(sd) = property {
239            for item in &sd.items {
240                let name = format!("{}{}", sd.prefix, item.suffix);
241                reject_taken_name(&name, ColumnRole::Property, property_columns(&columns))?;
242                columns.push(Cow::Owned(name));
243            }
244        } else {
245            reject_taken_name(
246                property.name(),
247                ColumnRole::Property,
248                property_columns(&columns),
249            )?;
250            columns.push(Cow::Borrowed(property.name()));
251        }
252    }
253    Ok((name, extent, columns))
254}
255
256/// Pair every property column's name with its kind, as the uniqueness check reads them.
257fn property_columns<'a>(
258    columns: &'a [Cow<'a, str>],
259) -> impl Iterator<Item = (&'a str, ColumnRole)> + 'a {
260    columns
261        .iter()
262        .map(|name| (name.as_ref(), ColumnRole::Property))
263}
264
265/// Check every m-value column against the geometry it runs over: a unique name, a mask of one
266/// bit per feature, and one value per vertex of every feature it marks present.
267#[cfg(feature = "unstable-v2")]
268fn validate_m_values(
269    geometry: &GeometryValues,
270    columns: &[Cow<'_, str>],
271    m_values: &[StagedMValue],
272) -> MltResult<()> {
273    let feature_count = geometry.feature_count();
274    let mut seen: Vec<&str> = Vec::with_capacity(m_values.len());
275    for column in m_values {
276        reject_taken_name(
277            column.name(),
278            ColumnRole::MValue,
279            property_columns(columns).chain(seen.iter().map(|n| (*n, ColumnRole::MValue))),
280        )?;
281        seen.push(column.name());
282        if let Some(actual) = column.feature_count()
283            && actual != feature_count
284        {
285            return Err(MltError::StagedFeatureCountMismatch {
286                column: column.name().to_string(),
287                expected: feature_count,
288                actual,
289            });
290        }
291        let mut expected = 0;
292        for index in 0..feature_count {
293            let present = column
294                .presence
295                .as_ref()
296                .is_none_or(|mask| mask.get(index).copied().unwrap_or(false));
297            if present {
298                expected += geometry.vertex_count(index)?;
299            }
300        }
301        if expected != column.values().count() {
302            return Err(MltError::MValueColumnLengthMismatch {
303                name: column.name().to_string(),
304                expected,
305                actual: column.values().count(),
306            });
307        }
308    }
309    Ok(())
310}
311
312/// Check every nested column against the layer: a unique name, a mask of one bit
313/// per feature, and a tree the wire can hold.
314#[cfg(feature = "unstable-v2")]
315fn validate_nested(
316    geometry: &GeometryValues,
317    columns: &[Cow<'_, str>],
318    m_values: &[StagedMValue],
319    nested: &[StagedNested],
320) -> MltResult<()> {
321    let feature_count = geometry.feature_count();
322    let mut seen: Vec<&str> = Vec::with_capacity(nested.len());
323    for column in nested {
324        reject_taken_name(
325            column.name(),
326            ColumnRole::Nested,
327            property_columns(columns)
328                .chain(m_values.iter().map(|c| (c.name(), ColumnRole::MValue)))
329                .chain(seen.iter().map(|n| (*n, ColumnRole::Nested))),
330        )?;
331        seen.push(column.name());
332        if column.root.parent_count() != feature_count {
333            return Err(MltError::StagedFeatureCountMismatch {
334                column: column.name().to_string(),
335                expected: feature_count,
336                actual: column.root.parent_count(),
337            });
338        }
339        validate_nested_interior(&column.root, 1)?;
340    }
341    Ok(())
342}
343
344/// Check one node of a nested tree against the depth the wire allows, then its own children.
345#[cfg(feature = "unstable-v2")]
346fn validate_nested_node(node: &StagedNode, depth: usize) -> MltResult<()> {
347    if depth > MAX_NESTED_DEPTH {
348        return Err(MltError::NestedTooDeep(depth));
349    }
350    match node {
351        StagedNode::Leaf(_) => Ok(()),
352        StagedNode::Interior(interior) => validate_nested_interior(interior, depth),
353    }
354}
355
356/// Check an interior node's children: unique field names, and no level the wire cannot reach.
357#[cfg(feature = "unstable-v2")]
358fn validate_nested_interior(interior: &StagedInterior, depth: usize) -> MltResult<()> {
359    match interior {
360        StagedInterior::Struct(node) => {
361            let mut seen: HashSet<&str> = HashSet::with_capacity(node.fields.len());
362            for (name, field) in &node.fields {
363                if !seen.insert(name.as_str()) {
364                    return Err(MltError::DuplicateFieldName(name.clone()));
365                }
366                validate_nested_node(field, depth + 1)?;
367            }
368            Ok(())
369        }
370        StagedInterior::List(node) => validate_nested_node(&node.element, depth + 1),
371        StagedInterior::Map(node) => validate_nested_node(&node.value, depth + 1),
372    }
373}
374
375/// Which wire format layers are encoded to.
376#[cfg(feature = "unstable-v2")]
377#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Default)]
378#[cfg_attr(feature = "arbitrary", derive(arbitrary::Arbitrary))]
379pub enum WireVersion {
380    /// Tag `0x01` - the stable v1 format.
381    #[default]
382    V01,
383    /// Tag `0x02` - the experimental v2 format (see `docs/migrating-to-v2.md`).
384    ///
385    /// Currently limited to ID, scalar, string, shared-dictionary and geometry columns.
386    V02,
387}
388
389#[cfg(feature = "unstable-v2")]
390impl WireVersion {
391    /// The layer tag byte identifying this format on the wire.
392    #[must_use]
393    pub(crate) fn tag(self) -> u8 {
394        match self {
395            Self::V01 => 1,
396            Self::V02 => 2,
397        }
398    }
399
400    /// The `FastPFor` block size and word order used by this format.
401    #[must_use]
402    pub(crate) fn fastpfor_kind(self) -> FastPForKind {
403        match self {
404            Self::V01 => FastPForKind::Block256Be,
405            Self::V02 => FastPForKind::Block128Le,
406        }
407    }
408
409    /// The RLE stream data layout used by this format.
410    #[must_use]
411    pub(crate) fn rle_layout(self) -> RleLayout {
412        match self {
413            Self::V01 => RleLayout::Split,
414            Self::V02 => RleLayout::Interleaved,
415        }
416    }
417
418    /// The header bytes a raw stream of fixed-width words saves over any other physical encoding.
419    /// v2 leaves such a stream's `byte_length` varint out, since its count and width give it.
420    #[must_use]
421    pub(crate) fn raw_header_saving(self) -> usize {
422        match self {
423            Self::V01 => 0,
424            Self::V02 => 1,
425        }
426    }
427}
428
429/// Global encoder settings controlling which optimization strategies are attempted.
430#[derive(Debug, Clone, Copy, PartialEq, Hash)]
431#[expect(
432    clippy::struct_excessive_bools,
433    reason = "enums would not model this better, not a state machine"
434)]
435pub struct EncoderConfig {
436    /// The wire format to encode layers to.
437    #[cfg(feature = "unstable-v2")]
438    wire_version: WireVersion,
439    /// Generate tessellation data for polygons and multi-polygons.
440    tessellate: bool,
441    /// Try sorting features by the Z-order (Morton) curve index of their first vertex.
442    attempt_spatial_morton_sort: bool,
443    /// Try sorting features by the Hilbert curve index of their first vertex.
444    attempt_spatial_hilbert_sort: bool,
445    /// Try sorting features by their feature ID in ascending order.
446    attempt_id_sort: bool,
447    /// Allow `FSST` string compression
448    allow_fsst: bool,
449    /// Allow `FastPFOR` integer compression
450    allow_fastpfor: bool,
451    /// Allow string grouping into shared dictionaries
452    allow_shared_dict: bool,
453    /// Allow the v2-only float dictionary encoding
454    #[cfg(feature = "unstable-v2")]
455    allow_float_dict: bool,
456    /// Allow the v2-only Framed, Exception-Free ALP float encoding
457    #[cfg(feature = "unstable-v2")]
458    allow_float_alp: bool,
459    /// Allow the v2-only bit-packed physical encoding for dictionary code streams
460    #[cfg(feature = "unstable-v2")]
461    allow_packed_dict_codes: bool,
462    /// Code a nested struct's or map's row shapes instead of per-field presence or per-entry keys.
463    #[cfg(feature = "unstable-v2")]
464    allow_row_shapes: bool,
465    /// Let a tessellated all-polygon layer store its triangles without the outlines.
466    #[cfg(feature = "unstable-v2")]
467    allow_triangles_only: bool,
468    /// Allow the v2-only second-order delta encodings of integer and vertex streams
469    #[cfg(feature = "unstable-v2")]
470    allow_delta2: bool,
471    /// Allow the v2-only rANS vertex encoding
472    #[cfg(feature = "unstable-v2")]
473    allow_rans_vertices: bool,
474}
475impl Default for EncoderConfig {
476    fn default() -> Self {
477        Self {
478            #[cfg(feature = "unstable-v2")]
479            wire_version: WireVersion::V01,
480            tessellate: false,
481            attempt_spatial_morton_sort: true,
482            attempt_spatial_hilbert_sort: true,
483            attempt_id_sort: true,
484            allow_fsst: true,
485            allow_fastpfor: true,
486            allow_shared_dict: true,
487            // Off by default while the encoding is still being measured.
488            #[cfg(feature = "unstable-v2")]
489            allow_float_dict: false,
490            #[cfg(feature = "unstable-v2")]
491            allow_float_alp: false,
492            #[cfg(feature = "unstable-v2")]
493            allow_row_shapes: false,
494            #[cfg(feature = "unstable-v2")]
495            allow_packed_dict_codes: false,
496            #[cfg(feature = "unstable-v2")]
497            allow_triangles_only: false,
498            #[cfg(feature = "unstable-v2")]
499            allow_delta2: false,
500            #[cfg(feature = "unstable-v2")]
501            allow_rans_vertices: false,
502        }
503    }
504}
505
506impl EncoderConfig {
507    #[cfg(feature = "unstable-v2")]
508    #[must_use]
509    pub fn wire_version(self) -> WireVersion {
510        self.wire_version
511    }
512
513    #[must_use]
514    pub fn tessellate(self) -> bool {
515        self.tessellate
516    }
517
518    #[must_use]
519    pub fn attempt_spatial_morton_sort(self) -> bool {
520        self.attempt_spatial_morton_sort
521    }
522
523    #[must_use]
524    pub fn attempt_spatial_hilbert_sort(self) -> bool {
525        self.attempt_spatial_hilbert_sort
526    }
527
528    #[must_use]
529    pub fn attempt_id_sort(self) -> bool {
530        self.attempt_id_sort
531    }
532
533    #[must_use]
534    pub fn allow_fsst(self) -> bool {
535        self.allow_fsst
536    }
537
538    #[must_use]
539    pub fn allow_fastpfor(self) -> bool {
540        self.allow_fastpfor
541    }
542
543    /// The `FastPFor` encoding to race, or `None` when `FastPFor` is switched off.
544    #[must_use]
545    pub(crate) fn fastpfor(self) -> Option<PhysicalEncoding> {
546        #[cfg(feature = "unstable-v2")]
547        let kind = self.wire_version.fastpfor_kind();
548        #[cfg(not(feature = "unstable-v2"))]
549        let kind = FastPForKind::Block256Be;
550        self.allow_fastpfor
551            .then_some(PhysicalEncoding::FastPFor(kind))
552    }
553
554    #[must_use]
555    pub fn allow_shared_dict(self) -> bool {
556        self.allow_shared_dict
557    }
558
559    /// Whether a nested struct or map may code its row shapes, which only v2 can express.
560    #[cfg(feature = "unstable-v2")]
561    #[must_use]
562    pub fn allow_row_shapes(self) -> bool {
563        self.allow_row_shapes
564    }
565
566    /// Let a nested struct or map replace per-field presence and per-entry keys with row shapes.
567    #[cfg(feature = "unstable-v2")]
568    #[must_use]
569    pub fn with_row_shapes(mut self, enabled: bool) -> Self {
570        self.allow_row_shapes = enabled;
571        self
572    }
573
574    /// Whether 8-bit integer columns stay 8-bit, which only v2 can express.
575    #[must_use]
576    pub fn allow_8bit_ints(self) -> bool {
577        #[cfg(feature = "unstable-v2")]
578        return self.wire_version != WireVersion::V01;
579        #[cfg(not(feature = "unstable-v2"))]
580        return false;
581    }
582
583    /// Whether float columns may use a dictionary, which only v2 can express.
584    #[cfg(feature = "unstable-v2")]
585    #[must_use]
586    pub fn allow_float_dict(self) -> bool {
587        self.allow_float_dict && self.wire_version != WireVersion::V01
588    }
589
590    /// Whether float columns may use Framed, Exception-Free ALP, which only v2 can express.
591    #[cfg(feature = "unstable-v2")]
592    #[must_use]
593    pub fn allow_float_alp(self) -> bool {
594        self.allow_float_alp && self.wire_version != WireVersion::V01
595    }
596
597    #[cfg(feature = "unstable-v2")]
598    #[must_use]
599    pub fn with_wire_version(mut self, version: WireVersion) -> Self {
600        self.wire_version = version;
601        self
602    }
603
604    #[must_use]
605    pub fn with_tessellation(mut self, enabled: bool) -> Self {
606        self.tessellate = enabled;
607        self
608    }
609
610    /// Whether a tessellated layer may drop its outlines, which only v2 can express.
611    #[cfg(feature = "unstable-v2")]
612    #[must_use]
613    pub fn allow_triangles_only(self) -> bool {
614        self.allow_triangles_only && self.wire_version != WireVersion::V01
615    }
616
617    /// Let a tessellated layer of only polygons, with no m-values, store its triangles without the outlines.
618    /// Each of its polygons then decodes as the triangles it was cut into.
619    #[cfg(feature = "unstable-v2")]
620    #[must_use]
621    pub fn with_triangles_only(mut self, enabled: bool) -> Self {
622        self.allow_triangles_only = enabled;
623        self
624    }
625
626    /// Whether integer and vertex streams may store the deltas of their deltas, which only v2 can express.
627    #[cfg(feature = "unstable-v2")]
628    #[must_use]
629    pub fn allow_delta2(self) -> bool {
630        self.allow_delta2 && self.wire_version != WireVersion::V01
631    }
632
633    /// Allow integer and vertex streams to store the deltas of their deltas.
634    #[cfg(feature = "unstable-v2")]
635    #[must_use]
636    pub fn with_delta2(mut self, enabled: bool) -> Self {
637        self.allow_delta2 = enabled;
638        self
639    }
640
641    #[must_use]
642    pub fn with_spatial_morton_sort(mut self, enabled: bool) -> Self {
643        self.attempt_spatial_morton_sort = enabled;
644        self
645    }
646
647    #[must_use]
648    pub fn with_spatial_hilbert_sort(mut self, enabled: bool) -> Self {
649        self.attempt_spatial_hilbert_sort = enabled;
650        self
651    }
652
653    #[must_use]
654    pub fn with_id_sort(mut self, enabled: bool) -> Self {
655        self.attempt_id_sort = enabled;
656        self
657    }
658
659    #[must_use]
660    pub fn with_fsst(mut self, enabled: bool) -> Self {
661        self.allow_fsst = enabled;
662        self
663    }
664
665    #[must_use]
666    pub fn with_fastpfor(mut self, enabled: bool) -> Self {
667        self.allow_fastpfor = enabled;
668        self
669    }
670
671    #[must_use]
672    pub fn with_shared_dict(mut self, enabled: bool) -> Self {
673        self.allow_shared_dict = enabled;
674        self
675    }
676
677    /// Allow float columns to store one code per value into a dictionary of the distinct ones.
678    /// Off by default, and only v2 can express it.
679    /// A column takes it only when it comes out strictly smaller in stored bytes.
680    #[cfg(feature = "unstable-v2")]
681    #[must_use]
682    pub fn with_float_dict(mut self, enabled: bool) -> Self {
683        self.allow_float_dict = enabled;
684        self
685    }
686
687    /// Allow float columns to store each value as a decimal-scaled integer.
688    #[cfg(feature = "unstable-v2")]
689    #[must_use]
690    pub fn with_float_alp(mut self, enabled: bool) -> Self {
691        self.allow_float_alp = enabled;
692        self
693    }
694
695    /// Whether dictionary code streams may be stored bit-packed.
696    #[cfg(feature = "unstable-v2")]
697    #[must_use]
698    pub fn allow_packed_dict_codes(self) -> bool {
699        self.allow_packed_dict_codes && self.wire_version != WireVersion::V01
700    }
701
702    /// Allow dictionary code streams to store every code in the same
703    /// `ceil(log2(dict_len))` bits instead of a varint each.
704    #[cfg(feature = "unstable-v2")]
705    #[must_use]
706    pub fn with_packed_dict_codes(mut self, enabled: bool) -> Self {
707        self.allow_packed_dict_codes = enabled;
708        self
709    }
710
711    /// Whether vertex streams may be rANS-coded.
712    #[cfg(feature = "unstable-v2")]
713    #[must_use]
714    pub fn allow_rans_vertices(self) -> bool {
715        self.allow_rans_vertices && self.wire_version != WireVersion::V01
716    }
717
718    /// Allow plain vertex streams to be rANS-coded over the layer's vertex runs.
719    #[cfg(feature = "unstable-v2")]
720    #[must_use]
721    pub fn with_rans_vertices(mut self, enabled: bool) -> Self {
722        self.allow_rans_vertices = enabled;
723        self
724    }
725}
726
727/// How to encode a string column.
728#[derive(Debug, Clone, Copy, PartialEq, Eq)]
729pub enum StrEncoding {
730    Plain,
731    Dict,
732    Fsst,
733    FsstDict,
734    FrontDict,
735    FsstFrontDict,
736}
737
738/// How to encode a float column, pinned rather than costed against the alternatives.
739#[derive(Debug, Clone, Copy, PartialEq, Eq)]
740pub enum FloatEncoding {
741    None,
742    #[cfg(feature = "unstable-v2")]
743    Dict,
744    #[cfg(feature = "unstable-v2")]
745    Alp,
746}
747
748#[derive(Debug, Clone, Copy, PartialEq)]
749pub enum ColumnKind {
750    Id,
751    Geometry,
752    Property,
753}
754
755/// Where one string stream set sits: the column that holds it, and the path
756/// through a nested column's tree that leads to it.
757///
758/// A flat column's path is empty, so its streams are named exactly as before.
759#[cfg(feature = "unstable-v2")]
760#[derive(Clone, Copy, Debug)]
761pub struct StrAt<'a> {
762    pub name: &'a str,
763    pub subname: &'a str,
764}
765
766#[cfg(feature = "unstable-v2")]
767impl<'a> StrAt<'a> {
768    #[must_use]
769    pub const fn flat(name: &'a str) -> Self {
770        Self { name, subname: "" }
771    }
772
773    #[must_use]
774    pub const fn nested(name: &'a str, subname: &'a str) -> Self {
775        Self { name, subname }
776    }
777
778    pub(crate) fn ctx(self, stream_type: StreamType) -> StreamCtx<'a> {
779        StreamCtx::prop2(stream_type, self.name, self.subname)
780    }
781
782    /// The name an [`ExplicitEncoder`] pins an encoding by, the column's plus the path.
783    pub(crate) fn qualified(self) -> Cow<'a, str> {
784        StreamCtx::prop2(StreamType::Present, self.name, self.subname).qualified()
785    }
786}
787
788/// Context for per-stream encoding decisions in [`ExplicitEncoder`] callbacks.
789#[derive(Clone, Copy, Debug, PartialEq)]
790pub struct StreamCtx<'a> {
791    pub kind: ColumnKind,
792    pub stream_type: StreamType,
793    pub name: &'a str,
794    pub subname: &'a str,
795}
796impl<'a> StreamCtx<'a> {
797    /// Stream with a logical sub-part (e.g. string column `"lengths"` / `"offsets"`, shared-dict child suffix).
798    #[inline]
799    #[must_use]
800    pub const fn new(
801        kind: ColumnKind,
802        stream_type: StreamType,
803        name: &'a str,
804        subname: &'a str,
805    ) -> Self {
806        Self {
807            kind,
808            stream_type,
809            name,
810            subname,
811        }
812    }
813
814    #[inline]
815    #[must_use]
816    pub const fn id(stream_type: StreamType) -> Self {
817        Self::new(ColumnKind::Id, stream_type, "", "")
818    }
819
820    #[inline]
821    #[must_use]
822    pub const fn geom(stream_type: StreamType, name: &'a str) -> Self {
823        Self::new(ColumnKind::Geometry, stream_type, name, "")
824    }
825
826    #[inline]
827    #[must_use]
828    pub const fn prop(stream_type: StreamType, name: &'a str) -> Self {
829        Self::new(ColumnKind::Property, stream_type, name, "")
830    }
831
832    #[inline]
833    #[must_use]
834    pub const fn prop_data(name: &'a str) -> Self {
835        let stream_type = StreamType::Data(DictionaryType::None);
836        Self::new(ColumnKind::Property, stream_type, name, "")
837    }
838
839    #[inline]
840    #[must_use]
841    pub const fn prop2(stream_type: StreamType, prefix: &'a str, suffix: &'a str) -> Self {
842        Self::new(ColumnKind::Property, stream_type, prefix, suffix)
843    }
844
845    /// The name an [`ExplicitEncoder`] pins an encoding by, the column's plus any path within it.
846    #[must_use]
847    pub fn qualified(&self) -> Cow<'a, str> {
848        if self.subname.is_empty() {
849            Cow::Borrowed(self.name)
850        } else {
851            Cow::Owned(format!("{}{}", self.name, self.subname))
852        }
853    }
854}
855
856/// Explicit, deterministic encoding configuration for synthetics and tests.
857///
858/// All encoding choices are caller-specified via callbacks so one struct can cover any combination without per-stream boilerplate.
859#[derive(Dbg)]
860pub struct ExplicitEncoder {
861    /// Vertex buffer layout for geometry streams.
862    pub vertex_buffer_type: VertexBufferType,
863    /// Per-stream override for the skip-empty-stream rule used by `write_geo_u32_stream`.
864    #[dbg(skip)]
865    pub force_stream: Box<dyn for<'a> Fn(&'a StreamCtx<'a>) -> bool>,
866    /// Return the [`IntEncoder`] for a stream identified by [`StreamCtx`].
867    #[dbg(skip)]
868    pub get_int_encoder: Box<dyn for<'a> Fn(&'a StreamCtx<'a>) -> IntEncoder>,
869    /// Return the string encoding strategy for a string property column.
870    #[dbg(skip)]
871    pub get_str_encoding: Box<dyn Fn(&str) -> StrEncoding>,
872    /// Return the logical encoding for a float property column.
873    #[dbg(skip)]
874    pub get_float_encoding: Box<dyn Fn(&str) -> FloatEncoding>,
875}
876
877#[cfg(all(test, feature = "unstable-v2"))]
878mod tests {
879    use insta::assert_snapshot;
880
881    use super::*;
882    use crate::encoder::{StagedLeaf, StagedList, StagedStruct, StagedValues};
883
884    fn leaf() -> StagedNode {
885        StagedNode::Leaf(StagedLeaf::new(None, StagedValues::I32(vec![1])))
886    }
887
888    /// A tree of `depth` levels: nested lists all the way down to one leaf.
889    fn lists(depth: usize) -> StagedInterior {
890        let mut node = leaf();
891        for _ in 2..depth {
892            node = StagedNode::Interior(StagedInterior::List(StagedList::new(None, vec![1], node)));
893        }
894        StagedInterior::List(StagedList::new(None, vec![1], node))
895    }
896
897    #[test]
898    fn a_struct_repeating_a_field_name() {
899        let root =
900            StagedInterior::Struct(StagedStruct::new(None, vec![("a", leaf()), ("a", leaf())]));
901        assert_snapshot!(
902            validate_nested_interior(&root, 1).unwrap_err(),
903            @"duplicate field name in a struct node: a"
904        );
905    }
906
907    #[test]
908    fn a_tree_one_level_past_the_limit() {
909        assert_snapshot!(
910            validate_nested_interior(&lists(MAX_NESTED_DEPTH + 1), 1).unwrap_err(),
911            @"a nested column is 9 levels deep, more than the 8 the format allows"
912        );
913    }
914
915    #[test]
916    fn a_tree_at_the_limit() {
917        validate_nested_interior(&lists(MAX_NESTED_DEPTH), 1).expect("a tree at the limit");
918    }
919}
mlt_core/encoder/model.rs

mlt_core/encoder/
model.rs
