mlt_core/decoder/
iterators.rs

1//! Zero-copy per-feature view into a fully-decoded [`Layer01<Parsed>`].
2//!
3//! [`ParsedLayer01::iter_features`] yields one [`FeatureRef`] per feature via
4//! [`LendingIterator`].  [`FeatureRef::iter_properties`] exposes per-feature
5//! property values as flat [`ColumnRef`] items; `SharedDict` columns are
6//! transparently expanded and null values are skipped.
7//!
8//! # Iterator model
9//!
10//! Feature iteration uses [`LendingIterator`] rather than [`std::iter::Iterator`].
11//! This allows the iterator to reuse an internal buffer across steps - the
12//! [`FeatureRef`] borrows its property values from that buffer - eliminating a
13//! per-feature `Vec` allocation.
14//!
15//! The consequence is that each [`FeatureRef`] must be dropped before calling
16//! [`LendingIterator::next`] again, so standard adapters like `.map()` and
17//! `.collect()` are **not** available directly.  Use a `while let` loop instead:
18
19use std::fmt;
20use std::iter::FusedIterator;
21use std::ops::Range;
22
23use geo_types::Geometry;
24use usize_cast::IntoUsize as _;
25
26use crate::decoder::{Layer01, ParsedLayer01, ParsedProperty, ParsedScalar, Property, RawProperty};
27use crate::{Lazy, LazyParsed, MltResult, Parsed};
28
29/// A minimal lending (streaming) iterator trait.
30///
31/// Unlike [`std::iter::Iterator`], the item type may borrow from the iterator
32/// itself, enabling zero-allocation iteration where the inner buffer is reused
33/// across steps.
34///
35/// Use a `while let` loop to drive the iterator:
36/// ```ignore
37/// let mut iter = layer.iter_features();
38/// while let Some(feat) = iter.next() {
39///     let feat = feat?;
40///     /* use feat here - it borrows from iter */
41/// }
42/// ```
43pub trait LendingIterator {
44    /// The type of each element, which may borrow from `self`.
45    type Item<'this>
46    where
47        Self: 'this;
48
49    /// Advance the iterator, returning the next element or `None` when exhausted.
50    fn next(&mut self) -> Option<Self::Item<'_>>;
51}
52
53impl<'a> Layer01<'a, Lazy> {
54    /// Iterate over the property column names of this layer, in order.
55    ///
56    /// Regular columns yield one [`PropName`]; `SharedDict` columns yield one name per
57    /// sub-item.  Names are available even before any column data has been decoded.
58    ///
59    /// Pair with [`FeatureRef::iter_all_properties`] to associate per-feature
60    /// values with their column names.
61    pub fn iterate_prop_names(&self) -> PropNamesIter<'_, Property<'a, Lazy>> {
62        PropNamesIter::new(&self.properties)
63    }
64}
65
66impl<'a> ParsedLayer01<'a> {
67    /// Iterate over all features in this fully-decoded layer via a [`LendingIterator`].
68    ///
69    /// Yields one `MltResult<`[`FeatureRef`]`>` per feature. Geometry decoding can
70    /// fail, hence the `Result` wrapper.
71    ///
72    /// ```text
73    /// let mut iter = parsed.iter_features();
74    /// while let Some(feat) = iter.next() {
75    ///     let feat = feat?;
76    ///     for col in feat.iter_properties() {
77    ///        // or use iter_all_properties() to include Nones
78    ///     }
79    /// }
80    /// ```
81    ///
82    /// All inner iterators - [`FeatureRef::iter_properties`],
83    /// [`FeatureRef::iter_all_properties`], and the name iterators - implement the
84    /// standard [`std::iter::Iterator`] trait and compose normally.
85    #[must_use]
86    pub fn iter_features(&self) -> Layer01FeatureIter<'_, 'a> {
87        Layer01FeatureIter::new(self)
88    }
89
90    /// Iterate over the property column names of this layer, in order.
91    /// See [`Layer01::iterate_prop_names`] for details.
92    pub fn iterate_prop_names(&self) -> PropNamesIter<'_, ParsedProperty<'a>> {
93        PropNamesIter::new(&self.properties)
94    }
95}
96
97/// A zero-allocation two-part property name yielded by [`FeatureRef::iter_properties`].
98///
99/// The two parts concatenate on [`Display`](fmt::Display) as `"{}{}"`:
100/// - For regular columns: `(column_name, "")` - zero allocation, second part always empty.
101/// - For `SharedDict` sub-items: `(prefix, suffix)` - both borrow directly from layer data.
102///
103/// Structural [`PartialEq`] compares both parts independently.  Use [`PartialEq<str>`] or
104/// [`PartialEq<&str>`] (also implemented) to compare against a plain `&str` as if the two
105/// parts were concatenated.
106#[derive(Debug, Clone, Copy)] // WARN: do not auto-derive PartialEq,Eq,Hash as it won't be correct
107pub struct PropName<'a>(&'a str, &'a str);
108
109impl fmt::Display for PropName<'_> {
110    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
111        f.write_str(self.0)?;
112        f.write_str(self.1)
113    }
114}
115
116impl PartialEq<PropName<'_>> for PropName<'_> {
117    fn eq(&self, other: &PropName<'_>) -> bool {
118        // Compare the concatenated strings byte-by-byte without allocating.
119        let (a0, a1) = (self.0.as_bytes(), self.1.as_bytes());
120        let a = a0.iter().chain(a1);
121        let (b0, b1) = (other.0.as_bytes(), other.1.as_bytes());
122        let b = b0.iter().chain(b1);
123        let combined_len_eq = a0.len() + a1.len() == b0.len() + b1.len();
124        combined_len_eq && a.eq(b)
125    }
126}
127
128impl PartialEq<str> for PropName<'_> {
129    /// Returns `true` if `other == self.0 + self.1`.
130    fn eq(&self, other: &str) -> bool {
131        other.strip_prefix(self.0) == Some(self.1)
132    }
133}
134
135impl PartialEq<PropName<'_>> for str {
136    fn eq(&self, other: &PropName<'_>) -> bool {
137        other == self
138    }
139}
140
141impl PartialEq<&str> for PropName<'_> {
142    fn eq(&self, other: &&str) -> bool {
143        self == *other
144    }
145}
146
147impl PartialEq<PropName<'_>> for &str {
148    fn eq(&self, other: &PropName<'_>) -> bool {
149        other == *self
150    }
151}
152
153/// A borrowed, non-null per-feature property value.
154///
155/// Nullability is lifted to [`ColumnRef`]: only non-null values appear in
156/// [`FeatureRef::iter_properties`].
157#[derive(Debug, Clone, Copy, PartialEq)]
158pub enum PropValueRef<'a> {
159    Bool(bool),
160    I8(i8),
161    U8(u8),
162    I32(i32),
163    U32(u32),
164    I64(i64),
165    U64(u64),
166    F32(f32),
167    F64(f64),
168    Str(&'a str),
169}
170
171macro_rules! impl_from_for_prop_value_ref {
172    ($($ty:ty => $variant:ident),+ $(,)?) => {
173        $(impl From<$ty> for PropValueRef<'_> {
174            fn from(v: $ty) -> Self { Self::$variant(v) }
175        })+
176    };
177}
178impl_from_for_prop_value_ref!(
179    bool => Bool, i8 => I8, u8 => U8,
180    i32 => I32, u32 => U32,
181    i64 => I64, u64 => U64,
182    f32 => F32, f64 => F64,
183);
184
185/// A single non-null property value for one feature, yielded by [`FeatureRef::iter_properties`].
186///
187/// `name` is a [`PropName`] that displays as `"{prefix}{suffix}"`.
188/// All borrows are zero-copy from the layer data.
189#[derive(Debug, Clone, Copy, PartialEq)]
190pub struct ColumnRef<'a> {
191    name: PropName<'a>,
192    value: PropValueRef<'a>,
193}
194
195impl<'a> ColumnRef<'a> {
196    #[must_use]
197    pub fn name(&self) -> PropName<'a> {
198        self.name
199    }
200
201    #[must_use]
202    pub fn value(&self) -> PropValueRef<'a> {
203        self.value
204    }
205}
206
207/// A single map feature returned by [`ParsedLayer01::iter_features`].
208///
209/// Borrows `values` from the outer [`Layer01FeatureIter`] buffer - it must be
210/// dropped before calling [`LendingIterator::next`] again.
211#[derive(Debug)]
212pub struct FeatureRef<'feat, 'layer: 'feat> {
213    /// Optional feature ID.
214    id: Option<u64>,
215    /// Geometry in [`Geometry<i32>`] form (owned, decoded on demand by the iterator).
216    geometry: Geometry<i32>,
217    /// Borrowed slice of column descriptors from the layer; used to yield column names.
218    columns: &'layer [ParsedProperty<'layer>],
219    /// Per-feature values in column order, one per slot (scalar, string, or `SharedDict`
220    /// sub-item).  Borrowed from the iterator's reused buffer - no allocation per feature.
221    values: &'feat [Option<PropValueRef<'layer>>],
222}
223
224impl<'feat, 'layer: 'feat> FeatureRef<'feat, 'layer> {
225    #[must_use]
226    pub fn id(&self) -> Option<u64> {
227        self.id
228    }
229
230    #[must_use]
231    pub fn geometry(&self) -> &Geometry<i32> {
232        &self.geometry
233    }
234
235    /// Iterate over every property slot for this feature, **values only**, in column order.
236    ///
237    /// Yields `Option<PropValueRef>`:
238    /// - `Some(value)` - the slot contains a non-null value.
239    /// - `None` - the slot is null / absent.
240    ///
241    /// Use [`Layer01::iterate_prop_names`] to pair values with their column names.
242    pub fn iter_all_properties(
243        &self,
244    ) -> impl ExactSizeIterator<Item = Option<PropValueRef<'layer>>>
245    + DoubleEndedIterator
246    + FusedIterator
247    + '_ {
248        self.values.iter().copied()
249    }
250
251    /// Iterate over all non-null properties for this feature.
252    ///
253    /// `SharedDict` columns are transparently expanded into one [`ColumnRef`] per sub-item.
254    /// Null / absent values are skipped entirely. The iterator is infallible.
255    pub fn iter_properties(
256        &self,
257    ) -> impl DoubleEndedIterator<Item = ColumnRef<'layer>> + FusedIterator + '_ {
258        PropNamesIter::new(self.columns)
259            .zip(self.values.iter().copied())
260            .filter_map(|(name, opt_val)| opt_val.map(|value| ColumnRef { name, value }))
261    }
262
263    /// Look up a property by name, returning its value if present and non-null.
264    ///
265    /// For `SharedDict` columns the expected name is `"{prefix}{suffix}"`, matching
266    /// the key used by [`iter_properties`](Self::iter_properties).
267    #[must_use]
268    pub fn get_property(&self, name: &str) -> Option<PropValueRef<'layer>> {
269        self.iter_properties()
270            .find(|col| col.name() == name)
271            .map(|col| col.value())
272    }
273}
274
275// ── Column name helpers ───────────────────────────────────────────────────────
276
277/// A property column that contributes one or more [`PropName`]s.
278///
279/// Scalar and string columns contribute exactly one name; `SharedDict` columns
280/// contribute one per sub-item.
281pub trait ColNames {
282    /// Always `PropName<'tile>`.  It cannot be written as `PropName<'_>` directly:
283    /// that would tie names to the `&self` borrow rather than to the tile buffer,
284    /// so they could no longer outlive the layer.
285    type Name;
286
287    /// Number of names this column contributes.
288    fn name_count(&self) -> usize;
289
290    /// The name at sub-index `idx`, which must be less than [`Self::name_count`].
291    fn name_at(&self, idx: usize) -> Self::Name;
292}
293
294impl<'p> ColNames for ParsedProperty<'p> {
295    type Name = PropName<'p>;
296
297    fn name_count(&self) -> usize {
298        match self {
299            Self::SharedDict(sd) => sd.items.len(),
300            Self::Bool(_)
301            | Self::I8(_)
302            | Self::U8(_)
303            | Self::I32(_)
304            | Self::U32(_)
305            | Self::I64(_)
306            | Self::U64(_)
307            | Self::F32(_)
308            | Self::F64(_)
309            | Self::Str(_) => 1,
310        }
311    }
312
313    fn name_at(&self, idx: usize) -> PropName<'p> {
314        use ParsedProperty as P;
315        match self {
316            P::Bool(s) => PropName(s.name, ""),
317            P::I8(s) => PropName(s.name, ""),
318            P::U8(s) => PropName(s.name, ""),
319            P::I32(s) => PropName(s.name, ""),
320            P::U32(s) => PropName(s.name, ""),
321            P::I64(s) => PropName(s.name, ""),
322            P::U64(s) => PropName(s.name, ""),
323            P::F32(s) => PropName(s.name, ""),
324            P::F64(s) => PropName(s.name, ""),
325            P::Str(s) => PropName(s.name, ""),
326            P::SharedDict(sd) => PropName(sd.prefix, sd.items[idx].suffix),
327        }
328    }
329}
330
331impl<'p> ColNames for RawProperty<'p> {
332    type Name = PropName<'p>;
333
334    fn name_count(&self) -> usize {
335        match self {
336            Self::SharedDict(sd) => sd.children.len(),
337            Self::Bool(_)
338            | Self::I8(_)
339            | Self::U8(_)
340            | Self::I32(_)
341            | Self::U32(_)
342            | Self::I64(_)
343            | Self::U64(_)
344            | Self::F32(_)
345            | Self::F64(_)
346            | Self::Str(_) => 1,
347        }
348    }
349
350    fn name_at(&self, idx: usize) -> PropName<'p> {
351        use RawProperty as P;
352        match self {
353            P::Bool(s) | P::I8(s) | P::U8(s) | P::I32(s) | P::U32(s) | P::I64(s) | P::U64(s) => {
354                PropName(s.name, "")
355            }
356            P::F32(s) | P::F64(s) => PropName(s.name, ""),
357            P::Str(s) => PropName(s.name, ""),
358            P::SharedDict(sd) => PropName(sd.name, sd.children[idx].name),
359        }
360    }
361}
362
363/// A column that failed to parse contributes no names at all.
364impl<'p> ColNames for LazyParsed<RawProperty<'p>, ParsedProperty<'p>> {
365    type Name = PropName<'p>;
366
367    fn name_count(&self) -> usize {
368        match self {
369            Self::Raw(r) => r.name_count(),
370            Self::Parsed(p) => p.name_count(),
371            Self::ParsingFailed => 0,
372        }
373    }
374
375    fn name_at(&self, idx: usize) -> PropName<'p> {
376        match self {
377            Self::Raw(r) => r.name_at(idx),
378            Self::Parsed(p) => p.name_at(idx),
379            Self::ParsingFailed => unreachable!("ParsingFailed contributes no names"),
380        }
381    }
382}
383
384/// Iterates the property column names of a layer, in column order.
385///
386/// Regular columns yield one [`PropName`]; `SharedDict` columns yield one name per
387/// sub-item (`(prefix, suffix)`).
388#[must_use]
389pub struct PropNamesIter<'a, C> {
390    props: &'a [C],
391    /// Columns that may still yield a name: `cols.start` is the front column,
392    /// `cols.end - 1` the back one.  The two coincide once they meet.
393    cols: Range<usize>,
394    /// Next sub-index to yield from `props[cols.start]`.
395    front_sub: usize,
396    /// One past the next sub-index to yield from `props[cols.end - 1]`.
397    back_sub: usize,
398    /// Names not yet yielded from either end.  Both ends decrement it, so it is
399    /// what stops them crossing while they share a column.
400    remaining: usize,
401}
402
403impl<'a, C: ColNames> PropNamesIter<'a, C> {
404    pub(crate) fn new(props: &'a [C]) -> Self {
405        Self {
406            props,
407            cols: 0..props.len(),
408            front_sub: 0,
409            back_sub: props.last().map_or(0, ColNames::name_count),
410            remaining: props.iter().map(ColNames::name_count).sum(),
411        }
412    }
413}
414
415impl<C: ColNames> Iterator for PropNamesIter<'_, C> {
416    type Item = C::Name;
417
418    fn next(&mut self) -> Option<C::Name> {
419        if self.remaining == 0 {
420            return None;
421        }
422        self.remaining -= 1;
423        loop {
424            // `remaining` was non-zero, so some column in `cols` still has a name.
425            let col = &self.props[self.cols.start];
426            if self.front_sub < col.name_count() {
427                let name = col.name_at(self.front_sub);
428                self.front_sub += 1;
429                return Some(name);
430            }
431            self.cols.start += 1;
432            self.front_sub = 0;
433        }
434    }
435
436    fn size_hint(&self) -> (usize, Option<usize>) {
437        (self.remaining, Some(self.remaining))
438    }
439}
440
441impl<C: ColNames> DoubleEndedIterator for PropNamesIter<'_, C> {
442    fn next_back(&mut self) -> Option<C::Name> {
443        if self.remaining == 0 {
444            return None;
445        }
446        self.remaining -= 1;
447        loop {
448            if self.back_sub > 0 {
449                self.back_sub -= 1;
450                return Some(self.props[self.cols.end - 1].name_at(self.back_sub));
451            }
452            self.cols.end -= 1;
453            self.back_sub = self.props[self.cols.end - 1].name_count();
454        }
455    }
456}
457
458impl<C: ColNames> ExactSizeIterator for PropNamesIter<'_, C> {
459    fn len(&self) -> usize {
460        self.remaining
461    }
462}
463
464impl<C: ColNames> FusedIterator for PropNamesIter<'_, C> {}
465
466/// A boxed per-column-slot value iterator yielding one `Option<`[`PropValueRef`]`>` per feature.
467type ColValIter<'l> = Box<dyn Iterator<Item = Option<PropValueRef<'l>>> + 'l>;
468
469/// Build one [`ColValIter`] per property column "slot" from a decoded column slice.
470///
471/// - Scalar and string columns contribute one slot each.
472/// - `SharedDict` columns contribute one slot per sub-item.
473fn build_col_iters<'p>(columns: &'p [ParsedProperty<'p>]) -> Vec<ColValIter<'p>> {
474    use ParsedProperty as PP;
475    let mut iters: Vec<ColValIter<'p>> = Vec::new();
476    for col in columns {
477        match col {
478            PP::Bool(s) => iters.push(scalar_col_iter(s)),
479            PP::I8(s) => iters.push(scalar_col_iter(s)),
480            PP::U8(s) => iters.push(scalar_col_iter(s)),
481            PP::I32(s) => iters.push(scalar_col_iter(s)),
482            PP::U32(s) => iters.push(scalar_col_iter(s)),
483            PP::I64(s) => iters.push(scalar_col_iter(s)),
484            PP::U64(s) => iters.push(scalar_col_iter(s)),
485            PP::F32(s) => iters.push(scalar_col_iter(s)),
486            PP::F64(s) => iters.push(scalar_col_iter(s)),
487            PP::Str(strings) => {
488                let data: &'p str = strings.data.as_ref();
489                let lengths: &'p [i32] = &strings.lengths;
490                let mut curr_end: usize = 0;
491                let mut feat_idx = 0usize;
492                iters.push(Box::new(std::iter::from_fn(move || {
493                    let &end_i32 = lengths.get(feat_idx)?;
494                    feat_idx += 1;
495                    if end_i32 >= 0 {
496                        let start = curr_end;
497                        curr_end = end_i32.cast_unsigned().into_usize();
498                        Some(data.get(start..curr_end).map(PropValueRef::Str))
499                    } else {
500                        // Null slot: curr_end unchanged (null encodes the current byte offset).
501                        Some(None)
502                    }
503                })));
504            }
505            PP::SharedDict(dict) => {
506                for item in &dict.items {
507                    let dict_ref: &'p _ = dict;
508                    let item_ref: &'p _ = item;
509                    let mut feat_idx = 0usize;
510                    iters.push(Box::new(std::iter::from_fn(move || {
511                        if feat_idx >= item_ref.ranges.len() {
512                            return None;
513                        }
514                        let idx = feat_idx;
515                        feat_idx += 1;
516                        Some(item_ref.get(dict_ref, idx).map(PropValueRef::Str))
517                    })));
518                }
519            }
520        }
521    }
522    iters
523}
524
525/// Build a boxed value iterator for a single scalar property column.
526fn scalar_col_iter<'p, T>(scalar: &'p ParsedScalar<'p, T>) -> ColValIter<'p>
527where
528    T: Copy + PartialEq,
529    PropValueRef<'p>: From<T>,
530{
531    Box::new(scalar.iter_optional().map(|o| o.map(PropValueRef::from)))
532}
533
534/// Iterator over the features of a fully-decoded [`Layer01<Parsed>`].
535///
536/// Returned by [`ParsedLayer01::iter_features`]. Implements [`LendingIterator`]:
537/// advance with `while let Some(feat) = iter.next()`.
538///
539/// Holds one O(1)-per-step cursor per property column slot. On each step the
540/// per-column cursors are advanced and their results written into a reused
541/// `values_buf` - yielding a [`FeatureRef`] that borrows that buffer with no
542/// per-feature heap allocation.
543pub struct Layer01FeatureIter<'layer, 'data: 'layer> {
544    layer: &'layer Layer01<'data, Parsed>,
545    index: usize,
546    feature_count: usize,
547    /// ID iterator, `None` when the layer has no ID column.
548    id_iter: Option<crate::utils::PresenceOptIter<'layer, u64>>,
549    /// One boxed value iterator per column slot (scalar, string, or `SharedDict` sub-item).
550    col_iters: Vec<ColValIter<'layer>>,
551    /// Reused buffer: filled on each `next()` call, borrowed by the yielded [`FeatureRef`].
552    values_buf: Vec<Option<PropValueRef<'layer>>>,
553}
554
555impl<'layer, 'data: 'layer> Layer01FeatureIter<'layer, 'data> {
556    fn new(layer: &'layer Layer01<'data, Parsed>) -> Self {
557        let col_iters = build_col_iters(&layer.properties);
558        let cap = col_iters.len();
559        Self {
560            layer,
561            index: 0,
562            feature_count: layer.feature_count(),
563            id_iter: layer.id.as_ref().map(|id| id.iter_optional()),
564            col_iters,
565            values_buf: Vec::with_capacity(cap),
566        }
567    }
568
569    /// Number of features not yet yielded.
570    #[must_use]
571    pub fn len(&self) -> usize {
572        self.feature_count - self.index
573    }
574
575    /// Returns `true` if all features have been yielded.
576    #[must_use]
577    pub fn is_empty(&self) -> bool {
578        self.index >= self.feature_count
579    }
580}
581
582impl<'layer> LendingIterator for Layer01FeatureIter<'layer, '_> {
583    type Item<'this>
584        = MltResult<FeatureRef<'this, 'layer>>
585    where
586        Self: 'this;
587
588    fn next(&mut self) -> Option<Self::Item<'_>> {
589        let index = self.index;
590        if index >= self.feature_count {
591            return None;
592        }
593        self.index += 1;
594
595        // Advance all per-feature cursors unconditionally, even if geometry decode fails,
596        // so that IDs and property values remain aligned with geometry indices.
597        let id = self.id_iter.as_mut().and_then(Iterator::next).flatten();
598        self.values_buf.clear();
599        self.values_buf
600            .extend(self.col_iters.iter_mut().map(|it| it.next().flatten()));
601
602        Some(
603            self.layer
604                .geometry
605                .to_geojson(index)
606                .map(|geometry| FeatureRef {
607                    id,
608                    geometry,
609                    columns: &self.layer.properties,
610                    values: &self.values_buf,
611                }),
612        )
613    }
614}
615
616#[cfg(test)]
617mod tests {
618    use geo_types::Point;
619    use serde_json::Value;
620
621    use super::*;
622    use crate::Layer;
623    use crate::decoder::GeometryValues;
624    use crate::encoder::model::StagedLayer;
625    use crate::encoder::{Codecs, Encoder, Presence, StagedId, StagedProperty, StagedSharedDict};
626    use crate::test_helpers::{assert_size_hint_exact, dec, parser};
627
628    fn layer_buf(staged: StagedLayer) -> Vec<u8> {
629        staged
630            .encode_into(Encoder::default(), &mut Codecs::default())
631            .unwrap()
632            .into_layer_bytes()
633            .unwrap()
634    }
635
636    fn three_points() -> GeometryValues {
637        let mut g = GeometryValues::default();
638        g.push_geom(&Geometry::<i32>::Point(Point::new(1, 2)));
639        g.push_geom(&Geometry::<i32>::Point(Point::new(3, 4)));
640        g.push_geom(&Geometry::<i32>::Point(Point::new(5, 6)));
641        g
642    }
643
644    fn empty_layer(name: &str) -> StagedLayer {
645        staged_layer(name, StagedId::None, GeometryValues::default(), vec![])
646    }
647
648    fn staged_layer(
649        name: &str,
650        id: StagedId,
651        geometry: GeometryValues,
652        properties: Vec<StagedProperty>,
653    ) -> StagedLayer {
654        StagedLayer::new(name, 4096, id, geometry, properties).unwrap()
655    }
656
657    #[test]
658    fn prop_name_display_concatenates_parts() {
659        assert_eq!(PropName("addr:", "city").to_string(), "addr:city");
660        assert_eq!(PropName("name", "").to_string(), "name");
661        assert_eq!(PropName("", "").to_string(), "");
662    }
663
664    #[test]
665    fn prop_name_eq_str_matches_concatenation() {
666        assert_eq!(PropName("addr:", "city"), "addr:city");
667        assert_eq!("addr:city", PropName("addr:", "city"));
668        assert_ne!(PropName("addr:", "city"), "addr:");
669        assert_ne!(PropName("addr:", "city"), "city");
670        assert_eq!(PropName("name", ""), "name");
671    }
672
673    #[test]
674    fn prop_name_structural_eq_is_part_wise() {
675        assert_eq!(PropName("a", "b"), PropName("a", "b"));
676        assert_eq!(PropName("ab", ""), PropName("a", "b"));
677    }
678
679    #[test]
680    fn prop_name_eq_prop_name_semantic_equality() {
681        assert_eq!(PropName("ab", ""), PropName("a", "b"));
682        assert_eq!(PropName("", "ab"), PropName("a", "b"));
683        assert_eq!(PropName("abc", "def"), PropName("ab", "cdef"));
684        assert_eq!(PropName("a", "bcdef"), PropName("abcde", "f"));
685
686        assert_ne!(PropName("a", "b"), PropName("a", "c"));
687        assert_ne!(PropName("a", "b"), PropName("ab", "c"));
688        assert_ne!(PropName("abc", ""), PropName("ab", ""));
689    }
690
691    #[test]
692    fn prop_value_ref_scalars_convert_to_json() {
693        assert_eq!(Value::from(PropValueRef::Bool(true)), Value::Bool(true));
694        assert_eq!(Value::from(PropValueRef::Bool(false)), Value::Bool(false));
695        assert_eq!(Value::from(PropValueRef::I8(-1)), Value::from(-1_i8));
696        assert_eq!(Value::from(PropValueRef::U8(255)), Value::from(255_u8));
697        assert_eq!(
698            Value::from(PropValueRef::I32(-1000)),
699            Value::from(-1000_i32)
700        );
701        assert_eq!(Value::from(PropValueRef::U32(1000)), Value::from(1000_u32));
702        assert_eq!(
703            Value::from(PropValueRef::I64(i64::MIN)),
704            Value::from(i64::MIN)
705        );
706        assert_eq!(
707            Value::from(PropValueRef::U64(u64::MAX)),
708            Value::from(u64::MAX)
709        );
710        assert_eq!(
711            Value::from(PropValueRef::Str("hello")),
712            Value::String("hello".into())
713        );
714    }
715
716    #[test]
717    fn prop_value_ref_float_finite_is_number() {
718        assert!(matches!(
719            Value::from(PropValueRef::F32(1.5)),
720            Value::Number(_)
721        ));
722        assert!(matches!(
723            Value::from(PropValueRef::F64(2.5)),
724            Value::Number(_)
725        ));
726    }
727
728    #[test]
729    fn prop_value_ref_float_non_finite_becomes_string_sentinel() {
730        assert_eq!(
731            Value::from(PropValueRef::F32(f32::NAN)),
732            Value::String("f32::NAN".into())
733        );
734        assert_eq!(
735            Value::from(PropValueRef::F32(f32::INFINITY)),
736            Value::String("f32::INFINITY".into())
737        );
738        assert_eq!(
739            Value::from(PropValueRef::F64(f64::NAN)),
740            Value::String("f64::NAN".into())
741        );
742        assert_eq!(
743            Value::from(PropValueRef::F64(f64::NEG_INFINITY)),
744            Value::String("f64::NEG_INFINITY".into())
745        );
746    }
747
748    #[test]
749    fn empty_layer_yields_no_features() {
750        let buf = layer_buf(empty_layer("empty"));
751        let (_, layer) = Layer::from_bytes(&buf, &mut parser()).unwrap();
752        let Layer::Tag01(lazy) = layer else {
753            panic!("expected Tag01")
754        };
755        let parsed = lazy.decode_all(&mut dec()).unwrap();
756
757        let iter = parsed.iter_features();
758        assert_eq!(iter.len(), 0);
759        assert!(iter.is_empty());
760        assert_eq!(parsed.iter_features().len(), 0);
761    }
762
763    #[test]
764    fn len_decreases_with_each_next() {
765        let buf = layer_buf(staged_layer("test", StagedId::None, three_points(), vec![]));
766        let (_, layer) = Layer::from_bytes(&buf, &mut parser()).unwrap();
767        let Layer::Tag01(lazy) = layer else { panic!() };
768        let parsed = lazy.decode_all(&mut dec()).unwrap();
769
770        let mut iter = parsed.iter_features();
771        assert_eq!(iter.len(), 3);
772        iter.next().unwrap().unwrap();
773        assert_eq!(iter.len(), 2);
774        iter.next().unwrap().unwrap();
775        assert_eq!(iter.len(), 1);
776        iter.next().unwrap().unwrap();
777        assert_eq!(iter.len(), 0);
778        assert!(iter.is_empty());
779        assert!(iter.next().is_none());
780    }
781
782    #[test]
783    fn feature_ids_are_preserved() {
784        let buf = layer_buf(staged_layer(
785            "test",
786            StagedId::from_optional(vec![Some(100), None, Some(200)]),
787            three_points(),
788            vec![],
789        ));
790        let (_, layer) = Layer::from_bytes(&buf, &mut parser()).unwrap();
791        let Layer::Tag01(lazy) = layer else { panic!() };
792        let parsed = lazy.decode_all(&mut dec()).unwrap();
793
794        let mut ids = Vec::new();
795        let mut iter = parsed.iter_features();
796        while let Some(r) = iter.next() {
797            ids.push(r.unwrap().id);
798        }
799        assert_eq!(ids, [Some(100), None, Some(200)]);
800    }
801
802    #[test]
803    fn geometry_values_match_input() {
804        let buf = layer_buf(staged_layer("test", StagedId::None, three_points(), vec![]));
805        let (_, layer) = Layer::from_bytes(&buf, &mut parser()).unwrap();
806        let Layer::Tag01(lazy) = layer else { panic!() };
807        let parsed = lazy.decode_all(&mut dec()).unwrap();
808
809        let mut geoms = Vec::new();
810        let mut iter = parsed.iter_features();
811        while let Some(r) = iter.next() {
812            geoms.push(r.unwrap().geometry);
813        }
814        assert_eq!(geoms[0], Geometry::<i32>::Point(Point::new(1, 2)));
815        assert_eq!(geoms[1], Geometry::<i32>::Point(Point::new(3, 4)));
816        assert_eq!(geoms[2], Geometry::<i32>::Point(Point::new(5, 6)));
817    }
818
819    #[test]
820    fn null_scalar_values_are_skipped() {
821        let buf = layer_buf(staged_layer(
822            "test",
823            StagedId::None,
824            three_points(),
825            vec![StagedProperty::opt_u32("n", vec![Some(1), None, Some(3)])],
826        ));
827        let (_, layer) = Layer::from_bytes(&buf, &mut parser()).unwrap();
828        let Layer::Tag01(lazy) = layer else { panic!() };
829        let parsed = lazy.decode_all(&mut dec()).unwrap();
830
831        let mut iter = parsed.iter_features();
832
833        {
834            let feat = iter.next().unwrap().unwrap();
835            let cols: Vec<_> = feat.iter_properties().collect();
836            assert_eq!(cols.len(), 1);
837            assert_eq!(cols[0].name, PropName("n", ""));
838            assert_eq!(cols[0].name, "n");
839            assert_eq!(cols[0].value, PropValueRef::U32(1));
840            let all: Vec<_> = feat.iter_all_properties().collect();
841            assert_eq!(all, [Some(PropValueRef::U32(1))]);
842        }
843        {
844            let feat = iter.next().unwrap().unwrap();
845            assert!(feat.iter_properties().next().is_none());
846            let all: Vec<_> = feat.iter_all_properties().collect();
847            assert_eq!(all, [None]);
848        }
849        {
850            let feat = iter.next().unwrap().unwrap();
851            assert_eq!(feat.get_property("n"), Some(PropValueRef::U32(3)));
852            let all: Vec<_> = feat.iter_all_properties().collect();
853            assert_eq!(all, [Some(PropValueRef::U32(3))]);
854        }
855
856        let names: Vec<_> = parsed.iterate_prop_names().map(|n| n.to_string()).collect();
857        assert_eq!(names, ["n"]);
858    }
859
860    #[test]
861    fn null_string_values_are_skipped() {
862        let buf = layer_buf(staged_layer(
863            "test",
864            StagedId::None,
865            three_points(),
866            vec![StagedProperty::opt_str(
867                "label",
868                vec![Some("foo"), None, Some("bar")],
869            )],
870        ));
871        let (_, layer) = Layer::from_bytes(&buf, &mut parser()).unwrap();
872        let Layer::Tag01(lazy) = layer else { panic!() };
873        let parsed = lazy.decode_all(&mut dec()).unwrap();
874
875        let mut iter = parsed.iter_features();
876        {
877            let feat = iter.next().unwrap().unwrap();
878            assert_eq!(feat.get_property("label"), Some(PropValueRef::Str("foo")));
879        }
880        {
881            let feat = iter.next().unwrap().unwrap();
882            assert_eq!(feat.get_property("label"), None);
883        }
884        {
885            let feat = iter.next().unwrap().unwrap();
886            assert_eq!(feat.get_property("label"), Some(PropValueRef::Str("bar")));
887        }
888    }
889
890    #[test]
891    fn multiple_columns_independently_nullable() {
892        let buf = layer_buf(staged_layer(
893            "test",
894            StagedId::None,
895            three_points(),
896            vec![
897                StagedProperty::opt_bool("flag", vec![Some(true), Some(false), None]),
898                StagedProperty::opt_i32("score", vec![None, Some(-5), Some(7)]),
899            ],
900        ));
901        let (_, layer) = Layer::from_bytes(&buf, &mut parser()).unwrap();
902        let Layer::Tag01(lazy) = layer else { panic!() };
903        let parsed = lazy.decode_all(&mut dec()).unwrap();
904
905        let mut iter = parsed.iter_features();
906
907        // feat 0: flag=true, score=null -> 1 property
908        {
909            let feat = iter.next().unwrap().unwrap();
910            assert_eq!(feat.iter_properties().count(), 1);
911            assert_eq!(feat.get_property("flag"), Some(PropValueRef::Bool(true)));
912            assert_eq!(feat.get_property("score"), None);
913        }
914        // feat 1: flag=false, score=-5 -> 2 properties
915        {
916            let feat = iter.next().unwrap().unwrap();
917            assert_eq!(feat.iter_properties().count(), 2);
918            assert_eq!(feat.get_property("flag"), Some(PropValueRef::Bool(false)));
919            assert_eq!(feat.get_property("score"), Some(PropValueRef::I32(-5)));
920        }
921        // feat 2: flag=null, score=7 -> 1 property
922        {
923            let feat = iter.next().unwrap().unwrap();
924            assert_eq!(feat.iter_properties().count(), 1);
925            assert_eq!(feat.get_property("flag"), None);
926            assert_eq!(feat.get_property("score"), Some(PropValueRef::I32(7)));
927        }
928    }
929
930    #[test]
931    fn geometry_error_does_not_misalign_ids() {
932        use crate::decoder::GeometryType;
933
934        let buf = layer_buf(staged_layer(
935            "test",
936            StagedId::from_optional(vec![Some(10), Some(20), Some(30)]),
937            three_points(),
938            vec![],
939        ));
940        let (_, layer) = Layer::from_bytes(&buf, &mut parser()).unwrap();
941        let Layer::Tag01(lazy) = layer else { panic!() };
942        let mut parsed = lazy.decode_all(&mut dec()).unwrap();
943
944        // Corrupt feature 1's geometry type: Point -> LineString.
945        // A LineString requires part_offsets, which are absent here, so
946        // to_geojson(1) will return Err(NoPartOffsets).
947        parsed.geometry.vector_types[1] = GeometryType::LineString;
948
949        let mut iter = parsed.iter_features();
950
951        // Feature 0: valid Point, id = Some(10)
952        let feat0 = iter.next().unwrap().unwrap();
953        assert_eq!(feat0.id, Some(10));
954
955        // Feature 1: geometry error - iterator still advances ID cursor
956        assert!(iter.next().unwrap().is_err());
957
958        // Feature 2: valid Point, id must be Some(30), not Some(20)
959        let feat2 = iter.next().unwrap().unwrap();
960        assert_eq!(
961            feat2.id,
962            Some(30),
963            "id cursor was not advanced on geometry error"
964        );
965
966        assert!(iter.next().is_none());
967    }
968
969    #[test]
970    fn get_property_absent_column_returns_none() {
971        let buf = layer_buf(staged_layer(
972            "test",
973            StagedId::None,
974            three_points(),
975            vec![StagedProperty::u32("x", vec![1, 2, 3])],
976        ));
977        let (_, layer) = Layer::from_bytes(&buf, &mut parser()).unwrap();
978        let Layer::Tag01(lazy) = layer else { panic!() };
979        let parsed = lazy.decode_all(&mut dec()).unwrap();
980
981        let mut iter = parsed.iter_features();
982        let feat = iter.next().unwrap().unwrap();
983        assert_eq!(feat.get_property("no_such_column"), None);
984    }
985
986    #[test]
987    fn shared_dict_columns_are_expanded() {
988        let shared_dict = StagedSharedDict::new(
989            "addr:",
990            [
991                (
992                    "city",
993                    vec![Some("Paris"), Some("Rome"), None],
994                    Presence::Mixed,
995                ),
996                (
997                    "zip",
998                    vec![Some("75001"), None, Some("00100")],
999                    Presence::Mixed,
1000                ),
1001            ],
1002        )
1003        .unwrap();
1004
1005        let buf = layer_buf(staged_layer(
1006            "test",
1007            StagedId::None,
1008            three_points(),
1009            vec![StagedProperty::SharedDict(shared_dict)],
1010        ));
1011        let (_, layer) = Layer::from_bytes(&buf, &mut parser()).unwrap();
1012        let Layer::Tag01(lazy) = layer else { panic!() };
1013        let parsed = lazy.decode_all(&mut dec()).unwrap();
1014
1015        let mut iter = parsed.iter_features();
1016
1017        // feat 0: city=Paris, zip=75001
1018        {
1019            let feat = iter.next().unwrap().unwrap();
1020            assert_eq!(
1021                feat.get_property("addr:city"),
1022                Some(PropValueRef::Str("Paris"))
1023            );
1024            assert_eq!(
1025                feat.get_property("addr:zip"),
1026                Some(PropValueRef::Str("75001"))
1027            );
1028            assert_eq!(feat.iter_properties().count(), 2);
1029        }
1030        // feat 1: city=Rome, zip=null
1031        {
1032            let feat = iter.next().unwrap().unwrap();
1033            assert_eq!(
1034                feat.get_property("addr:city"),
1035                Some(PropValueRef::Str("Rome"))
1036            );
1037            assert_eq!(feat.get_property("addr:zip"), None);
1038            assert_eq!(feat.iter_properties().count(), 1);
1039            // iter_all_properties: values only (no names); SharedDict expands to two slots
1040            let all: Vec<_> = feat.iter_all_properties().collect();
1041            assert_eq!(all, [Some(PropValueRef::Str("Rome")), None]);
1042        }
1043        // feat 2: city=null, zip=00100
1044        {
1045            let feat = iter.next().unwrap().unwrap();
1046            assert_eq!(feat.get_property("addr:city"), None);
1047            assert_eq!(
1048                feat.get_property("addr:zip"),
1049                Some(PropValueRef::Str("00100"))
1050            );
1051        }
1052
1053        let names: Vec<_> = parsed.iterate_prop_names().map(|n| n.to_string()).collect();
1054        assert_eq!(names, ["addr:city", "addr:zip"]);
1055    }
1056
1057    fn strs<'p>(iter: impl Iterator<Item = PropName<'p>>) -> Vec<String> {
1058        iter.map(|n| n.to_string()).collect()
1059    }
1060
1061    fn reversed<'p>(iter: impl DoubleEndedIterator<Item = PropName<'p>>) -> Vec<String> {
1062        let mut names = strs(iter.rev());
1063        names.reverse();
1064        names
1065    }
1066
1067    fn both_ends_layer() -> Vec<u8> {
1068        let shared_dict = StagedSharedDict::new(
1069            "addr:",
1070            [
1071                ("city", vec![Some("Paris"); 3], Presence::AllPresent),
1072                ("zip", vec![Some("75001"); 3], Presence::AllPresent),
1073                ("street", vec![Some("Rue"); 3], Presence::AllPresent),
1074            ],
1075        )
1076        .unwrap();
1077
1078        layer_buf(staged_layer(
1079            "test",
1080            StagedId::None,
1081            three_points(),
1082            vec![
1083                StagedProperty::str("before", ["a", "a", "a"]),
1084                StagedProperty::u32("count", vec![1, 2, 3]),
1085                StagedProperty::SharedDict(shared_dict),
1086                StagedProperty::str("after", ["b", "b", "b"]),
1087            ],
1088        ))
1089    }
1090
1091    const BOTH_ENDS_NAMES: [&str; 6] = [
1092        "before",
1093        "count",
1094        "addr:city",
1095        "addr:zip",
1096        "addr:street",
1097        "after",
1098    ];
1099
1100    fn prop_names() -> Vec<PropName<'static>> {
1101        BOTH_ENDS_NAMES.iter().map(|n| PropName(n, "")).collect()
1102    }
1103
1104    #[test]
1105    fn prop_names_iterate_from_both_ends_lazy() {
1106        let buf = both_ends_layer();
1107        let (_, layer) = Layer::from_bytes(&buf, &mut parser()).unwrap();
1108        let Layer::Tag01(lazy) = layer else { panic!() };
1109
1110        assert_eq!(strs(lazy.iterate_prop_names()), BOTH_ENDS_NAMES);
1111        assert_eq!(reversed(lazy.iterate_prop_names()), BOTH_ENDS_NAMES);
1112        assert_size_hint_exact(|| lazy.iterate_prop_names(), &prop_names());
1113    }
1114
1115    #[test]
1116    fn prop_names_iterate_from_both_ends_parsed() {
1117        let buf = both_ends_layer();
1118        let (_, layer) = Layer::from_bytes(&buf, &mut parser()).unwrap();
1119        let Layer::Tag01(lazy) = layer else { panic!() };
1120        let parsed = lazy.decode_all(&mut dec()).unwrap();
1121
1122        assert_eq!(strs(parsed.iterate_prop_names()), BOTH_ENDS_NAMES);
1123        assert_eq!(reversed(parsed.iterate_prop_names()), BOTH_ENDS_NAMES);
1124
1125        assert_size_hint_exact(|| parsed.iterate_prop_names(), &prop_names());
1126    }
1127
1128    #[test]
1129    fn prop_names_of_layer_without_properties() {
1130        let buf = layer_buf(empty_layer("test"));
1131        let (_, layer) = Layer::from_bytes(&buf, &mut parser()).unwrap();
1132        let Layer::Tag01(lazy) = layer else { panic!() };
1133
1134        assert_eq!(lazy.iterate_prop_names().size_hint(), (0, Some(0)));
1135        assert_eq!(lazy.iterate_prop_names().next(), None);
1136        assert_eq!(lazy.iterate_prop_names().next_back(), None);
1137
1138        let parsed = lazy.decode_all(&mut dec()).unwrap();
1139        assert_eq!(parsed.iterate_prop_names().size_hint(), (0, Some(0)));
1140        assert_eq!(parsed.iterate_prop_names().next(), None);
1141        assert_eq!(parsed.iterate_prop_names().next_back(), None);
1142    }
1143
1144    /// `Layer01<Lazy>` only ever holds `Raw` columns today, so the `Parsed` and
1145    /// `ParsingFailed` arms are exercised against a hand-built column slice.
1146    #[test]
1147    fn prop_names_skip_columns_that_failed_to_parse() {
1148        let buf = both_ends_layer();
1149        let (_, layer) = Layer::from_bytes(&buf, &mut parser()).unwrap();
1150        let Layer::Tag01(lazy) = layer else { panic!() };
1151        let raw = lazy.properties.clone();
1152
1153        let (_, layer) = Layer::from_bytes(&buf, &mut parser()).unwrap();
1154        let Layer::Tag01(lazy) = layer else { panic!() };
1155        let parsed = lazy.decode_all(&mut dec()).unwrap().properties;
1156
1157        let mixed = vec![
1158            raw[0].clone(),
1159            LazyParsed::ParsingFailed,
1160            LazyParsed::Parsed(parsed[2].clone()),
1161            LazyParsed::ParsingFailed,
1162            raw[3].clone(),
1163        ];
1164        let expected = ["before", "addr:city", "addr:zip", "addr:street", "after"];
1165
1166        assert_eq!(strs(PropNamesIter::new(&mixed)), expected);
1167        assert_eq!(reversed(PropNamesIter::new(&mixed)), expected);
1168        assert_size_hint_exact(
1169            || PropNamesIter::new(&mixed),
1170            &expected.map(|n| PropName(n, "")),
1171        );
1172
1173        let all_failed = vec![LazyParsed::ParsingFailed; 3];
1174        assert_eq!(PropNamesIter::new(&all_failed).size_hint(), (0, Some(0)));
1175        assert_eq!(PropNamesIter::new(&all_failed).next(), None);
1176        assert_eq!(PropNamesIter::new(&all_failed).next_back(), None);
1177    }
1178
1179    #[test]
1180    fn feature_property_iterators_run_backwards() {
1181        let buf = both_ends_layer();
1182        let (_, layer) = Layer::from_bytes(&buf, &mut parser()).unwrap();
1183        let Layer::Tag01(lazy) = layer else { panic!() };
1184        let parsed = lazy.decode_all(&mut dec()).unwrap();
1185
1186        let mut iter = parsed.iter_features();
1187        let feat = iter.next().unwrap().unwrap();
1188
1189        let all: Vec<_> = feat.iter_all_properties().collect();
1190        assert_size_hint_exact(|| feat.iter_all_properties(), &all);
1191        let mut all_back: Vec<_> = feat.iter_all_properties().rev().collect();
1192        all_back.reverse();
1193        assert_eq!(all_back, all);
1194
1195        let mut props_back: Vec<_> = feat
1196            .iter_properties()
1197            .rev()
1198            .map(|c| c.name().to_string())
1199            .collect();
1200        props_back.reverse();
1201        assert_eq!(props_back, BOTH_ENDS_NAMES);
1202    }
1203}
mlt_core/decoder/iterators.rs

mlt_core/decoder/
iterators.rs
