mlt_core/decoder/
iterators.rs

1//! Zero-copy per-feature view into a fully-decoded [`Layer01<Parsed>`].
2//!
3//! [`ParsedLayer01::iter_features`] yields one [`FeatureRef`] per feature via
4//! [`LendingIterator`].  [`FeatureRef::iter_properties`] exposes per-feature
5//! property values as flat [`ColumnRef`] items; `SharedDict` columns are
6//! transparently expanded and null values are skipped.
7//!
8//! # Iterator model
9//!
10//! Feature iteration uses [`LendingIterator`] rather than [`std::iter::Iterator`].
11//! This allows the iterator to reuse an internal buffer across steps - the
12//! [`FeatureRef`] borrows its property values from that buffer - eliminating a
13//! per-feature `Vec` allocation.
14//!
15//! The consequence is that each [`FeatureRef`] must be dropped before calling
16//! [`LendingIterator::next`] again, so standard adapters like `.map()` and
17//! `.collect()` are **not** available directly.  Use a `while let` loop instead:
18
19use std::fmt;
20use std::iter::FusedIterator;
21use std::ops::Range;
22
23use geo_types::Geometry;
24use usize_cast::IntoUsize as _;
25
26use crate::decoder::{Layer01, ParsedLayer01, ParsedProperty, ParsedScalar, Property, RawProperty};
27use crate::{Lazy, LazyParsed, MltResult, Parsed};
28
29/// A minimal lending (streaming) iterator trait.
30///
31/// Unlike [`std::iter::Iterator`], the item type may borrow from the iterator
32/// itself, enabling zero-allocation iteration where the inner buffer is reused
33/// across steps.
34///
35/// Use a `while let` loop to drive the iterator:
36/// ```ignore
37/// let mut iter = layer.iter_features();
38/// while let Some(feat) = iter.next() {
39///     let feat = feat?;
40///     /* use feat here - it borrows from iter */
41/// }
42/// ```
43pub trait LendingIterator {
44    /// The type of each element, which may borrow from `self`.
45    type Item<'this>
46    where
47        Self: 'this;
48
49    /// Advance the iterator, returning the next element or `None` when exhausted.
50    fn next(&mut self) -> Option<Self::Item<'_>>;
51}
52
53impl<'a> Layer01<'a, Lazy> {
54    /// Iterate over the property column names of this layer, in order.
55    ///
56    /// Regular columns yield one [`PropName`]; `SharedDict` columns yield one name per
57    /// sub-item.  Names are available even before any column data has been decoded.
58    ///
59    /// Pair with [`FeatureRef::iter_all_properties`] to associate per-feature
60    /// values with their column names.
61    pub fn iterate_prop_names(&self) -> PropNamesIter<'_, Property<'a, Lazy>> {
62        PropNamesIter::new(&self.properties)
63    }
64}
65
66impl<'a> ParsedLayer01<'a> {
67    /// Iterate over all features in this fully-decoded layer via a [`LendingIterator`].
68    ///
69    /// Yields one `MltResult<`[`FeatureRef`]`>` per feature. Geometry decoding can
70    /// fail, hence the `Result` wrapper.
71    ///
72    /// ```text
73    /// let mut iter = parsed.iter_features();
74    /// while let Some(feat) = iter.next() {
75    ///     let feat = feat?;
76    ///     for col in feat.iter_properties() {
77    ///        // or use iter_all_properties() to include Nones
78    ///     }
79    /// }
80    /// ```
81    ///
82    /// All inner iterators - [`FeatureRef::iter_properties`],
83    /// [`FeatureRef::iter_all_properties`], and the name iterators - implement the
84    /// standard [`std::iter::Iterator`] trait and compose normally.
85    #[must_use]
86    pub fn iter_features(&self) -> Layer01FeatureIter<'_, 'a> {
87        Layer01FeatureIter::new(self)
88    }
89
90    /// Iterate over the property column names of this layer, in order.
91    /// See [`Layer01::iterate_prop_names`] for details.
92    pub fn iterate_prop_names(&self) -> PropNamesIter<'_, ParsedProperty<'a>> {
93        PropNamesIter::new(&self.properties)
94    }
95}
96
97/// A zero-allocation two-part property name yielded by [`FeatureRef::iter_properties`].
98///
99/// The two parts concatenate on [`Display`](fmt::Display) as `"{}{}"`:
100/// - For regular columns: `(column_name, "")` - zero allocation, second part always empty.
101/// - For `SharedDict` sub-items: `(prefix, suffix)` - both borrow directly from layer data.
102///
103/// Structural [`PartialEq`] compares both parts independently.  Use [`PartialEq<str>`] or
104/// [`PartialEq<&str>`] (also implemented) to compare against a plain `&str` as if the two
105/// parts were concatenated.
106#[derive(Debug, Clone, Copy)] // WARN: do not auto-derive PartialEq,Eq,Hash as it won't be correct
107pub struct PropName<'a>(&'a str, &'a str);
108
109impl fmt::Display for PropName<'_> {
110    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
111        f.write_str(self.0)?;
112        f.write_str(self.1)
113    }
114}
115
116impl PartialEq<PropName<'_>> for PropName<'_> {
117    fn eq(&self, other: &PropName<'_>) -> bool {
118        // Compare the concatenated strings byte-by-byte without allocating.
119        let (a0, a1) = (self.0.as_bytes(), self.1.as_bytes());
120        let a = a0.iter().chain(a1);
121        let (b0, b1) = (other.0.as_bytes(), other.1.as_bytes());
122        let b = b0.iter().chain(b1);
123        let combined_len_eq = a0.len() + a1.len() == b0.len() + b1.len();
124        combined_len_eq && a.eq(b)
125    }
126}
127
128impl PartialEq<str> for PropName<'_> {
129    /// Returns `true` if `other == self.0 + self.1`.
130    fn eq(&self, other: &str) -> bool {
131        other.strip_prefix(self.0) == Some(self.1)
132    }
133}
134
135impl PartialEq<PropName<'_>> for str {
136    fn eq(&self, other: &PropName<'_>) -> bool {
137        other == self
138    }
139}
140
141impl PartialEq<&str> for PropName<'_> {
142    fn eq(&self, other: &&str) -> bool {
143        self == *other
144    }
145}
146
147impl PartialEq<PropName<'_>> for &str {
148    fn eq(&self, other: &PropName<'_>) -> bool {
149        other == *self
150    }
151}
152
153/// A borrowed, non-null per-feature property value.
154///
155/// Nullability is lifted to [`ColumnRef`]: only non-null values appear in
156/// [`FeatureRef::iter_properties`].
157#[derive(Debug, Clone, Copy, PartialEq)]
158pub enum PropValueRef<'a> {
159    Bool(bool),
160    I8(i8),
161    U8(u8),
162    I32(i32),
163    U32(u32),
164    I64(i64),
165    U64(u64),
166    F32(f32),
167    F64(f64),
168    Str(&'a str),
169}
170
171macro_rules! impl_from_for_prop_value_ref {
172    ($($ty:ty => $variant:ident),+ $(,)?) => {
173        $(impl From<$ty> for PropValueRef<'_> {
174            fn from(v: $ty) -> Self { Self::$variant(v) }
175        })+
176    };
177}
178impl_from_for_prop_value_ref!(
179    bool => Bool, i8 => I8, u8 => U8,
180    i32 => I32, u32 => U32,
181    i64 => I64, u64 => U64,
182    f32 => F32, f64 => F64,
183);
184
185/// A single non-null property value for one feature, yielded by [`FeatureRef::iter_properties`].
186///
187/// `name` is a [`PropName`] that displays as `"{prefix}{suffix}"`.
188/// All borrows are zero-copy from the layer data.
189#[derive(Debug, Clone, Copy, PartialEq)]
190pub struct ColumnRef<'a> {
191    name: PropName<'a>,
192    value: PropValueRef<'a>,
193}
194
195impl<'a> ColumnRef<'a> {
196    #[must_use]
197    pub fn name(&self) -> PropName<'a> {
198        self.name
199    }
200
201    #[must_use]
202    pub fn value(&self) -> PropValueRef<'a> {
203        self.value
204    }
205}
206
207/// A single map feature returned by [`ParsedLayer01::iter_features`].
208///
209/// Borrows `values` from the outer [`Layer01FeatureIter`] buffer - it must be
210/// dropped before calling [`LendingIterator::next`] again.
211#[derive(Debug)]
212pub struct FeatureRef<'feat, 'layer: 'feat> {
213    /// Optional feature ID.
214    id: Option<u64>,
215    /// Geometry in [`Geometry<i32>`] form (owned, decoded on demand by the iterator).
216    geometry: Geometry<i32>,
217    /// Borrowed slice of column descriptors from the layer; used to yield column names.
218    columns: &'layer [ParsedProperty<'layer>],
219    /// Per-feature values in column order, one per slot (scalar, string, or `SharedDict`
220    /// sub-item).  Borrowed from the iterator's reused buffer - no allocation per feature.
221    values: &'feat [Option<PropValueRef<'layer>>],
222}
223
224impl<'feat, 'layer: 'feat> FeatureRef<'feat, 'layer> {
225    #[must_use]
226    pub fn id(&self) -> Option<u64> {
227        self.id
228    }
229
230    #[must_use]
231    pub fn geometry(&self) -> &Geometry<i32> {
232        &self.geometry
233    }
234
235    /// Iterate over every property slot for this feature, **values only**, in column order.
236    ///
237    /// Yields `Option<PropValueRef>`:
238    /// - `Some(value)` - the slot contains a non-null value.
239    /// - `None` - the slot is null / absent.
240    ///
241    /// Use [`Layer01::iterate_prop_names`] to pair values with their column names.
242    #[must_use]
243    pub fn iter_all_properties(
244        &self,
245    ) -> impl ExactSizeIterator<Item = Option<PropValueRef<'layer>>>
246    + DoubleEndedIterator
247    + FusedIterator
248    + '_ {
249        self.values.iter().copied()
250    }
251
252    /// Iterate over all non-null properties for this feature.
253    ///
254    /// `SharedDict` columns are transparently expanded into one [`ColumnRef`] per sub-item.
255    /// Null / absent values are skipped entirely. The iterator is infallible.
256    #[must_use]
257    pub fn iter_properties(
258        &self,
259    ) -> impl DoubleEndedIterator<Item = ColumnRef<'layer>> + FusedIterator + '_ {
260        PropNamesIter::new(self.columns)
261            .zip(self.values.iter().copied())
262            .filter_map(|(name, opt_val)| opt_val.map(|value| ColumnRef { name, value }))
263    }
264
265    /// Look up a property by name, returning its value if present and non-null.
266    ///
267    /// For `SharedDict` columns the expected name is `"{prefix}{suffix}"`, matching
268    /// the key used by [`iter_properties`](Self::iter_properties).
269    #[must_use]
270    pub fn get_property(&self, name: &str) -> Option<PropValueRef<'layer>> {
271        self.iter_properties()
272            .find(|col| col.name() == name)
273            .map(|col| col.value())
274    }
275}
276
277// ── Column name helpers ───────────────────────────────────────────────────────
278
279/// A property column that contributes one or more [`PropName`]s.
280///
281/// Scalar and string columns contribute exactly one name; `SharedDict` columns
282/// contribute one per sub-item.
283pub trait ColNames {
284    /// Always `PropName<'tile>`.  It cannot be written as `PropName<'_>` directly:
285    /// that would tie names to the `&self` borrow rather than to the tile buffer,
286    /// so they could no longer outlive the layer.
287    type Name;
288
289    /// Number of names this column contributes.
290    fn name_count(&self) -> usize;
291
292    /// The name at sub-index `idx`, which must be less than [`Self::name_count`].
293    fn name_at(&self, idx: usize) -> Self::Name;
294}
295
296impl<'p> ColNames for ParsedProperty<'p> {
297    type Name = PropName<'p>;
298
299    fn name_count(&self) -> usize {
300        match self {
301            Self::SharedDict(sd) => sd.items.len(),
302            Self::Bool(_)
303            | Self::I8(_)
304            | Self::U8(_)
305            | Self::I32(_)
306            | Self::U32(_)
307            | Self::I64(_)
308            | Self::U64(_)
309            | Self::F32(_)
310            | Self::F64(_)
311            | Self::Str(_) => 1,
312        }
313    }
314
315    fn name_at(&self, idx: usize) -> PropName<'p> {
316        use ParsedProperty as P;
317        match self {
318            P::Bool(s) => PropName(s.name, ""),
319            P::I8(s) => PropName(s.name, ""),
320            P::U8(s) => PropName(s.name, ""),
321            P::I32(s) => PropName(s.name, ""),
322            P::U32(s) => PropName(s.name, ""),
323            P::I64(s) => PropName(s.name, ""),
324            P::U64(s) => PropName(s.name, ""),
325            P::F32(s) => PropName(s.name, ""),
326            P::F64(s) => PropName(s.name, ""),
327            P::Str(s) => PropName(s.name, ""),
328            P::SharedDict(sd) => PropName(sd.prefix, sd.items[idx].suffix),
329        }
330    }
331}
332
333impl<'p> ColNames for RawProperty<'p> {
334    type Name = PropName<'p>;
335
336    fn name_count(&self) -> usize {
337        match self {
338            Self::SharedDict(sd) => sd.children.len(),
339            Self::Bool(_)
340            | Self::I8(_)
341            | Self::U8(_)
342            | Self::I32(_)
343            | Self::U32(_)
344            | Self::I64(_)
345            | Self::U64(_)
346            | Self::F32(_)
347            | Self::F64(_)
348            | Self::Str(_) => 1,
349        }
350    }
351
352    fn name_at(&self, idx: usize) -> PropName<'p> {
353        use RawProperty as P;
354        match self {
355            P::Bool(s) | P::I8(s) | P::U8(s) | P::I32(s) | P::U32(s) | P::I64(s) | P::U64(s) => {
356                PropName(s.name, "")
357            }
358            P::F32(s) | P::F64(s) => PropName(s.name, ""),
359            P::Str(s) => PropName(s.name, ""),
360            P::SharedDict(sd) => PropName(sd.name, sd.children[idx].name),
361        }
362    }
363}
364
365/// A column that failed to parse contributes no names at all.
366impl<'p> ColNames for LazyParsed<RawProperty<'p>, ParsedProperty<'p>> {
367    type Name = PropName<'p>;
368
369    fn name_count(&self) -> usize {
370        match self {
371            Self::Raw(r) => r.name_count(),
372            Self::Parsed(p) => p.name_count(),
373            Self::ParsingFailed => 0,
374        }
375    }
376
377    fn name_at(&self, idx: usize) -> PropName<'p> {
378        match self {
379            Self::Raw(r) => r.name_at(idx),
380            Self::Parsed(p) => p.name_at(idx),
381            Self::ParsingFailed => unreachable!("ParsingFailed contributes no names"),
382        }
383    }
384}
385
386/// Iterates the property column names of a layer, in column order.
387///
388/// Regular columns yield one [`PropName`]; `SharedDict` columns yield one name per
389/// sub-item (`(prefix, suffix)`).
390#[must_use]
391pub struct PropNamesIter<'a, C> {
392    props: &'a [C],
393    /// Columns that may still yield a name: `cols.start` is the front column,
394    /// `cols.end - 1` the back one.  The two coincide once they meet.
395    cols: Range<usize>,
396    /// Next sub-index to yield from `props[cols.start]`.
397    front_sub: usize,
398    /// One past the next sub-index to yield from `props[cols.end - 1]`.
399    back_sub: usize,
400    /// Names not yet yielded from either end.  Both ends decrement it, so it is
401    /// what stops them crossing while they share a column.
402    remaining: usize,
403}
404
405impl<'a, C: ColNames> PropNamesIter<'a, C> {
406    pub(crate) fn new(props: &'a [C]) -> Self {
407        Self {
408            props,
409            cols: 0..props.len(),
410            front_sub: 0,
411            back_sub: props.last().map_or(0, ColNames::name_count),
412            remaining: props.iter().map(ColNames::name_count).sum(),
413        }
414    }
415}
416
417impl<C: ColNames> Iterator for PropNamesIter<'_, C> {
418    type Item = C::Name;
419
420    fn next(&mut self) -> Option<C::Name> {
421        if self.remaining == 0 {
422            return None;
423        }
424        self.remaining -= 1;
425        loop {
426            // `remaining` was non-zero, so some column in `cols` still has a name.
427            let col = &self.props[self.cols.start];
428            if self.front_sub < col.name_count() {
429                let name = col.name_at(self.front_sub);
430                self.front_sub += 1;
431                return Some(name);
432            }
433            self.cols.start += 1;
434            self.front_sub = 0;
435        }
436    }
437
438    fn size_hint(&self) -> (usize, Option<usize>) {
439        (self.remaining, Some(self.remaining))
440    }
441}
442
443impl<C: ColNames> DoubleEndedIterator for PropNamesIter<'_, C> {
444    fn next_back(&mut self) -> Option<C::Name> {
445        if self.remaining == 0 {
446            return None;
447        }
448        self.remaining -= 1;
449        loop {
450            if self.back_sub > 0 {
451                self.back_sub -= 1;
452                return Some(self.props[self.cols.end - 1].name_at(self.back_sub));
453            }
454            self.cols.end -= 1;
455            self.back_sub = self.props[self.cols.end - 1].name_count();
456        }
457    }
458}
459
460impl<C: ColNames> ExactSizeIterator for PropNamesIter<'_, C> {
461    fn len(&self) -> usize {
462        self.remaining
463    }
464}
465
466impl<C: ColNames> FusedIterator for PropNamesIter<'_, C> {}
467
468/// A boxed per-column-slot value iterator yielding one `Option<`[`PropValueRef`]`>` per feature.
469type ColValIter<'l> = Box<dyn Iterator<Item = Option<PropValueRef<'l>>> + 'l>;
470
471/// Build one [`ColValIter`] per property column "slot" from a decoded column slice.
472///
473/// - Scalar and string columns contribute one slot each.
474/// - `SharedDict` columns contribute one slot per sub-item.
475fn build_col_iters<'p>(columns: &'p [ParsedProperty<'p>]) -> Vec<ColValIter<'p>> {
476    use ParsedProperty as PP;
477    let mut iters: Vec<ColValIter<'p>> = Vec::new();
478    for col in columns {
479        match col {
480            PP::Bool(s) => iters.push(scalar_col_iter(s)),
481            PP::I8(s) => iters.push(scalar_col_iter(s)),
482            PP::U8(s) => iters.push(scalar_col_iter(s)),
483            PP::I32(s) => iters.push(scalar_col_iter(s)),
484            PP::U32(s) => iters.push(scalar_col_iter(s)),
485            PP::I64(s) => iters.push(scalar_col_iter(s)),
486            PP::U64(s) => iters.push(scalar_col_iter(s)),
487            PP::F32(s) => iters.push(scalar_col_iter(s)),
488            PP::F64(s) => iters.push(scalar_col_iter(s)),
489            PP::Str(strings) => {
490                let data: &'p str = strings.data.as_ref();
491                let lengths: &'p [i32] = &strings.lengths;
492                let mut curr_end: usize = 0;
493                let mut feat_idx = 0usize;
494                iters.push(Box::new(std::iter::from_fn(move || {
495                    let &end_i32 = lengths.get(feat_idx)?;
496                    feat_idx += 1;
497                    if end_i32 >= 0 {
498                        let start = curr_end;
499                        curr_end = end_i32.cast_unsigned().into_usize();
500                        Some(data.get(start..curr_end).map(PropValueRef::Str))
501                    } else {
502                        // Null slot: curr_end unchanged (null encodes the current byte offset).
503                        Some(None)
504                    }
505                })));
506            }
507            PP::SharedDict(dict) => {
508                for item in &dict.items {
509                    let dict_ref: &'p _ = dict;
510                    let item_ref: &'p _ = item;
511                    let mut feat_idx = 0usize;
512                    iters.push(Box::new(std::iter::from_fn(move || {
513                        if feat_idx >= item_ref.ranges.len() {
514                            return None;
515                        }
516                        let idx = feat_idx;
517                        feat_idx += 1;
518                        Some(item_ref.get(dict_ref, idx).map(PropValueRef::Str))
519                    })));
520                }
521            }
522        }
523    }
524    iters
525}
526
527/// Build a boxed value iterator for a single scalar property column.
528fn scalar_col_iter<'p, T>(scalar: &'p ParsedScalar<'p, T>) -> ColValIter<'p>
529where
530    T: Copy + PartialEq,
531    PropValueRef<'p>: From<T>,
532{
533    Box::new(scalar.iter_optional().map(|o| o.map(PropValueRef::from)))
534}
535
536/// Iterator over the features of a fully-decoded [`Layer01<Parsed>`].
537///
538/// Returned by [`ParsedLayer01::iter_features`]. Implements [`LendingIterator`]:
539/// advance with `while let Some(feat) = iter.next()`.
540///
541/// Holds one O(1)-per-step cursor per property column slot. On each step the
542/// per-column cursors are advanced and their results written into a reused
543/// `values_buf` - yielding a [`FeatureRef`] that borrows that buffer with no
544/// per-feature heap allocation.
545pub struct Layer01FeatureIter<'layer, 'data: 'layer> {
546    layer: &'layer Layer01<'data, Parsed>,
547    index: usize,
548    feature_count: usize,
549    /// ID iterator, `None` when the layer has no ID column.
550    id_iter: Option<crate::utils::PresenceOptIter<'layer, u64>>,
551    /// One boxed value iterator per column slot (scalar, string, or `SharedDict` sub-item).
552    col_iters: Vec<ColValIter<'layer>>,
553    /// Reused buffer: filled on each `next()` call, borrowed by the yielded [`FeatureRef`].
554    values_buf: Vec<Option<PropValueRef<'layer>>>,
555}
556
557impl<'layer, 'data: 'layer> Layer01FeatureIter<'layer, 'data> {
558    fn new(layer: &'layer Layer01<'data, Parsed>) -> Self {
559        let col_iters = build_col_iters(&layer.properties);
560        let cap = col_iters.len();
561        Self {
562            layer,
563            index: 0,
564            feature_count: layer.feature_count(),
565            id_iter: layer.id.as_ref().map(|id| id.iter_optional()),
566            col_iters,
567            values_buf: Vec::with_capacity(cap),
568        }
569    }
570
571    /// Number of features not yet yielded.
572    #[must_use]
573    pub fn len(&self) -> usize {
574        self.feature_count - self.index
575    }
576
577    /// Returns `true` if all features have been yielded.
578    #[must_use]
579    pub fn is_empty(&self) -> bool {
580        self.index >= self.feature_count
581    }
582}
583
584impl<'layer> LendingIterator for Layer01FeatureIter<'layer, '_> {
585    type Item<'this>
586        = MltResult<FeatureRef<'this, 'layer>>
587    where
588        Self: 'this;
589
590    fn next(&mut self) -> Option<Self::Item<'_>> {
591        let index = self.index;
592        if index >= self.feature_count {
593            return None;
594        }
595        self.index += 1;
596
597        // Advance all per-feature cursors unconditionally, even if geometry decode fails,
598        // so that IDs and property values remain aligned with geometry indices.
599        let id = self.id_iter.as_mut().and_then(Iterator::next).flatten();
600        self.values_buf.clear();
601        self.values_buf
602            .extend(self.col_iters.iter_mut().map(|it| it.next().flatten()));
603
604        Some(
605            self.layer
606                .geometry
607                .to_geojson(index)
608                .map(|geometry| FeatureRef {
609                    id,
610                    geometry,
611                    columns: &self.layer.properties,
612                    values: &self.values_buf,
613                }),
614        )
615    }
616}
617
618#[cfg(test)]
619mod tests {
620    use geo_types::Point;
621    use serde_json::Value;
622
623    use super::*;
624    use crate::Layer;
625    use crate::decoder::GeometryValues;
626    use crate::encoder::model::StagedLayer;
627    use crate::encoder::{Codecs, Encoder, Presence, StagedId, StagedProperty, StagedSharedDict};
628    use crate::test_helpers::{assert_size_hint_exact, dec, parser};
629
630    fn layer_buf(staged: StagedLayer) -> Vec<u8> {
631        staged
632            .encode_into(Encoder::default(), &mut Codecs::default())
633            .unwrap()
634            .into_layer_bytes()
635            .unwrap()
636    }
637
638    fn three_points() -> GeometryValues {
639        let mut g = GeometryValues::default();
640        g.push_geom(&Geometry::<i32>::Point(Point::new(1, 2)));
641        g.push_geom(&Geometry::<i32>::Point(Point::new(3, 4)));
642        g.push_geom(&Geometry::<i32>::Point(Point::new(5, 6)));
643        g
644    }
645
646    fn empty_layer(name: &str) -> StagedLayer {
647        staged_layer(name, StagedId::None, GeometryValues::default(), vec![])
648    }
649
650    fn staged_layer(
651        name: &str,
652        id: StagedId,
653        geometry: GeometryValues,
654        properties: Vec<StagedProperty>,
655    ) -> StagedLayer {
656        StagedLayer::new(name, 4096, id, geometry, properties).unwrap()
657    }
658
659    #[test]
660    fn prop_name_display_concatenates_parts() {
661        assert_eq!(PropName("addr:", "city").to_string(), "addr:city");
662        assert_eq!(PropName("name", "").to_string(), "name");
663        assert_eq!(PropName("", "").to_string(), "");
664    }
665
666    #[test]
667    fn prop_name_eq_str_matches_concatenation() {
668        assert_eq!(PropName("addr:", "city"), "addr:city");
669        assert_eq!("addr:city", PropName("addr:", "city"));
670        assert_ne!(PropName("addr:", "city"), "addr:");
671        assert_ne!(PropName("addr:", "city"), "city");
672        assert_eq!(PropName("name", ""), "name");
673    }
674
675    #[test]
676    fn prop_name_structural_eq_is_part_wise() {
677        assert_eq!(PropName("a", "b"), PropName("a", "b"));
678        assert_eq!(PropName("ab", ""), PropName("a", "b"));
679    }
680
681    #[test]
682    fn prop_name_eq_prop_name_semantic_equality() {
683        assert_eq!(PropName("ab", ""), PropName("a", "b"));
684        assert_eq!(PropName("", "ab"), PropName("a", "b"));
685        assert_eq!(PropName("abc", "def"), PropName("ab", "cdef"));
686        assert_eq!(PropName("a", "bcdef"), PropName("abcde", "f"));
687
688        assert_ne!(PropName("a", "b"), PropName("a", "c"));
689        assert_ne!(PropName("a", "b"), PropName("ab", "c"));
690        assert_ne!(PropName("abc", ""), PropName("ab", ""));
691    }
692
693    #[test]
694    fn prop_value_ref_scalars_convert_to_json() {
695        assert_eq!(Value::from(PropValueRef::Bool(true)), Value::Bool(true));
696        assert_eq!(Value::from(PropValueRef::Bool(false)), Value::Bool(false));
697        assert_eq!(Value::from(PropValueRef::I8(-1)), Value::from(-1_i8));
698        assert_eq!(Value::from(PropValueRef::U8(255)), Value::from(255_u8));
699        assert_eq!(
700            Value::from(PropValueRef::I32(-1000)),
701            Value::from(-1000_i32)
702        );
703        assert_eq!(Value::from(PropValueRef::U32(1000)), Value::from(1000_u32));
704        assert_eq!(
705            Value::from(PropValueRef::I64(i64::MIN)),
706            Value::from(i64::MIN)
707        );
708        assert_eq!(
709            Value::from(PropValueRef::U64(u64::MAX)),
710            Value::from(u64::MAX)
711        );
712        assert_eq!(
713            Value::from(PropValueRef::Str("hello")),
714            Value::String("hello".into())
715        );
716    }
717
718    #[test]
719    fn prop_value_ref_float_finite_is_number() {
720        assert!(matches!(
721            Value::from(PropValueRef::F32(1.5)),
722            Value::Number(_)
723        ));
724        assert!(matches!(
725            Value::from(PropValueRef::F64(2.5)),
726            Value::Number(_)
727        ));
728    }
729
730    #[test]
731    fn prop_value_ref_float_non_finite_becomes_string_sentinel() {
732        assert_eq!(
733            Value::from(PropValueRef::F32(f32::NAN)),
734            Value::String("f32::NAN".into())
735        );
736        assert_eq!(
737            Value::from(PropValueRef::F32(f32::INFINITY)),
738            Value::String("f32::INFINITY".into())
739        );
740        assert_eq!(
741            Value::from(PropValueRef::F64(f64::NAN)),
742            Value::String("f64::NAN".into())
743        );
744        assert_eq!(
745            Value::from(PropValueRef::F64(f64::NEG_INFINITY)),
746            Value::String("f64::NEG_INFINITY".into())
747        );
748    }
749
750    #[test]
751    fn empty_layer_yields_no_features() {
752        let buf = layer_buf(empty_layer("empty"));
753        let (_, layer) = Layer::from_bytes(&buf, &mut parser()).unwrap();
754        let Layer::Tag01(lazy) = layer else {
755            panic!("expected Tag01")
756        };
757        let parsed = lazy.decode_all(&mut dec()).unwrap();
758
759        let iter = parsed.iter_features();
760        assert_eq!(iter.len(), 0);
761        assert!(iter.is_empty());
762        assert_eq!(parsed.iter_features().len(), 0);
763    }
764
765    #[test]
766    fn len_decreases_with_each_next() {
767        let buf = layer_buf(staged_layer("test", StagedId::None, three_points(), vec![]));
768        let (_, layer) = Layer::from_bytes(&buf, &mut parser()).unwrap();
769        let Layer::Tag01(lazy) = layer else { panic!() };
770        let parsed = lazy.decode_all(&mut dec()).unwrap();
771
772        let mut iter = parsed.iter_features();
773        assert_eq!(iter.len(), 3);
774        iter.next().unwrap().unwrap();
775        assert_eq!(iter.len(), 2);
776        iter.next().unwrap().unwrap();
777        assert_eq!(iter.len(), 1);
778        iter.next().unwrap().unwrap();
779        assert_eq!(iter.len(), 0);
780        assert!(iter.is_empty());
781        assert!(iter.next().is_none());
782    }
783
784    #[test]
785    fn feature_ids_are_preserved() {
786        let buf = layer_buf(staged_layer(
787            "test",
788            StagedId::from_optional(vec![Some(100), None, Some(200)]),
789            three_points(),
790            vec![],
791        ));
792        let (_, layer) = Layer::from_bytes(&buf, &mut parser()).unwrap();
793        let Layer::Tag01(lazy) = layer else { panic!() };
794        let parsed = lazy.decode_all(&mut dec()).unwrap();
795
796        let mut ids = Vec::new();
797        let mut iter = parsed.iter_features();
798        while let Some(r) = iter.next() {
799            ids.push(r.unwrap().id);
800        }
801        assert_eq!(ids, [Some(100), None, Some(200)]);
802    }
803
804    #[test]
805    fn geometry_values_match_input() {
806        let buf = layer_buf(staged_layer("test", StagedId::None, three_points(), vec![]));
807        let (_, layer) = Layer::from_bytes(&buf, &mut parser()).unwrap();
808        let Layer::Tag01(lazy) = layer else { panic!() };
809        let parsed = lazy.decode_all(&mut dec()).unwrap();
810
811        let mut geoms = Vec::new();
812        let mut iter = parsed.iter_features();
813        while let Some(r) = iter.next() {
814            geoms.push(r.unwrap().geometry);
815        }
816        assert_eq!(geoms[0], Geometry::<i32>::Point(Point::new(1, 2)));
817        assert_eq!(geoms[1], Geometry::<i32>::Point(Point::new(3, 4)));
818        assert_eq!(geoms[2], Geometry::<i32>::Point(Point::new(5, 6)));
819    }
820
821    #[test]
822    fn null_scalar_values_are_skipped() {
823        let buf = layer_buf(staged_layer(
824            "test",
825            StagedId::None,
826            three_points(),
827            vec![StagedProperty::opt_u32("n", vec![Some(1), None, Some(3)])],
828        ));
829        let (_, layer) = Layer::from_bytes(&buf, &mut parser()).unwrap();
830        let Layer::Tag01(lazy) = layer else { panic!() };
831        let parsed = lazy.decode_all(&mut dec()).unwrap();
832
833        let mut iter = parsed.iter_features();
834
835        {
836            let feat = iter.next().unwrap().unwrap();
837            let cols: Vec<_> = feat.iter_properties().collect();
838            assert_eq!(cols.len(), 1);
839            assert_eq!(cols[0].name, PropName("n", ""));
840            assert_eq!(cols[0].name, "n");
841            assert_eq!(cols[0].value, PropValueRef::U32(1));
842            let all: Vec<_> = feat.iter_all_properties().collect();
843            assert_eq!(all, [Some(PropValueRef::U32(1))]);
844        }
845        {
846            let feat = iter.next().unwrap().unwrap();
847            assert!(feat.iter_properties().next().is_none());
848            let all: Vec<_> = feat.iter_all_properties().collect();
849            assert_eq!(all, [None]);
850        }
851        {
852            let feat = iter.next().unwrap().unwrap();
853            assert_eq!(feat.get_property("n"), Some(PropValueRef::U32(3)));
854            let all: Vec<_> = feat.iter_all_properties().collect();
855            assert_eq!(all, [Some(PropValueRef::U32(3))]);
856        }
857
858        let names: Vec<_> = parsed.iterate_prop_names().map(|n| n.to_string()).collect();
859        assert_eq!(names, ["n"]);
860    }
861
862    #[test]
863    fn null_string_values_are_skipped() {
864        let buf = layer_buf(staged_layer(
865            "test",
866            StagedId::None,
867            three_points(),
868            vec![StagedProperty::opt_str(
869                "label",
870                vec![Some("foo"), None, Some("bar")],
871            )],
872        ));
873        let (_, layer) = Layer::from_bytes(&buf, &mut parser()).unwrap();
874        let Layer::Tag01(lazy) = layer else { panic!() };
875        let parsed = lazy.decode_all(&mut dec()).unwrap();
876
877        let mut iter = parsed.iter_features();
878        {
879            let feat = iter.next().unwrap().unwrap();
880            assert_eq!(feat.get_property("label"), Some(PropValueRef::Str("foo")));
881        }
882        {
883            let feat = iter.next().unwrap().unwrap();
884            assert_eq!(feat.get_property("label"), None);
885        }
886        {
887            let feat = iter.next().unwrap().unwrap();
888            assert_eq!(feat.get_property("label"), Some(PropValueRef::Str("bar")));
889        }
890    }
891
892    #[test]
893    fn multiple_columns_independently_nullable() {
894        let buf = layer_buf(staged_layer(
895            "test",
896            StagedId::None,
897            three_points(),
898            vec![
899                StagedProperty::opt_bool("flag", vec![Some(true), Some(false), None]),
900                StagedProperty::opt_i32("score", vec![None, Some(-5), Some(7)]),
901            ],
902        ));
903        let (_, layer) = Layer::from_bytes(&buf, &mut parser()).unwrap();
904        let Layer::Tag01(lazy) = layer else { panic!() };
905        let parsed = lazy.decode_all(&mut dec()).unwrap();
906
907        let mut iter = parsed.iter_features();
908
909        // feat 0: flag=true, score=null -> 1 property
910        {
911            let feat = iter.next().unwrap().unwrap();
912            assert_eq!(feat.iter_properties().count(), 1);
913            assert_eq!(feat.get_property("flag"), Some(PropValueRef::Bool(true)));
914            assert_eq!(feat.get_property("score"), None);
915        }
916        // feat 1: flag=false, score=-5 -> 2 properties
917        {
918            let feat = iter.next().unwrap().unwrap();
919            assert_eq!(feat.iter_properties().count(), 2);
920            assert_eq!(feat.get_property("flag"), Some(PropValueRef::Bool(false)));
921            assert_eq!(feat.get_property("score"), Some(PropValueRef::I32(-5)));
922        }
923        // feat 2: flag=null, score=7 -> 1 property
924        {
925            let feat = iter.next().unwrap().unwrap();
926            assert_eq!(feat.iter_properties().count(), 1);
927            assert_eq!(feat.get_property("flag"), None);
928            assert_eq!(feat.get_property("score"), Some(PropValueRef::I32(7)));
929        }
930    }
931
932    #[test]
933    fn geometry_error_does_not_misalign_ids() {
934        use crate::decoder::GeometryType;
935
936        let buf = layer_buf(staged_layer(
937            "test",
938            StagedId::from_optional(vec![Some(10), Some(20), Some(30)]),
939            three_points(),
940            vec![],
941        ));
942        let (_, layer) = Layer::from_bytes(&buf, &mut parser()).unwrap();
943        let Layer::Tag01(lazy) = layer else { panic!() };
944        let mut parsed = lazy.decode_all(&mut dec()).unwrap();
945
946        // Corrupt feature 1's geometry type: Point -> LineString.
947        // A LineString requires part_offsets, which are absent here, so
948        // to_geojson(1) will return Err(NoPartOffsets).
949        parsed.geometry.vector_types[1] = GeometryType::LineString;
950
951        let mut iter = parsed.iter_features();
952
953        // Feature 0: valid Point, id = Some(10)
954        let feat0 = iter.next().unwrap().unwrap();
955        assert_eq!(feat0.id, Some(10));
956
957        // Feature 1: geometry error - iterator still advances ID cursor
958        assert!(iter.next().unwrap().is_err());
959
960        // Feature 2: valid Point, id must be Some(30), not Some(20)
961        let feat2 = iter.next().unwrap().unwrap();
962        assert_eq!(
963            feat2.id,
964            Some(30),
965            "id cursor was not advanced on geometry error"
966        );
967
968        assert!(iter.next().is_none());
969    }
970
971    #[test]
972    fn get_property_absent_column_returns_none() {
973        let buf = layer_buf(staged_layer(
974            "test",
975            StagedId::None,
976            three_points(),
977            vec![StagedProperty::u32("x", vec![1, 2, 3])],
978        ));
979        let (_, layer) = Layer::from_bytes(&buf, &mut parser()).unwrap();
980        let Layer::Tag01(lazy) = layer else { panic!() };
981        let parsed = lazy.decode_all(&mut dec()).unwrap();
982
983        let mut iter = parsed.iter_features();
984        let feat = iter.next().unwrap().unwrap();
985        assert_eq!(feat.get_property("no_such_column"), None);
986    }
987
988    #[test]
989    fn shared_dict_columns_are_expanded() {
990        let shared_dict = StagedSharedDict::new(
991            "addr:",
992            [
993                (
994                    "city",
995                    vec![Some("Paris"), Some("Rome"), None],
996                    Presence::Mixed,
997                ),
998                (
999                    "zip",
1000                    vec![Some("75001"), None, Some("00100")],
1001                    Presence::Mixed,
1002                ),
1003            ],
1004        )
1005        .unwrap();
1006
1007        let buf = layer_buf(staged_layer(
1008            "test",
1009            StagedId::None,
1010            three_points(),
1011            vec![StagedProperty::SharedDict(shared_dict)],
1012        ));
1013        let (_, layer) = Layer::from_bytes(&buf, &mut parser()).unwrap();
1014        let Layer::Tag01(lazy) = layer else { panic!() };
1015        let parsed = lazy.decode_all(&mut dec()).unwrap();
1016
1017        let mut iter = parsed.iter_features();
1018
1019        // feat 0: city=Paris, zip=75001
1020        {
1021            let feat = iter.next().unwrap().unwrap();
1022            assert_eq!(
1023                feat.get_property("addr:city"),
1024                Some(PropValueRef::Str("Paris"))
1025            );
1026            assert_eq!(
1027                feat.get_property("addr:zip"),
1028                Some(PropValueRef::Str("75001"))
1029            );
1030            assert_eq!(feat.iter_properties().count(), 2);
1031        }
1032        // feat 1: city=Rome, zip=null
1033        {
1034            let feat = iter.next().unwrap().unwrap();
1035            assert_eq!(
1036                feat.get_property("addr:city"),
1037                Some(PropValueRef::Str("Rome"))
1038            );
1039            assert_eq!(feat.get_property("addr:zip"), None);
1040            assert_eq!(feat.iter_properties().count(), 1);
1041            // iter_all_properties: values only (no names); SharedDict expands to two slots
1042            let all: Vec<_> = feat.iter_all_properties().collect();
1043            assert_eq!(all, [Some(PropValueRef::Str("Rome")), None]);
1044        }
1045        // feat 2: city=null, zip=00100
1046        {
1047            let feat = iter.next().unwrap().unwrap();
1048            assert_eq!(feat.get_property("addr:city"), None);
1049            assert_eq!(
1050                feat.get_property("addr:zip"),
1051                Some(PropValueRef::Str("00100"))
1052            );
1053        }
1054
1055        let names: Vec<_> = parsed.iterate_prop_names().map(|n| n.to_string()).collect();
1056        assert_eq!(names, ["addr:city", "addr:zip"]);
1057    }
1058
1059    fn strs<'p>(iter: impl Iterator<Item = PropName<'p>>) -> Vec<String> {
1060        iter.map(|n| n.to_string()).collect()
1061    }
1062
1063    fn reversed<'p>(iter: impl DoubleEndedIterator<Item = PropName<'p>>) -> Vec<String> {
1064        let mut names = strs(iter.rev());
1065        names.reverse();
1066        names
1067    }
1068
1069    fn both_ends_layer() -> Vec<u8> {
1070        let shared_dict = StagedSharedDict::new(
1071            "addr:",
1072            [
1073                ("city", vec![Some("Paris"); 3], Presence::AllPresent),
1074                ("zip", vec![Some("75001"); 3], Presence::AllPresent),
1075                ("street", vec![Some("Rue"); 3], Presence::AllPresent),
1076            ],
1077        )
1078        .unwrap();
1079
1080        layer_buf(staged_layer(
1081            "test",
1082            StagedId::None,
1083            three_points(),
1084            vec![
1085                StagedProperty::str("before", ["a", "a", "a"]),
1086                StagedProperty::u32("count", vec![1, 2, 3]),
1087                StagedProperty::SharedDict(shared_dict),
1088                StagedProperty::str("after", ["b", "b", "b"]),
1089            ],
1090        ))
1091    }
1092
1093    const BOTH_ENDS_NAMES: [&str; 6] = [
1094        "before",
1095        "count",
1096        "addr:city",
1097        "addr:zip",
1098        "addr:street",
1099        "after",
1100    ];
1101
1102    fn prop_names() -> Vec<PropName<'static>> {
1103        BOTH_ENDS_NAMES.iter().map(|n| PropName(n, "")).collect()
1104    }
1105
1106    #[test]
1107    fn prop_names_iterate_from_both_ends_lazy() {
1108        let buf = both_ends_layer();
1109        let (_, layer) = Layer::from_bytes(&buf, &mut parser()).unwrap();
1110        let Layer::Tag01(lazy) = layer else { panic!() };
1111
1112        assert_eq!(strs(lazy.iterate_prop_names()), BOTH_ENDS_NAMES);
1113        assert_eq!(reversed(lazy.iterate_prop_names()), BOTH_ENDS_NAMES);
1114        assert_size_hint_exact(|| lazy.iterate_prop_names(), &prop_names());
1115    }
1116
1117    #[test]
1118    fn prop_names_iterate_from_both_ends_parsed() {
1119        let buf = both_ends_layer();
1120        let (_, layer) = Layer::from_bytes(&buf, &mut parser()).unwrap();
1121        let Layer::Tag01(lazy) = layer else { panic!() };
1122        let parsed = lazy.decode_all(&mut dec()).unwrap();
1123
1124        assert_eq!(strs(parsed.iterate_prop_names()), BOTH_ENDS_NAMES);
1125        assert_eq!(reversed(parsed.iterate_prop_names()), BOTH_ENDS_NAMES);
1126
1127        assert_size_hint_exact(|| parsed.iterate_prop_names(), &prop_names());
1128    }
1129
1130    #[test]
1131    fn prop_names_of_layer_without_properties() {
1132        let buf = layer_buf(empty_layer("test"));
1133        let (_, layer) = Layer::from_bytes(&buf, &mut parser()).unwrap();
1134        let Layer::Tag01(lazy) = layer else { panic!() };
1135
1136        assert_eq!(lazy.iterate_prop_names().size_hint(), (0, Some(0)));
1137        assert_eq!(lazy.iterate_prop_names().next(), None);
1138        assert_eq!(lazy.iterate_prop_names().next_back(), None);
1139
1140        let parsed = lazy.decode_all(&mut dec()).unwrap();
1141        assert_eq!(parsed.iterate_prop_names().size_hint(), (0, Some(0)));
1142        assert_eq!(parsed.iterate_prop_names().next(), None);
1143        assert_eq!(parsed.iterate_prop_names().next_back(), None);
1144    }
1145
1146    /// `Layer01<Lazy>` only ever holds `Raw` columns today, so the `Parsed` and
1147    /// `ParsingFailed` arms are exercised against a hand-built column slice.
1148    #[test]
1149    fn prop_names_skip_columns_that_failed_to_parse() {
1150        let buf = both_ends_layer();
1151        let (_, layer) = Layer::from_bytes(&buf, &mut parser()).unwrap();
1152        let Layer::Tag01(lazy) = layer else { panic!() };
1153        let raw = lazy.properties.clone();
1154
1155        let (_, layer) = Layer::from_bytes(&buf, &mut parser()).unwrap();
1156        let Layer::Tag01(lazy) = layer else { panic!() };
1157        let parsed = lazy.decode_all(&mut dec()).unwrap().properties;
1158
1159        let mixed = vec![
1160            raw[0].clone(),
1161            LazyParsed::ParsingFailed,
1162            LazyParsed::Parsed(parsed[2].clone()),
1163            LazyParsed::ParsingFailed,
1164            raw[3].clone(),
1165        ];
1166        let expected = ["before", "addr:city", "addr:zip", "addr:street", "after"];
1167
1168        assert_eq!(strs(PropNamesIter::new(&mixed)), expected);
1169        assert_eq!(reversed(PropNamesIter::new(&mixed)), expected);
1170        assert_size_hint_exact(
1171            || PropNamesIter::new(&mixed),
1172            &expected.map(|n| PropName(n, "")),
1173        );
1174
1175        let all_failed = vec![LazyParsed::ParsingFailed; 3];
1176        assert_eq!(PropNamesIter::new(&all_failed).size_hint(), (0, Some(0)));
1177        assert_eq!(PropNamesIter::new(&all_failed).next(), None);
1178        assert_eq!(PropNamesIter::new(&all_failed).next_back(), None);
1179    }
1180
1181    #[test]
1182    fn feature_property_iterators_run_backwards() {
1183        let buf = both_ends_layer();
1184        let (_, layer) = Layer::from_bytes(&buf, &mut parser()).unwrap();
1185        let Layer::Tag01(lazy) = layer else { panic!() };
1186        let parsed = lazy.decode_all(&mut dec()).unwrap();
1187
1188        let mut iter = parsed.iter_features();
1189        let feat = iter.next().unwrap().unwrap();
1190
1191        let all: Vec<_> = feat.iter_all_properties().collect();
1192        assert_size_hint_exact(|| feat.iter_all_properties(), &all);
1193        let mut all_back: Vec<_> = feat.iter_all_properties().rev().collect();
1194        all_back.reverse();
1195        assert_eq!(all_back, all);
1196
1197        let mut props_back: Vec<_> = feat
1198            .iter_properties()
1199            .rev()
1200            .map(|c| c.name().to_string())
1201            .collect();
1202        props_back.reverse();
1203        assert_eq!(props_back, BOTH_ENDS_NAMES);
1204    }
1205}
mlt_core/decoder/iterators.rs

mlt_core/decoder/
iterators.rs
