mlt_core/decoder/property/
strings.rs

1use std::borrow::Cow;
2
3use bitvec::order::Lsb0;
4use bitvec::slice::BitSlice;
5use usize_cast::IntoUsize as _;
6
7use crate::MltError::{BufferUnderflow, DictIndexOutOfBounds};
8#[cfg(feature = "unstable-v2")]
9use crate::codecs::front_coding::{FrontLengths, front_decode};
10use crate::codecs::fsst::{decode_fsst, decode_fsst_bytes};
11use crate::decoder::{
12    DictLayout, DictionaryType, LengthType, OffsetType, ParsedSharedDict, ParsedSharedDictItem,
13    ParsedStrings, RawFsstData, RawPlainData, RawPresence, RawSharedDictEncoding, RawStream,
14    RawStrings, RawStringsEncoding, StreamType,
15};
16use crate::errors::AsMltError as _;
17use crate::{Decoder, DictRange, MltError, MltResult, RawSharedDict, RawSharedDictItem};
18
19impl<'a> ParsedStrings<'a> {
20    #[must_use]
21    pub fn new(name: &'a str, lengths: Vec<i32>, data: Cow<'a, str>) -> Self {
22        ParsedStrings {
23            name,
24            lengths,
25            data,
26        }
27    }
28
29    #[must_use]
30    pub fn feature_count(&self) -> usize {
31        self.lengths.len()
32    }
33
34    #[must_use]
35    pub fn presence_bools(&self) -> Vec<bool> {
36        self.lengths.iter().map(|&end| end >= 0).collect()
37    }
38
39    #[must_use]
40    pub fn get(&self, idx: u32) -> Option<&str> {
41        let idx = idx.into_usize();
42
43        let end = *self.lengths.get(idx)?;
44        if end < 0 {
45            return None;
46        }
47        let end = decode_end(end).into_usize();
48
49        let start = idx
50            .checked_sub(1)
51            .and_then(|prev| self.lengths.get(prev).copied())
52            .map_or(0, decode_end)
53            .into_usize();
54
55        self.data.get(start..end)
56    }
57
58    #[must_use]
59    pub fn dense_values(&self) -> Vec<String> {
60        let mut values = Vec::new();
61        let mut start = 0_u32;
62        for &end in &self.lengths {
63            let end_u32 = decode_end(end);
64            let start_idx = start.into_usize();
65            let end_idx = end_u32.into_usize();
66            if end >= 0
67                && let Some(value) = self.data.get(start_idx..end_idx)
68            {
69                values.push(value.to_string());
70            }
71            start = end_u32;
72        }
73        values
74    }
75
76    #[must_use]
77    pub fn materialize(&self) -> Vec<Option<String>> {
78        (0..u32::try_from(self.feature_count()).unwrap_or(u32::MAX))
79            .map(|i| self.get(i).map(str::to_string))
80            .collect()
81    }
82}
83
84pub(crate) fn decode_shared_dict_range(range: DictRange) -> Option<(u32, u32)> {
85    if let (Ok(start), Ok(end)) = (u32::try_from(range.start), u32::try_from(range.end)) {
86        Some((start, end))
87    } else {
88        None
89    }
90}
91
92pub(crate) fn shared_dict_spans(lengths: &[u32], dec: &mut Decoder) -> MltResult<Vec<(u32, u32)>> {
93    let mut spans = dec.alloc(lengths.len())?;
94    let mut offset = 0_u32;
95    for &len in lengths {
96        let start = offset;
97        offset = offset.saturating_add(len);
98        spans.push((start, offset));
99    }
100    Ok(spans)
101}
102
103pub(crate) fn resolve_dict_spans(
104    offsets: &[u32],
105    presence: Option<&BitSlice<u8, Lsb0>>,
106    dict_spans: &[(u32, u32)],
107    dec: &mut Decoder,
108) -> MltResult<Vec<Option<(u32, u32)>>> {
109    let present_count = presence.map_or(offsets.len(), BitSlice::len);
110    let mut resolved = dec.alloc(present_count)?;
111    let mut next = offsets.iter().copied();
112
113    if let Some(presence) = presence {
114        let fail = || MltError::PresenceValueCountMismatch(presence.count_ones(), offsets.len());
115        for present in presence.iter().by_vals() {
116            if !present {
117                resolved.push(None);
118                continue;
119            }
120            let idx = next.next().ok_or_else(fail)?;
121            let span = dict_spans
122                .get(idx.into_usize())
123                .copied()
124                .ok_or(DictIndexOutOfBounds(idx, dict_spans.len()))?;
125            resolved.push(Some(span));
126        }
127
128        if next.next().is_some() {
129            return Err(fail());
130        }
131    } else {
132        for &idx in offsets {
133            let span = dict_spans
134                .get(idx.into_usize())
135                .copied()
136                .ok_or(DictIndexOutOfBounds(idx, dict_spans.len()))?;
137            resolved.push(Some(span));
138        }
139    }
140
141    Ok(resolved)
142}
143
144fn dict_span_str(dict_data: &str, span: (u32, u32)) -> MltResult<&str> {
145    let start = span.0.into_usize();
146    let end = span.1.into_usize();
147    // A slice of valid UTF-8 is valid itself wherever both ends fall on character boundaries.
148    if let Some(value) = dict_data.get(start..end) {
149        return Ok(value);
150    }
151    let bytes = dict_data.as_bytes();
152    let Some(value) = bytes.get(start..end) else {
153        let len = span.1.saturating_sub(span.0);
154        return Err(BufferUnderflow(len, bytes.len().saturating_sub(start)));
155    };
156    Ok(str::from_utf8(value)?)
157}
158
159impl ParsedSharedDict<'_> {
160    /// The child columns reading this dictionary, in wire order.
161    #[must_use]
162    pub fn items(&self) -> &[ParsedSharedDictItem<'_>] {
163        &self.items
164    }
165
166    #[must_use]
167    pub fn corpus(&self) -> &str {
168        &self.data
169    }
170
171    #[must_use]
172    pub fn get(&self, span: (u32, u32)) -> Option<&str> {
173        let start = span.0.into_usize();
174        let end = span.1.into_usize();
175        self.corpus().get(start..end)
176    }
177}
178
179impl ParsedSharedDictItem<'_> {
180    /// Number of features this child spans, present or not.
181    #[must_use]
182    pub fn feature_count(&self) -> usize {
183        self.ranges.len()
184    }
185
186    #[must_use]
187    pub fn get<'a>(&self, shared_dict: &'a ParsedSharedDict<'_>, i: usize) -> Option<&'a str> {
188        self.ranges
189            .get(i)
190            .copied()
191            .and_then(decode_shared_dict_range)
192            .and_then(|span| shared_dict.get(span))
193    }
194}
195
196impl<'a> RawPlainData<'a> {
197    pub fn new(lengths: RawStream<'a>, data: RawStream<'a>) -> MltResult<Self> {
198        validate_stream!(
199            lengths,
200            StreamType::Length(LengthType::VarBinary | LengthType::Dictionary)
201        );
202        validate_stream!(
203            data,
204            StreamType::Data(
205                DictionaryType::None | DictionaryType::Single | DictionaryType::Shared
206            )
207        );
208        Ok(Self { lengths, data })
209    }
210
211    pub fn decode(self, dec: &mut Decoder) -> MltResult<(&'a str, Vec<u32>)> {
212        let (data, lengths) = self.decode_bytes(dec)?;
213        Ok((str::from_utf8(data)?, lengths))
214    }
215
216    /// As [`Self::decode`], for a blob that is only valid UTF-8 once its caller reassembles it.
217    pub fn decode_bytes(self, dec: &mut Decoder) -> MltResult<(&'a [u8], Vec<u32>)> {
218        Ok((self.data.data, self.lengths.decode_ints(dec)?))
219    }
220}
221
222impl<'a> RawFsstData<'a> {
223    pub fn new(
224        symbol_lengths: RawStream<'a>,
225        symbol_table: RawStream<'a>,
226        lengths: RawStream<'a>,
227        corpus: RawStream<'a>,
228    ) -> MltResult<Self> {
229        validate_stream!(symbol_lengths, StreamType::Length(LengthType::Symbol));
230        validate_stream!(symbol_table, StreamType::Data(DictionaryType::Fsst));
231        validate_stream!(lengths, StreamType::Length(LengthType::Dictionary));
232        validate_stream!(
233            corpus,
234            StreamType::Data(DictionaryType::Single | DictionaryType::Shared)
235        );
236        Ok(Self {
237            symbol_lengths,
238            symbol_table,
239            lengths,
240            corpus,
241        })
242    }
243
244    pub fn decode(self, dec: &mut Decoder) -> MltResult<(String, Vec<u32>)> {
245        decode_fsst(self, dec)
246    }
247
248    /// As [`Self::decode`], for a corpus that is only valid UTF-8 once its caller reassembles it.
249    pub fn decode_bytes(self, dec: &mut Decoder) -> MltResult<(Vec<u8>, Vec<u32>)> {
250        decode_fsst_bytes(self, dec)
251    }
252}
253
254impl<'a> RawStringsEncoding<'a> {
255    #[must_use]
256    pub fn plain(plain_data: RawPlainData<'a>) -> Self {
257        Self::Plain(plain_data)
258    }
259
260    pub fn dictionary(
261        plain_data: RawPlainData<'a>,
262        offsets: RawStream<'a>,
263        dict: DictLayout,
264    ) -> MltResult<Self> {
265        validate_stream!(offsets, StreamType::Offset(OffsetType::String));
266        Ok(Self::Dictionary {
267            plain_data,
268            offsets,
269            dict,
270        })
271    }
272
273    #[must_use]
274    pub fn fsst_plain(fsst_data: RawFsstData<'a>) -> Self {
275        Self::FsstPlain(fsst_data)
276    }
277
278    pub fn fsst_dictionary(
279        fsst_data: RawFsstData<'a>,
280        offsets: RawStream<'a>,
281        dict: DictLayout,
282    ) -> MltResult<Self> {
283        validate_stream!(offsets, StreamType::Offset(OffsetType::String));
284        Ok(Self::FsstDictionary {
285            fsst_data,
286            offsets,
287            dict,
288        })
289    }
290}
291
292impl<'a> RawSharedDictEncoding<'a> {
293    /// Plain shared dict (2 streams): lengths + data.
294    #[must_use]
295    pub fn plain(plain_data: RawPlainData<'a>) -> Self {
296        Self::Plain(plain_data)
297    }
298
299    /// FSST plain shared dict (4 streams): symbol lengths, symbol table, lengths, corpus.
300    #[must_use]
301    pub fn fsst_plain(fsst_data: RawFsstData<'a>) -> Self {
302        Self::FsstPlain(fsst_data)
303    }
304}
305
306impl<'a> RawStrings<'a> {
307    #[must_use]
308    pub fn new(name: &'a str, presence: RawPresence<'a>, encoding: RawStringsEncoding<'a>) -> Self {
309        Self {
310            name,
311            presence,
312            encoding,
313        }
314    }
315
316    /// How many values this column holds, off whichever of its streams counts them.
317    ///
318    /// The leading stream is one value per string in every layout: its lengths
319    /// where the strings are stored one each, its codes where they are pooled.
320    #[cfg(feature = "unstable-v2")]
321    #[must_use]
322    pub(crate) fn value_count(&self) -> u32 {
323        match &self.encoding {
324            RawStringsEncoding::Plain(plain) => plain.lengths.meta.num_values,
325            RawStringsEncoding::FsstPlain(fsst) => fsst.lengths.meta.num_values,
326            RawStringsEncoding::Dictionary { offsets, .. }
327            | RawStringsEncoding::FsstDictionary { offsets, .. } => offsets.meta.num_values,
328        }
329    }
330
331    /// Decode string property from its encoded column.
332    pub fn decode(self, dec: &mut Decoder) -> MltResult<ParsedStrings<'a>> {
333        let name = self.name;
334        let presence = self.presence.decode_bits(dec)?;
335
336        let parsed = match self.encoding {
337            RawStringsEncoding::Plain(plain_data) => {
338                let (data, lengths) = plain_data.decode(dec)?;
339                ParsedStrings {
340                    name,
341                    lengths: to_absolute_lengths(&lengths, presence.as_deref(), dec)?,
342                    data: data.into(),
343                }
344            }
345            RawStringsEncoding::Dictionary {
346                plain_data,
347                offsets,
348                dict,
349            } => {
350                let (data, lengths) = plain_data.decode_bytes(dec)?;
351                let offsets: Vec<u32> = offsets.decode_ints(dec)?;
352                let (dictionary, lengths) = rebuild_dictionary(dict, &lengths, data)?;
353                decode_dictionary_strings(
354                    name,
355                    &lengths,
356                    &offsets,
357                    presence.as_deref(),
358                    &dictionary,
359                    dec,
360                )?
361            }
362            RawStringsEncoding::FsstPlain(fsst_data) => {
363                let (data, dict_lens) = fsst_data.decode(dec)?;
364                ParsedStrings {
365                    name,
366                    lengths: to_absolute_lengths(&dict_lens, presence.as_deref(), dec)?,
367                    data: data.into(),
368                }
369            }
370            RawStringsEncoding::FsstDictionary {
371                fsst_data,
372                offsets,
373                dict,
374            } => {
375                let (data, lengths) = fsst_data.decode_bytes(dec)?;
376                let offsets: Vec<u32> = offsets.decode_ints(dec)?;
377                let (dictionary, lengths) = rebuild_dictionary(dict, &lengths, &data)?;
378                decode_dictionary_strings(
379                    name,
380                    &lengths,
381                    &offsets,
382                    presence.as_deref(),
383                    &dictionary,
384                    dec,
385                )?
386            }
387        };
388        Ok(parsed)
389    }
390}
391
392fn to_absolute_lengths(
393    lengths: &[u32],
394    presence: Option<&BitSlice<u8, Lsb0>>,
395    dec: &mut Decoder,
396) -> MltResult<Vec<i32>> {
397    let capacity = presence.map_or(lengths.len(), BitSlice::len);
398    let mut absolute = dec.alloc(capacity)?;
399    let mut iter = lengths.iter().copied();
400    let mut end = 0_i32;
401    if let Some(presence) = presence {
402        for present in presence.iter().by_vals() {
403            if present {
404                let len = iter.next().ok_or(MltError::PresenceValueCountMismatch(
405                    presence.len(),
406                    lengths.len(),
407                ))?;
408                end = checked_absolute_end(end, len)?;
409                absolute.push(end);
410            } else {
411                absolute.push(encode_null_end(end));
412            }
413        }
414        if iter.next().is_some() {
415            return Err(MltError::PresenceValueCountMismatch(
416                presence.count_ones(),
417                lengths.len(),
418            ));
419        }
420    } else {
421        for &len in lengths {
422            end = checked_absolute_end(end, len)?;
423            absolute.push(end);
424        }
425    }
426    Ok(absolute)
427}
428
429/// Turn a dictionary blob into its entries back to back with one length each, whatever its layout.
430///
431/// A front-coded blob holds only suffixes, which are not valid UTF-8 on their own, so the
432/// conversion happens here rather than when the blob is read.
433pub(crate) fn rebuild_dictionary<'d, 'l>(
434    dict: DictLayout,
435    lengths: &'l [u32],
436    data: &'d [u8],
437) -> MltResult<(Cow<'d, str>, Cow<'l, [u32]>)> {
438    match dict {
439        DictLayout::Plain => Ok((Cow::Borrowed(str::from_utf8(data)?), Cow::Borrowed(lengths))),
440        #[cfg(feature = "unstable-v2")]
441        DictLayout::FrontCoded => {
442            let (entries, entry_lengths) = front_decode(FrontLengths::split(lengths)?, data)?;
443            Ok((Cow::Owned(entries), Cow::Owned(entry_lengths)))
444        }
445    }
446}
447
448fn decode_dictionary_strings<'a>(
449    name: &'a str,
450    dict_lengths: &[u32],
451    offsets: &[u32],
452    presence: Option<&BitSlice<u8, Lsb0>>,
453    dict_data: &str,
454    dec: &mut Decoder,
455) -> MltResult<ParsedStrings<'a>> {
456    let dict_spans = shared_dict_spans(dict_lengths, dec)?;
457    let resolved_spans = resolve_dict_spans(offsets, presence, &dict_spans, dec)?;
458    let mut lengths = dec.alloc(resolved_spans.len())?;
459    // Charged up front, since a few entries repeated over many rows can outgrow the input by far.
460    let dict_len = dict_data.len();
461    let total = resolved_spans
462        .iter()
463        .flatten()
464        .fold(0_usize, |sum, &(start, end)| {
465            sum.saturating_add(
466                end.into_usize()
467                    .min(dict_len)
468                    .saturating_sub(start.into_usize()),
469            )
470        });
471    dec.consume_items::<u8>(total)?;
472    let mut data = String::with_capacity(total);
473    let mut end = 0_i32;
474    for span in resolved_spans {
475        if let Some(span) = span {
476            let value = dict_span_str(dict_data, span)?;
477            data.push_str(value);
478            end = checked_string_end(end, value.len())?;
479            lengths.push(end);
480        } else {
481            lengths.push(encode_null_end(end));
482        }
483    }
484    Ok(ParsedStrings {
485        name,
486        lengths,
487        data: Cow::Owned(data),
488    })
489}
490
491pub(crate) fn encode_null_end(end: i32) -> i32 {
492    -end - 1
493}
494
495fn decode_end(end: i32) -> u32 {
496    if end >= 0 {
497        u32::try_from(end).expect("non-negative decoded string end must fit in u32")
498    } else {
499        u32::try_from(-i64::from(end) - 1).expect("encoded null boundary must fit in u32")
500    }
501}
502
503pub(crate) fn checked_string_end(current_end: i32, byte_len: usize) -> MltResult<i32> {
504    let byte_len = u32::try_from(byte_len)?;
505    checked_absolute_end(current_end, byte_len)
506}
507
508pub(crate) fn checked_absolute_end(current_end: i32, delta: u32) -> MltResult<i32> {
509    let delta = i32::try_from(delta)?;
510    current_end.checked_add(delta).or_overflow()
511}
512
513impl<'a> RawSharedDict<'a> {
514    #[must_use]
515    pub fn new(
516        name: &'a str,
517        encoding: RawSharedDictEncoding<'a>,
518        dict: DictLayout,
519        children: Vec<RawSharedDictItem<'a>>,
520    ) -> Self {
521        Self {
522            name,
523            encoding,
524            dict,
525            children,
526        }
527    }
528
529    /// Decode a shared-dictionary column into its decoded form.
530    pub fn decode(self, dec: &mut Decoder) -> MltResult<ParsedSharedDict<'a>> {
531        let prefix = self.name;
532        let (data, dict_spans) = match self.encoding {
533            RawSharedDictEncoding::Plain(plain_data) => {
534                let (blob, lengths) = plain_data.decode_bytes(dec)?;
535                let (entries, lengths) = rebuild_dictionary(self.dict, &lengths, blob)?;
536                let dict_spans = shared_dict_spans(&lengths, dec)?;
537                (entries, dict_spans)
538            }
539            RawSharedDictEncoding::FsstPlain(fsst_data) => {
540                let (blob, lengths) = fsst_data.decode_bytes(dec)?;
541                // `entries` borrows the decompressed corpus, which is local, so it is taken owned.
542                let (entries, lengths) = rebuild_dictionary(self.dict, &lengths, &blob)?;
543                let (entries, lengths) = (entries.into_owned(), lengths.into_owned());
544                let dict_spans = shared_dict_spans(&lengths, dec)?;
545                (Cow::Owned(entries), dict_spans)
546            }
547        };
548        let mut items = Vec::with_capacity(self.children.len());
549        for child in self.children {
550            let offsets: Vec<u32> = child.data.decode_ints(dec)?;
551            let presence = child.presence.decode_bits(dec)?;
552            let ranges = resolve_dict_spans(&offsets, presence.as_deref(), &dict_spans, dec)?
553                .into_iter()
554                .map(|span| match span {
555                    Some(span) => encode_shared_dict_range(span.0, span.1),
556                    None => Ok(DictRange::NULL),
557                })
558                .collect::<Result<Vec<_>, _>>()?;
559            items.push(ParsedSharedDictItem {
560                suffix: child.name,
561                ranges,
562            });
563        }
564
565        let parsed = ParsedSharedDict {
566            prefix,
567            data,
568            items,
569        };
570        // Corpus size is only known after decompression; charge after.
571        let bytes = parsed.items.iter().try_fold(
572            u32::try_from(parsed.data.len()).or_overflow()?,
573            |acc, item| {
574                let n = u32::try_from(item.ranges.len() * size_of::<DictRange>()).or_overflow()?;
575                acc.checked_add(n).or_overflow()
576            },
577        )?;
578        dec.consume(bytes)?;
579        Ok(parsed)
580    }
581}
582
583pub(crate) fn encode_shared_dict_range(start: u32, end: u32) -> MltResult<DictRange> {
584    Ok(DictRange {
585        start: i32::try_from(start)?,
586        end: i32::try_from(end)?,
587    })
588}
589
590#[cfg(test)]
591mod tests {
592    use rstest::rstest;
593
594    use super::*;
595    use crate::test_helpers::dec;
596
597    #[rstest]
598    #[case::empty(&[], "", &[])]
599    #[case::all_present(&[1, 3, 6], "abbccc", &[Some("a"), Some("bb"), Some("ccc")])]
600    #[case::leading_null(&[-1, 1], "a", &[None, Some("a")])]
601    #[case::trailing_null(&[1, -2], "a", &[Some("a"), None])]
602    #[case::consecutive_nulls(&[1, -2, -2, 2], "ab", &[Some("a"), None, None, Some("b")])]
603    #[case::only_nulls(&[-1, -1], "", &[None, None])]
604    #[case::empty_string_after_a_value(&[1, 1, -2], "a", &[Some("a"), Some(""), None])]
605    #[case::empty_string_after_a_null(&[1, -2, 1], "a", &[Some("a"), None, Some("")])]
606    #[case::multibyte(&[2, 7], "µ€µ", &[Some("µ"), Some("€µ")])]
607    fn rows_read_back_from_the_length_stream(
608        #[case] lengths: &[i32],
609        #[case] data: &str,
610        #[case] expected: &[Option<&str>],
611    ) {
612        let strings = ParsedStrings::new("col", lengths.to_vec(), Cow::Borrowed(data));
613
614        assert_eq!(strings.feature_count(), expected.len());
615        assert_eq!(
616            strings.materialize(),
617            expected
618                .iter()
619                .map(|v| v.map(str::to_string))
620                .collect::<Vec<_>>()
621        );
622        assert_eq!(
623            strings.dense_values(),
624            expected
625                .iter()
626                .flatten()
627                .map(|v| (*v).to_string())
628                .collect::<Vec<_>>()
629        );
630        assert_eq!(
631            strings.presence_bools(),
632            expected.iter().map(Option::is_some).collect::<Vec<_>>()
633        );
634        for (idx, want) in expected.iter().enumerate() {
635            assert_eq!(strings.get(u32::try_from(idx).expect("small")), *want);
636        }
637        assert_eq!(
638            strings.get(u32::try_from(expected.len()).expect("small")),
639            None
640        );
641    }
642
643    #[rstest]
644    #[case::end_past_the_corpus(&[9], "ab")]
645    #[case::end_before_the_start(&[2, 1], "ab")]
646    #[case::split_multibyte(&[1], "µ")]
647    fn a_row_outside_the_corpus_reads_as_absent(#[case] lengths: &[i32], #[case] data: &str) {
648        let strings = ParsedStrings::new("col", lengths.to_vec(), Cow::Borrowed(data));
649        let last = u32::try_from(lengths.len() - 1).expect("small");
650
651        assert_eq!(strings.get(last), None);
652        assert!(strings.presence_bools().iter().all(|&p| p));
653        assert_eq!(strings.materialize().last(), Some(&None));
654    }
655
656    fn dictionary_rows(
657        dict_lengths: &[u32],
658        dict: &str,
659        offsets: &[u32],
660    ) -> MltResult<Vec<Option<String>>> {
661        decode_dictionary_strings("col", dict_lengths, offsets, None, dict, &mut dec())
662            .map(|strings| strings.materialize())
663    }
664
665    #[test]
666    fn dictionary_rows_repeat_multibyte_entries() {
667        let rows = dictionary_rows(&[2, 3], "µ€", &[1, 0, 1]).expect("valid dictionary");
668        assert_eq!(
669            rows,
670            [
671                Some("€".to_string()),
672                Some("µ".to_string()),
673                Some("€".to_string())
674            ]
675        );
676    }
677
678    #[test]
679    fn dictionary_entry_past_the_dictionary() {
680        let err = dictionary_rows(&[5], "ab", &[0]).expect_err("entry runs past the dictionary");
681        assert!(matches!(err, BufferUnderflow(5, 2)), "{err:?}");
682    }
683
684    #[test]
685    fn dictionary_entry_splitting_a_character() {
686        let err = dictionary_rows(&[1, 1], "µ", &[0]).expect_err("entry splits a character");
687        assert!(matches!(err, MltError::Utf8(_)), "{err:?}");
688    }
689
690    #[test]
691    fn dictionary_rows_are_charged_to_the_budget() {
692        let consumed = |dict: &str| {
693            let mut dec = dec();
694            let len = u32::try_from(dict.len()).expect("small");
695            decode_dictionary_strings("col", &[len], &[0; 100], None, dict, &mut dec)
696                .expect("valid dictionary");
697            dec.consumed()
698        };
699        assert_eq!(consumed("abc") - consumed("a"), 200);
700    }
701}
mlt_core/decoder/property/strings.rs

mlt_core/decoder/property/
strings.rs
