mlt_core/dump/
decode.rs

1//! Typed decode of a single data blob, for callers that render the values themselves.
2
3use std::borrow::Cow;
4
5use serde::Serialize;
6use usize_cast::IntoUsize as _;
7
8use super::model::{BlobInfo, DecodeHint, DumpTree, Region};
9use crate::codecs::fsst::decode_fsst_bytes;
10use crate::decoder::{DictionaryType, LengthType, RawFsstData, RawStream, StreamType};
11use crate::{Decoder, GeometryType, MltError, MltResult};
12
13/// One data blob decoded for display, capped at the caller's value count.
14///
15/// [`render`](super::render()) formats the same payloads as text.
16/// This is the structured form, for the wasm binding and anything else that lays out its own view.
17#[derive(Debug, Clone, PartialEq, Serialize)]
18#[serde(
19    tag = "kind",
20    rename_all = "camelCase",
21    rename_all_fields = "camelCase"
22)]
23pub enum DecodedBlob {
24    /// Values that survive an `f64` round-trip.
25    Numbers {
26        values: Vec<f64>,
27        /// Value count before the cap, or `None` when nothing was dropped.
28        truncated_from: Option<u32>,
29    },
30    /// 64-bit values, which an `f64` would round.
31    #[serde(rename = "bigints")]
32    BigInts {
33        values: Vec<i128>,
34        truncated_from: Option<u32>,
35    },
36    Bools {
37        values: Vec<bool>,
38        truncated_from: Option<u32>,
39    },
40    /// Values that name something: each `names` entry is what the `values` entry stands for.
41    Enum {
42        values: Vec<u32>,
43        names: Vec<Cow<'static, str>>,
44        truncated_from: Option<u32>,
45    },
46    /// The strings an FSST corpus stands for, which its own bytes do not say.
47    Strings {
48        values: Vec<String>,
49        truncated_from: Option<u32>,
50    },
51    /// A byte payload that is valid UTF-8.
52    Text { value: String },
53    /// A byte payload that is not.
54    Binary { len: u32 },
55    /// The payload did not decode.
56    Error { message: String },
57}
58
59/// Decode `data` the way `info`'s hint says to, keeping at most `max_values` values.
60///
61/// `max_values` of `0` keeps all of them.
62/// A bytes payload is one value rather than a list, so it crosses whole.
63/// Never panics: a decode failure comes back as [`DecodedBlob::Error`].
64pub fn decode_blob(
65    info: BlobInfo,
66    data: &[u8],
67    max_values: usize,
68    dec: &mut Decoder,
69) -> DecodedBlob {
70    // Bound memory/time per blob, as `render` does.
71    dec.reset_budget();
72    let meta = info.meta;
73    match info.hint {
74        DecodeHint::Presence => match RawStream::new(meta, data).decode_bitvec(dec) {
75            Ok(bits) => bools(bits.iter().by_vals().collect(), max_values),
76            Err(e) => error(&e),
77        },
78        #[cfg(feature = "unstable-v2")]
79        DecodeHint::PresenceCoded(coding) => {
80            use crate::codecs::presence_coding::{Unmetered, read};
81            match read(data, meta.num_values, coding, &mut Unmetered) {
82                Ok((_, bits)) => bools(bits.iter().by_vals().collect(), max_values),
83                Err(e) => DecodedBlob::Error {
84                    message: e.to_string(),
85                },
86            }
87        }
88        #[cfg(feature = "unstable-v2")]
89        DecodeHint::PackedBits => {
90            let n = meta.num_values.into_usize();
91            let available = data.len() * 8;
92            if available < n {
93                return DecodedBlob::Error {
94                    message: format!("needs {n} bits, got {available}"),
95                };
96            }
97            bools(
98                (0..n).map(|i| data[i / 8] >> (i % 8) & 1 == 1).collect(),
99                max_values,
100            )
101        }
102        #[cfg(feature = "unstable-v2")]
103        DecodeHint::Rans => DecodedBlob::Binary {
104            len: count(data.len()),
105        },
106        DecodeHint::Bool => match RawStream::new(meta, data).decode_bools(dec) {
107            Ok(v) => bools(v, max_values),
108            Err(e) => error(&e),
109        },
110        DecodeHint::GeometryType => geometry_types(
111            RawStream::new(meta, data).decode_ints::<u32>(dec),
112            max_values,
113        ),
114        DecodeHint::I32 => numbers(
115            RawStream::new(meta, data).decode_ints::<i32>(dec),
116            max_values,
117        ),
118        DecodeHint::U32 => numbers(
119            RawStream::new(meta, data).decode_ints::<u32>(dec),
120            max_values,
121        ),
122        DecodeHint::F32 => numbers(
123            RawStream::new(meta, data).decode_floats::<f32>(dec),
124            max_values,
125        ),
126        DecodeHint::F64 => numbers(
127            RawStream::new(meta, data).decode_floats::<f64>(dec),
128            max_values,
129        ),
130        DecodeHint::I64 => bigints(
131            RawStream::new(meta, data).decode_ints::<i64>(dec),
132            max_values,
133        ),
134        DecodeHint::U64 => bigints(
135            RawStream::new(meta, data).decode_ints::<u64>(dec),
136            max_values,
137        ),
138        #[cfg(feature = "unstable-v2")]
139        DecodeHint::Alp(params) => bigints(
140            RawStream::new(meta, data).decode_alp_codes(params, dec),
141            max_values,
142        ),
143        DecodeHint::Bytes => match std::str::from_utf8(data) {
144            Ok(s) => DecodedBlob::Text {
145                value: s.to_string(),
146            },
147            Err(_) => DecodedBlob::Binary {
148                len: count(data.len()),
149            },
150        },
151    }
152}
153
154/// As [`decode_blob`] for the payload at `index` of `tree`, which may need the streams beside it.
155///
156/// An FSST corpus is only codes into the symbol table next to it, so it decodes to the strings
157/// those codes expand to. Any other payload decodes exactly as [`decode_blob`] does.
158pub fn decode_region(
159    tree: &DumpTree,
160    buf: &[u8],
161    index: usize,
162    max_values: usize,
163    dec: &mut Decoder,
164) -> DecodedBlob {
165    let Some(region) = tree.regions.get(index) else {
166        return DecodedBlob::Error {
167            message: format!("the tree has no region {index}"),
168        };
169    };
170    let Some(info) = region.blob else {
171        return DecodedBlob::Error {
172            message: format!("region {index} carries no stream metadata"),
173        };
174    };
175    if let Some(streams) = fsst_streams(&tree.regions, index) {
176        dec.reset_budget();
177        return fsst_strings(streams, buf, max_values, dec).unwrap_or_else(|e| error(&e));
178    }
179    match buf.get(region.offset..region.offset + region.len) {
180        Some(bytes) => decode_blob(info, bytes, max_values, dec),
181        None => DecodedBlob::Error {
182            message: format!("region {index} lies outside the tile buffer"),
183        },
184    }
185}
186
187/// The four streams an FSST corpus is read against, in the order `RawFsstData::new` takes them.
188///
189/// They are the streams of the corpus's own column, so a second of any kind there would make
190/// a pick a guess, and the corpus is then left to decode as bytes.
191fn fsst_streams(regions: &[Region], at: usize) -> Option<[&Region; 4]> {
192    let corpus = &regions[at];
193    if !matches!(
194        corpus.blob?.meta.stream_type,
195        StreamType::Data(DictionaryType::Single | DictionaryType::Shared)
196    ) {
197        return None;
198    }
199    let parent = |of: usize| {
200        regions[..of]
201            .iter()
202            .rposition(|r| r.depth < regions[of].depth)
203    };
204    // The corpus sits in its stream, which sits in the column.
205    let column = parent(parent(at)?)?;
206    let depth = regions[column].depth;
207    let after = &regions[column + 1..];
208    let column_streams = &after[..after
209        .iter()
210        .position(|r| r.depth <= depth)
211        .unwrap_or(after.len())];
212    let only = |want: StreamType| {
213        let mut hits = column_streams
214            .iter()
215            .filter(|r| r.blob.is_some_and(|b| b.meta.stream_type == want));
216        let first = hits.next()?;
217        hits.next().is_none().then_some(first)
218    };
219    Some([
220        only(StreamType::Length(LengthType::Symbol))?,
221        only(StreamType::Data(DictionaryType::Fsst))?,
222        only(StreamType::Length(LengthType::Dictionary))?,
223        corpus,
224    ])
225}
226
227fn fsst_strings(
228    [symbol_lengths, symbol_table, lengths, corpus]: [&Region; 4],
229    buf: &[u8],
230    max_values: usize,
231    dec: &mut Decoder,
232) -> MltResult<DecodedBlob> {
233    let stream = |r: &Region| -> MltResult<RawStream<'_>> {
234        let meta = r
235            .blob
236            .ok_or(MltError::MalformedFsst("a stream without metadata"))?
237            .meta;
238        let bytes = buf
239            .get(r.offset..r.offset + r.len)
240            .ok_or(MltError::MalformedFsst("a stream outside the tile"))?;
241        Ok(RawStream::new(meta, bytes))
242    };
243    let raw = RawFsstData::new(
244        stream(symbol_lengths)?,
245        stream(symbol_table)?,
246        stream(lengths)?,
247        stream(corpus)?,
248    )?;
249    let (bytes, lengths) = decode_fsst_bytes(raw, dec)?;
250    let (kept, truncated_from) = cap(lengths, max_values);
251
252    let mut rest = bytes.as_slice();
253    let mut values = Vec::with_capacity(kept.len());
254    for len in kept {
255        let (string, tail) = rest
256            .split_at_checked(len.into_usize())
257            .ok_or(MltError::MalformedFsst("string lengths overrun the corpus"))?;
258        values.push(String::from_utf8_lossy(string).into_owned());
259        rest = tail;
260    }
261    Ok(DecodedBlob::Strings {
262        values,
263        truncated_from,
264    })
265}
266
267fn numbers<T: Into<f64>>(res: MltResult<Vec<T>>, max_values: usize) -> DecodedBlob {
268    match res {
269        Ok(v) => {
270            let (v, truncated_from) = cap(v, max_values);
271            DecodedBlob::Numbers {
272                values: v.into_iter().map(Into::into).collect(),
273                truncated_from,
274            }
275        }
276        Err(e) => error(&e),
277    }
278}
279
280fn geometry_types(res: MltResult<Vec<u32>>, max_values: usize) -> DecodedBlob {
281    match res {
282        Ok(v) => {
283            let (values, truncated_from) = cap(v, max_values);
284            let names = values
285                .iter()
286                .map(|&v| {
287                    u8::try_from(v)
288                        .ok()
289                        .and_then(|b| GeometryType::try_from(b).ok())
290                        .map_or_else(
291                            || format!("unknown({v})").into(),
292                            |t| <&str>::from(t).into(),
293                        )
294                })
295                .collect();
296            DecodedBlob::Enum {
297                values,
298                names,
299                truncated_from,
300            }
301        }
302        Err(e) => error(&e),
303    }
304}
305
306fn bigints<T: Into<i128>>(res: MltResult<Vec<T>>, max_values: usize) -> DecodedBlob {
307    match res {
308        Ok(v) => {
309            let (v, truncated_from) = cap(v, max_values);
310            DecodedBlob::BigInts {
311                values: v.into_iter().map(Into::into).collect(),
312                truncated_from,
313            }
314        }
315        Err(e) => error(&e),
316    }
317}
318
319fn bools(v: Vec<bool>, max_values: usize) -> DecodedBlob {
320    let (values, truncated_from) = cap(v, max_values);
321    DecodedBlob::Bools {
322        values,
323        truncated_from,
324    }
325}
326
327fn error(e: &MltError) -> DecodedBlob {
328    DecodedBlob::Error {
329        message: e.to_string(),
330    }
331}
332
333/// Keep the first `max_values`, reporting the original count when that drops any.
334fn cap<T>(mut values: Vec<T>, max_values: usize) -> (Vec<T>, Option<u32>) {
335    let total = values.len();
336    if max_values == 0 || total <= max_values {
337        return (values, None);
338    }
339    values.truncate(max_values);
340    (values, Some(count(total)))
341}
342
343fn count(n: usize) -> u32 {
344    u32::try_from(n).unwrap_or(u32::MAX)
345}
mlt_core/dump/decode.rs

mlt_core/dump/
decode.rs
