use std::borrow::Cow;
use bitvec::order::Lsb0;
use bitvec::slice::BitSlice;
use bitvec::view::BitView as _;
use usize_cast::IntoUsize as _;
use crate::LazyParsed::Raw;
use crate::MltError::{BufferUnderflow, MissingLayerName, TrailingLayerData};
use crate::codecs::varint::parse_varint;
use crate::decoder::nested::parse_nested;
use crate::decoder::stream::header02;
use crate::decoder::stream::header02::{Count02, HAS_EXPLICIT_COUNT, StrLayout, StreamCtx02};
use crate::decoder::{
Column02, ColumnCounts, ColumnKind02, ColumnType02, DataType02, Decoder, DictLayout,
DictionaryType, FloatLogical, GeoLayout, GeoTypes, Id, IdWidth02, Layer01, LayerHeader02,
LayerLayout, LengthType, LogicalEncoding, MValues, Nested, Presence02, RawFloats,
RawFloatsEncoding, RawFsstData, RawGeometry, RawId, RawIdValue, RawMValue, RawPlainData,
RawPresence, RawProperty, RawScalar, RawSharedDict, RawSharedDictEncoding, RawSharedDictItem,
RawStream, RawStrings, RawStringsEncoding, SharedDictKind, ValueType02, ValuesColumn02,
};
use crate::tile::{ColumnRole, Extent, reject_taken_name};
use crate::utils::{SetOptionOnce as _, parse_string, parse_u8, take};
use crate::{Lazy, MltError, MltRefResult, MltResult, Parser};
pub(crate) fn parse_layer02<'a>(
input: &'a [u8],
parser: &mut Parser,
) -> MltResult<Layer01<'a, Lazy>> {
let (input, layer_name) = parse_string(input)?;
if layer_name.is_empty() {
return Err(MissingLayerName);
}
let (input, header_byte) = parse_u8(input)?;
let header = LayerHeader02::parse(header_byte)?;
let extent = Extent::new(header.extent.get())?;
let (input, feature_count) = parse_varint::<u32>(input)?;
let (input, layout_byte) = parse_u8(input)?;
let layout = LayerLayout::parse(layout_byte)?;
if header.m_values && !layout.geometry.allows_m_values() {
return Err(MltError::MValuesNeedVertexCounts(layout.geometry.into()));
}
let (input, cols) = parse_shared_presence(input, layout, feature_count)?;
let (input, geometry) = parse_geometry(input, header, layout.geometry, feature_count, parser)?;
let (mut input, counts) = ColumnCounts::parse(input, header.m_values)?;
let column_count = counts.columns;
if input.len() < column_count.into_usize() {
return Err(BufferUnderflow(column_count, input.len()));
}
let mut id_column: Option<Id> = None;
let mut properties = Vec::with_capacity(column_count.into_usize());
let mut nested: Vec<Nested<'a, Lazy>> = Vec::new();
let mut column_names: Vec<(Cow<'a, str>, ColumnRole)> = Vec::new();
#[cfg(fuzzing)]
let mut layer_order = vec![crate::decoder::fuzzing::LayerOrdering::Geometry];
for _ in 0..column_count {
let typ_byte;
(input, typ_byte) = parse_u8(input)?;
let typ = match cols.column(typ_byte)? {
Column02::SharedDict(kind) => {
let shared_dict;
(input, shared_dict) = parse_shared_dict02(input, kind, &cols, parser)?;
for child in &shared_dict.children {
let name = format!("{}{}", shared_dict.name, child.name);
reject_column_name(&column_names, &name, ColumnRole::Property)?;
column_names.push((Cow::Owned(name), ColumnRole::Property));
}
properties.push(Raw(RawProperty::SharedDict(shared_dict)));
#[cfg(fuzzing)]
layer_order.push(crate::decoder::fuzzing::LayerOrdering::Property);
continue;
}
Column02::Values(typ) => typ,
};
let name;
let presence;
(input, name, presence) = parse_column_header(input, typ, &cols)?;
let data_count = cols.count(&presence)?;
match typ.split() {
ColumnKind02::Id(width) => {
#[cfg(fuzzing)]
layer_order.push(crate::decoder::fuzzing::LayerOrdering::Id);
let ctx = StreamCtx02::Property(width.into());
let data;
let count = Count02::Implied(data_count);
(input, data) = header02::parse_stream(input, ctx, count, parser)?;
let value = match width {
IdWidth02::Id32 => RawIdValue::Id32(data),
IdWidth02::Id64 => RawIdValue::Id64(data),
};
id_column.set_once(Raw(RawId { presence, value }))?;
}
ColumnKind02::Values(column) => {
#[cfg(fuzzing)]
layer_order.push(crate::decoder::fuzzing::LayerOrdering::Property);
reject_column_name(&column_names, name, ColumnRole::Property)?;
column_names.push((Cow::Borrowed(name), ColumnRole::Property));
let values;
(input, values) = parse_column_values(
input,
column.values,
name,
presence,
Count02::Implied(data_count),
parser,
)?;
properties.push(Raw(values.into()));
}
ColumnKind02::Nested(column) => {
#[cfg(fuzzing)]
layer_order.push(crate::decoder::fuzzing::LayerOrdering::Property);
reject_column_name(&column_names, name, ColumnRole::Nested)?;
column_names.push((Cow::Borrowed(name), ColumnRole::Nested));
let tree;
(input, tree) =
parse_nested(input, name, presence, column.root, data_count, parser)?;
nested.push(Raw(tree));
}
}
}
let m_values;
(input, m_values) = if header.m_values {
parse_m_values(input, counts.m_values, &cols, &mut column_names, parser)?
} else {
(input, Vec::new())
};
if !input.is_empty() {
return Err(TrailingLayerData(input.len()));
}
Ok(Layer01 {
name: layer_name,
extent,
id: id_column,
geometry: Raw(geometry),
properties,
nested,
m_values,
#[cfg(fuzzing)]
layer_order,
})
}
fn reject_column_name(
columns: &[(Cow<'_, str>, ColumnRole)],
name: &str,
role: ColumnRole,
) -> MltResult<()> {
reject_taken_name(
name,
role,
columns.iter().map(|(seen, taken_by)| (&**seen, *taken_by)),
)
}
#[derive(Debug, Clone, PartialEq)]
pub(crate) enum ColumnValues<'a> {
Bool(RawScalar<'a>),
I8(RawScalar<'a>),
U8(RawScalar<'a>),
I32(RawScalar<'a>),
U32(RawScalar<'a>),
I64(RawScalar<'a>),
U64(RawScalar<'a>),
F32(RawFloats<'a>),
F64(RawFloats<'a>),
Str(RawStrings<'a>),
}
impl<'a> From<ColumnValues<'a>> for RawProperty<'a> {
fn from(values: ColumnValues<'a>) -> Self {
match values {
ColumnValues::Bool(v) => Self::Bool(v),
ColumnValues::I8(v) => Self::I8(v),
ColumnValues::U8(v) => Self::U8(v),
ColumnValues::I32(v) => Self::I32(v),
ColumnValues::U32(v) => Self::U32(v),
ColumnValues::I64(v) => Self::I64(v),
ColumnValues::U64(v) => Self::U64(v),
ColumnValues::F32(v) => Self::F32(v),
ColumnValues::F64(v) => Self::F64(v),
ColumnValues::Str(v) => Self::Str(v),
}
}
}
impl<'a> ColumnValues<'a> {
pub(super) fn decode(self, dec: &mut Decoder) -> MltResult<MValues<'a>> {
Ok(crate::Decode::decode(RawMValue::from(self), dec)?.into_values())
}
}
impl<'a> From<ColumnValues<'a>> for RawMValue<'a> {
fn from(values: ColumnValues<'a>) -> Self {
match values {
ColumnValues::Bool(v) => Self::Bool(v),
ColumnValues::I8(v) => Self::I8(v),
ColumnValues::U8(v) => Self::U8(v),
ColumnValues::I32(v) => Self::I32(v),
ColumnValues::U32(v) => Self::U32(v),
ColumnValues::I64(v) => Self::I64(v),
ColumnValues::U64(v) => Self::U64(v),
ColumnValues::F32(v) => Self::F32(v),
ColumnValues::F64(v) => Self::F64(v),
ColumnValues::Str(v) => Self::Str(v),
}
}
}
fn parse_column_header<'a>(
input: &'a [u8],
typ: ColumnType02,
cols: &LayerCols<'a>,
) -> MltResult<(&'a [u8], &'a str, RawPresence<'a>)> {
let (input, name) = if typ.data.has_name() {
parse_string(input)?
} else {
(input, "")
};
let (input, presence) = cols.presence(typ, input)?;
Ok((input, name, presence))
}
pub(super) fn parse_column_values<'a>(
input: &'a [u8],
values: ValueType02,
name: &'a str,
presence: RawPresence<'a>,
count: Count02,
parser: &mut Parser,
) -> MltRefResult<'a, ColumnValues<'a>> {
if count == Count02::Explicit {
let (_, enc_byte) = parse_u8(input)?;
if enc_byte & HAS_EXPLICIT_COUNT == 0 {
return Err(MltError::MValueImplicitCount {
name: name.to_string(),
byte: enc_byte,
});
}
}
macro_rules! scalar {
($type:ident) => {{
let ctx = StreamCtx02::Property(DataType02::$type);
let (input, data) = header02::parse_stream(input, ctx, count, parser)?;
let scalar = RawScalar::new(name, presence, data);
(input, ColumnValues::$type(scalar))
}};
}
macro_rules! floats {
($type:ident) => {{
let typ = DataType02::$type;
let ctx = StreamCtx02::Property(typ);
let (input, data) = header02::parse_stream(input, ctx, count, parser)?;
let (input, floats) = parse_floats(input, typ, name, presence, data, parser)?;
(input, ColumnValues::$type(floats))
}};
}
Ok(match values {
ValueType02::Bool => scalar!(Bool),
ValueType02::I8 => scalar!(I8),
ValueType02::U8 => scalar!(U8),
ValueType02::I32 => scalar!(I32),
ValueType02::U32 => scalar!(U32),
ValueType02::I64 => scalar!(I64),
ValueType02::U64 => scalar!(U64),
ValueType02::F32 => floats!(F32),
ValueType02::F64 => floats!(F64),
ValueType02::Str => {
let (input, strings) = parse_strings(input, name, presence, count, parser)?;
(input, ColumnValues::Str(strings))
}
})
}
fn parse_m_values<'a>(
mut input: &'a [u8],
count: u32,
cols: &LayerCols<'a>,
column_names: &mut Vec<(Cow<'a, str>, ColumnRole)>,
parser: &mut Parser,
) -> MltRefResult<'a, Vec<crate::decoder::MValueColumn<'a, Lazy>>> {
if input.len() < count.into_usize() {
return Err(BufferUnderflow(count, input.len()));
}
parser.reserve(count)?;
let mut m_values: Vec<crate::decoder::MValueColumn<'a, Lazy>> =
Vec::with_capacity(count.into_usize());
for _ in 0..count {
let typ_byte;
(input, typ_byte) = parse_u8(input)?;
let column = ValuesColumn02::parse_m_value(typ_byte, cols.shared_count()?)?;
let name;
let presence;
(input, name, presence) = parse_column_header(input, column.into(), cols)?;
reject_column_name(column_names, name, ColumnRole::MValue)?;
column_names.push((Cow::Borrowed(name), ColumnRole::MValue));
let values;
(input, values) = parse_column_values(
input,
column.values,
name,
presence,
Count02::Explicit,
parser,
)?;
m_values.push(Raw(values.into()));
}
Ok((input, m_values))
}
fn parse_floats<'a>(
input: &'a [u8],
typ: DataType02,
name: &'a str,
presence: RawPresence<'a>,
data: RawStream<'a>,
parser: &mut Parser,
) -> MltRefResult<'a, RawFloats<'a>> {
let (input, encoding) = match data.meta.encoding.logical {
LogicalEncoding::Float(FloatLogical::Alp(params)) => {
(input, RawFloatsEncoding::Alp { params, data })
}
LogicalEncoding::Float(FloatLogical::Dict) => {
let ctx = StreamCtx02::PropertyDictionary(typ);
let count = Count02::Implied(data.meta.num_values);
let (input, dictionary) = header02::parse_stream(input, ctx, count, parser)?;
let encoding = RawFloatsEncoding::Dictionary {
codes: data,
dictionary,
};
(input, encoding)
}
LogicalEncoding::Float(FloatLogical::None)
| LogicalEncoding::Int(_)
| LogicalEncoding::Bool(_)
| LogicalEncoding::Vertex(_) => (input, RawFloatsEncoding::Single(data)),
};
Ok((
input,
RawFloats {
name,
presence,
encoding,
},
))
}
fn parse_shared_dict02<'a>(
input: &'a [u8],
kind: SharedDictKind,
cols: &LayerCols<'a>,
parser: &mut Parser,
) -> MltRefResult<'a, RawSharedDict<'a>> {
let (input, name) = parse_string(input)?;
let (input, child_count) = parse_varint::<u32>(input)?;
if input.len() < child_count.into_usize() {
return Err(BufferUnderflow(child_count, input.len()));
}
parser.reserve(child_count)?;
let stream = |input: &'a [u8], ctx, parser: &mut Parser| {
header02::parse_stream(input, ctx, Count02::Explicit, parser)
};
let (input, encoding, dict) = match kind {
SharedDictKind::Plain => {
let (input, lengths) = stream(input, StreamCtx02::StrDictLengths, parser)?;
let dict = blob_layout(input)?;
let (input, data) =
stream(input, StreamCtx02::StrBlob(DictionaryType::Shared), parser)?;
let plain = RawPlainData::new(lengths, data)?;
(input, RawSharedDictEncoding::plain(plain), dict)
}
SharedDictKind::Fsst => {
let (input, lengths) = stream(input, StreamCtx02::StrDictLengths, parser)?;
let (input, symbol_lengths) = stream(input, StreamCtx02::StrSymbolLengths, parser)?;
let (input, symbols) =
stream(input, StreamCtx02::StrBlob(DictionaryType::Fsst), parser)?;
let dict = blob_layout(input)?;
let (input, corpus) =
stream(input, StreamCtx02::StrBlob(DictionaryType::Shared), parser)?;
let fsst = RawFsstData::new(symbol_lengths, symbols, lengths, corpus)?;
(input, RawSharedDictEncoding::fsst_plain(fsst), dict)
}
};
let mut input = input;
let mut children = Vec::with_capacity(child_count.into_usize());
for _ in 0..child_count {
let child_byte;
(input, child_byte) = parse_u8(input)?;
let child_typ = cols.column_type(child_byte)?;
if child_typ.data != DataType02::Str {
return Err(MltError::ParsingColumnType(child_byte));
}
let child_name;
(input, child_name) = parse_string(input)?;
let presence;
(input, presence) = cols.presence(child_typ, input)?;
let count = cols.count(&presence)?;
let data;
(input, data) = header02::parse_stream(
input,
StreamCtx02::StrData(StrLayout::Dict),
Count02::Implied(count),
parser,
)?;
children.push(RawSharedDictItem {
name: child_name,
presence,
data,
});
}
Ok((input, RawSharedDict::new(name, encoding, dict, children)))
}
fn blob_layout(input: &[u8]) -> MltResult<DictLayout> {
let (_, enc_byte) = parse_u8(input)?;
Ok(DictLayout::from_bits(enc_byte).unwrap_or(DictLayout::Plain))
}
pub(super) fn parse_strings<'a>(
input: &'a [u8],
name: &'a str,
presence: RawPresence<'a>,
count: Count02,
parser: &mut Parser,
) -> MltRefResult<'a, RawStrings<'a>> {
let (_, enc_byte) = parse_u8(input)?;
let layout = StrLayout::from_bits(enc_byte);
let (input, leading) =
header02::parse_stream(input, StreamCtx02::StrData(layout), count, parser)?;
let count = Count02::Implied(leading.meta.num_values);
let stream = |input: &'a [u8], ctx, parser: &mut Parser| {
header02::parse_stream(input, ctx, count, parser)
};
let (input, encoding) = match layout {
StrLayout::Plain => {
let (input, data) = stream(input, StreamCtx02::StrBlob(DictionaryType::None), parser)?;
let plain = RawPlainData::new(leading, data)?;
(input, RawStringsEncoding::plain(plain))
}
StrLayout::Dict => {
let (input, lengths) = stream(input, StreamCtx02::StrDictLengths, parser)?;
let dict = blob_layout(input)?;
let (input, data) =
stream(input, StreamCtx02::StrBlob(DictionaryType::Single), parser)?;
let plain = RawPlainData::new(lengths, data)?;
(input, RawStringsEncoding::dictionary(plain, leading, dict)?)
}
StrLayout::Fsst => {
let (input, symbol_lengths) = stream(input, StreamCtx02::StrSymbolLengths, parser)?;
let (input, symbols) =
stream(input, StreamCtx02::StrBlob(DictionaryType::Fsst), parser)?;
let (input, corpus) =
stream(input, StreamCtx02::StrBlob(DictionaryType::Single), parser)?;
let fsst = RawFsstData::new(symbol_lengths, symbols, leading, corpus)?;
(input, RawStringsEncoding::fsst_plain(fsst))
}
StrLayout::FsstDict => {
let (input, lengths) = stream(input, StreamCtx02::StrDictLengths, parser)?;
let (input, symbol_lengths) = stream(input, StreamCtx02::StrSymbolLengths, parser)?;
let (input, symbols) =
stream(input, StreamCtx02::StrBlob(DictionaryType::Fsst), parser)?;
let dict = blob_layout(input)?;
let (input, corpus) =
stream(input, StreamCtx02::StrBlob(DictionaryType::Single), parser)?;
let fsst = RawFsstData::new(symbol_lengths, symbols, lengths, corpus)?;
(
input,
RawStringsEncoding::fsst_dictionary(fsst, leading, dict)?,
)
}
};
Ok((
input,
RawStrings {
name,
presence,
encoding,
},
))
}
struct LayerCols<'a> {
shared: Vec<&'a BitSlice<u8, Lsb0>>,
feature_count: u32,
}
impl<'a> LayerCols<'a> {
fn shared_count(&self) -> MltResult<u8> {
Ok(u8::try_from(self.shared.len())?)
}
fn column(&self, byte: u8) -> MltResult<Column02> {
Column02::parse(byte, self.shared_count()?)
}
fn column_type(&self, byte: u8) -> MltResult<ColumnType02> {
ColumnType02::parse(byte, self.shared_count()?)
}
fn presence(&self, typ: ColumnType02, input: &'a [u8]) -> MltRefResult<'a, RawPresence<'a>> {
match typ.presence {
Presence02::AllPresent => Ok((input, RawPresence::AllPresent)),
Presence02::Inline => {
let (input, bits) = parse_bitfield(input, self.feature_count)?;
Ok((input, RawPresence::Bitfield(bits)))
}
Presence02::Shared(index) => self
.shared
.get(usize::from(index))
.map(|&bits| (input, RawPresence::Bitfield(bits)))
.ok_or_else(|| MltError::ParsingColumnType(typ.to_byte())),
}
}
fn count(&self, presence: &RawPresence<'_>) -> MltResult<u32> {
Ok(match presence {
RawPresence::Bitfield(bits) => u32::try_from(bits.count_ones())?,
RawPresence::AllPresent | RawPresence::Stream(_) => self.feature_count,
})
}
}
fn parse_shared_presence(
input: &[u8],
layout: LayerLayout,
feature_count: u32,
) -> MltRefResult<'_, LayerCols<'_>> {
let mut input = input;
let mut shared = Vec::with_capacity(usize::from(layout.shared_presence));
for _ in 0..layout.shared_presence {
let bits;
(input, bits) = parse_bitfield(input, feature_count)?;
shared.push(bits);
}
Ok((
input,
LayerCols {
shared,
feature_count,
},
))
}
fn parse_bitfield(input: &[u8], feature_count: u32) -> MltRefResult<'_, &BitSlice<u8, Lsb0>> {
let (input, bytes) = take(input, feature_count.div_ceil(8))?;
Ok((
input,
&bytes.view_bits::<Lsb0>()[..feature_count.into_usize()],
))
}
fn parse_geometry<'a>(
input: &'a [u8],
header: LayerHeader02,
layout: GeoLayout,
feature_count: u32,
parser: &mut Parser,
) -> MltRefResult<'a, RawGeometry<'a>> {
let count = Count02::Implied(feature_count);
let mut input = input;
let types = if let Some(geometry_type) = header.uniform_type {
GeoTypes::Uniform {
geometry_type,
feature_count,
}
} else {
let stream;
(input, stream) = header02::parse_stream(input, StreamCtx02::GeomTypes, count, parser)?;
GeoTypes::Stream(stream)
};
let mut items = Vec::with_capacity(6);
let mut stream = |input: &'a [u8], ctx, items: &mut Vec<_>| -> MltResult<&'a [u8]> {
let (rest, parsed) = header02::parse_stream(input, ctx, count, parser)?;
items.push(parsed);
Ok(rest)
};
let lengths = [
(layout.has_geo_lengths(), LengthType::Geometries),
(layout.has_part_lengths(), LengthType::Parts),
(layout.has_ring_lengths(), LengthType::Rings),
];
for (present, length_type) in lengths {
if present {
input = stream(input, StreamCtx02::GeomOffsets(length_type), &mut items)?;
}
}
if layout.is_tess() {
let triangles = StreamCtx02::GeomOffsets(LengthType::Triangles);
input = stream(input, triangles, &mut items)?;
input = stream(input, StreamCtx02::GeomIndices, &mut items)?;
}
input = stream(input, StreamCtx02::GeomVertices, &mut items)?;
if layout.is_dict() {
input = stream(input, StreamCtx02::GeomVertexOffsets, &mut items)?;
}
Ok((input, RawGeometry { types, items }))
}