use std::borrow::Cow;
#[cfg(feature = "unstable-v2")]
use std::collections::HashSet;
use derive_debug::Dbg;
#[cfg(feature = "unstable-v2")]
use crate::decoder::RleLayout;
use crate::decoder::{DictionaryType, FastPForKind, GeometryValues, PhysicalEncoding, StreamType};
use crate::encoder::geometry::VertexBufferType;
use crate::encoder::{IntEncoder, StagedId, StagedProperty};
#[cfg(feature = "unstable-v2")]
use crate::encoder::{StagedInterior, StagedMValue, StagedNested, StagedNode};
#[cfg(feature = "unstable-v2")]
use crate::tile::MAX_NESTED_DEPTH;
use crate::tile::{ColumnRole, Extent, reject_taken_name};
use crate::{MltError, MltResult};
#[derive(Debug, Clone, Default, PartialEq)]
pub struct EncodedUnknown {
pub(crate) tag: u8,
pub(crate) value: Vec<u8>,
}
impl EncodedUnknown {
pub fn new(tag: u8, value: Vec<u8>) -> MltResult<Self> {
if tag == 1 {
return Err(MltError::ParsingColumnType(tag));
}
Ok(Self { tag, value })
}
#[must_use]
pub fn tag(&self) -> u32 {
u32::from(self.tag)
}
#[must_use]
pub fn data(&self) -> &[u8] {
&self.value
}
}
#[derive(Debug, Clone, Copy, PartialEq)]
pub struct CurveParams {
pub shift: u32,
pub bits: u32,
}
impl Default for CurveParams {
fn default() -> Self {
Self { shift: 0, bits: 1 }
}
}
impl CurveParams {
#[must_use]
pub fn from_vertices(vertices: &[i32]) -> Self {
if vertices.is_empty() {
return Self::default();
}
let (min, max) = vertices
.iter()
.fold((i32::MAX, i32::MIN), |(mn, mx), &v| (mn.min(v), mx.max(v)));
crate::codecs::hilbert::hilbert_curve_params_from_bounds(min, max)
}
}
#[derive(Debug, PartialEq, Clone)]
pub struct StagedLayer {
pub(crate) name: String,
pub(crate) extent: Extent,
pub(crate) id: StagedId,
pub(crate) geometry: GeometryValues,
pub(crate) properties: Vec<StagedProperty>,
#[cfg(feature = "unstable-v2")]
pub(crate) m_values: Vec<StagedMValue>,
#[cfg(feature = "unstable-v2")]
pub(crate) nested: Vec<StagedNested>,
}
#[cfg_attr(not(feature = "__private"), allow(dead_code))]
impl StagedLayer {
pub fn new(
name: impl Into<String>,
extent: u32,
id: StagedId,
geometry: GeometryValues,
properties: Vec<StagedProperty>,
) -> MltResult<Self> {
let (name, extent, _) = validate_staged(name, extent, &id, &geometry, &properties)?;
Ok(Self {
name,
extent,
id,
geometry,
properties,
#[cfg(feature = "unstable-v2")]
m_values: Vec::new(),
#[cfg(feature = "unstable-v2")]
nested: Vec::new(),
})
}
#[cfg(feature = "unstable-v2")]
pub fn with_m_values(
name: impl Into<String>,
extent: u32,
id: StagedId,
geometry: GeometryValues,
properties: Vec<StagedProperty>,
m_values: Vec<StagedMValue>,
) -> MltResult<Self> {
let (name, extent, columns) = validate_staged(name, extent, &id, &geometry, &properties)?;
validate_m_values(&geometry, &columns, &m_values)?;
Ok(Self {
name,
extent,
id,
geometry,
properties,
m_values,
nested: Vec::new(),
})
}
#[cfg(feature = "unstable-v2")]
pub fn with_nested(
name: impl Into<String>,
extent: u32,
id: StagedId,
geometry: GeometryValues,
properties: Vec<StagedProperty>,
m_values: Vec<StagedMValue>,
nested: Vec<StagedNested>,
) -> MltResult<Self> {
let (name, extent, columns) = validate_staged(name, extent, &id, &geometry, &properties)?;
validate_m_values(&geometry, &columns, &m_values)?;
validate_nested(&geometry, &columns, &m_values, &nested)?;
Ok(Self {
name,
extent,
id,
geometry,
properties,
m_values,
nested,
})
}
#[must_use]
pub fn name(&self) -> &str {
&self.name
}
#[must_use]
pub fn extent(&self) -> Extent {
self.extent
}
#[must_use]
pub fn id(&self) -> &StagedId {
&self.id
}
#[must_use]
pub fn geometry(&self) -> &GeometryValues {
&self.geometry
}
#[must_use]
pub fn properties(&self) -> &[StagedProperty] {
&self.properties
}
#[cfg(feature = "unstable-v2")]
#[must_use]
pub fn m_values(&self) -> &[StagedMValue] {
&self.m_values
}
#[cfg(feature = "unstable-v2")]
#[must_use]
pub fn nested(&self) -> &[StagedNested] {
&self.nested
}
}
fn validate_staged<'p>(
name: impl Into<String>,
extent: u32,
id: &StagedId,
geometry: &GeometryValues,
properties: &'p [StagedProperty],
) -> MltResult<(String, Extent, Vec<Cow<'p, str>>)> {
let name = name.into();
if name.is_empty() {
return Err(MltError::MissingLayerName);
}
let extent = Extent::new(extent)?;
let feature_count = geometry.feature_count();
if let Some(actual) = id.feature_count()
&& actual != feature_count
{
return Err(MltError::StagedFeatureCountMismatch {
column: "id".into(),
expected: feature_count,
actual,
});
}
let mut columns: Vec<Cow<'p, str>> = Vec::with_capacity(properties.len());
for property in properties {
let actual = property.feature_count();
if actual != feature_count {
return Err(MltError::StagedFeatureCountMismatch {
column: property.name().to_string(),
expected: feature_count,
actual,
});
}
if let StagedProperty::SharedDict(sd) = property {
for item in &sd.items {
let name = format!("{}{}", sd.prefix, item.suffix);
reject_taken_name(&name, ColumnRole::Property, property_columns(&columns))?;
columns.push(Cow::Owned(name));
}
} else {
reject_taken_name(
property.name(),
ColumnRole::Property,
property_columns(&columns),
)?;
columns.push(Cow::Borrowed(property.name()));
}
}
Ok((name, extent, columns))
}
fn property_columns<'a>(
columns: &'a [Cow<'a, str>],
) -> impl Iterator<Item = (&'a str, ColumnRole)> + 'a {
columns
.iter()
.map(|name| (name.as_ref(), ColumnRole::Property))
}
#[cfg(feature = "unstable-v2")]
fn validate_m_values(
geometry: &GeometryValues,
columns: &[Cow<'_, str>],
m_values: &[StagedMValue],
) -> MltResult<()> {
let feature_count = geometry.feature_count();
let mut seen: Vec<&str> = Vec::with_capacity(m_values.len());
for column in m_values {
reject_taken_name(
column.name(),
ColumnRole::MValue,
property_columns(columns).chain(seen.iter().map(|n| (*n, ColumnRole::MValue))),
)?;
seen.push(column.name());
if let Some(actual) = column.feature_count()
&& actual != feature_count
{
return Err(MltError::StagedFeatureCountMismatch {
column: column.name().to_string(),
expected: feature_count,
actual,
});
}
let mut expected = 0;
for index in 0..feature_count {
let present = column
.presence
.as_ref()
.is_none_or(|mask| mask.get(index).copied().unwrap_or(false));
if present {
expected += geometry.vertex_count(index)?;
}
}
if expected != column.values().count() {
return Err(MltError::MValueColumnLengthMismatch {
name: column.name().to_string(),
expected,
actual: column.values().count(),
});
}
}
Ok(())
}
#[cfg(feature = "unstable-v2")]
fn validate_nested(
geometry: &GeometryValues,
columns: &[Cow<'_, str>],
m_values: &[StagedMValue],
nested: &[StagedNested],
) -> MltResult<()> {
let feature_count = geometry.feature_count();
let mut seen: Vec<&str> = Vec::with_capacity(nested.len());
for column in nested {
reject_taken_name(
column.name(),
ColumnRole::Nested,
property_columns(columns)
.chain(m_values.iter().map(|c| (c.name(), ColumnRole::MValue)))
.chain(seen.iter().map(|n| (*n, ColumnRole::Nested))),
)?;
seen.push(column.name());
if column.root.parent_count() != feature_count {
return Err(MltError::StagedFeatureCountMismatch {
column: column.name().to_string(),
expected: feature_count,
actual: column.root.parent_count(),
});
}
validate_nested_interior(&column.root, 1)?;
}
Ok(())
}
#[cfg(feature = "unstable-v2")]
fn validate_nested_node(node: &StagedNode, depth: usize) -> MltResult<()> {
if depth > MAX_NESTED_DEPTH {
return Err(MltError::NestedTooDeep(depth));
}
match node {
StagedNode::Leaf(_) => Ok(()),
StagedNode::Interior(interior) => validate_nested_interior(interior, depth),
}
}
#[cfg(feature = "unstable-v2")]
fn validate_nested_interior(interior: &StagedInterior, depth: usize) -> MltResult<()> {
match interior {
StagedInterior::Struct(node) => {
let mut seen: HashSet<&str> = HashSet::with_capacity(node.fields.len());
for (name, field) in &node.fields {
if !seen.insert(name.as_str()) {
return Err(MltError::DuplicateFieldName(name.clone()));
}
validate_nested_node(field, depth + 1)?;
}
Ok(())
}
StagedInterior::List(node) => validate_nested_node(&node.element, depth + 1),
StagedInterior::Map(node) => validate_nested_node(&node.value, depth + 1),
}
}
#[cfg(feature = "unstable-v2")]
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Default)]
#[cfg_attr(feature = "arbitrary", derive(arbitrary::Arbitrary))]
pub enum WireVersion {
#[default]
V01,
V02,
}
#[cfg(feature = "unstable-v2")]
impl WireVersion {
#[must_use]
pub(crate) fn tag(self) -> u8 {
match self {
Self::V01 => 1,
Self::V02 => 2,
}
}
#[must_use]
pub(crate) fn fastpfor_kind(self) -> FastPForKind {
match self {
Self::V01 => FastPForKind::Block256Be,
Self::V02 => FastPForKind::Block128Le,
}
}
#[must_use]
pub(crate) fn rle_layout(self) -> RleLayout {
match self {
Self::V01 => RleLayout::Split,
Self::V02 => RleLayout::Interleaved,
}
}
#[must_use]
pub(crate) fn raw_header_saving(self) -> usize {
match self {
Self::V01 => 0,
Self::V02 => 1,
}
}
}
#[derive(Debug, Clone, Copy, PartialEq, Hash)]
#[expect(
clippy::struct_excessive_bools,
reason = "enums would not model this better, not a state machine"
)]
pub struct EncoderConfig {
#[cfg(feature = "unstable-v2")]
wire_version: WireVersion,
tessellate: bool,
attempt_spatial_morton_sort: bool,
attempt_spatial_hilbert_sort: bool,
attempt_id_sort: bool,
allow_fsst: bool,
allow_fastpfor: bool,
allow_shared_dict: bool,
#[cfg(feature = "unstable-v2")]
allow_float_dict: bool,
#[cfg(feature = "unstable-v2")]
allow_float_alp: bool,
#[cfg(feature = "unstable-v2")]
allow_packed_dict_codes: bool,
#[cfg(feature = "unstable-v2")]
allow_row_shapes: bool,
}
impl Default for EncoderConfig {
fn default() -> Self {
Self {
#[cfg(feature = "unstable-v2")]
wire_version: WireVersion::V01,
tessellate: false,
attempt_spatial_morton_sort: true,
attempt_spatial_hilbert_sort: true,
attempt_id_sort: true,
allow_fsst: true,
allow_fastpfor: true,
allow_shared_dict: true,
#[cfg(feature = "unstable-v2")]
allow_float_dict: false,
#[cfg(feature = "unstable-v2")]
allow_float_alp: false,
#[cfg(feature = "unstable-v2")]
allow_row_shapes: false,
#[cfg(feature = "unstable-v2")]
allow_packed_dict_codes: false,
}
}
}
impl EncoderConfig {
#[cfg(feature = "unstable-v2")]
#[must_use]
pub fn wire_version(self) -> WireVersion {
self.wire_version
}
#[must_use]
pub fn tessellate(self) -> bool {
self.tessellate
}
#[must_use]
pub fn attempt_spatial_morton_sort(self) -> bool {
self.attempt_spatial_morton_sort
}
#[must_use]
pub fn attempt_spatial_hilbert_sort(self) -> bool {
self.attempt_spatial_hilbert_sort
}
#[must_use]
pub fn attempt_id_sort(self) -> bool {
self.attempt_id_sort
}
#[must_use]
pub fn allow_fsst(self) -> bool {
self.allow_fsst
}
#[must_use]
pub fn allow_fastpfor(self) -> bool {
self.allow_fastpfor
}
#[must_use]
pub(crate) fn fastpfor(self) -> Option<PhysicalEncoding> {
#[cfg(feature = "unstable-v2")]
let kind = self.wire_version.fastpfor_kind();
#[cfg(not(feature = "unstable-v2"))]
let kind = FastPForKind::Block256Be;
self.allow_fastpfor
.then_some(PhysicalEncoding::FastPFor(kind))
}
#[must_use]
pub fn allow_shared_dict(self) -> bool {
self.allow_shared_dict
}
#[cfg(feature = "unstable-v2")]
#[must_use]
pub fn allow_row_shapes(self) -> bool {
self.allow_row_shapes
}
#[cfg(feature = "unstable-v2")]
#[must_use]
pub fn with_row_shapes(mut self, enabled: bool) -> Self {
self.allow_row_shapes = enabled;
self
}
#[must_use]
pub fn allow_8bit_ints(self) -> bool {
#[cfg(feature = "unstable-v2")]
return self.wire_version != WireVersion::V01;
#[cfg(not(feature = "unstable-v2"))]
return false;
}
#[cfg(feature = "unstable-v2")]
#[must_use]
pub fn allow_float_dict(self) -> bool {
self.allow_float_dict && self.wire_version != WireVersion::V01
}
#[cfg(feature = "unstable-v2")]
#[must_use]
pub fn allow_float_alp(self) -> bool {
self.allow_float_alp && self.wire_version != WireVersion::V01
}
#[cfg(feature = "unstable-v2")]
#[must_use]
pub fn with_wire_version(mut self, version: WireVersion) -> Self {
self.wire_version = version;
self
}
#[must_use]
pub fn with_tessellation(mut self, enabled: bool) -> Self {
self.tessellate = enabled;
self
}
#[must_use]
pub fn with_spatial_morton_sort(mut self, enabled: bool) -> Self {
self.attempt_spatial_morton_sort = enabled;
self
}
#[must_use]
pub fn with_spatial_hilbert_sort(mut self, enabled: bool) -> Self {
self.attempt_spatial_hilbert_sort = enabled;
self
}
#[must_use]
pub fn with_id_sort(mut self, enabled: bool) -> Self {
self.attempt_id_sort = enabled;
self
}
#[must_use]
pub fn with_fsst(mut self, enabled: bool) -> Self {
self.allow_fsst = enabled;
self
}
#[must_use]
pub fn with_fastpfor(mut self, enabled: bool) -> Self {
self.allow_fastpfor = enabled;
self
}
#[must_use]
pub fn with_shared_dict(mut self, enabled: bool) -> Self {
self.allow_shared_dict = enabled;
self
}
#[cfg(feature = "unstable-v2")]
#[must_use]
pub fn with_float_dict(mut self, enabled: bool) -> Self {
self.allow_float_dict = enabled;
self
}
#[cfg(feature = "unstable-v2")]
#[must_use]
pub fn with_float_alp(mut self, enabled: bool) -> Self {
self.allow_float_alp = enabled;
self
}
#[cfg(feature = "unstable-v2")]
#[must_use]
pub fn allow_packed_dict_codes(self) -> bool {
self.allow_packed_dict_codes && self.wire_version != WireVersion::V01
}
#[cfg(feature = "unstable-v2")]
#[must_use]
pub fn with_packed_dict_codes(mut self, enabled: bool) -> Self {
self.allow_packed_dict_codes = enabled;
self
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum StrEncoding {
Plain,
Dict,
Fsst,
FsstDict,
FrontDict,
FsstFrontDict,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum FloatEncoding {
None,
#[cfg(feature = "unstable-v2")]
Dict,
#[cfg(feature = "unstable-v2")]
Alp,
}
#[derive(Debug, Clone, Copy, PartialEq)]
pub enum ColumnKind {
Id,
Geometry,
Property,
}
#[cfg(feature = "unstable-v2")]
#[derive(Clone, Copy, Debug)]
pub struct StrAt<'a> {
pub name: &'a str,
pub subname: &'a str,
}
#[cfg(feature = "unstable-v2")]
impl<'a> StrAt<'a> {
#[must_use]
pub const fn flat(name: &'a str) -> Self {
Self { name, subname: "" }
}
#[must_use]
pub const fn nested(name: &'a str, subname: &'a str) -> Self {
Self { name, subname }
}
pub(crate) fn ctx(self, stream_type: StreamType) -> StreamCtx<'a> {
StreamCtx::prop2(stream_type, self.name, self.subname)
}
pub(crate) fn qualified(self) -> Cow<'a, str> {
StreamCtx::prop2(StreamType::Present, self.name, self.subname).qualified()
}
}
#[derive(Clone, Copy, Debug, PartialEq)]
pub struct StreamCtx<'a> {
pub kind: ColumnKind,
pub stream_type: StreamType,
pub name: &'a str,
pub subname: &'a str,
}
impl<'a> StreamCtx<'a> {
#[inline]
#[must_use]
pub const fn new(
kind: ColumnKind,
stream_type: StreamType,
name: &'a str,
subname: &'a str,
) -> Self {
Self {
kind,
stream_type,
name,
subname,
}
}
#[inline]
#[must_use]
pub const fn id(stream_type: StreamType) -> Self {
Self::new(ColumnKind::Id, stream_type, "", "")
}
#[inline]
#[must_use]
pub const fn geom(stream_type: StreamType, name: &'a str) -> Self {
Self::new(ColumnKind::Geometry, stream_type, name, "")
}
#[inline]
#[must_use]
pub const fn prop(stream_type: StreamType, name: &'a str) -> Self {
Self::new(ColumnKind::Property, stream_type, name, "")
}
#[inline]
#[must_use]
pub const fn prop_data(name: &'a str) -> Self {
let stream_type = StreamType::Data(DictionaryType::None);
Self::new(ColumnKind::Property, stream_type, name, "")
}
#[inline]
#[must_use]
pub const fn prop2(stream_type: StreamType, prefix: &'a str, suffix: &'a str) -> Self {
Self::new(ColumnKind::Property, stream_type, prefix, suffix)
}
#[must_use]
pub fn qualified(&self) -> Cow<'a, str> {
if self.subname.is_empty() {
Cow::Borrowed(self.name)
} else {
Cow::Owned(format!("{}{}", self.name, self.subname))
}
}
}
#[derive(Dbg)]
pub struct ExplicitEncoder {
pub vertex_buffer_type: VertexBufferType,
#[dbg(skip)]
pub force_stream: Box<dyn for<'a> Fn(&'a StreamCtx<'a>) -> bool>,
#[dbg(skip)]
pub get_int_encoder: Box<dyn for<'a> Fn(&'a StreamCtx<'a>) -> IntEncoder>,
#[dbg(skip)]
pub get_str_encoding: Box<dyn Fn(&str) -> StrEncoding>,
#[dbg(skip)]
pub get_float_encoding: Box<dyn Fn(&str) -> FloatEncoding>,
}
#[cfg(all(test, feature = "unstable-v2"))]
mod tests {
use insta::assert_snapshot;
use super::*;
use crate::encoder::{StagedLeaf, StagedList, StagedStruct, StagedValues};
fn leaf() -> StagedNode {
StagedNode::Leaf(StagedLeaf::new(None, StagedValues::I32(vec![1])))
}
fn lists(depth: usize) -> StagedInterior {
let mut node = leaf();
for _ in 2..depth {
node = StagedNode::Interior(StagedInterior::List(StagedList::new(None, vec![1], node)));
}
StagedInterior::List(StagedList::new(None, vec![1], node))
}
#[test]
fn a_struct_repeating_a_field_name() {
let root =
StagedInterior::Struct(StagedStruct::new(None, vec![("a", leaf()), ("a", leaf())]));
assert_snapshot!(
validate_nested_interior(&root, 1).unwrap_err(),
@"duplicate field name in a struct node: a"
);
}
#[test]
fn a_tree_one_level_past_the_limit() {
assert_snapshot!(
validate_nested_interior(&lists(MAX_NESTED_DEPTH + 1), 1).unwrap_err(),
@"a nested column is 9 levels deep, more than the 8 the format allows"
);
}
#[test]
fn a_tree_at_the_limit() {
validate_nested_interior(&lists(MAX_NESTED_DEPTH), 1).expect("a tree at the limit");
}
}