use crate::mvt::{self, GeomType, LayerBuilder, Value};
use crate::shortbread::{self, AttrValue};
const _: () = assert!(
cfg!(target_endian = "little"),
"wire format assumes little-endian"
);
const KEY_NAMES: &[&str] = &[
"kind", "name", "name_en", "name_de", "admin_level", "bridge", "tunnel", "link", "rail", "surface", "tracktype", "service", "oneway", "oneway_reverse", "bicycle", "horse", "ref", "ref_rows", "ref_cols", "population", "maritime", "disputed", "iata", "housenumber", "housename", "amenity", "highway", "office", "leisure", "man_made", "tourism", "cuisine", "vending", "religion", "denomination", "sport", "tower:type", "information", "atm", "recycling:glass_bottles", "recycling:paper", "recycling:clothes", "recycling:scrap_metal", "shop", "emergency", "historic", "way_area", "height", "ele", "min_height", "building:levels", "natural", ];
#[allow(clippy::cast_possible_truncation)]
fn key_to_id(key: &str) -> u8 {
KEY_NAMES
.iter()
.position(|&k| k == key)
.unwrap_or_else(|| panic!("unknown wire format key: {key:?}")) as u8
}
const KIND_VALUES: &[&str] = &[
"water", "glacier", "riverbank", "dock", "canal", "reservoir", "basin", "river", "stream", "ditch", "dam", "pier", "breakwater", "groyne", "forest", "farmland", "farmyard", "meadow", "orchard", "vineyard", "allotments", "brownfield", "cemetery", "commercial", "garages", "grass", "greenfield", "greenhouse_horticulture", "industrial", "landfill", "plant_nursery", "quarry", "railway", "recreation_ground", "residential", "retail", "village_green", "garden", "golf_course", "miniature_golf", "park", "playground", "bare_rock", "beach", "grassland", "heath", "sand", "scree", "scrub", "shingle", "bog", "marsh", "string_bog", "swamp", "wet_meadow", "grave_yard", "danger_area", "sports_centre", "construction", "bicycle_parking", "college", "hospital", "parking", "prison", "university", "motorway", "trunk", "primary", "secondary", "tertiary", "unclassified", "busway", "bus_guideway", "living_street", "service", "pedestrian", "track", "footway", "steps", "path", "cycleway", "runway", "taxiway", "rail", "narrow_gauge", "light_rail", "subway", "tram", "funicular", "monorail", "motorway_junction", "bridge", "motorway_link", "trunk_link", "primary_link", "secondary_link", "tertiary_link", "cable_car", "gondola", "goods", "chair_lift", "drag_lift", "t-bar", "j-bar", "platter", "rope_tow", "ferry", "aerialway_station", "aerodrome", "helipad", "bus_station", "ferry_terminal", "station", "halt", "tram_stop", "bus_stop", "capital", "state_capital", "city", "town", "village", "hamlet", "suburb", "quarter", "neighbourhood", "isolated_dwelling", "farm", "island", "locality", ];
#[allow(clippy::cast_possible_truncation)]
fn kind_value_to_id(s: &str) -> Option<u8> {
KIND_VALUES.iter().position(|&v| v == s).map(|i| i as u8)
}
#[allow(clippy::cast_possible_truncation)]
#[hotpath::measure]
pub(crate) fn encode_attrs_bytes(buf: &mut Vec<u8>, attrs: &[shortbread::Attr], zoom: u8) {
buf.clear();
let filtered_count = attrs
.iter()
.filter(|(_, _, az)| zoom >= *az)
.count()
.min(u8::MAX as usize);
buf.push(filtered_count as u8);
for (key, val, attr_zoom) in attrs {
if zoom < *attr_zoom {
continue;
}
let kid = key_to_id(key);
buf.push(kid);
match val {
AttrValue::Str(s) => {
if kid == 0 {
if let Some(vid) = kind_value_to_id(s) {
buf.push(4);
buf.push(vid);
} else {
buf.push(0);
let sb = s.as_bytes();
let slen = sb.len().min(u16::MAX as usize);
buf.extend_from_slice(&(slen as u16).to_le_bytes());
buf.extend_from_slice(&sb[..slen]);
}
} else {
buf.push(0);
let sb = s.as_bytes();
let slen = sb.len().min(u16::MAX as usize);
buf.extend_from_slice(&(slen as u16).to_le_bytes());
buf.extend_from_slice(&sb[..slen]);
}
}
AttrValue::Int(i) => {
buf.push(1);
buf.extend_from_slice(&i.to_le_bytes());
}
AttrValue::Bool(b) => {
buf.push(2);
buf.push(u8::from(*b));
}
AttrValue::Float(f) => {
buf.push(3);
buf.extend_from_slice(&f.to_le_bytes());
}
}
}
}
pub(crate) fn encode_feature_data_with_attrs(
osm_id: u64,
geom_type: GeomType,
geom_cmds: &[u32],
attrs_bytes: &[u8],
) -> Box<[u8]> {
let mut buf = Vec::with_capacity(13 + geom_cmds.len() * 4 + attrs_bytes.len());
append_feature_data_with_attrs(&mut buf, osm_id, geom_type, geom_cmds, attrs_bytes);
buf.into_boxed_slice()
}
pub(crate) fn append_feature_data_with_attrs(
buf: &mut Vec<u8>,
osm_id: u64,
geom_type: GeomType,
geom_cmds: &[u32],
attrs_bytes: &[u8],
) -> std::ops::Range<usize> {
let start = buf.len();
buf.reserve(13 + geom_cmds.len() * 4 + attrs_bytes.len());
buf.extend_from_slice(&osm_id.to_le_bytes());
buf.push(geom_type as u8);
#[allow(clippy::cast_possible_truncation)]
let cmd_count = geom_cmds.len() as u32;
buf.extend_from_slice(&cmd_count.to_le_bytes());
for &cmd in geom_cmds {
buf.extend_from_slice(&cmd.to_le_bytes());
}
buf.extend_from_slice(attrs_bytes);
start..buf.len()
}
#[allow(dead_code)]
pub(crate) fn encode_feature_data(
osm_id: u64,
geom_type: GeomType,
geom_cmds: &[u32],
attrs: &[shortbread::Attr],
zoom: u8,
) -> Box<[u8]> {
let mut attrs_bytes = Vec::with_capacity(64);
encode_attrs_bytes(&mut attrs_bytes, attrs, zoom);
encode_feature_data_with_attrs(osm_id, geom_type, geom_cmds, &attrs_bytes)
}
#[hotpath::measure]
#[allow(clippy::cast_possible_truncation)]
#[allow(clippy::too_many_lines)]
pub(crate) fn add_feature_to_layer(
layer: &mut LayerBuilder,
data: &[u8],
geom_pool: &mut Vec<Vec<u32>>,
tags_pool: &mut Vec<Vec<(u16, u16)>>,
) {
if data.len() < 13 {
return;
}
let osm_id = u64::from_le_bytes(data[0..8].try_into().expect("osm_id"));
let mut pos: usize = 8;
let gt_byte = data[pos];
pos += 1;
let geom_type = match gt_byte {
1 => GeomType::Point,
2 => GeomType::LineString,
3 => GeomType::Polygon,
_ => return,
};
let cmd_count = u32::from_le_bytes(data[pos..pos + 4].try_into().expect("cmd_count")) as usize;
pos += 4;
let cmd_bytes = cmd_count * 4;
if pos + cmd_bytes > data.len() {
return;
}
let mut geom_cmds = geom_pool.pop().unwrap_or_default();
geom_cmds.clear();
geom_cmds.resize(cmd_count, 0);
#[allow(unsafe_code)]
unsafe {
std::ptr::copy_nonoverlapping(
data[pos..].as_ptr(),
geom_cmds.as_mut_ptr().cast::<u8>(),
cmd_bytes,
);
}
pos += cmd_bytes;
if pos >= data.len() {
return;
}
let attr_count = data[pos] as usize;
pos += 1;
let mut tag_pairs = tags_pool.pop().unwrap_or_default();
tag_pairs.clear();
for _ in 0..attr_count {
if pos >= data.len() {
break;
}
let key_id = data[pos] as usize;
pos += 1;
if key_id >= KEY_NAMES.len() {
break;
}
let key = KEY_NAMES[key_id];
if pos >= data.len() {
break;
}
let val_type = data[pos];
pos += 1;
let ki = layer.intern_key(key);
let vi = match val_type {
0 => {
if pos + 2 > data.len() {
break;
}
let slen =
u16::from_le_bytes(data[pos..pos + 2].try_into().expect("slen")) as usize;
pos += 2;
if pos + slen > data.len() {
break;
}
let s = std::str::from_utf8(&data[pos..pos + slen]).unwrap_or("");
pos += slen;
layer.intern_string_value(s)
}
1 => {
if pos + 8 > data.len() {
break;
}
let i = i64::from_le_bytes(data[pos..pos + 8].try_into().expect("int"));
pos += 8;
layer.intern_value(Value::Int(i))
}
2 => {
if pos >= data.len() {
break;
}
let b = data[pos] != 0;
pos += 1;
layer.intern_value(Value::Bool(b))
}
3 => {
if pos + 8 > data.len() {
break;
}
let f = f64::from_le_bytes(data[pos..pos + 8].try_into().expect("float"));
pos += 8;
layer.intern_value(Value::Double(f))
}
4 => {
if pos >= data.len() {
break;
}
let vid = data[pos] as usize;
pos += 1;
if vid >= KIND_VALUES.len() {
break;
}
layer.intern_string_value(KIND_VALUES[vid])
}
_ => break,
};
tag_pairs.push((ki, vi));
}
layer.add_feature(mvt::Feature {
id: Some(osm_id),
geom_type,
geometry: geom_cmds,
tags: tag_pairs,
});
}
#[cfg(test)]
#[allow(clippy::unwrap_used)]
mod tests {
use super::*;
use crate::mvt::{GeomType, LayerBuilder, Value};
use crate::shortbread::AttrValue;
use std::borrow::Cow;
use std::collections::BTreeMap;
fn feature_tag_map(layer: &LayerBuilder, feature_idx: usize) -> BTreeMap<String, Value> {
let mut out = BTreeMap::new();
let f = layer.test_feature(feature_idx);
for (k, v) in &f.tags {
out.insert(layer.test_key(*k).to_string(), layer.test_value(*v).clone());
}
out
}
#[test]
fn roundtrip_mixed_attribute_types() {
let osm_id: u64 = 123_456_789;
let geom_type = GeomType::LineString;
let geom_cmds: Vec<u32> = vec![9, 2, 2, 18, 2, 2];
let attrs: Vec<shortbread::Attr> = vec![
("name", AttrValue::Str(Cow::Borrowed("Main Street")), 0),
("admin_level", AttrValue::Int(4), 0),
("bridge", AttrValue::Bool(true), 0),
("way_area", AttrValue::Float(1234.5), 0),
];
let encoded = encode_feature_data(osm_id, geom_type, &geom_cmds, &attrs, 14);
let mut layer = LayerBuilder::new("test");
let mut gp = Vec::new();
let mut tp = Vec::new();
add_feature_to_layer(&mut layer, &encoded, &mut gp, &mut tp);
assert_eq!(layer.test_feature_count(), 1);
let f = layer.test_feature(0);
assert_eq!(f.id, Some(osm_id));
assert_eq!(f.geom_type, geom_type);
assert_eq!(f.geometry, geom_cmds);
assert_eq!(f.tags.len(), 4);
let (k0, v0) = f.tags[0];
assert_eq!(layer.test_key(k0), "name");
assert_eq!(
*layer.test_value(v0),
Value::String("Main Street".to_string())
);
let (k1, v1) = f.tags[1];
assert_eq!(layer.test_key(k1), "admin_level");
assert_eq!(*layer.test_value(v1), Value::Int(4));
let (k2, v2) = f.tags[2];
assert_eq!(layer.test_key(k2), "bridge");
assert_eq!(*layer.test_value(v2), Value::Bool(true));
let (k3, v3) = f.tags[3];
assert_eq!(layer.test_key(k3), "way_area");
assert_eq!(*layer.test_value(v3), Value::Double(1234.5));
}
#[test]
fn zoom_dependent_attribute_filtering() {
let osm_id: u64 = 42;
let geom_type = GeomType::Point;
let geom_cmds: Vec<u32> = vec![9, 10, 20];
let attrs: Vec<shortbread::Attr> = vec![
("kind", AttrValue::Str(Cow::Borrowed("city")), 0),
("bridge", AttrValue::Bool(false), 0),
("tunnel", AttrValue::Bool(true), 12),
("surface", AttrValue::Str(Cow::Borrowed("asphalt")), 12),
];
let encoded = encode_feature_data(osm_id, geom_type, &geom_cmds, &attrs, 10);
let mut layer = LayerBuilder::new("test");
let mut gp = Vec::new();
let mut tp = Vec::new();
add_feature_to_layer(&mut layer, &encoded, &mut gp, &mut tp);
assert_eq!(layer.test_feature_count(), 1);
let f = layer.test_feature(0);
assert_eq!(f.tags.len(), 2);
let (k0, v0) = f.tags[0];
assert_eq!(layer.test_key(k0), "kind");
assert_eq!(*layer.test_value(v0), Value::String("city".to_string()));
let (k1, v1) = f.tags[1];
assert_eq!(layer.test_key(k1), "bridge");
assert_eq!(*layer.test_value(v1), Value::Bool(false));
let encoded_z12 = encode_feature_data(osm_id, geom_type, &geom_cmds, &attrs, 12);
let mut layer2 = LayerBuilder::new("test2");
let mut gp2 = Vec::new();
let mut tp2 = Vec::new();
add_feature_to_layer(&mut layer2, &encoded_z12, &mut gp2, &mut tp2);
let f2 = layer2.test_feature(0);
assert_eq!(f2.tags.len(), 4);
let (k2, v2) = f2.tags[2];
assert_eq!(layer2.test_key(k2), "tunnel");
assert_eq!(*layer2.test_value(v2), Value::Bool(true));
let (k3, v3) = f2.tags[3];
assert_eq!(layer2.test_key(k3), "surface");
assert_eq!(*layer2.test_value(v3), Value::String("asphalt".to_string()));
}
#[test]
fn interned_kind_value_roundtrip() {
let osm_id: u64 = 100;
let geom_type = GeomType::Polygon;
let geom_cmds: Vec<u32> = vec![9, 0, 0, 26, 20, 0, 0, 20, 19, 0, 15];
let attrs: Vec<shortbread::Attr> =
vec![("kind", AttrValue::Str(Cow::Borrowed("residential")), 0)];
let encoded = encode_feature_data(osm_id, geom_type, &geom_cmds, &attrs, 14);
let mut layer = LayerBuilder::new("test");
let mut gp = Vec::new();
let mut tp = Vec::new();
add_feature_to_layer(&mut layer, &encoded, &mut gp, &mut tp);
assert_eq!(layer.test_feature_count(), 1);
let f = layer.test_feature(0);
assert_eq!(f.tags.len(), 1);
let (k0, v0) = f.tags[0];
assert_eq!(layer.test_key(k0), "kind");
assert_eq!(
*layer.test_value(v0),
Value::String("residential".to_string())
);
}
#[test]
fn interned_kind_values_multiple_categories() {
let geom_cmds: Vec<u32> = vec![9, 10, 20];
let test_kinds = ["water", "forest", "motorway", "park", "river"];
for kind in test_kinds {
let attrs: Vec<shortbread::Attr> =
vec![("kind", AttrValue::Str(Cow::Borrowed(kind)), 0)];
let encoded = encode_feature_data(1, GeomType::Point, &geom_cmds, &attrs, 14);
let mut layer = LayerBuilder::new("test");
let mut gp = Vec::new();
let mut tp = Vec::new();
add_feature_to_layer(&mut layer, &encoded, &mut gp, &mut tp);
let f = layer.test_feature(0);
let (_, v0) = f.tags[0];
assert_eq!(
*layer.test_value(v0),
Value::String(kind.to_string()),
"kind={kind} should roundtrip through interned encoding"
);
}
}
#[test]
fn non_interned_kind_value_fallback() {
let geom_cmds: Vec<u32> = vec![9, 10, 20];
let attrs: Vec<shortbread::Attr> =
vec![("kind", AttrValue::Str(Cow::Borrowed("custom_kind")), 0)];
let encoded = encode_feature_data(1, GeomType::Point, &geom_cmds, &attrs, 14);
let mut layer = LayerBuilder::new("test");
let mut gp = Vec::new();
let mut tp = Vec::new();
add_feature_to_layer(&mut layer, &encoded, &mut gp, &mut tp);
let f = layer.test_feature(0);
let (k0, v0) = f.tags[0];
assert_eq!(layer.test_key(k0), "kind");
assert_eq!(
*layer.test_value(v0),
Value::String("custom_kind".to_string())
);
}
#[test]
fn minzoom_boundary_mixed_types() {
let osm_id: u64 = 999;
let geom_type = GeomType::Point;
let geom_cmds: Vec<u32> = vec![9, 10, 20];
let attrs: Vec<shortbread::Attr> = vec![
("kind", AttrValue::Str(Cow::Borrowed("city")), 0),
("bridge", AttrValue::Bool(true), 5),
("admin_level", AttrValue::Int(6), 10),
("height", AttrValue::Float(42.0), 10),
];
let encoded_z9 = encode_feature_data(osm_id, geom_type, &geom_cmds, &attrs, 9);
let mut layer_z9 = LayerBuilder::new("z9");
let mut gp = Vec::new();
let mut tp = Vec::new();
add_feature_to_layer(&mut layer_z9, &encoded_z9, &mut gp, &mut tp);
let z9_tags = feature_tag_map(&layer_z9, 0);
assert_eq!(z9_tags.len(), 2, "z9 must not include min_zoom=10 attrs");
assert_eq!(
z9_tags.get("kind"),
Some(&Value::String("city".to_string()))
);
assert_eq!(z9_tags.get("bridge"), Some(&Value::Bool(true)));
assert!(!z9_tags.contains_key("admin_level"));
assert!(!z9_tags.contains_key("height"));
let encoded_z10 = encode_feature_data(osm_id, geom_type, &geom_cmds, &attrs, 10);
let mut layer_z10 = LayerBuilder::new("z10");
let mut gp2 = Vec::new();
let mut tp2 = Vec::new();
add_feature_to_layer(&mut layer_z10, &encoded_z10, &mut gp2, &mut tp2);
let z10_tags = feature_tag_map(&layer_z10, 0);
assert_eq!(z10_tags.len(), 4, "z10 should include all attrs");
assert_eq!(
z10_tags.get("kind"),
Some(&Value::String("city".to_string()))
);
assert_eq!(z10_tags.get("bridge"), Some(&Value::Bool(true)));
assert_eq!(z10_tags.get("admin_level"), Some(&Value::Int(6)));
assert_eq!(z10_tags.get("height"), Some(&Value::Double(42.0)));
}
#[test]
fn minzoom_filtering_leading_attr_gated_out() {
let attrs: Vec<shortbread::Attr> = vec![
("surface", AttrValue::Str(Cow::Borrowed("asphalt")), 12),
("kind", AttrValue::Str(Cow::Borrowed("city")), 0),
("bridge", AttrValue::Bool(false), 0),
];
let geom_cmds: Vec<u32> = vec![9, 10, 20];
let encoded = encode_feature_data(1, GeomType::Point, &geom_cmds, &attrs, 11);
let mut layer = LayerBuilder::new("test");
let mut gp = Vec::new();
let mut tp = Vec::new();
add_feature_to_layer(&mut layer, &encoded, &mut gp, &mut tp);
let tags = feature_tag_map(&layer, 0);
assert_eq!(tags.len(), 2, "gated leading attr should be excluded");
assert_eq!(tags.get("kind"), Some(&Value::String("city".to_string())));
assert_eq!(tags.get("bridge"), Some(&Value::Bool(false)));
assert!(!tags.contains_key("surface"));
}
#[test]
fn preencoded_attrs_respect_minzoom() {
let attrs: Vec<shortbread::Attr> = vec![
("kind", AttrValue::Str(Cow::Borrowed("city")), 0),
("tunnel", AttrValue::Bool(true), 12),
];
let geom_cmds: Vec<u32> = vec![9, 10, 20];
let mut attrs_bytes = Vec::new();
encode_attrs_bytes(&mut attrs_bytes, &attrs, 11);
let encoded = encode_feature_data_with_attrs(7, GeomType::Point, &geom_cmds, &attrs_bytes);
let mut layer = LayerBuilder::new("test");
let mut gp = Vec::new();
let mut tp = Vec::new();
add_feature_to_layer(&mut layer, &encoded, &mut gp, &mut tp);
let f = layer.test_feature(0);
assert_eq!(f.tags.len(), 1);
assert_eq!(layer.test_key(f.tags[0].0), "kind");
}
#[test]
fn minzoom_filtering_all_attrs_gated_out() {
let attrs: Vec<shortbread::Attr> = vec![
("kind", AttrValue::Str(Cow::Borrowed("city")), 5),
("bridge", AttrValue::Bool(true), 5),
];
let geom_cmds: Vec<u32> = vec![9, 10, 20];
let encoded = encode_feature_data(42, GeomType::Point, &geom_cmds, &attrs, 4);
let mut layer = LayerBuilder::new("test");
let mut gp = Vec::new();
let mut tp = Vec::new();
add_feature_to_layer(&mut layer, &encoded, &mut gp, &mut tp);
let f = layer.test_feature(0);
assert_eq!(
f.tags.len(),
0,
"z4 should include no attrs when all min_zoom=5"
);
}
#[test]
fn attrs_count_is_capped_to_u8_max() {
let geom_cmds: Vec<u32> = vec![9, 10, 20];
let attrs: Vec<shortbread::Attr> = (0..300)
.map(|_| ("kind", AttrValue::Str(Cow::Borrowed("city")), 0))
.collect();
let encoded = encode_feature_data(7, GeomType::Point, &geom_cmds, &attrs, 14);
let mut layer = LayerBuilder::new("test");
let mut gp = Vec::new();
let mut tp = Vec::new();
add_feature_to_layer(&mut layer, &encoded, &mut gp, &mut tp);
let f = layer.test_feature(0);
assert_eq!(f.tags.len(), 255, "attrs must be capped to u8::MAX entries");
}
#[test]
fn attrs_cap_boundaries_remain_decodeable_with_mixed_values() {
let geom_cmds: Vec<u32> = vec![9, 10, 20];
let attrs: Vec<shortbread::Attr> = (0..300)
.map(|i| {
if i < 255 {
("bridge", AttrValue::Bool(false), 0)
} else {
("bridge", AttrValue::Bool(true), 0)
}
})
.collect();
let encoded = encode_feature_data(77, GeomType::Point, &geom_cmds, &attrs, 14);
let mut layer = LayerBuilder::new("test");
let mut gp = Vec::new();
let mut tp = Vec::new();
add_feature_to_layer(&mut layer, &encoded, &mut gp, &mut tp);
let f = layer.test_feature(0);
assert_eq!(f.tags.len(), 255);
for (_, v) in &f.tags {
assert_eq!(*layer.test_value(*v), Value::Bool(false));
}
}
#[test]
fn attrs_cap_and_minzoom_interaction_keeps_expected_prefix() {
let geom_cmds: Vec<u32> = vec![9, 10, 20];
let mut attrs: Vec<shortbread::Attr> = Vec::new();
for _ in 0..220 {
attrs.push(("kind", AttrValue::Str(Cow::Borrowed("city")), 0));
}
for _ in 0..120 {
attrs.push(("name", AttrValue::Str(Cow::Borrowed("late")), 12));
}
let encoded_z11 = encode_feature_data(10, GeomType::Point, &geom_cmds, &attrs, 11);
let mut layer_z11 = LayerBuilder::new("z11");
let mut gp = Vec::new();
let mut tp = Vec::new();
add_feature_to_layer(&mut layer_z11, &encoded_z11, &mut gp, &mut tp);
let f11 = layer_z11.test_feature(0);
assert_eq!(f11.tags.len(), 220);
for (k, _) in &f11.tags {
assert_eq!(layer_z11.test_key(*k), "kind");
}
let encoded_z12 = encode_feature_data(11, GeomType::Point, &geom_cmds, &attrs, 12);
let mut layer_z12 = LayerBuilder::new("z12");
let mut gp2 = Vec::new();
let mut tp2 = Vec::new();
add_feature_to_layer(&mut layer_z12, &encoded_z12, &mut gp2, &mut tp2);
let f12 = layer_z12.test_feature(0);
assert_eq!(f12.tags.len(), 255);
let kind_count = f12
.tags
.iter()
.filter(|(k, _)| layer_z12.test_key(*k) == "kind")
.count();
let name_count = f12
.tags
.iter()
.filter(|(k, _)| layer_z12.test_key(*k) == "name")
.count();
assert_eq!(kind_count, 220);
assert_eq!(name_count, 35);
}
}