use crate::input::ParsedFeature;
use crate::quadbin;
use crate::tiler::{GeneratedTile, TileWriter};
use anyhow::Result;
use h3o::{LatLng, Resolution};
use std::collections::{BTreeMap, HashMap};
use stt_core::arrow_tile::{ColumnarLayer, Coord, GeometryColumn, PropertyColumn};
use stt_core::metadata::{
SummaryAggregation, SummaryColumn, SummaryScheme, SummaryTier,
};
use stt_core::projection;
use stt_core::tile::TileId;
#[derive(Debug, Clone)]
pub struct SummaryConfig {
pub scheme: SummaryScheme,
pub min_zoom: u8,
pub max_zoom: u8,
pub temporal_bucket_ms: u64,
pub columns: Vec<SummaryColumn>,
pub layer_name: String,
pub sub_buckets: u32,
}
impl SummaryConfig {
pub fn to_tier(&self) -> SummaryTier {
let mut resolutions: Vec<u8> = Vec::with_capacity(
(self.max_zoom - self.min_zoom + 1) as usize,
);
for z in self.min_zoom..=self.max_zoom {
resolutions.push(match self.scheme {
SummaryScheme::H3 => h3_resolution_for_zoom(z),
SummaryScheme::Quadbin => quadbin_zoom_for_zoom(z),
});
}
SummaryTier {
scheme: self.scheme,
min_zoom: self.min_zoom,
max_zoom: self.max_zoom,
cell_resolution_per_zoom: resolutions,
columns: self.columns.clone(),
layer_name: self.layer_name.clone(),
sub_buckets: self.sub_buckets.max(1),
}
}
}
pub fn h3_resolution_for_zoom(zoom: u8) -> u8 {
zoom.min(15)
}
pub const QUADBIN_RES_OFFSET: u8 = 3;
pub fn quadbin_zoom_for_zoom(zoom: u8) -> u8 {
zoom.saturating_add(QUADBIN_RES_OFFSET).min(26)
}
#[derive(Debug, Default)]
struct CellAggregate {
count: u64,
time_start: i64,
time_end: i64,
sources: HashMap<String, Accumulator>,
sub_bucket_counts: Vec<u32>,
}
#[derive(Debug, Default, Clone, Copy)]
struct Accumulator {
sum: f64,
sum_count: u64,
min: f64,
max: f64,
has_any: bool,
}
impl Accumulator {
fn observe(&mut self, v: f64) {
if !v.is_finite() {
return;
}
if !self.has_any {
self.min = v;
self.max = v;
self.has_any = true;
} else {
if v < self.min {
self.min = v;
}
if v > self.max {
self.max = v;
}
}
self.sum += v;
self.sum_count += 1;
}
}
pub fn build_summary_tier<W: TileWriter>(
features: &[ParsedFeature],
config: &SummaryConfig,
writer: &mut W,
) -> Result<usize> {
let bucket_ms = config.temporal_bucket_ms.max(1);
let sub_buckets = config.sub_buckets.max(1) as usize;
let sub_bucket_ms = if sub_buckets > 1 {
(bucket_ms as usize / sub_buckets).max(1) as u64
} else {
bucket_ms
};
let mut total = 0usize;
for zoom in config.min_zoom..=config.max_zoom {
let h3_res = match config.scheme {
SummaryScheme::H3 => {
let h3_res_u8 = h3_resolution_for_zoom(zoom);
Some(Resolution::try_from(h3_res_u8).map_err(|e| {
anyhow::anyhow!("invalid H3 resolution {h3_res_u8} for zoom {zoom}: {e:?}")
})?)
}
SummaryScheme::Quadbin => None,
};
let quad_z = quadbin_zoom_for_zoom(zoom);
let mut buckets: BTreeMap<(u32, u32, u64), HashMap<u64, CellAggregate>> =
BTreeMap::new();
for feature in features {
let (tx, ty) =
match projection::lonlat_to_tile(feature.lon, feature.lat, zoom) {
Ok(xy) => xy,
Err(_) => continue,
};
let bucket_start = (feature.timestamp / bucket_ms) * bucket_ms;
let cell_id: u64 = match config.scheme {
SummaryScheme::H3 => {
let res = h3_res.expect("H3 resolution resolved above");
match LatLng::new(feature.lat, feature.lon) {
Ok(ll) => ll.to_cell(res).into(),
Err(_) => continue, }
}
SummaryScheme::Quadbin => {
if !feature.lon.is_finite() || !feature.lat.is_finite() {
continue;
}
quadbin::lonlat_to_quadbin(feature.lon, feature.lat, quad_z)
}
};
let bucket = buckets
.entry((tx, ty, bucket_start))
.or_default();
let agg = bucket.entry(cell_id).or_default();
if sub_buckets > 1 {
if agg.sub_bucket_counts.is_empty() {
agg.sub_bucket_counts = vec![0u32; sub_buckets];
}
let sub_idx = (((feature.timestamp - bucket_start) / sub_bucket_ms)
as usize)
.min(sub_buckets - 1);
agg.sub_bucket_counts[sub_idx] = agg.sub_bucket_counts[sub_idx]
.saturating_add(1);
}
agg.count += 1;
let t = feature.timestamp as i64;
let t_end = feature.end_timestamp.unwrap_or(feature.timestamp) as i64;
if agg.count == 1 {
agg.time_start = t;
agg.time_end = t_end;
} else {
if t < agg.time_start {
agg.time_start = t;
}
if t_end > agg.time_end {
agg.time_end = t_end;
}
}
if let Some(props) = feature.shared_properties.as_deref() {
for col in &config.columns {
if matches!(col.agg, SummaryAggregation::Count) {
continue;
}
let v = props.get(&col.name).and_then(|v| {
v.as_f64()
.or_else(|| v.as_str().and_then(|s| s.parse::<f64>().ok()))
});
if let Some(v) = v {
agg.sources
.entry(col.name.clone())
.or_default()
.observe(v);
}
}
}
}
for ((tx, ty, bucket_start), cells) in buckets {
if cells.is_empty() {
continue;
}
let layer =
build_summary_layer(config.scheme, &config.layer_name, &config.columns, &cells);
let time_end = cells
.values()
.map(|c| c.time_end)
.max()
.unwrap_or((bucket_start + bucket_ms) as i64);
let cover_t_min = cells
.values()
.map(|c| c.time_start)
.min()
.unwrap_or(bucket_start as i64);
let tile = GeneratedTile {
id: TileId::new(zoom, tx, ty, bucket_start),
time_start: bucket_start as i64,
time_end,
cover_t_min,
layers: vec![layer],
};
writer.write_tile(&tile)?;
total += 1;
}
}
Ok(total)
}
fn build_summary_layer(
scheme: SummaryScheme,
name: &str,
columns: &[SummaryColumn],
cells: &HashMap<u64, CellAggregate>,
) -> ColumnarLayer {
let n = cells.len();
let mut entries: Vec<(u64, &CellAggregate)> =
cells.iter().map(|(k, v)| (*k, v)).collect();
entries.sort_by_key(|(k, _)| *k);
let mut feature_ids: Vec<u64> = Vec::with_capacity(n);
let mut start_times: Vec<i64> = Vec::with_capacity(n);
let mut end_times: Vec<i64> = Vec::with_capacity(n);
let mut centroids: Vec<Coord> = Vec::with_capacity(n);
let mut counts: Vec<Option<f64>> = Vec::with_capacity(n);
let mut per_column: Vec<Vec<Option<f64>>> =
columns.iter().map(|_| Vec::with_capacity(n)).collect();
for (cell_id, agg) in &entries {
feature_ids.push(*cell_id);
start_times.push(agg.time_start);
end_times.push(agg.time_end);
counts.push(Some(agg.count as f64));
let centroid: Coord = match scheme {
SummaryScheme::H3 => match h3o::CellIndex::try_from(*cell_id) {
Ok(c) => {
let ll: LatLng = c.into();
[ll.lng(), ll.lat()]
}
Err(_) => [0.0, 0.0],
},
SummaryScheme::Quadbin => {
let (z, x, y) = quadbin::quadbin_to_tile(*cell_id);
quadbin::quadbin_centroid(z, x, y)
}
};
centroids.push(centroid);
for (i, col) in columns.iter().enumerate() {
let acc = agg.sources.get(&col.name);
let v: Option<f64> = match col.agg {
SummaryAggregation::Count => Some(agg.count as f64),
SummaryAggregation::Sum => acc.and_then(|a| {
if a.has_any { Some(a.sum) } else { None }
}),
SummaryAggregation::Mean => acc.and_then(|a| {
if a.sum_count > 0 {
Some(a.sum / a.sum_count as f64)
} else {
None
}
}),
SummaryAggregation::Min => acc.and_then(|a| {
if a.has_any { Some(a.min) } else { None }
}),
SummaryAggregation::Max => acc.and_then(|a| {
if a.has_any { Some(a.max) } else { None }
}),
};
per_column[i].push(v);
}
}
let mut properties: Vec<(String, PropertyColumn)> = Vec::new();
properties.push(("count".to_string(), PropertyColumn::Numeric(counts)));
let sub_bucket_n = entries
.iter()
.map(|(_, a)| a.sub_bucket_counts.len())
.max()
.unwrap_or(0);
if sub_bucket_n > 0 {
for sub_idx in 0..sub_bucket_n {
let mut col_vals: Vec<Option<f64>> = Vec::with_capacity(entries.len());
for (_, agg) in &entries {
let v = agg
.sub_bucket_counts
.get(sub_idx)
.copied()
.unwrap_or(0) as f64;
col_vals.push(Some(v));
}
properties.push((
format!("bucket_{}", sub_idx),
PropertyColumn::Numeric(col_vals),
));
}
}
for (i, col) in columns.iter().enumerate() {
if matches!(col.agg, SummaryAggregation::Count) {
continue;
}
let prop_name = output_column_name(col);
properties.push((prop_name, PropertyColumn::Numeric(per_column[i].clone())));
}
ColumnarLayer {
name: name.to_string(),
feature_ids,
start_times,
end_times,
geometry: GeometryColumn::Point(centroids),
vertex_times: None,
vertex_values: None,
properties,
triangles: None,
vertex_value_matrix: None,
}
}
pub fn output_column_name(col: &SummaryColumn) -> String {
let prefix = match col.agg {
SummaryAggregation::Count => "count",
SummaryAggregation::Sum => "sum",
SummaryAggregation::Mean => "mean",
SummaryAggregation::Min => "min",
SummaryAggregation::Max => "max",
};
if matches!(col.agg, SummaryAggregation::Count) {
"count".to_string()
} else {
format!("{prefix}_{}", col.name)
}
}
pub fn parse_summary_columns(spec: &str) -> Result<Vec<SummaryColumn>> {
let mut out = Vec::new();
for entry in spec.split(',') {
let entry = entry.trim();
if entry.is_empty() {
continue;
}
if entry.eq_ignore_ascii_case("count") {
out.push(SummaryColumn {
name: "_count".to_string(),
agg: SummaryAggregation::Count,
});
continue;
}
let (name, agg) = entry.split_once(':').ok_or_else(|| {
anyhow::anyhow!(
"summary column '{entry}' must be 'name:agg' (agg in sum|mean|min|max|count)"
)
})?;
let agg = match agg.trim().to_ascii_lowercase().as_str() {
"sum" => SummaryAggregation::Sum,
"mean" | "avg" | "average" => SummaryAggregation::Mean,
"min" => SummaryAggregation::Min,
"max" => SummaryAggregation::Max,
"count" => SummaryAggregation::Count,
other => anyhow::bail!(
"unknown summary aggregation '{other}' (use sum|mean|min|max|count)"
),
};
out.push(SummaryColumn {
name: name.trim().to_string(),
agg,
});
}
Ok(out)
}
#[cfg(test)]
mod tests {
use super::*;
use crate::tiler::GeneratedTile;
use geojson::{Feature, Geometry, Value as GeomValue};
fn point(lon: f64, lat: f64, ts: u64, mag: f64) -> ParsedFeature {
let props = serde_json::json!({ "magnitude": mag })
.as_object()
.cloned()
.map(std::sync::Arc::new);
ParsedFeature {
geojson: Feature {
bbox: None,
geometry: Some(Geometry::new(GeomValue::Point(vec![lon, lat]))),
id: None,
properties: None,
foreign_members: None,
},
shared_properties: props,
timestamp: ts,
end_timestamp: None,
vertex_timestamps: None,
vertex_values: None,
vertex_value_matrix: None,
lon,
lat,
}
}
struct CollectingWriter {
tiles: Vec<GeneratedTile>,
}
impl TileWriter for CollectingWriter {
fn write_tile(&mut self, tile: &GeneratedTile) -> anyhow::Result<()> {
self.tiles.push(GeneratedTile {
id: tile.id,
time_start: tile.time_start,
time_end: tile.time_end,
cover_t_min: tile.cover_t_min,
layers: tile.layers.clone(),
});
Ok(())
}
}
#[test]
fn h3_resolution_per_zoom_clamps() {
assert_eq!(h3_resolution_for_zoom(0), 0);
assert_eq!(h3_resolution_for_zoom(5), 5);
assert_eq!(h3_resolution_for_zoom(16), 15); }
#[test]
fn parses_column_spec() {
let cols = parse_summary_columns("magnitude:mean, magnitude:max, count").unwrap();
assert_eq!(cols.len(), 3);
assert_eq!(cols[0].name, "magnitude");
assert!(matches!(cols[0].agg, SummaryAggregation::Mean));
assert!(matches!(cols[2].agg, SummaryAggregation::Count));
}
#[test]
fn build_summary_tier_emits_aggregate_tiles() {
let mut features = Vec::new();
for i in 0..5 {
features.push(point(-122.45 + 0.001 * i as f64, 37.77, 1_000_000, 5.0 + i as f64));
}
features.push(point(-0.1278, 51.5074, 1_000_000, 6.0));
let config = SummaryConfig {
scheme: SummaryScheme::H3,
min_zoom: 0,
max_zoom: 2,
temporal_bucket_ms: 3_600_000,
columns: vec![
SummaryColumn {
name: "magnitude".into(),
agg: SummaryAggregation::Mean,
},
SummaryColumn {
name: "magnitude".into(),
agg: SummaryAggregation::Max,
},
SummaryColumn {
name: "_count".into(),
agg: SummaryAggregation::Count,
},
],
layer_name: "summary".into(),
sub_buckets: 1,
};
let mut writer = CollectingWriter { tiles: Vec::new() };
let n = build_summary_tier(&features, &config, &mut writer).unwrap();
assert!(n > 0, "expected at least one summary tile");
for tile in &writer.tiles {
assert_eq!(tile.layers.len(), 1);
let layer = &tile.layers[0];
assert_eq!(layer.name, "summary");
let prop_names: Vec<&str> =
layer.properties.iter().map(|(n, _)| n.as_str()).collect();
assert!(prop_names.contains(&"count"));
assert!(prop_names.contains(&"mean_magnitude"));
assert!(prop_names.contains(&"max_magnitude"));
}
let z0_total: usize = writer
.tiles
.iter()
.filter(|t| t.id.z == 0)
.map(|t| t.feature_count() as usize)
.sum();
assert!(z0_total <= 5, "expected ≤5 cells at zoom 0, got {z0_total}");
}
#[test]
fn build_summary_tier_quadbin_aggregates_into_one_cell() {
let n_cluster = 4u64;
let mut features = Vec::new();
for i in 0..n_cluster {
features.push(point(-122.4194 + 0.0001 * i as f64, 37.7749, 1_000_000, 5.0));
}
features.push(point(139.6917, 35.6895, 1_000_000, 9.0));
let config = SummaryConfig {
scheme: SummaryScheme::Quadbin,
min_zoom: 8,
max_zoom: 8,
temporal_bucket_ms: 3_600_000,
columns: vec![SummaryColumn {
name: "_count".into(),
agg: SummaryAggregation::Count,
}],
layer_name: "summary".into(),
sub_buckets: 1,
};
let mut writer = CollectingWriter { tiles: Vec::new() };
let n = build_summary_tier(&features, &config, &mut writer).unwrap();
assert!(n > 0, "expected at least one quadbin summary tile");
let mut saw_cluster_count = false;
let mut total_cells = 0usize;
for tile in &writer.tiles {
assert_eq!(tile.layers.len(), 1);
let layer = &tile.layers[0];
assert_eq!(layer.name, "summary");
total_cells += layer.feature_ids.len();
for &id in &layer.feature_ids {
assert_eq!(id >> 60, 0b100, "id not a Quadbin header: {id:#018x}");
assert_eq!((id >> 59) & 1, 1, "Quadbin mode bit unset: {id:#018x}");
assert_eq!(
((id >> 52) & 0x1f) as u8,
8 + QUADBIN_RES_OFFSET,
"wrong zoom field: {id:#018x}"
);
}
let counts = layer
.properties
.iter()
.find(|(n, _)| n == "count")
.map(|(_, c)| c)
.expect("count column present");
if let PropertyColumn::Numeric(vals) = counts {
if vals
.iter()
.any(|v| matches!(v, Some(c) if (*c - n_cluster as f64).abs() < f64::EPSILON))
{
saw_cluster_count = true;
}
}
}
assert!(
saw_cluster_count,
"expected a cell with count == {n_cluster}"
);
assert_eq!(total_cells, 2, "expected exactly two quadbin cells");
}
#[test]
fn build_summary_tier_emits_per_sub_bucket_columns_when_configured() {
let mut features = Vec::new();
let bucket_ms: u64 = 60 * 60 * 1000; let sub_buckets: u32 = 6; let sub_ms = bucket_ms / sub_buckets as u64;
for i in 0..sub_buckets {
features.push(point(
-122.45,
37.77,
1_000_000_000_000 + (i as u64 * sub_ms),
5.0,
));
}
let config = SummaryConfig {
scheme: SummaryScheme::H3,
min_zoom: 5,
max_zoom: 5,
temporal_bucket_ms: bucket_ms,
columns: vec![],
layer_name: "summary".into(),
sub_buckets,
};
let mut writer = CollectingWriter { tiles: Vec::new() };
let n = build_summary_tier(&features, &config, &mut writer).unwrap();
assert!(n > 0);
for tile in &writer.tiles {
let names: Vec<&str> = tile.layers[0]
.properties
.iter()
.map(|(n, _)| n.as_str())
.collect();
for i in 0..sub_buckets {
assert!(
names.contains(&format!("bucket_{}", i).as_str()),
"tile missing bucket_{}: {names:?}",
i
);
}
}
}
#[test]
fn to_tier_emits_one_resolution_per_zoom() {
let config = SummaryConfig {
scheme: SummaryScheme::H3,
min_zoom: 0,
max_zoom: 4,
temporal_bucket_ms: 3_600_000,
columns: vec![],
layer_name: "summary".into(),
sub_buckets: 1,
};
let tier = config.to_tier();
assert_eq!(tier.cell_resolution_per_zoom, vec![0, 1, 2, 3, 4]);
assert_eq!(tier.resolution_for_zoom(3), 3);
}
}