use super::*;
use source::ByteSource;
enum Inner {
Pm(Box<pmtiles::Reader>),
Mb(mbtiles::MbTiles),
}
pub struct TileArchive {
inner: Inner,
_local: Option<crate::vfs::LocalPath>,
}
impl std::fmt::Debug for TileArchive {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.debug_struct("TileArchive")
.field("kind", &self.kind())
.field("format", &self.format())
.field("zooms", &self.zoom_range())
.finish_non_exhaustive()
}
}
impl TileArchive {
pub fn open(path: &str) -> Result<Self> {
let kind = archive_kind(&crate::vfs::Location::parse(path).file_name()).or_else(|| archive_kind(path));
match kind {
Some(ArchiveKind::MBTiles) => {
let local = crate::vfs::resolve(path)?;
let bytes = std::fs::read(local.path())?;
Ok(Self { inner: Inner::Mb(mbtiles::MbTiles::from_bytes(bytes)?), _local: None })
}
_ => {
let (mut source, local) = ByteSource::open(path)?;
let head = source.read_at(0, source.len().min(16))?;
if head.starts_with(b"SQLite format 3\0") {
drop(source);
let local = crate::vfs::resolve(path)?;
return Ok(Self { inner: Inner::Mb(mbtiles::MbTiles::from_bytes(std::fs::read(local.path())?)?), _local: None });
}
Ok(Self { inner: Inner::Pm(Box::new(pmtiles::Reader::open(source)?)), _local: local })
}
}
}
pub fn from_bytes(bytes: Vec<u8>) -> Result<Self> {
if bytes.starts_with(b"SQLite format 3\0") {
return Ok(Self { inner: Inner::Mb(mbtiles::MbTiles::from_bytes(bytes)?), _local: None });
}
Ok(Self { inner: Inner::Pm(Box::new(pmtiles::Reader::open(ByteSource::Memory(bytes))?)), _local: None })
}
pub fn from_sparse(file: crate::sparse::SparseFile) -> Result<Self> {
Ok(Self { inner: Inner::Pm(Box::new(pmtiles::Reader::open(ByteSource::Sparse(file))?)), _local: None })
}
pub fn into_dataset(self, name: &str, options: &crate::VectorOpenOptions) -> Result<crate::Dataset> {
archive_dataset(self, name, options)
}
pub fn kind(&self) -> ArchiveKind {
match self.inner {
Inner::Pm(_) => ArchiveKind::PMTiles,
Inner::Mb(_) => ArchiveKind::MBTiles,
}
}
pub fn is_remote(&self) -> bool {
matches!(&self.inner, Inner::Pm(r) if r.is_remote())
}
pub(crate) fn is_sparse(&self) -> bool {
matches!(&self.inner, Inner::Pm(r) if r.is_sparse())
}
pub fn format(&self) -> TileFormat {
match &self.inner {
Inner::Pm(r) => match r.header.tile_type {
pmtiles::TILE_MVT => TileFormat::Mvt,
pmtiles::TILE_PNG => TileFormat::Png,
pmtiles::TILE_JPEG => TileFormat::Jpeg,
pmtiles::TILE_WEBP => TileFormat::Webp,
pmtiles::TILE_AVIF => TileFormat::Avif,
_ => TileFormat::Unknown,
},
Inner::Mb(m) => match m.item("format").map(TileFormat::from_mbtiles) {
Some(f) if f != TileFormat::Unknown => f,
_ => m.zoom_range().and_then(|(_, z)| m.any_tile(z)).map(TileFormat::sniff).unwrap_or(TileFormat::Unknown),
},
}
}
pub fn is_vector(&self) -> bool {
self.format() == TileFormat::Mvt
}
pub fn zoom_range(&self) -> (u8, u8) {
match &self.inner {
Inner::Pm(r) => (r.header.min_zoom.min(30), r.header.max_zoom.min(30)),
Inner::Mb(m) => m.zoom_range().unwrap_or((0, 0)),
}
}
pub fn bounds(&self) -> Option<[f64; 4]> {
match &self.inner {
Inner::Pm(r) => Some(r.header.bounds()),
Inner::Mb(m) => {
let v: Vec<f64> = m.item("bounds")?.split(',').filter_map(|s| s.trim().parse().ok()).collect();
let ok = v.len() == 4
&& v[0].abs() <= 180.0
&& v[2].abs() <= 180.0
&& v[1].abs() < 89.99
&& v[3].abs() < 89.99
&& v[0] <= v[2]
&& v[1] <= v[3];
ok.then(|| [v[0], v[1], v[2], v[3]])
}
}
}
pub fn metadata(&self) -> Vec<(String, String)> {
match &self.inner {
Inner::Mb(m) => m.metadata.clone(),
Inner::Pm(r) => match &r.metadata {
serde_json::Value::Object(o) => o
.iter()
.map(|(k, v)| {
let s = match v {
serde_json::Value::String(s) => s.clone(),
v => v.to_string(),
};
(k.clone(), s)
})
.collect(),
_ => Vec::new(),
},
}
}
pub(crate) fn json(&self) -> serde_json::Value {
match &self.inner {
Inner::Pm(r) => {
let mut v = r.metadata.clone();
if v.get("vector_layers").is_none()
&& let Some(inner) = v.get("json").and_then(|j| j.as_str()).and_then(|s| serde_json::from_str(s).ok())
{
v = inner;
}
v
}
Inner::Mb(m) => m.item("json").and_then(|s| serde_json::from_str(s).ok()).unwrap_or(serde_json::Value::Null),
}
}
pub fn tile(&mut self, z: u8, x: u32, y: u32) -> Result<Option<Vec<u8>>> {
match &mut self.inner {
Inner::Pm(r) => r.tile(z, x, y),
Inner::Mb(m) => Ok(m.tile(z, x, y).map(|b| b.to_vec())),
}
}
pub(crate) fn tiles(&mut self, wanted: &[(u8, u32, u32)]) -> Result<Vec<Option<Vec<u8>>>> {
match &mut self.inner {
Inner::Pm(r) => r.tiles(wanted),
Inner::Mb(m) => Ok(wanted.iter().map(|&(z, x, y)| m.tile(z, x, y).map(|b| b.to_vec())).collect()),
}
}
pub fn tiles_at_zoom(&mut self, z: u8) -> Result<Vec<(u32, u32)>> {
match &mut self.inner {
Inner::Pm(r) => r.tiles_at_zoom(z),
Inner::Mb(m) => Ok(m.tiles_at_zoom(z)),
}
}
pub fn driver_name(&self) -> &'static str {
match self.kind() {
ArchiveKind::PMTiles => "PMTiles",
ArchiveKind::MBTiles => "MBTiles",
}
}
}
pub fn is_archive(path: &str) -> bool {
let loc = crate::vfs::Location::parse(path);
!matches!(loc, crate::vfs::Location::Zip { .. }) && archive_kind(&loc.file_name()).is_some()
}
pub fn open_dataset(path: &str, options: &crate::VectorOpenOptions) -> Result<crate::Dataset> {
let archive = TileArchive::open(path)?;
let name = crate::vfs::Location::parse(path).file_name();
archive_dataset(archive, &name, options)
}
pub(crate) fn archive_dataset(mut archive: TileArchive, name: &str, options: &crate::VectorOpenOptions) -> Result<crate::Dataset> {
if archive.is_vector() {
let layers = vector::read_layers(&mut archive, &options.tiles, &options.layers, options.spatial_filter)?;
let driver = match archive.kind() {
ArchiveKind::PMTiles => crate::formats::VectorDriver::PMTiles,
ArchiveKind::MBTiles => crate::formats::VectorDriver::MBTiles,
};
let names: Vec<String> = layers.iter().map(|l| l.name.clone()).collect();
let mut ds = crate::dataset::finish_vector(layers, name, driver);
for (l, n) in ds.layers.iter_mut().zip(names) {
l.name = n;
}
return Ok(crate::Dataset::Vector(ds));
}
let mut r = raster::TileRaster::new(archive, options.tiles.zoom_level)?;
Ok(crate::Dataset::Raster(r.read_all()?))
}
pub(crate) fn open_dataset_bytes(bytes: Vec<u8>, name: &str, options: &crate::VectorOpenOptions) -> Result<crate::Dataset> {
archive_dataset(TileArchive::from_bytes(bytes)?, name, options)
}
pub(crate) fn write_raster_archive(rt: &raster::RasterTiles, o: &TileWriteOptions, name: &str, kind: ArchiveKind) -> Result<Vec<u8>> {
write_raster_archive_with(rt, o, name, kind, &[])
}
fn write_raster_archive_with(
rt: &raster::RasterTiles,
o: &TileWriteOptions,
name: &str,
kind: ArchiveKind,
kept: &[(String, String)],
) -> Result<Vec<u8>> {
let b = rt.bounds;
let name = o.name.clone().unwrap_or_else(|| name.to_string());
let format = rt.format.name();
let mut md: Vec<(String, String)> = vec![
("name".into(), name.clone()),
("type".into(), o.tileset_type.clone()),
("description".into(), o.description.clone().unwrap_or(name)),
("version".into(), "1.1".into()),
("format".into(), format.into()),
("bounds".into(), format!("{},{},{},{}", g17(b[0]), g17(b[1]), g17(b[2]), g17(b[3]))),
("minzoom".into(), rt.min_zoom.to_string()),
("maxzoom".into(), rt.max_zoom.to_string()),
];
keep_items(&mut md, kept);
match kind {
ArchiveKind::MBTiles => mbtiles::write(&md, rt.tiles.clone()),
ArchiveKind::PMTiles => {
let json: serde_json::Map<String, serde_json::Value> = md.into_iter().map(|(k, v)| (k, serde_json::Value::String(v))).collect();
let tile_type = match rt.format {
TileFormat::Jpeg => pmtiles::TILE_JPEG,
TileFormat::Webp => pmtiles::TILE_WEBP,
_ => pmtiles::TILE_PNG,
};
let tiles = rt.tiles.iter().map(|(z, x, y, d)| pmtiles::OutTile { z: *z, x: *x, y: *y, data: d.clone() }).collect();
let params = pmtiles::ArchiveParams {
tile_type,
tile_compression: pmtiles::COMPRESSION_NONE,
min_zoom: rt.min_zoom,
max_zoom: rt.max_zoom,
bounds: b,
center: ((b[0] + b[2]) / 2.0, (b[1] + b[3]) / 2.0, rt.max_zoom),
};
pmtiles::write_archive(tiles, serde_json::Value::Object(json).to_string().as_bytes(), ¶ms)
}
}
}
fn g17(v: f64) -> String {
format!("{v}")
}
pub(crate) fn write_vector_archive(vt: &vector::VectorTiles, o: &TileWriteOptions, name: &str, kind: ArchiveKind) -> Result<Vec<u8>> {
write_vector_archive_with(vt, o, name, kind, &[])
}
fn write_vector_archive_with(
vt: &vector::VectorTiles,
o: &TileWriteOptions,
name: &str,
kind: ArchiveKind,
kept: &[(String, String)],
) -> Result<Vec<u8>> {
let mut md = vector::metadata_items(vt, o, name);
keep_items(&mut md, kept);
let gz = |d: &[u8]| pmtiles::compress(d, pmtiles::COMPRESSION_GZIP);
match kind {
ArchiveKind::MBTiles => {
md.push(("scheme".into(), "tms".into()));
md.push(("json".into(), serde_json::to_string_pretty(&vt.json).unwrap_or_default()));
let mut tiles = Vec::with_capacity(vt.tiles.len());
for (z, x, y, d) in &vt.tiles {
tiles.push((*z, *x, *y, gz(d)?));
}
mbtiles::write(&md, tiles)
}
ArchiveKind::PMTiles => {
let mut json: serde_json::Map<String, serde_json::Value> =
md.into_iter().map(|(k, v)| (k, serde_json::Value::String(v))).collect();
if let serde_json::Value::Object(o) = &vt.json {
for (k, v) in o {
json.insert(k.clone(), v.clone());
}
}
let mut tiles = Vec::with_capacity(vt.tiles.len());
for (z, x, y, d) in &vt.tiles {
tiles.push(pmtiles::OutTile { z: *z, x: *x, y: *y, data: gz(d)? });
}
let b = vt.bounds;
let params = pmtiles::ArchiveParams {
tile_type: pmtiles::TILE_MVT,
tile_compression: pmtiles::COMPRESSION_GZIP,
min_zoom: vt.min_zoom,
max_zoom: vt.max_zoom,
bounds: b,
center: ((b[0] + b[2]) / 2.0, (b[1] + b[3]) / 2.0, vt.min_zoom),
};
pmtiles::write_archive(tiles, serde_json::Value::Object(json).to_string().as_bytes(), ¶ms)
}
}
}
fn keep_items(md: &mut Vec<(String, String)>, kept: &[(String, String)]) {
const DERIVED: [&str; 4] = ["json", "scheme", "vector_layers", "tilestats"];
for (k, v) in kept {
if !DERIVED.contains(&k.as_str()) && !md.iter().any(|(m, _)| m == k) {
md.push((k.clone(), v.clone()));
}
}
}
fn options_of(archive: &TileArchive, o: &TileWriteOptions) -> (TileWriteOptions, Vec<(String, String)>) {
let items = archive.metadata();
let item = |k: &str| items.iter().find(|(n, _)| n == k).map(|(_, v)| v.clone());
let mut o = o.clone();
o.name = o.name.or_else(|| item("name"));
o.description = o.description.or_else(|| item("description"));
if o.tileset_type == TileWriteOptions::default().tileset_type
&& let Some(t) = item("type")
{
o.tileset_type = t;
}
(o, items)
}
pub fn update_raster_bytes(bytes: Vec<u8>, ds: &crate::RasterDataset, o: &TileWriteOptions) -> Result<Vec<u8>> {
let archive = TileArchive::from_bytes(bytes)?;
let kind = archive.kind();
let (o, items) = options_of(&archive, o);
let rt = raster::update_tiles(archive, ds, &o)?;
write_raster_archive_with(&rt, &o, "tiles", kind, &items)
}
pub fn update_vector_bytes(bytes: Vec<u8>, layers: &[wbvector::Layer], o: &TileWriteOptions, overwrite: bool) -> Result<Vec<u8>> {
let mut archive = TileArchive::from_bytes(bytes)?;
let kind = archive.kind();
let (o, items) = options_of(&archive, o);
let vt = vector::update_tiles(&mut archive, layers, &o, overwrite)?;
write_vector_archive_with(&vt, &o, "tiles", kind, &items)
}
#[cfg(not(target_arch = "wasm32"))]
fn replace_file(path: &str, bytes: &[u8]) -> Result<()> {
let tmp = format!("{path}.terrio-update");
std::fs::write(&tmp, bytes)?;
std::fs::rename(&tmp, path).inspect_err(|_| {
let _ = std::fs::remove_file(&tmp);
})?;
Ok(())
}
#[cfg(not(target_arch = "wasm32"))]
pub fn update_raster(path: &str, ds: &crate::RasterDataset, o: &TileWriteOptions) -> Result<()> {
let bytes = update_raster_bytes(std::fs::read(path)?, ds, o)?;
replace_file(path, &bytes)
}
#[cfg(not(target_arch = "wasm32"))]
pub fn update_vector(path: &str, layers: &[wbvector::Layer], o: &TileWriteOptions, overwrite: bool) -> Result<()> {
let bytes = update_vector_bytes(std::fs::read(path)?, layers, o, overwrite)?;
replace_file(path, &bytes)
}
pub(crate) fn raster_to_bytes(ds: &crate::RasterDataset, o: &TileWriteOptions, name: &str, kind: ArchiveKind) -> Result<Vec<u8>> {
let rt = raster::make_tiles(ds, o)?;
write_raster_archive(&rt, o, name, kind)
}
pub(crate) fn vector_to_bytes(layers: &[wbvector::Layer], o: &TileWriteOptions, name: &str, kind: ArchiveKind) -> Result<Vec<u8>> {
let vt = vector::make_tiles(layers, o)?;
write_vector_archive(&vt, o, name, kind)
}
#[cfg(not(target_arch = "wasm32"))]
pub(crate) fn write_mvt_dir(layers: &[wbvector::Layer], o: &TileWriteOptions, path: &std::path::Path) -> Result<()> {
let vt = vector::make_tiles(layers, o)?;
let name = path.file_name().and_then(|s| s.to_str()).unwrap_or("tiles");
if path.exists() {
return Err(Error::InvalidArgument(format!("'{}' already exists", path.display())));
}
std::fs::create_dir_all(path)?;
let mut md: serde_json::Map<String, serde_json::Value> =
vector::metadata_items(&vt, o, name).into_iter().map(|(k, v)| (k, serde_json::Value::String(v))).collect();
md.insert("json".into(), serde_json::Value::String(serde_json::to_string_pretty(&vt.json).unwrap_or_default()));
std::fs::write(path.join("metadata.json"), serde_json::to_string_pretty(&serde_json::Value::Object(md)).unwrap_or_default())?;
for (z, x, y, d) in &vt.tiles {
let dir = path.join(z.to_string()).join(x.to_string());
std::fs::create_dir_all(&dir)?;
std::fs::write(dir.join(format!("{y}.pbf")), pmtiles::compress(d, pmtiles::COMPRESSION_GZIP)?)?;
}
Ok(())
}
#[cfg(not(target_arch = "wasm32"))]
pub(crate) fn read_mvt_dir(path: &std::path::Path, options: &crate::VectorOpenOptions) -> Result<Vec<wbvector::Layer>> {
let text = std::fs::read_to_string(path.join("metadata.json"))?;
let mut md: serde_json::Value = serde_json::from_str(&text).map_err(|e| Error::InvalidData(format!("metadata.json: {e}")))?;
let zooms: Vec<u8> = std::fs::read_dir(path)?
.filter_map(|e| e.ok())
.filter(|e| e.path().is_dir())
.filter_map(|e| e.file_name().to_str().and_then(|s| s.parse().ok()))
.collect();
let (min_z, max_z) = (zooms.iter().copied().min().unwrap_or(0), zooms.iter().copied().max().unwrap_or(0));
let mut tiles = Vec::new();
let z = options.tiles.zoom_level.unwrap_or(max_z);
let zdir = path.join(z.to_string());
if zdir.is_dir() {
for xe in std::fs::read_dir(&zdir)?.filter_map(|e| e.ok()) {
let Some(x) = xe.file_name().to_str().and_then(|s| s.parse::<u32>().ok()) else { continue };
for ye in std::fs::read_dir(xe.path())?.filter_map(|e| e.ok()) {
let name = ye.file_name();
let Some(y) = name.to_str().and_then(|s| s.split('.').next()).and_then(|s| s.parse::<u32>().ok()) else { continue };
tiles.push((z, x, y, std::fs::read(ye.path())?));
}
}
}
let mut items: Vec<(String, String)> = Vec::new();
if let serde_json::Value::Object(o) = &mut md {
for (k, v) in o.iter() {
items.push((k.clone(), v.as_str().map(str::to_string).unwrap_or_else(|| v.to_string())));
}
}
items.retain(|(k, _)| k != "minzoom" && k != "maxzoom");
items.push(("minzoom".into(), min_z.to_string()));
items.push(("maxzoom".into(), max_z.to_string()));
let mut archive = TileArchive::from_bytes(mbtiles::write(&items, tiles)?)?;
vector::read_layers(&mut archive, &options.tiles, &options.layers, options.spatial_filter)
}
pub(crate) fn raster_write_options(options: &crate::RasterWriteOptions) -> Result<TileWriteOptions> {
let mut o = TileWriteOptions::default();
if let Some(c) = &options.compression {
o.tile_format = Some(c.to_ascii_uppercase());
}
for (k, v) in &options.creation_options {
if !o.set(k, v)? {
return Err(Error::InvalidArgument(format!("unknown creation option '{k}' for tile archives")));
}
}
Ok(o)
}
fn stem(path: &str) -> String {
std::path::Path::new(path).file_stem().and_then(|s| s.to_str()).unwrap_or("tiles").to_string()
}
pub(crate) fn write_raster(
ds: &crate::RasterDataset,
path: &str,
driver: crate::formats::RasterDriver,
options: &crate::RasterWriteOptions,
) -> Result<()> {
let kind = if driver == crate::formats::RasterDriver::MBTiles { ArchiveKind::MBTiles } else { ArchiveKind::PMTiles };
let bytes = raster_to_bytes(ds, &raster_write_options(options)?, &stem(path), kind)?;
std::fs::write(path, bytes)?;
Ok(())
}
pub(crate) fn write_vector(
layers: &[wbvector::Layer],
path: &str,
driver: crate::formats::VectorDriver,
o: &TileWriteOptions,
) -> Result<()> {
use crate::formats::VectorDriver as V;
let kind = match driver {
V::MBTiles => ArchiveKind::MBTiles,
V::PMTiles => ArchiveKind::PMTiles,
_ => {
#[cfg(not(target_arch = "wasm32"))]
return write_mvt_dir(layers, o, std::path::Path::new(path));
#[cfg(target_arch = "wasm32")]
return Err(Error::Unsupported("directories of MVT tiles cannot be written in WebAssembly builds".into()));
}
};
std::fs::write(path, vector_to_bytes(layers, o, &stem(path), kind)?)?;
Ok(())
}
pub(crate) fn is_mvt(b: &[u8]) -> bool {
b.starts_with(&[0x1f, 0x8b]) || b.first() == Some(&0x1a)
}
pub(crate) fn open_mvt_local(path: &str, options: &crate::VectorOpenOptions) -> Option<Result<Vec<wbvector::Layer>>> {
let p = std::path::Path::new(path);
#[cfg(not(target_arch = "wasm32"))]
if p.is_dir() {
if p.join("metadata.json").is_file() {
return Some(read_mvt_dir(p, options));
}
let zoom = p.file_name().and_then(|n| n.to_str()).and_then(|n| n.parse::<u8>().ok());
if let (Some(z), Some(parent)) = (zoom, p.parent())
&& parent.join("metadata.json").is_file()
{
let mut o = options.clone();
o.tiles.zoom_level = Some(z);
return Some(read_mvt_dir(parent, &o));
}
}
let lower = path.to_ascii_lowercase();
if !(lower.ends_with(".mvt") || lower.ends_with(".mvt.gz") || (lower.ends_with(".pbf") && !lower.ends_with(".osm.pbf"))) || !p.is_file()
{
return None;
}
let bytes = match std::fs::read(p) {
Ok(b) => b,
Err(e) => return Some(Err(e.into())),
};
if !is_mvt(&bytes) {
return None;
}
Some(vector::read_tile_file(&bytes, path, &options.tiles))
}
pub(crate) fn layer_names(path: &str) -> Result<Vec<String>> {
let a = TileArchive::open(path)?;
let names: Vec<String> = a
.json()
.get("vector_layers")
.and_then(|v| v.as_array())
.map(|v| v.iter().filter_map(|l| l.get("id").and_then(|i| i.as_str()).map(str::to_string)).collect())
.unwrap_or_default();
if names.is_empty() {
return match archive_dataset(a, path, &Default::default())? {
crate::Dataset::Vector(v) => Ok(v.layers.into_iter().map(|l| l.name).collect()),
crate::Dataset::Raster(_) => Ok(Vec::new()),
};
}
Ok(names)
}