use std::collections::HashSet;
use std::ffi::{CStr, c_double, c_int};
use std::fs;
use std::sync::mpsc::{Receiver, SyncSender, sync_channel};
use std::sync::{Arc, Mutex};
use std::thread::JoinHandle;
use rusqlite::Connection;
use tippecanoe::errors::{EXIT_CLOSE, EXIT_EXISTS, EXIT_IMPOSSIBLE, EXIT_OPEN, EXIT_SQLITE};
use tippecanoe::mbtiles::{Metadata, c_str, sqlite3_errmsg, sqlite3_exec, sqlite3_open};
use tippecanoe::text::fnv1a;
use tippecanoe::write_json::JsonWriter;
use super::{SinkSettings, TileSink, check_tile, tileset_metadata};
use crate::{Error, Result};
const BATCH_TILES: usize = 100_000;
const BATCH_BYTES: usize = 256 << 20;
macro_rules! mprintf {
($fmt:literal $(, $arg:expr)* $(,)?) => {
unsafe {
tippecanoe::mbtiles::sqlite3_take(rusqlite::ffi::sqlite3_mprintf(
($fmt as &CStr).as_ptr() $(, $arg)*
))
}
};
}
type Extra = Vec<(String, String)>;
enum Message {
Tile {
z: u8,
x: u32,
y: u32,
hash: u64,
data: Vec<u8>,
},
Erase {
z: u8,
reply: SyncSender<Result<()>>,
},
Finish {
metadata: Box<Metadata>,
extra: Extra,
},
}
pub struct MbtilesSink {
sender: Option<SyncSender<Message>>,
writer: Option<JoinHandle<Result<()>>>,
here: Option<Mutex<Writer>>,
failure: Arc<Mutex<Option<(i32, String)>>>,
tile_format: crate::mlt::TileFormat,
}
fn sqlite_error(message: String) -> Error {
Error::new(EXIT_SQLITE, message)
}
impl MbtilesSink {
pub fn create(path: &str, settings: &SinkSettings) -> Result<MbtilesSink> {
if settings.force {
let _ = fs::remove_file(path);
}
let db = mbtiles_open(path, settings)?;
let failure = Arc::new(Mutex::new(None));
let tile_format = settings.tile_format;
let allow_existing = settings.allow_existing;
if cfg!(target_family = "wasm") {
return Ok(MbtilesSink {
sender: None,
writer: None,
here: Some(Mutex::new(Writer::new(db, allow_existing)?)),
failure,
tile_format,
});
}
let (sender, receiver) = sync_channel(4096);
let writer_failure = failure.clone();
let writer = std::thread::Builder::new()
.name("mbtiles writer".to_string())
.spawn(move || {
let result = write_tiles(db, receiver, allow_existing);
if let Err(e) = &result {
*writer_failure.lock().unwrap_or_else(|p| p.into_inner()) =
Some((e.exit_code, e.message.clone()));
}
result
})
.map_err(|e| Error::new(EXIT_IMPOSSIBLE, format!("mbtiles writer thread: {}", e)))?;
Ok(MbtilesSink {
sender: Some(sender),
writer: Some(writer),
here: None,
failure,
tile_format,
})
}
fn send(&self, message: Message) -> Result<()> {
if let Some(here) = &self.here {
if let Some((code, message)) = &*self.failure.lock().unwrap_or_else(|p| p.into_inner())
{
return Err(Error::new(*code, message.clone()));
}
let taken = here.lock().unwrap_or_else(|p| p.into_inner()).take(message);
if let Err(e) = &taken {
*self.failure.lock().unwrap_or_else(|p| p.into_inner()) =
Some((e.exit_code, e.message.clone()));
}
return taken.map(|_| ());
}
let sent = self.sender.as_ref().map(|s| s.send(message));
match sent {
Some(Ok(())) => Ok(()),
_ => Err(self.writer_error()),
}
}
fn writer_error(&self) -> Error {
match &*self.failure.lock().unwrap_or_else(|p| p.into_inner()) {
Some((code, message)) => Error::new(*code, message.clone()),
None => Error::new(EXIT_IMPOSSIBLE, "mbtiles writer stopped"),
}
}
fn close_here(&mut self) -> Result<()> {
match self.here.take() {
Some(here)
if self
.failure
.lock()
.unwrap_or_else(|p| p.into_inner())
.is_none() =>
{
here.into_inner().unwrap_or_else(|p| p.into_inner()).close()
}
_ => Ok(()),
}
}
}
impl Drop for MbtilesSink {
fn drop(&mut self) {
self.sender.take();
if let Some(writer) = self.writer.take() {
let _ = writer.join();
}
let _ = self.close_here();
}
}
impl TileSink for MbtilesSink {
fn write_tile(&self, z: u8, x: u32, y: u32, data: &[u8]) -> Result<()> {
check_tile(z, x, y)?;
self.send(Message::Tile {
z,
x,
y,
hash: fnv1a(data),
data: data.to_vec(),
})
}
fn erase_zoom(&self, z: u8) -> Result<()> {
let (reply, answer) = sync_channel(1);
self.send(Message::Erase { z, reply })?;
answer.recv().unwrap_or_else(|_| Err(self.writer_error()))
}
fn finish(self: Box<Self>, metadata: &Metadata) -> Result<()> {
self.finish_with(metadata, &[])
}
fn finish_with(
mut self: Box<Self>,
metadata: &Metadata,
extra: &[(String, String)],
) -> Result<()> {
let sent = self.send(Message::Finish {
metadata: Box::new(tileset_metadata(metadata, self.tile_format).into_owned()),
extra: extra.to_vec(),
});
self.sender.take();
let joined = match self.writer.take() {
Some(writer) => writer
.join()
.unwrap_or_else(|_| Err(Error::new(EXIT_IMPOSSIBLE, "mbtiles writer panicked"))),
None => self.close_here(),
};
sent.and(joined)
}
}
pub fn mbtiles_open(path: &str, settings: &SinkSettings) -> Result<Connection> {
let program = &settings.program;
let db = sqlite3_open(path)
.map_err(|msg| Error::new(EXIT_OPEN, format!("{}: {}: {}", program, path, msg)))?;
for pragma in [
c"PRAGMA synchronous=0",
c"PRAGMA locking_mode=EXCLUSIVE",
c"PRAGMA journal_mode=DELETE",
] {
sqlite3_exec(&db, pragma)
.map_err(|err| sqlite_error(format!("{}: async: {}", program, err)))?;
}
let creates: [(&CStr, &str); 7] = [
(c"CREATE TABLE metadata (name text, value text);", ""),
(c"create unique index name on metadata (name);", "index metadata"),
(
c"CREATE TABLE map (zoom_level INTEGER, tile_column INTEGER, tile_row INTEGER, tile_id TEXT);",
"create map table",
),
(
c"CREATE UNIQUE INDEX map_index ON map (zoom_level, tile_column, tile_row);",
"create map index",
),
(
c"CREATE TABLE images (zoom_level integer, tile_data blob, tile_id text);",
"create images table",
),
(
c"CREATE UNIQUE INDEX images_id ON images (zoom_level, tile_id);",
"create images index",
),
(
c"CREATE VIEW tiles AS SELECT map.zoom_level AS zoom_level, map.tile_column AS tile_column, map.tile_row AS tile_row, images.tile_data AS tile_data FROM map JOIN images ON images.tile_id = map.tile_id and images.zoom_level = map.zoom_level;",
"create tiles view",
),
];
for (sql, what) in creates {
if let Err(err) = sqlite3_exec(&db, sql) {
let message = if what.is_empty() {
format!(
"{}: Tileset \"{}\" already exists. You can use --force if you want to delete the old tileset.\n{}: {}",
program, path, program, err
)
} else {
format!("{}: {}: {}", program, what, err)
};
if !settings.allow_existing {
return Err(Error::new(EXIT_EXISTS, message));
}
eprintln!("{}", message);
}
}
sqlite3_exec(&db, c"PRAGMA cache_size=-262144")
.map_err(|err| sqlite_error(format!("cache_size: {}", err)))?;
Ok(db)
}
fn write_tiles(db: Connection, receiver: Receiver<Message>, allow_existing: bool) -> Result<()> {
let mut writer = Writer::new(db, allow_existing)?;
for message in receiver.iter() {
if !writer.take(message)? {
break;
}
}
writer.close()
}
struct Writer {
db: Connection,
allow_existing: bool,
stored: HashSet<(u8, u64)>,
tiles_in_batch: usize,
bytes_in_batch: usize,
finish: Option<(Box<Metadata>, Extra)>,
}
impl Writer {
fn new(db: Connection, allow_existing: bool) -> Result<Writer> {
let writer = Writer {
db,
allow_existing,
stored: HashSet::new(),
tiles_in_batch: 0,
bytes_in_batch: 0,
finish: None,
};
for (sql, what) in [
(INSERT_IMAGE, "images"),
(INSERT_MAP, "map"),
(DELETE_MAP, "delete map"),
(DELETE_IMAGES, "delete images"),
] {
writer.prepare(sql, what)?;
}
writer.begin()?;
Ok(writer)
}
fn prepare(&self, sql: &str, what: &str) -> Result<rusqlite::CachedStatement<'_>> {
self.db
.prepare_cached(sql)
.map_err(|_| sqlite_error(format!("sqlite3 {} prep failed", what)))
}
fn begin(&self) -> Result<()> {
sqlite3_exec(&self.db, c"BEGIN").map_err(|e| sqlite_error(format!("BEGIN: {}", e)))
}
fn commit(&self) -> Result<()> {
sqlite3_exec(&self.db, c"COMMIT").map_err(|e| sqlite_error(format!("COMMIT: {}", e)))
}
fn failed(&self, what: &str) -> Error {
sqlite_error(format!(
"sqlite3 {} failed: {}",
what,
sqlite3_errmsg(&self.db)
))
}
fn take(&mut self, message: Message) -> Result<bool> {
match message {
Message::Tile {
z,
x,
y,
hash,
data,
} => {
let hash_text = hash.to_string();
if self.stored.insert((z, hash)) {
self.prepare(INSERT_IMAGE, "images")?
.execute(rusqlite::params![z as i32, hash_text, data])
.map_err(|_| self.failed("images insert"))?;
}
let tile_row = ((1i64 << z) - 1 - y as i64) as i32;
self.prepare(INSERT_MAP, "map")?
.execute(rusqlite::params![z as i32, x as i32, tile_row, hash_text])
.map_err(|_| self.failed("map insert"))?;
self.tiles_in_batch += 1;
self.bytes_in_batch += data.len();
if self.tiles_in_batch >= BATCH_TILES || self.bytes_in_batch >= BATCH_BYTES {
self.commit()?;
self.begin()?;
self.tiles_in_batch = 0;
self.bytes_in_batch = 0;
}
Ok(true)
}
Message::Erase { z, reply } => {
let erased = self
.prepare(DELETE_MAP, "delete map")
.and_then(|mut s| s.execute([z as i32]).map_err(|_| self.failed("delete map")))
.and_then(|_| {
self.prepare(DELETE_IMAGES, "delete images")
.and_then(|mut s| {
s.execute([z as i32])
.map_err(|_| self.failed("delete images"))
})
})
.map(|_| ());
self.stored.retain(|&(zoom, _)| zoom != z);
let failed = erased
.as_ref()
.err()
.map(|e| Error::new(e.exit_code, e.message.clone()));
let _ = reply.send(erased);
match failed {
Some(e) => Err(e),
None => Ok(true),
}
}
Message::Finish { metadata, extra } => {
self.finish = Some((metadata, extra));
Ok(false)
}
}
}
fn close(self) -> Result<()> {
self.commit()?;
let Writer {
db,
allow_existing,
finish,
..
} = self;
db.flush_prepared_statement_cache();
if let Some((m, extra)) = finish {
mbtiles_write_metadata(&db, &m, allow_existing)?;
for (k, v) in extra {
db.execute(
"INSERT INTO metadata (name, value) VALUES (?1, ?2)",
(&k, &v),
)
.map_err(|e| sqlite_error(format!("set {} in metadata: {}", k, e)))?;
}
}
mbtiles_close(db)
}
}
const INSERT_IMAGE: &str = "replace into images (zoom_level, tile_id, tile_data) values (?, ?, ?)";
const INSERT_MAP: &str =
"insert into map (zoom_level, tile_column, tile_row, tile_id) values (?, ?, ?, ?)";
const DELETE_MAP: &str = "delete from map where zoom_level = ?";
const DELETE_IMAGES: &str = "delete from images where zoom_level = ?";
fn mbtiles_close(db: Connection) -> Result<()> {
sqlite3_exec(&db, c"ANALYZE;")
.map_err(|err| sqlite_error(format!("ANALYZE failed: {}", err)))?;
db.close().map_err(|(db, _)| {
Error::new(
EXIT_CLOSE,
format!("could not close database: {}", sqlite3_errmsg(&db)),
)
})
}
fn exec_metadata(db: &Connection, sql: &CStr, what: &str, forcetable: bool) -> Result<()> {
if let Err(err) = sqlite3_exec(db, sql) {
let message = format!("{}: {}", what, err);
if !forcetable {
return Err(sqlite_error(message));
}
eprintln!("{}", message);
}
Ok(())
}
pub fn mbtiles_write_metadata(db: &Connection, m: &Metadata, forcetable: bool) -> Result<()> {
let name = c_str(&m.name);
let sql = mprintf!(
c"INSERT INTO metadata (name, value) VALUES ('name', %Q);",
name.as_ptr()
);
exec_metadata(db, &sql, "set name in metadata", forcetable)?;
let description = c_str(&m.description);
let sql = mprintf!(
c"INSERT INTO metadata (name, value) VALUES ('description', %Q);",
description.as_ptr()
);
exec_metadata(db, &sql, "set description in metadata", forcetable)?;
let sql = mprintf!(
c"INSERT INTO metadata (name, value) VALUES ('version', %d);",
m.version as c_int
);
exec_metadata(db, &sql, "set version ", forcetable)?;
let sql = mprintf!(
c"INSERT INTO metadata (name, value) VALUES ('minzoom', %d);",
m.minzoom as c_int
);
exec_metadata(db, &sql, "set minzoom", forcetable)?;
let sql = mprintf!(
c"INSERT INTO metadata (name, value) VALUES ('maxzoom', %d);",
m.maxzoom as c_int
);
exec_metadata(db, &sql, "set maxzoom", forcetable)?;
let sql = mprintf!(
c"INSERT INTO metadata (name, value) VALUES ('center', '%f,%f,%d');",
m.center_lon as c_double,
m.center_lat as c_double,
m.center_z as c_int
);
exec_metadata(db, &sql, "set center", forcetable)?;
let sql = mprintf!(
c"INSERT INTO metadata (name, value) VALUES ('bounds', '%f,%f,%f,%f');",
m.minlon as c_double,
m.minlat as c_double,
m.maxlon as c_double,
m.maxlat as c_double
);
exec_metadata(db, &sql, "set bounds", forcetable)?;
let sql = mprintf!(
c"INSERT INTO metadata (name, value) VALUES ('antimeridian_adjusted_bounds', '%f,%f,%f,%f');",
m.minlon2 as c_double,
m.minlat2 as c_double,
m.maxlon2 as c_double,
m.maxlat2 as c_double
);
exec_metadata(db, &sql, "set bounds", forcetable)?;
let type_ = c_str(&m.type_);
let sql = mprintf!(
c"INSERT INTO metadata (name, value) VALUES ('type', %Q);",
type_.as_ptr()
);
exec_metadata(db, &sql, "set type", forcetable)?;
if !m.attribution.is_empty() {
let attribution = c_str(&m.attribution);
let sql = mprintf!(
c"INSERT INTO metadata (name, value) VALUES ('attribution', %Q);",
attribution.as_ptr()
);
exec_metadata(db, &sql, "set attribution", forcetable)?;
}
let format = c_str(&m.format);
let sql = mprintf!(
c"INSERT INTO metadata (name, value) VALUES ('format', %Q);",
format.as_ptr()
);
exec_metadata(db, &sql, "set format", forcetable)?;
let generator = c_str(&m.generator);
let sql = mprintf!(
c"INSERT INTO metadata (name, value) VALUES ('generator', %Q);",
generator.as_ptr()
);
exec_metadata(db, &sql, "set generator", forcetable)?;
let generator_options = c_str(&m.generator_options);
let sql = mprintf!(
c"INSERT INTO metadata (name, value) VALUES ('generator_options', %Q);",
generator_options.as_ptr()
);
exec_metadata(db, &sql, "set commandline", forcetable)?;
if !m.strategies_json.is_empty() {
let strategies = c_str(&m.strategies_json);
let sql = mprintf!(
c"INSERT INTO metadata (name, value) VALUES ('strategies', %Q);",
strategies.as_ptr()
);
exec_metadata(db, &sql, "set strategies", forcetable)?;
}
if !m.decisions_json.is_empty() {
let decisions = c_str(&m.decisions_json);
let sql = mprintf!(
c"INSERT INTO metadata (name, value) VALUES ('tippecanoe_decisions', %Q);",
decisions.as_ptr()
);
exec_metadata(db, &sql, "set decisions", forcetable)?;
}
if !m.vector_layers_json.is_empty() || !m.tilestats_json.is_empty() {
let mut json = String::new();
{
let mut state = JsonWriter::new_string(&mut json);
state.nospace = true;
state.json_write_hash();
if !m.vector_layers_json.is_empty() {
state.nospace = true;
state.json_write_string("vector_layers");
state.nospace = true;
state.json_write_json(&m.vector_layers_json);
}
if !m.tilestats_json.is_empty() {
state.nospace = true;
state.json_write_string("tilestats");
state.nospace = true;
state.json_write_json(&m.tilestats_json);
}
state.nospace = true;
state.json_end_hash();
}
let json = c_str(&json);
let sql = mprintf!(
c"INSERT INTO metadata (name, value) VALUES ('json', %Q);",
json.as_ptr()
);
exec_metadata(db, &sql, "set json", forcetable)?;
}
Ok(())
}