use geo_traits::GeometryTrait;
use geopackage_core::ident::quote;
use geopackage_core::triggers;
use geopackage_core::types::{GeometryTypeSet, ZmFlag};
use rusqlite::Connection;
use crate::bulk::{self, BulkIndexOptions};
use crate::index::drop_all_rtree_triggers;
use crate::transaction::WriteTransaction;
use crate::{Error, Layer, Result};
use super::constraints::ColumnConstraints;
use super::feature_writer::{
FeatureWriter, Shape, ValueColumn, build_insert_sql, build_partial_update_sql, build_update_sql,
};
use super::row::{NewFeature, WritableRow};
const MERGE_REBUILD_RATIO: usize = 10;
#[derive(Debug)]
pub(crate) struct GeomTarget {
pub(crate) name: String,
pub(crate) quoted_name: String,
pub(crate) srs_id: i32,
pub(crate) z: ZmFlag,
pub(crate) m: ZmFlag,
}
#[derive(Debug, Clone, Copy)]
pub(crate) struct BboxFold {
pub(crate) min_x: f64,
pub(crate) max_x: f64,
pub(crate) min_y: f64,
pub(crate) max_y: f64,
pub(crate) seen: bool,
}
impl BboxFold {
pub(crate) fn new() -> Self {
Self {
min_x: f64::INFINITY,
max_x: f64::NEG_INFINITY,
min_y: f64::INFINITY,
max_y: f64::NEG_INFINITY,
seen: false,
}
}
pub(crate) fn seed(&mut self, existing: Option<[f64; 4]>) {
if let Some([min_x, max_x, min_y, max_y]) = existing {
self.min_x = min_x;
self.max_x = max_x;
self.min_y = min_y;
self.max_y = max_y;
self.seen = true;
}
}
pub(crate) fn add(&mut self, [min_x, max_x, min_y, max_y]: [f64; 4]) {
self.min_x = self.min_x.min(min_x);
self.max_x = self.max_x.max(max_x);
self.min_y = self.min_y.min(min_y);
self.max_y = self.max_y.max(max_y);
self.seen = true;
}
pub(crate) fn bounds(&self) -> Option<[f64; 4]> {
self.seen
.then_some([self.min_x, self.max_x, self.min_y, self.max_y])
}
}
impl<'a> Layer<'a> {
pub fn writer(&self) -> Result<FeatureWriter<'a>> {
self.gpkg().check_writable(self.table_name())?;
let conn: &Connection = self.gpkg().connection();
let tx = WriteTransaction::begin(conn)?;
let existing = self.stored_extent()?;
let bbox_covers_layer = existing.is_some() || !self.has_rows()?;
let pk_name = self.primary_key_column();
let pk_expr = match pk_name {
Some(pk) => quote(pk)?,
None => "rowid".to_owned(),
};
let value_columns = self
.value_columns()
.iter()
.map(|c| {
Ok(ValueColumn {
quoted: quote(&c.name)?,
name: c.name.clone(),
})
})
.collect::<Result<Vec<_>>>()?;
let geometry = match self.geometry_column() {
Some(g) => Some(GeomTarget {
name: g.column_name.clone(),
quoted_name: quote(&g.column_name)?,
srs_id: g.srs_id,
z: g.z,
m: g.m,
}),
None => None,
};
let mut bbox = BboxFold::new();
bbox.seed(existing.map(|b| [b.min_x, b.max_x, b.min_y, b.max_y]));
let quoted_table = quote(self.table_name())?;
let shape = Shape {
table_name: self.table_name(),
quoted_table: "ed_table,
pk_expr: &pk_expr,
value_columns: &value_columns,
geometry: geometry.as_ref(),
};
let insert_stmts = [
conn.prepare_cached(&build_insert_sql(&shape, false, false))?,
conn.prepare_cached(&build_insert_sql(&shape, true, false))?,
conn.prepare_cached(&build_insert_sql(&shape, false, true))?,
conn.prepare_cached(&build_insert_sql(&shape, true, true))?,
];
let update_stmts = [
conn.prepare_cached(&build_update_sql(&shape, false))?,
conn.prepare_cached(&build_update_sql(&shape, true))?,
];
let delete_stmt =
conn.prepare_cached(&format!("DELETE FROM {quoted_table} WHERE {pk_expr} = ?1"))?;
let partial_stmt = conn.prepare_cached(&build_partial_update_sql(&shape, &[])?)?;
let constraints = ColumnConstraints::read(self, conn, &value_columns)?;
Ok(FeatureWriter {
tx,
conn,
table_name: self.table_name().to_owned(),
quoted_table,
pk_expr,
value_columns,
geometry,
bbox,
insert_stmts,
update_stmts,
delete_stmt,
partial_columns: Vec::new(),
partial_stmt,
dirty: false,
bbox_dirty: false,
bbox_covers_layer,
constraints,
geometry_types: GeometryTypeSet::new(),
})
}
pub fn write_all<G, I>(&self, features: I, batch_size: usize) -> Result<Vec<i64>>
where
G: GeometryTrait<T = f64>,
I: IntoIterator<Item = NewFeature<G>>,
{
self.write_all_with(features, batch_size, BulkIndexOptions::default())
}
pub fn write_all_with<G, I>(
&self,
features: I,
batch_size: usize,
options: BulkIndexOptions,
) -> Result<Vec<i64>>
where
G: GeometryTrait<T = f64>,
I: IntoIterator<Item = NewFeature<G>>,
{
self.write_all_impl(features, batch_size, options, bulk::no_fault)
}
pub(crate) fn write_all_impl<R, I>(
&self,
features: I,
batch_size: usize,
options: BulkIndexOptions,
fault: bulk::TestFault,
) -> Result<Vec<i64>>
where
R: WritableRow,
I: IntoIterator<Item = R>,
{
self.gpkg().check_writable(self.table_name())?;
let mut iter = features.into_iter();
let (bulk, buffered) = self.bulk_write_engages(&mut iter, options)?;
let features = buffered.into_iter().chain(iter);
if bulk {
self.write_all_bulk(features, options, fault)
} else {
self.write_all_batched(features, batch_size)
}
}
fn bulk_write_engages<R, I>(
&self,
features: &mut I,
options: BulkIndexOptions,
) -> Result<(bool, Vec<R>)>
where
I: Iterator<Item = R>,
{
let threshold = options.bulk_threshold;
if threshold == usize::MAX {
return Ok((false, Vec::new()));
}
let (lower, upper) = features.size_hint();
if upper.is_some_and(|upper| upper < threshold) {
return Ok((false, Vec::new()));
}
if lower >= threshold {
return Ok((self.has_spatial_index()?, Vec::new()));
}
if !self.has_spatial_index()? {
return Ok((false, Vec::new()));
}
let mut buffered = Vec::new();
while buffered.len() < threshold {
let Some(feature) = features.next() else {
return Ok((false, buffered));
};
buffered.push(feature);
}
Ok((true, buffered))
}
fn write_all_batched<R, I>(&self, features: I, batch_size: usize) -> Result<Vec<i64>>
where
R: WritableRow,
I: IntoIterator<Item = R>,
{
let mut fids = Vec::new();
let mut iter = features.into_iter();
let mut batch = self.writer()?;
let mut in_batch = 0usize;
let mut wrote_any = false;
for feature in iter.by_ref() {
let (fid, _) = feature.write(&mut batch)?;
fids.push(fid);
wrote_any = true;
in_batch += 1;
if batch_size != 0 && in_batch >= batch_size {
batch.commit()?;
batch = self.writer()?;
in_batch = 0;
wrote_any = false;
}
}
if wrote_any || in_batch > 0 {
batch.commit()?;
} else {
drop(batch);
}
Ok(fids)
}
fn write_all_bulk<R, I>(
&self,
features: I,
options: BulkIndexOptions,
fault: bulk::TestFault,
) -> Result<Vec<i64>>
where
R: WritableRow,
I: IntoIterator<Item = R>,
{
let geom = self
.geometry_column()
.ok_or_else(|| Error::NoGeometryColumn {
table_name: self.table_name().to_owned(),
})?;
let pk = self
.primary_key_column()
.ok_or_else(|| Error::NoPrimaryKey {
table_name: self.table_name().to_owned(),
})?;
let table = self.table_name();
let column = &geom.column_name;
let rtree = triggers::rtree_table_name(table, column);
(|| -> Result<Vec<i64>> {
let mut fids = Vec::new();
let mut entries = Vec::new();
let mut writer = self.writer()?;
drop_all_rtree_triggers(writer.connection(), table, column)?;
let conn = writer.connection();
let table_was_empty = bulk::table_row_count(conn, table)? == 0;
let indexed = rtree_entry_count(conn, &rtree)?;
for feature in features {
let (fid, envelope) = feature.write(&mut writer)?;
if let Some(envelope) = envelope {
entries.push((fid, envelope));
}
fids.push(fid);
}
let tx = writer.flush()?;
let reinstall = |conn: &Connection| -> Result<()> {
for sql in triggers::create_triggers_sql(table, column, pk)? {
conn.execute_batch(&sql)?;
}
Ok(())
};
if rebuild_beats_append(entries.len(), indexed) {
let precomputed = table_was_empty.then_some(entries);
bulk::fill_index_in_transaction(
conn,
table,
column,
pk,
&rtree,
options,
precomputed,
fault,
reinstall,
)?;
} else {
append_entries(conn, &rtree, &entries)?;
fault(conn, &rtree)?;
reinstall(conn)?;
}
tx.commit()?;
Ok(fids)
})()
}
}
pub(crate) fn rebuild_beats_append(new_entries: usize, indexed: usize) -> bool {
if indexed == 0 {
return true;
}
new_entries >= indexed / MERGE_REBUILD_RATIO
}
pub(crate) fn append_entries(
conn: &Connection,
rtree: &str,
entries: &[(i64, [f64; 4])],
) -> Result<()> {
if entries.is_empty() {
return Ok(());
}
let sql = format!(
"INSERT OR REPLACE INTO {} VALUES (?1, ?2, ?3, ?4, ?5)",
quote(rtree)?
);
let mut stmt = conn.prepare_cached(&sql)?;
for &(fid, [min_x, max_x, min_y, max_y]) in entries {
stmt.execute(rusqlite::params![fid, min_x, max_x, min_y, max_y])?;
}
Ok(())
}
pub(crate) fn rtree_entry_count(conn: &Connection, rtree: &str) -> Result<usize> {
let count: i64 = conn.query_row(
&format!("SELECT count(*) FROM {}", quote(rtree)?),
[],
|r| r.get(0),
)?;
Ok(usize::try_from(count).unwrap_or(usize::MAX))
}
#[cfg(test)]
mod tests {
use super::*;
use crate::{GeoPackage, GeometrySpec, TableSchemaBuilder};
use geo_types::Point;
use geopackage_core::types::GeometryType;
fn fail_the_build(_: &Connection, _: &str) -> Result<()> {
Err(Error::NoSpatialIndex {
table_name: "pts".to_owned(),
column_name: "geom".to_owned(),
})
}
fn indexed_empty_layer() -> (tempfile::TempDir, GeoPackage) {
let dir = tempfile::tempdir().unwrap();
let gpkg = GeoPackage::create(dir.path().join("t.gpkg")).unwrap();
let layer = gpkg
.create_layer(
&TableSchemaBuilder::new("pts")
.geometry(GeometrySpec::new(GeometryType::Point, 4326))
.spatial_index(false),
)
.unwrap();
layer.create_spatial_index().unwrap();
(dir, gpkg)
}
#[test]
fn failed_bulk_write_rolls_back_rows_and_index() {
let (_dir, gpkg) = indexed_empty_layer();
let layer = gpkg.layer("pts").unwrap();
let features: Vec<NewFeature<Point<f64>>> = (1..=50)
.map(|i| {
let f = f64::from(i);
NewFeature::new(Point::new(f, -f), Vec::new()).with_fid(i64::from(i))
})
.collect();
let result =
layer.write_all_impl(features, 0, BulkIndexOptions::always_bulk(), fail_the_build);
assert!(result.is_err(), "the build should have failed here");
let rows: i64 = gpkg
.connection()
.query_row("SELECT count(*) FROM pts", [], |r| r.get(0))
.unwrap();
assert_eq!(rows, 0, "rows survived a failed bulk build");
assert_eq!(
layer.spatial_index_status().unwrap(),
crate::SpatialIndexStatus::Current,
"index left desynchronised by a failed bulk build"
);
let mut writer = layer.writer().unwrap();
writer.insert(Some(1), &Point::new(5.0, 5.0), &[]).unwrap();
writer.commit().unwrap();
let indexed: i64 = gpkg
.connection()
.query_row("SELECT count(*) FROM rtree_pts_geom", [], |r| r.get(0))
.unwrap();
assert_eq!(indexed, 1, "triggers did not survive the rollback");
}
#[test]
fn failed_append_write_rolls_back_rows_and_index() {
let (_dir, gpkg) = indexed_empty_layer();
let layer = gpkg.layer("pts").unwrap();
{
let mut writer = layer.writer().unwrap();
for i in 1..=100 {
let f = f64::from(i);
writer
.insert(Some(i64::from(i)), &Point::new(f, -f), &[])
.unwrap();
}
writer.commit().unwrap();
}
let features: Vec<NewFeature<Point<f64>>> = (101..=105)
.map(|i| {
let f = f64::from(i);
NewFeature::new(Point::new(f, -f), Vec::new()).with_fid(i64::from(i))
})
.collect();
let result = layer.write_all_impl(
features,
0,
BulkIndexOptions::with_threshold(1),
fail_the_build,
);
assert!(result.is_err(), "the append should have failed here");
let conn = gpkg.connection();
let rows: i64 = conn
.query_row("SELECT count(*) FROM pts", [], |r| r.get(0))
.unwrap();
assert_eq!(rows, 100, "rows survived a failed append");
let indexed: i64 = conn
.query_row("SELECT count(*) FROM rtree_pts_geom", [], |r| r.get(0))
.unwrap();
assert_eq!(indexed, 100, "index entries survived a failed append");
assert_eq!(
layer.spatial_index_status().unwrap(),
crate::SpatialIndexStatus::Current,
"index left desynchronised by a failed append"
);
}
}