use anyhow::{Context, Result};
use rusqlite::{Connection, OpenFlags, ffi};
use std::{
ffi::{CString, c_char, c_int, c_void},
ops::{Deref, DerefMut},
path::Path,
ptr,
sync::{
Mutex,
atomic::{AtomicU64, Ordering},
},
};
static NEXT_VFS: AtomicU64 = AtomicU64::new(0);
struct State {
parent: *mut ffi::sqlite3_vfs,
directory: tempfile::TempDir,
filenames: Mutex<Vec<Box<[u8]>>>,
}
struct ScratchVfs {
vfs: Box<ffi::sqlite3_vfs>,
_name: CString,
#[allow(dead_code)] state: Box<State>,
}
unsafe impl Send for ScratchVfs {}
impl Drop for ScratchVfs {
fn drop(&mut self) {
unsafe {
ffi::sqlite3_vfs_unregister(&mut *self.vfs);
}
}
}
unsafe fn state<'a>(vfs: *mut ffi::sqlite3_vfs) -> &'a State {
unsafe { &*((*vfs).pAppData.cast::<State>()) }
}
unsafe extern "C" fn open(
vfs: *mut ffi::sqlite3_vfs,
name: ffi::sqlite3_filename,
file: *mut ffi::sqlite3_file,
flags: c_int,
out: *mut c_int,
) -> c_int {
std::panic::catch_unwind(|| unsafe {
let state = state(vfs);
if !name.is_null() {
return ((*state.parent).xOpen.unwrap())(state.parent, name, file, flags, out);
}
let temporary = match tempfile::Builder::new()
.prefix("spill-")
.tempfile_in(state.directory.path())
{
Ok(file) => file,
Err(_) => return ffi::SQLITE_CANTOPEN,
};
let path = temporary.path();
let Some(path) = path.to_str() else {
return ffi::SQLITE_CANTOPEN;
};
let mut bytes = path.as_bytes().to_vec();
bytes.extend_from_slice(&[0, 0]);
let filename = bytes.into_boxed_slice();
drop(temporary);
let mut names = match state.filenames.lock() {
Ok(names) => names,
Err(_) => return ffi::SQLITE_IOERR,
};
names.push(filename);
let name = names.last().unwrap().as_ptr().cast::<c_char>();
((*state.parent).xOpen.unwrap())(state.parent, name, file, flags, out)
})
.unwrap_or(ffi::SQLITE_IOERR)
}
macro_rules! forward {
($name:ident, $field:ident, ($($arg:ident: $ty:ty),*) -> $ret:ty) => {
unsafe extern "C" fn $name(vfs: *mut ffi::sqlite3_vfs, $($arg: $ty),*) -> $ret {
unsafe {
let parent = state(vfs).parent;
((*parent).$field.unwrap())(parent, $($arg),*)
}
}
};
}
type DlSymbol = Option<unsafe extern "C" fn(*mut ffi::sqlite3_vfs, *mut c_void, *const c_char)>;
forward!(delete, xDelete, (name: *const c_char, sync: c_int) -> c_int);
forward!(access, xAccess, (name: *const c_char, flags: c_int, out: *mut c_int) -> c_int);
forward!(full_path, xFullPathname, (name: *const c_char, len: c_int, out: *mut c_char) -> c_int);
forward!(dl_open, xDlOpen, (name: *const c_char) -> *mut c_void);
forward!(dl_error, xDlError, (len: c_int, out: *mut c_char) -> ());
forward!(dl_sym, xDlSym, (handle: *mut c_void, name: *const c_char) -> DlSymbol);
forward!(dl_close, xDlClose, (handle: *mut c_void) -> ());
forward!(random, xRandomness, (len: c_int, out: *mut c_char) -> c_int);
forward!(sleep, xSleep, (micros: c_int) -> c_int);
forward!(time, xCurrentTime, (out: *mut f64) -> c_int);
forward!(error, xGetLastError, (len: c_int, out: *mut c_char) -> c_int);
forward!(time64, xCurrentTimeInt64, (out: *mut i64) -> c_int);
forward!(set_call, xSetSystemCall, (name: *const c_char, call: ffi::sqlite3_syscall_ptr) -> c_int);
forward!(get_call, xGetSystemCall, (name: *const c_char) -> ffi::sqlite3_syscall_ptr);
forward!(next_call, xNextSystemCall, (name: *const c_char) -> *const c_char);
pub(crate) struct ScratchConnection {
connection: Option<Connection>,
_vfs: ScratchVfs,
}
impl Deref for ScratchConnection {
type Target = Connection;
fn deref(&self) -> &Connection {
self.connection.as_ref().unwrap()
}
}
impl DerefMut for ScratchConnection {
fn deref_mut(&mut self) -> &mut Connection {
self.connection.as_mut().unwrap()
}
}
impl ScratchConnection {
pub(crate) fn open(path: impl AsRef<Path>) -> Result<Self> {
let path = path.as_ref();
let folder = path
.parent()
.filter(|p| !p.as_os_str().is_empty())
.unwrap_or(Path::new("."));
let folder = std::fs::canonicalize(folder).with_context(|| {
format!("cannot resolve SQLite scratch folder {}", folder.display())
})?;
let directory = tempfile::Builder::new()
.prefix(".taxutils-sqlite-")
.tempdir_in(&folder)
.with_context(|| {
format!(
"cannot create SQLite scratch directory in {}",
folder.display()
)
})?;
let name = CString::new(format!(
"taxutils-scratch-{}",
NEXT_VFS.fetch_add(1, Ordering::Relaxed)
))?;
unsafe {
anyhow::ensure!(
ffi::sqlite3_initialize() == ffi::SQLITE_OK,
"SQLite initialization failed"
);
let parent = ffi::sqlite3_vfs_find(ptr::null());
anyhow::ensure!(!parent.is_null(), "SQLite has no default VFS");
let mut state = Box::new(State {
parent,
directory,
filenames: Mutex::new(Vec::new()),
});
let mut vfs = Box::new(ptr::read(parent));
vfs.pNext = ptr::null_mut();
vfs.zName = name.as_ptr();
vfs.pAppData = (&mut *state as *mut State).cast();
vfs.xOpen = Some(open);
macro_rules! bind {
($field:ident,$callback:ident) => {
if vfs.$field.is_some() {
vfs.$field = Some($callback);
}
};
}
bind!(xDelete, delete);
bind!(xAccess, access);
bind!(xFullPathname, full_path);
bind!(xDlOpen, dl_open);
bind!(xDlError, dl_error);
bind!(xDlSym, dl_sym);
bind!(xDlClose, dl_close);
bind!(xRandomness, random);
bind!(xSleep, sleep);
bind!(xCurrentTime, time);
bind!(xGetLastError, error);
bind!(xCurrentTimeInt64, time64);
bind!(xSetSystemCall, set_call);
bind!(xGetSystemCall, get_call);
bind!(xNextSystemCall, next_call);
anyhow::ensure!(
ffi::sqlite3_vfs_register(&mut *vfs, 0) == ffi::SQLITE_OK,
"cannot register scratch VFS"
);
let owner = ScratchVfs {
vfs,
_name: name,
state,
};
let connection = Connection::open_with_flags_and_vfs(
path,
OpenFlags::default(),
owner._name.to_str()?,
)
.with_context(|| format!("cannot open {} with local SQLite scratch", path.display()))?;
Ok(Self {
connection: Some(connection),
_vfs: owner,
})
}
}
pub(crate) fn close(mut self) -> rusqlite::Result<()> {
match self.connection.take().unwrap().close() {
Ok(()) => Ok(()),
Err((connection, error)) => {
self.connection = Some(connection);
Err(error)
}
}
}
}
#[cfg(test)]
mod tests {
use super::*;
fn spill(connection: &ScratchConnection) {
connection
.execute_batch(
"PRAGMA temp_store=FILE; PRAGMA cache_size=-64;
PRAGMA temp.cache_size=-64; PRAGMA threads=2;
CREATE TEMP TABLE stage(k INTEGER, payload BLOB);
WITH RECURSIVE seq(x) AS (VALUES(1) UNION ALL SELECT x+1 FROM seq WHERE x<60000)
INSERT INTO stage SELECT x,randomblob(256) FROM seq;",
)
.unwrap();
let staged = connection._vfs.state.filenames.lock().unwrap().len();
assert!(staged > 0, "TEMP table did not spill");
connection
.execute_batch(
"CREATE TABLE data AS SELECT * FROM stage;
CREATE INDEX idx_payload ON data(payload);",
)
.unwrap();
assert!(
connection._vfs.state.filenames.lock().unwrap().len() > staged,
"index sorter did not create scratch files"
);
let root = connection._vfs.state.directory.path();
for name in connection._vfs.state.filenames.lock().unwrap().iter() {
let path = std::ffi::CStr::from_bytes_until_nul(name)
.unwrap()
.to_str()
.unwrap();
assert!(Path::new(path).starts_with(root));
}
#[cfg(target_os = "linux")]
{
let targets: Vec<_> = std::fs::read_dir("/proc/self/fd")
.unwrap()
.filter_map(|f| std::fs::read_link(f.ok()?.path()).ok())
.collect();
assert!(
targets.iter().any(|p| p.starts_with(root)),
"no actual scratch descriptor in selected directory"
);
}
assert_eq!(
connection
.query_row("PRAGMA integrity_check", [], |r| r.get::<_, String>(0))
.unwrap(),
"ok"
);
}
#[test]
fn real_temp_table_and_index_spills_stay_local_and_cleanup() {
let root = tempfile::tempdir().unwrap();
let connection = ScratchConnection::open(root.path().join("db.sqlite")).unwrap();
let scratch = connection._vfs.state.directory.path().to_path_buf();
spill(&connection);
connection.close().unwrap();
assert!(!scratch.exists());
assert!(root.path().join("db.sqlite").exists());
}
#[test]
fn concurrent_connections_keep_independent_directories() {
let roots: Vec<_> = (0..2).map(|_| tempfile::tempdir().unwrap()).collect();
let barrier = std::sync::Arc::new(std::sync::Barrier::new(2));
let threads: Vec<_> = roots
.iter()
.map(|root| {
let folder = root.path().to_path_buf();
let barrier = barrier.clone();
std::thread::spawn(move || {
let connection = ScratchConnection::open(folder.join("db.sqlite")).unwrap();
barrier.wait();
spill(&connection);
})
})
.collect();
for thread in threads {
thread.join().unwrap();
}
for root in roots {
assert!(std::fs::read_dir(root.path()).unwrap().all(|e| {
!e.unwrap()
.file_name()
.to_string_lossy()
.starts_with(".taxutils-sqlite-")
}));
}
}
#[test]
fn unavailable_scratch_errors_without_falling_back() {
let root = tempfile::tempdir().unwrap();
let connection = ScratchConnection::open(root.path().join("db.sqlite")).unwrap();
std::fs::remove_dir(connection._vfs.state.directory.path()).unwrap();
let error = connection
.execute_batch(
"PRAGMA temp_store=FILE; PRAGMA temp.cache_size=-16;
CREATE TEMP TABLE stage(payload);
WITH RECURSIVE seq(x) AS (VALUES(1) UNION ALL SELECT x+1 FROM seq WHERE x<10000)
INSERT INTO stage SELECT randomblob(1024) FROM seq;",
)
.unwrap_err();
assert_eq!(
error.sqlite_error_code(),
Some(rusqlite::ErrorCode::CannotOpen)
);
assert!(connection._vfs.state.filenames.lock().unwrap().is_empty());
}
}