use std::collections::BTreeMap;
use std::path::PathBuf;
use std::sync::atomic::{AtomicU64, AtomicU8, Ordering};
use std::sync::mpsc::{sync_channel, Receiver, RecvTimeoutError};
use std::sync::{Arc, Mutex};
use std::time::Duration;
use littlegraph_core::{CancellationToken, TransactionOptions};
use pyo3::exceptions::{PyTypeError, PyValueError};
use pyo3::prelude::*;
use pyo3::types::{PyAny, PyBool, PyBytes, PyFloat, PyInt, PyString, PyTraceback, PyType};
use crate::error::{BindingError, DatabaseClosedError};
use crate::options::Limits;
use crate::result::{result_to_python, QueryResult};
use crate::value::parameters_from_python;
use crate::worker::{self, AutoExecute, Command, WorkerHandle};
const OPEN: u8 = 0;
const CLOSING: u8 = 1;
const CLOSED: u8 = 2;
const FAILED: u8 = 3;
struct DatabaseInner {
path: String,
read_only: bool,
process_id: u32,
state: AtomicU8,
worker: Mutex<Option<WorkerHandle>>,
limits: Limits,
next_operation_id: AtomicU64,
active_operations: Mutex<BTreeMap<u64, CancellationToken>>,
}
impl DatabaseInner {
fn ensure_process(&self) -> Result<(), BindingError> {
if std::process::id() != self.process_id {
Err(BindingError::forked())
} else {
Ok(())
}
}
fn close(&self) -> Result<(), BindingError> {
self.ensure_process()?;
let mut worker = self
.worker
.lock()
.map_err(|_| BindingError::worker("database worker state is poisoned"))?;
match self.state.load(Ordering::Acquire) {
CLOSED => return Ok(()),
FAILED => return Err(BindingError::worker("database worker is unavailable")),
OPEN => self.state.store(CLOSING, Ordering::Release),
CLOSING => {
return Err(BindingError::worker(
"database has an invalid concurrent close state",
))
}
_ => {
return Err(BindingError::worker(
"database has an invalid lifecycle state",
))
}
}
let active_operations = self
.active_operations
.lock()
.map_err(|_| BindingError::worker("database operation state is poisoned"))?;
for cancellation in active_operations.values() {
cancellation.cancel();
}
drop(active_operations);
let Some(worker) = worker.take() else {
self.state.store(CLOSED, Ordering::Release);
return Ok(());
};
let _ = worker.commands.send(Command::Shutdown);
match worker.thread.join() {
Ok(()) => {
self.state.store(CLOSED, Ordering::Release);
Ok(())
}
Err(_) => {
self.state.store(FAILED, Ordering::Release);
Err(BindingError::worker(
"database worker panicked during shutdown",
))
}
}
}
fn command_sender(&self) -> Result<std::sync::mpsc::SyncSender<Command>, BindingError> {
self.ensure_process()?;
if self.state.load(Ordering::Acquire) != OPEN {
return Err(BindingError::worker("database is closed"));
}
self.worker
.lock()
.map_err(|_| BindingError::worker("database worker state is poisoned"))?
.as_ref()
.map(|worker| worker.commands.clone())
.ok_or_else(|| BindingError::worker("database worker is unavailable"))
}
fn register_operation(
self: &Arc<Self>,
cancellation: CancellationToken,
) -> Result<OperationGuard, BindingError> {
let mut operations = self
.active_operations
.lock()
.map_err(|_| BindingError::worker("database operation state is poisoned"))?;
if self.state.load(Ordering::Acquire) != OPEN {
return Err(BindingError::worker("database is closed"));
}
let id = self
.next_operation_id
.fetch_update(Ordering::Relaxed, Ordering::Relaxed, |id| id.checked_add(1))
.map_err(|_| BindingError::worker("database operation identifier space exhausted"))?;
operations.insert(id, cancellation);
Ok(OperationGuard {
database: self.clone(),
id,
})
}
}
struct OperationGuard {
database: Arc<DatabaseInner>,
id: u64,
}
impl Drop for OperationGuard {
fn drop(&mut self) {
if let Ok(mut operations) = self.database.active_operations.lock() {
operations.remove(&self.id);
}
}
}
impl Drop for DatabaseInner {
fn drop(&mut self) {
if std::process::id() != self.process_id {
return;
}
if let Ok(worker) = self.worker.get_mut() {
if let Some(worker) = worker.take() {
let _ = worker.commands.try_send(Command::Shutdown);
}
}
}
}
#[pyclass(module = "littlegraph._littlegraph")]
pub(crate) struct Database {
inner: Arc<DatabaseInner>,
}
#[pymethods]
impl Database {
#[new]
#[pyo3(signature = (path, *, read_only=true, create=false, limits=None))]
fn new(
py: Python<'_>,
path: &Bound<'_, PyAny>,
read_only: bool,
create: bool,
limits: Option<PyRef<'_, Limits>>,
) -> PyResult<Self> {
open_impl(py, path, read_only, create, limits.as_deref())
}
#[getter]
fn path(&self, py: Python<'_>) -> PyResult<String> {
self.inner
.ensure_process()
.map_err(|error| error.into_pyerr(py))?;
Ok(self.inner.path.clone())
}
#[getter]
fn read_only(&self, py: Python<'_>) -> PyResult<bool> {
self.inner
.ensure_process()
.map_err(|error| error.into_pyerr(py))?;
Ok(self.inner.read_only)
}
#[getter]
fn closed(&self, py: Python<'_>) -> PyResult<bool> {
self.inner
.ensure_process()
.map_err(|error| error.into_pyerr(py))?;
Ok(self.inner.state.load(Ordering::Acquire) != OPEN)
}
fn close(&self, py: Python<'_>) -> PyResult<()> {
py.allow_threads(|| self.inner.close())
.map_err(|error| error.into_pyerr(py))
}
#[pyo3(signature = (query, parameters=None, *, timeout=None))]
fn execute(
&self,
py: Python<'_>,
query: String,
parameters: Option<&Bound<'_, PyAny>>,
timeout: Option<&Bound<'_, PyAny>>,
) -> PyResult<Py<QueryResult>> {
if query.is_empty() {
return Err(PyValueError::new_err("query must not be empty"));
}
self.inner
.ensure_process()
.map_err(|error| error.into_pyerr(py))?;
if self.inner.state.load(Ordering::Acquire) != OPEN {
return Err(closed_error(py));
}
let parameters = parameters_from_python(py, parameters, &self.inner.limits)
.map_err(|error| error.into_pyerr(py))?;
let timeout = parse_timeout(timeout)?;
let cancellation = CancellationToken::new();
let _operation = self
.inner
.register_operation(cancellation.clone())
.map_err(|_| closed_error(py))?;
let sender = self
.inner
.command_sender()
.map_err(|error| error.into_pyerr(py))?;
let (result_tx, result_rx) = sync_channel(1);
let (acknowledge_tx, acknowledge_rx) = sync_channel(1);
let (finish_tx, finish_rx) = sync_channel(1);
let options = TransactionOptions {
timeout,
max_workspace_bytes: Some(self.inner.limits.default_transaction_workspace_bytes),
max_records: Some(self.inner.limits.default_transaction_records),
max_query_memory_bytes: Some(self.inner.limits.default_query_memory_bytes),
spill_threshold_bytes: Some(self.inner.limits.default_spill_threshold_bytes),
max_spill_bytes: Some(self.inner.limits.default_max_spill_bytes),
max_journal_bytes: Some(self.inner.limits.default_max_journal_bytes),
cancellation_token: Some(cancellation.clone()),
read_only: self.inner.read_only,
..TransactionOptions::default()
};
let command = Command::AutoExecute(Box::new(AutoExecute {
query,
parameters,
options,
max_result_rows: self.inner.limits.max_result_rows,
max_result_bytes: self.inner.limits.max_result_bytes,
result: result_tx,
acknowledge: acknowledge_rx,
finish: finish_tx,
}));
py.allow_threads(move || sender.send(command))
.map_err(|_| BindingError::worker("database worker is unavailable").into_pyerr(py))?;
let core_result = match receive_with_signals(
py,
result_rx,
self.inner.limits.poll_interval(),
&cancellation,
)? {
Receive::Value(result) => result.map_err(|error| error.into_pyerr(py))?,
Receive::Interrupted { error, value } => {
if matches!(value, Some(Ok(_))) {
let _ = acknowledge_tx.send(false);
wait_without_signals(py, finish_rx);
}
return Err(error);
}
};
let result = match result_to_python(py, core_result) {
Ok(result) => result,
Err(error) => {
let _ = acknowledge_tx.send(false);
wait_without_signals(py, finish_rx);
return Err(error);
}
};
if let Err(error) = py.check_signals() {
cancellation.cancel();
let _ = acknowledge_tx.send(false);
wait_without_signals(py, finish_rx);
return Err(error);
}
acknowledge_tx
.send(true)
.map_err(|_| BindingError::worker("database worker is unavailable").into_pyerr(py))?;
match receive_with_signals(
py,
finish_rx,
self.inner.limits.poll_interval(),
&cancellation,
)? {
Receive::Value(completed) => {
completed.map_err(|error| error.into_pyerr(py))?;
Ok(result)
}
Receive::Interrupted { error, .. } => Err(error),
}
}
fn __enter__<'py>(slf: PyRef<'py, Self>, py: Python<'py>) -> PyResult<PyRef<'py, Self>> {
slf.inner
.ensure_process()
.map_err(|error| error.into_pyerr(py))?;
if slf.inner.state.load(Ordering::Acquire) != OPEN {
return Err(closed_error(py));
}
Ok(slf)
}
fn __exit__(
&self,
py: Python<'_>,
_exception_type: Option<&Bound<'_, PyType>>,
_exception: Option<&Bound<'_, PyAny>>,
_traceback: Option<&Bound<'_, PyTraceback>>,
) -> PyResult<bool> {
self.close(py)?;
Ok(false)
}
fn __repr__(&self, py: Python<'_>) -> PyResult<String> {
self.inner
.ensure_process()
.map_err(|error| error.into_pyerr(py))?;
Ok(format!(
"Database(path={:?}, read_only={}, closed={})",
self.inner.path,
self.inner.read_only,
self.inner.state.load(Ordering::Acquire) != OPEN,
))
}
}
#[pyfunction(name = "open", signature = (path, *, read_only=true, create=false, limits=None))]
pub(crate) fn open_database(
py: Python<'_>,
path: &Bound<'_, PyAny>,
read_only: bool,
create: bool,
limits: Option<PyRef<'_, Limits>>,
) -> PyResult<Database> {
open_impl(py, path, read_only, create, limits.as_deref())
}
fn open_impl(
py: Python<'_>,
path: &Bound<'_, PyAny>,
read_only: bool,
create: bool,
limits: Option<&Limits>,
) -> PyResult<Database> {
if create && read_only {
return Err(PyValueError::new_err(
"create=True requires read_only=False",
));
}
let path = extract_path(py, path)?;
let path_buf = PathBuf::from(&path);
let limits = limits.cloned().unwrap_or_default();
let (worker, startup) =
worker::start(path_buf, read_only, create, limits.worker_command_capacity)
.map_err(|error| error.into_pyerr(py))?;
let startup_result = py.allow_threads(move || startup.recv());
match startup_result {
Ok(Ok(())) => Ok(Database {
inner: Arc::new(DatabaseInner {
path,
read_only,
process_id: std::process::id(),
state: AtomicU8::new(OPEN),
worker: Mutex::new(Some(worker)),
limits,
next_operation_id: AtomicU64::new(1),
active_operations: Mutex::new(BTreeMap::new()),
}),
}),
Ok(Err(error)) => {
let _ = py.allow_threads(|| worker.thread.join());
Err(error.into_pyerr(py))
}
Err(_) => {
let join = py.allow_threads(|| worker.thread.join());
let message = if join.is_err() {
"database worker panicked during startup"
} else {
"database worker stopped during startup"
};
Err(BindingError::worker(message).into_pyerr(py))
}
}
}
enum Receive<T> {
Value(T),
Interrupted { error: PyErr, value: Option<T> },
}
fn receive_with_signals<T: Send>(
py: Python<'_>,
receiver: Receiver<T>,
interval: Duration,
cancellation: &CancellationToken,
) -> PyResult<Receive<T>> {
let receiver = Mutex::new(receiver);
loop {
let received = py.allow_threads(|| {
receiver
.lock()
.map_err(|_| RecvTimeoutError::Disconnected)?
.recv_timeout(interval)
});
match received {
Ok(value) => return Ok(Receive::Value(value)),
Err(RecvTimeoutError::Disconnected) => {
return Err(BindingError::worker("database worker is unavailable").into_pyerr(py));
}
Err(RecvTimeoutError::Timeout) => {
if let Err(error) = py.check_signals() {
cancellation.cancel();
let value = py.allow_threads(|| {
receiver
.lock()
.ok()
.and_then(|receiver| receiver.recv().ok())
});
return Ok(Receive::Interrupted { error, value });
}
}
}
}
}
fn wait_without_signals<T: Send>(py: Python<'_>, receiver: Receiver<T>) {
let _ = py.allow_threads(move || receiver.recv());
}
fn parse_timeout(value: Option<&Bound<'_, PyAny>>) -> PyResult<Option<Duration>> {
let Some(value) = value else {
return Ok(None);
};
if value.is_instance_of::<PyBool>()
|| !(value.is_instance_of::<PyInt>() || value.is_instance_of::<PyFloat>())
{
return Err(PyTypeError::new_err("timeout must be a number"));
}
let seconds = value
.extract::<f64>()
.map_err(|_| PyValueError::new_err("timeout is outside the supported range"))?;
if !seconds.is_finite() || seconds < 0.0 {
return Err(PyValueError::new_err(
"timeout must be a finite, non-negative number of seconds",
));
}
Duration::try_from_secs_f64(seconds)
.map(Some)
.map_err(|_| PyValueError::new_err("timeout is outside the supported range"))
}
fn extract_path(py: Python<'_>, path: &Bound<'_, PyAny>) -> PyResult<String> {
let fspath = py.import("os")?.getattr("fspath")?.call1((path,))?;
if fspath.is_instance_of::<PyBytes>() {
return Err(PyTypeError::new_err(
"database path must be str or os.PathLike[str], not bytes",
));
}
if !fspath.is_instance_of::<PyString>() {
return Err(PyTypeError::new_err(
"database path must be str or os.PathLike[str]",
));
}
fspath.extract::<String>().map_err(|_| {
PyValueError::new_err("database path contains characters unsupported by this platform")
})
}
fn closed_error(py: Python<'_>) -> PyErr {
let message = "database is closed";
let error = PyErr::new::<DatabaseClosedError, _>(message);
let value = error.value(py);
let _ = value.setattr("code", "Interface.DatabaseClosed");
let _ = value.setattr("message", message);
error
}
#[cfg(test)]
mod tests {
use littlegraph_core::Neo4jStorage;
use super::*;
#[test]
fn worker_opens_and_closes_created_store() {
let directory = tempfile::tempdir().unwrap();
let path = directory.path().join("database");
let (worker, startup) = worker::start(path.clone(), false, true, 1).unwrap();
startup.recv().unwrap().unwrap();
worker.commands.send(Command::Shutdown).unwrap();
worker.thread.join().unwrap();
Neo4jStorage::open(path).unwrap();
}
#[test]
fn read_only_worker_does_not_create_store() {
let directory = tempfile::tempdir().unwrap();
let path = directory.path().join("missing");
let (worker, startup) = worker::start(path.clone(), true, false, 1).unwrap();
assert!(startup.recv().unwrap().is_err());
worker.thread.join().unwrap();
assert!(!path.exists());
}
}