pub mod gql {
pub use graphfusion_gql_parser::*;
}
pub mod catalog;
mod error;
pub mod graph;
mod parquet;
mod persistence;
mod query;
mod session;
mod storage;
mod transaction;
pub mod types;
pub use datafusion::arrow;
pub use error::{Error, Result};
use persistence::Disk;
pub use query::QueryResult;
pub use session::{
ExecutionResult, Parameter, QueryLimits, Session, SessionState, StatementOutput,
StatementResult, TransactionAction, TransactionStatus, Value,
};
use std::{
collections::HashMap,
fs,
path::{Path, PathBuf},
sync::{
atomic::{AtomicBool, AtomicU64, AtomicUsize, Ordering},
Arc, Mutex, OnceLock, Weak,
},
time::Instant,
};
use transaction::{PublishedState, StatementTxn};
#[derive(Clone, Debug, Default)]
pub struct OpenOptions {
pub create_if_missing: bool,
}
#[derive(Debug, Default)]
struct Metrics {
commits: AtomicU64,
conflicts: AtomicU64,
commit_wait_micros: AtomicU64,
recovery_micros: AtomicU64,
checkpoint_micros: AtomicU64,
checkpoint_busy: AtomicU64,
}
#[derive(Clone, Debug, Default)]
pub struct Statistics {
pub commits: u64,
pub conflicts: u64,
pub commit_wait_micros: u64,
pub recovery_micros: u64,
pub checkpoint_micros: u64,
pub checkpoint_busy: u64,
pub log_bytes: u64,
}
#[derive(Debug)]
struct Inner {
disk: Option<Disk>,
state: Mutex<Arc<PublishedState>>,
active: AtomicUsize,
unhealthy: AtomicBool,
metrics: Metrics,
}
#[derive(Clone, Debug)]
pub struct Database {
inner: Arc<Inner>,
}
impl Default for Database {
fn default() -> Self {
Self::new()
}
}
type Registry = Mutex<HashMap<PathBuf, Weak<Inner>>>;
static DATABASES: OnceLock<Registry> = OnceLock::new();
impl Database {
pub fn new() -> Self {
Self {
inner: Arc::new(Inner {
disk: None,
state: Mutex::new(Arc::new(PublishedState::initial())),
active: AtomicUsize::new(0),
unhealthy: AtomicBool::new(false),
metrics: Metrics::default(),
}),
}
}
pub fn open(path: impl AsRef<Path>, options: OpenOptions) -> Result<Self> {
if !cfg!(any(target_os = "linux", target_os = "macos")) {
return Err(Error::UnsupportedFeature(
"persistent databases require Linux or macOS".into(),
));
}
let path = path.as_ref();
if options.create_if_missing {
fs::create_dir_all(path)?;
let absolute = fs::canonicalize(path)?;
for parent in absolute.ancestors().skip(1) {
fs::File::open(parent)?.sync_all()?;
}
}
let directory = fs::canonicalize(path)?;
let mut registry = DATABASES
.get_or_init(Mutex::default)
.lock()
.map_err(|_| Error::Poisoned)?;
if let Some(inner) = registry.get(&directory).and_then(Weak::upgrade) {
if !inner.unhealthy.load(Ordering::SeqCst) {
return Ok(Self { inner });
}
}
let disk = Disk {
directory: directory.clone(),
};
let _lease = disk.lifecycle(false)?;
let _commit = disk.commit_lock()?;
if !options.create_if_missing && !directory.join("MANIFEST").exists() {
return Err(Error::NotFound("database manifest".into()));
}
disk.initialize()?;
let (state, _) = disk.load()?;
let inner = Arc::new(Inner {
disk: Some(disk),
state: Mutex::new(Arc::new(state)),
active: AtomicUsize::new(0),
unhealthy: AtomicBool::new(false),
metrics: Metrics::default(),
});
registry.insert(directory, Arc::downgrade(&inner));
Ok(Self { inner })
}
pub fn parse_gql(&self, input: &str) -> gql::Result<gql::Program> {
gql::parse(input)
}
pub fn session(&self) -> Session {
Session::new(self.clone())
}
pub fn with_catalog<T>(
&self,
inspect: impl FnOnce(&catalog::CatalogSnapshot) -> T,
) -> Result<T> {
let tx = StatementTxn::begin(self)?;
Ok(inspect(tx.catalog()))
}
pub fn create_directory(&self, components: &[&str]) -> Result<catalog::ObjectId> {
let mut tx = StatementTxn::begin(self)?;
let mut parent = catalog::ROOT_DIRECTORY;
for name in components {
parent = match tx.lookup(parent, catalog::ObjectKind::Directory, name) {
Some(entry) => entry.id,
None => tx.create(parent, name, catalog::ObjectDefinition::Directory)?,
};
}
tx.commit()?;
Ok(parent)
}
pub fn checkpoint(&self) -> Result<()> {
self.check_healthy()?;
let started = Instant::now();
let lease = self
.inner
.disk
.as_ref()
.map(|d| d.lifecycle(true))
.transpose();
if matches!(lease, Err(Error::Busy)) {
self.inner
.metrics
.checkpoint_busy
.fetch_add(1, Ordering::Relaxed);
}
let _lease = lease?;
let mut current = self.inner.state.lock().map_err(|_| Error::Poisoned)?;
self.check_healthy()?;
if self.inner.active.load(Ordering::SeqCst) != 0 {
self.inner
.metrics
.checkpoint_busy
.fetch_add(1, Ordering::Relaxed);
return Err(Error::Busy);
}
let _commit = self
.inner
.disk
.as_ref()
.map(|d| d.commit_lock())
.transpose()?;
let generation = if let Some(disk) = &self.inner.disk {
let (state, generation) = disk.load()?;
*current = Arc::new(state);
generation
} else {
0
};
let mut next = (**current).clone();
Arc::make_mut(&mut next.storage).reclaim();
Arc::make_mut(&mut next.catalog)
.names
.retain(|_, binding| binding.object.is_some());
next.validate()?;
if let Some(disk) = &self.inner.disk {
disk.checkpoint(&next, generation)?;
}
*current = Arc::new(next);
self.inner
.metrics
.checkpoint_micros
.fetch_add(started.elapsed().as_micros() as u64, Ordering::Relaxed);
Ok(())
}
pub fn statistics(&self) -> Result<Statistics> {
let m = &self.inner.metrics;
let mut stats = Statistics {
commits: m.commits.load(Ordering::Relaxed),
conflicts: m.conflicts.load(Ordering::Relaxed),
commit_wait_micros: m.commit_wait_micros.load(Ordering::Relaxed),
recovery_micros: m.recovery_micros.load(Ordering::Relaxed),
checkpoint_micros: m.checkpoint_micros.load(Ordering::Relaxed),
checkpoint_busy: m.checkpoint_busy.load(Ordering::Relaxed),
log_bytes: 0,
};
if let Some(disk) = &self.inner.disk {
for entry in fs::read_dir(&disk.directory)? {
let entry = entry?;
if entry.file_name().to_string_lossy().starts_with("wal-") {
match entry.metadata() {
Ok(m) => stats.log_bytes += m.len(),
Err(e) if e.kind() == std::io::ErrorKind::NotFound => (),
Err(e) => return Err(e.into()),
}
}
}
}
Ok(stats)
}
fn check_healthy(&self) -> Result<()> {
if self.inner.unhealthy.load(Ordering::SeqCst) {
return Err(Error::Poisoned);
}
Ok(())
}
}
#[cfg(test)]
mod tests;