use std::fs;
use std::path::{Path, PathBuf};
use std::sync::atomic::{AtomicBool, Ordering as AtomicOrdering};
use std::sync::{Arc, Condvar, Mutex, MutexGuard};
use std::thread::JoinHandle;
use std::time::{Duration, Instant, SystemTime};
use serde_json::{Map, Value};
use crate::format::OutputFormat;
use super::backend::{Backend, PutOutcome};
use super::digest::{
assert_digest, digest_file, digest_of, drain_digest, engine_io_err, stream_to_verified_file,
Digest,
};
use super::engine::{assert_execute_allowed, assert_query_allowed, Engine, EngineStartOptions};
use super::errors::{backend_err, err, Category, Result};
use super::head::{parse_head, serialize_head, HeadSnapshot};
use super::keys::{checkpoint_key, uuid8, wal_key, HEAD_KEY};
use super::lease::{
acquire_lease, create_cold, new_instance_id, now_seconds, read_head, release_lease, ColdParams,
LeaseParams,
};
use super::negotiate::{
assert_engine_compatible, assert_readable, assert_writable, raise_compatibility_floor,
RunningEngine,
};
use super::types::{Head, Lease, Manifest, ObjectRef, MAX_WAL_SEGMENT_BYTES};
use super::wal::{assert_statement_within_limit, decode_segment, encode_segment, line_bytes};
use super::EngineFactory;
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct Tuning {
pub lease_ttl: Duration,
pub heartbeat_interval: Duration,
pub clock_skew_allowance: Duration,
pub commit_deadline: Duration,
pub max_commit_attempts: u32,
}
impl Default for Tuning {
fn default() -> Self {
Self {
lease_ttl: Duration::from_secs(30),
heartbeat_interval: Duration::from_secs(10),
clock_skew_allowance: Duration::from_secs(5),
commit_deadline: Duration::from_secs(30),
max_commit_attempts: 5,
}
}
}
impl Tuning {
fn validate(&self) -> Result<()> {
for (name, value) in [
("lease_ttl", self.lease_ttl),
("heartbeat_interval", self.heartbeat_interval),
("clock_skew_allowance", self.clock_skew_allowance),
("commit_deadline", self.commit_deadline),
] {
if value.is_zero() {
return Err(err(
Category::Backend,
format!("durable: Tuning::{name} must be a positive duration"),
));
}
}
if self.max_commit_attempts == 0 {
return Err(err(
Category::Backend,
"durable: Tuning::max_commit_attempts must be positive",
));
}
if self.heartbeat_interval * 3 > self.lease_ttl {
return Err(err(
Category::Backend,
format!(
"durable: Tuning::heartbeat_interval ({:?}) must be at most a third of \
lease_ttl ({:?}); the contract requires room for a retry before expiry",
self.heartbeat_interval, self.lease_ttl
),
));
}
Ok(())
}
}
#[derive(Debug, Clone, Default)]
pub struct OpenOptions {
pub read_only: bool,
pub force: bool,
pub existing_only: bool,
pub owner: Option<String>,
pub database: Option<String>,
pub scratch_root: Option<PathBuf>,
pub tuning: Tuning,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]
#[non_exhaustive]
pub struct WriteTicket {
pub statement: u64,
}
#[derive(Debug, Clone)]
#[non_exhaustive]
pub struct Stats {
pub id: String,
pub database: String,
pub read_only: bool,
pub state: &'static str,
pub fenced: bool,
pub generation: u64,
pub owner: String,
pub instance: String,
pub committed_seq: u64,
pub base_key: Option<String>,
pub wal_segments: usize,
pub executed_statements: u64,
pub committed_statements: u64,
pub pending_statements: usize,
pub pending_bytes: u64,
pub last_flush_at: Option<SystemTime>,
pub last_checkpoint_at: Option<SystemTime>,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
enum Lifecycle {
Open,
Closing,
Closed,
}
impl Lifecycle {
fn as_str(self) -> &'static str {
match self {
Self::Open => "open",
Self::Closing => "closing",
Self::Closed => "closed",
}
}
}
#[derive(Debug, Clone)]
struct Scratch {
root: PathBuf,
data: PathBuf,
backups: PathBuf,
staging: PathBuf,
}
fn make_scratch(root: Option<&Path>) -> std::io::Result<Scratch> {
let parent = root.map_or_else(std::env::temp_dir, Path::to_path_buf);
fs::create_dir_all(&parent)?;
let base = parent.join(format!("chdb-durable-{}", uuid8()));
fs::create_dir(&base)?;
let scratch = Scratch {
data: base.join("data"),
backups: base.join("backups"),
staging: base.join("staging"),
root: base,
};
for dir in [&scratch.data, &scratch.backups, &scratch.staging] {
if let Err(e) = fs::create_dir_all(dir) {
let _ = fs::remove_dir_all(&scratch.root);
return Err(e);
}
}
Ok(scratch)
}
struct State {
head: Head,
etag: String,
raw: Map<String, Value>,
wal_buffer: Vec<String>,
wal_buffer_bytes: u64,
statement_counter: u64,
committed_statements: u64,
last_flush_at: Option<SystemTime>,
last_checkpoint_at: Option<SystemTime>,
lifecycle: Lifecycle,
fenced: bool,
lease_deadline: Option<Instant>,
}
struct Core {
id: String,
read_only: bool,
backend: Arc<dyn Backend>,
engine: Mutex<Box<dyn Engine>>,
scratch: Scratch,
tuning: Tuning,
instance: String,
owner: String,
running: RunningEngine,
ops: Mutex<()>,
head_gate: Mutex<()>,
state: Mutex<State>,
stop: (Mutex<bool>, Condvar),
heartbeat: Mutex<Option<JoinHandle<()>>>,
reclaimed: AtomicBool,
}
fn lock<T>(mutex: &Mutex<T>) -> MutexGuard<'_, T> {
mutex
.lock()
.unwrap_or_else(|poisoned| poisoned.into_inner())
}
impl Core {
fn assert_usable(&self, allow_closing: bool) -> Result<()> {
let state = lock(&self.state);
let usable = state.lifecycle == Lifecycle::Open
|| (allow_closing && state.lifecycle == Lifecycle::Closing);
if !usable {
return Err(err(
Category::Closed,
format!("durable: object {} is closed", self.id),
));
}
if state.fenced {
return Err(err(
Category::LeaseFenced,
format!(
"durable: object {} lost its lease (generation {}); this handle cannot be \
used again",
self.id, state.head.lease.generation
),
));
}
Ok(())
}
fn assert_writer(&self, allow_closing: bool) -> Result<()> {
self.assert_usable(allow_closing)?;
if self.read_only {
return Err(err(
Category::ClassificationRefused,
format!(
"durable: object {} is open read-only; only READ_ONLY statements are accepted",
self.id
),
));
}
let lapsed = lock(&self.state)
.lease_deadline
.is_none_or(|deadline| Instant::now() >= deadline);
if lapsed {
self.fence();
return Err(err(
Category::LeaseFenced,
format!(
"durable: object {} could not confirm its lease before it lapsed; the writer \
has fenced itself",
self.id
),
));
}
Ok(())
}
fn fence(&self) {
lock(&self.state).fenced = true;
self.signal_stop();
}
fn signal_stop(&self) {
let (flag, condvar) = &self.stop;
*lock(flag) = true;
condvar.notify_all();
}
fn restore(&self) -> Result<()> {
let head = lock(&self.state).head.clone();
let db = head.manifest.db.clone();
let mut engine = lock(&self.engine);
match &head.manifest.base {
None => engine.create_database(&db)?,
Some(base) => {
let mut reader = self.backend.open_reader(&base.key)?.ok_or_else(|| {
err(
Category::Corrupt,
format!(
"durable: manifest names base {}, which is not present at {}",
base.key,
self.backend.describe()
),
)
})?;
let staged = self.scratch.staging.join(format!("base-{}.part", uuid8()));
let archive = self
.scratch
.backups
.join(format!("base-{}.tar.gz", uuid8()));
stream_to_verified_file(&mut reader, base, &staged, &archive, "base checkpoint")?;
drop(reader);
engine.restore_database(&db, &archive).map_err(|cause| {
err(
Category::EngineIncompatible,
format!(
"durable: restoring base {} into {db:?} failed on chdb {}. The \
archive was produced by {} and the object declares min_reader {} at \
archive format {}, so this restore was expected to be supported: \
{cause}",
base.key,
self.running.version,
head.engine.version,
head.engine.min_reader,
head.engine.backup_format
),
)
.with_bounds(head.engine.version.clone(), self.running.version.clone())
})?;
let _ = fs::remove_file(&archive);
}
}
engine.use_database(&db)?;
for reference in &head.manifest.wal {
let segment = self.backend.get_bytes(&reference.key)?.ok_or_else(|| {
err(
Category::Corrupt,
format!(
"durable: manifest names WAL segment {}, which is not present at {}",
reference.key,
self.backend.describe()
),
)
})?;
assert_digest(reference, &digest_of(&segment), "WAL segment")?;
for sql in decode_segment(&segment, &reference.key)? {
engine.run(&sql)?;
}
}
Ok(())
}
fn query(&self, sql: &str, format: OutputFormat) -> Result<Vec<u8>> {
let _ops = lock(&self.ops);
if lock(&self.state).lifecycle != Lifecycle::Open {
return Err(err(
Category::Closed,
format!("durable: object {} is closed", self.id),
));
}
let database = self.database();
let mut engine = lock(&self.engine);
let analysis = engine.analyze(sql, &database)?;
assert_query_allowed(&analysis)?;
engine.query(sql, format)
}
fn execute(&self, sql: &str) -> Result<WriteTicket> {
let _ops = lock(&self.ops);
self.assert_writer(false)?;
let database = self.database();
let analysis = lock(&self.engine).analyze(sql, &database)?;
assert_execute_allowed(&analysis, &database)?;
assert_statement_within_limit(sql)?;
let line = line_bytes(sql);
let projected = lock(&self.state).wal_buffer_bytes + line;
if projected > MAX_WAL_SEGMENT_BYTES {
return Err(err(
Category::LimitExceeded,
format!(
"durable: this statement would take the unflushed buffer to {projected} \
bytes, over the {MAX_WAL_SEGMENT_BYTES}-byte WAL segment limit; flush or \
checkpoint first"
),
)
.with_limit(MAX_WAL_SEGMENT_BYTES, projected));
}
lock(&self.engine).run(sql)?;
let mut state = lock(&self.state);
state.wal_buffer.push(sql.to_string());
state.wal_buffer_bytes += line;
state.statement_counter += 1;
Ok(WriteTicket {
statement: state.statement_counter,
})
}
fn flush(&self) -> Result<Option<ObjectRef>> {
let _ops = lock(&self.ops);
self.flush_locked(false)
}
fn flush_through(&self, ticket: WriteTicket) -> Result<()> {
if ticket.statement <= lock(&self.state).committed_statements {
return Ok(());
}
let _ops = lock(&self.ops);
if ticket.statement <= lock(&self.state).committed_statements {
return Ok(());
}
self.flush_locked(false).map(|_| ())
}
fn flush_locked(&self, closing: bool) -> Result<Option<ObjectRef>> {
self.assert_writer(closing)?;
let (statements, generation, next_seq) = {
let state = lock(&self.state);
(
state.wal_buffer.clone(),
state.head.lease.generation,
state.head.manifest.seq + 1,
)
};
if statements.is_empty() {
return Ok(None);
}
let data = encode_segment(&statements)?;
let digest = digest_of(&data);
let reference = ObjectRef {
key: wal_key(generation, next_seq),
size: digest.size,
sha256: digest.sha256,
};
self.publish_bytes(&reference, &data)?;
self.assert_writer(closing)?;
let published = reference.clone();
self.commit_head(
&reference.key,
|current| {
let mut next = current.clone();
next.manifest.wal.push(published.clone());
next.manifest.seq = current.manifest.seq + 1;
next
},
|observed| observed.manifest.wal.iter().any(|w| w.key == reference.key),
false,
)?;
let mut state = lock(&self.state);
state.wal_buffer.drain(..statements.len());
state.wal_buffer_bytes = state.wal_buffer.iter().map(|sql| line_bytes(sql)).sum();
state.committed_statements += statements.len() as u64;
state.last_flush_at = Some(SystemTime::now());
Ok(Some(reference))
}
fn checkpoint(&self) -> Result<ObjectRef> {
let _ops = lock(&self.ops);
self.checkpoint_locked()
}
fn checkpoint_locked(&self) -> Result<ObjectRef> {
self.assert_writer(false)?;
let (database, generation, next_seq) = {
let state = lock(&self.state);
(
state.head.manifest.db.clone(),
state.head.lease.generation,
state.head.manifest.seq + 1,
)
};
let archive = self
.scratch
.backups
.join(format!("checkpoint-{}.tar.gz", uuid8()));
lock(&self.engine)
.backup_database(&database, &archive)
.inspect_err(|_| {
let _ = fs::remove_file(&archive);
})?;
let outcome = self.publish_checkpoint(&archive, generation, next_seq);
let _ = fs::remove_file(&archive);
let reference = outcome?;
self.assert_writer(false)?;
let covered = lock(&self.state).wal_buffer.len();
let published = reference.clone();
self.commit_head(
&reference.key,
|current| {
let mut next = current.clone();
next.manifest.base = Some(published.clone());
next.manifest.wal = Vec::new();
next.manifest.seq = current.manifest.seq + 1;
next
},
|observed| {
observed
.manifest
.base
.as_ref()
.is_some_and(|base| base.key == reference.key)
},
false,
)?;
let mut state = lock(&self.state);
state.wal_buffer.drain(..covered);
state.wal_buffer_bytes = state.wal_buffer.iter().map(|sql| line_bytes(sql)).sum();
state.committed_statements += covered as u64;
state.last_checkpoint_at = Some(SystemTime::now());
Ok(reference)
}
fn publish_checkpoint(
&self,
archive: &Path,
generation: u64,
next_seq: u64,
) -> Result<ObjectRef> {
let digest = digest_file(archive)
.map_err(|e| engine_io_err(e, "durable: cannot read the archive just written"))?;
let reference = ObjectRef {
key: checkpoint_key(generation, next_seq),
size: digest.size,
sha256: digest.sha256.clone(),
};
self.publish_file(&reference, archive, &digest)?;
Ok(reference)
}
fn publish_bytes(&self, reference: &ObjectRef, data: &[u8]) -> Result<()> {
match self.backend.put_bytes_if_absent(&reference.key, data)? {
PutOutcome::Created => Ok(()),
outcome => self.reconcile_upload(reference, outcome),
}
}
fn publish_file(&self, reference: &ObjectRef, local: &Path, digest: &Digest) -> Result<()> {
match self
.backend
.put_file_if_absent(&reference.key, local, digest)?
{
PutOutcome::Created => Ok(()),
outcome => self.reconcile_upload(reference, outcome),
}
}
fn reconcile_upload(&self, reference: &ObjectRef, outcome: PutOutcome) -> Result<()> {
let Some(mut reader) = self.backend.open_reader(&reference.key)? else {
if outcome == PutOutcome::AlreadyExists {
return Err(err(
Category::Corrupt,
format!(
"durable: {} was reported as existing but cannot be read from {}",
reference.key,
self.backend.describe()
),
));
}
return Err(err(
Category::CommitAmbiguous,
format!(
"durable: could not determine whether {} was uploaded to {}",
reference.key,
self.backend.describe()
),
)
.with_key(reference.key.clone()));
};
let observed = drain_digest(&mut reader)
.map_err(|e| backend_err(e, format!("durable: cannot re-read {}", reference.key)))?;
assert_digest(reference, &observed, "published object")
}
fn commit_head(
&self,
key: &str,
build: impl Fn(&Head) -> Head,
committed: impl Fn(&Head) -> bool,
skip_ownership_after: bool,
) -> Result<()> {
let _gate = lock(&self.head_gate);
let deadline = Instant::now() + self.tuning.commit_deadline;
let mut saw_ambiguous = false;
for attempt in 1.. {
let (current, etag, raw, generation) = {
let state = lock(&self.state);
(
state.head.clone(),
state.etag.clone(),
state.raw.clone(),
state.head.lease.generation,
)
};
let candidate = raise_compatibility_floor(build(¤t), &self.running)?;
let body = serialize_head(&candidate, Some(&raw))?;
match self.backend.replace_if_match(HEAD_KEY, &body, &etag)? {
super::backend::ReplaceOutcome::Done { etag } => {
self.adopt(candidate, etag, None);
return Ok(());
}
super::backend::ReplaceOutcome::Ambiguous => saw_ambiguous = true,
super::backend::ReplaceOutcome::NotMatched => {}
}
let fresh = read_head(&*self.backend)?.ok_or_else(|| {
err(
Category::Corrupt,
format!(
"durable: {HEAD_KEY} disappeared from {} while committing",
self.backend.describe()
),
)
})?;
let still_ours = fresh.head.lease.instance_is(&self.instance)
&& fresh.head.lease.generation == generation;
if committed(&fresh.head) && (still_ours || skip_ownership_after) {
self.adopt(fresh.head, fresh.etag, Some(fresh.raw));
return Ok(());
}
if !still_ours {
self.fence();
return Err(err(
Category::LeaseFenced,
format!(
"durable: object {} was taken over (generation {}); this writer can no \
longer commit",
self.id, fresh.head.lease.generation
),
));
}
self.adopt(fresh.head, fresh.etag, Some(fresh.raw));
if attempt >= self.tuning.max_commit_attempts || Instant::now() >= deadline {
if saw_ambiguous {
return Err(err(
Category::CommitAmbiguous,
format!(
"durable: gave up committing {key} after {attempt} attempts without \
proving the outcome"
),
)
.with_key(key.to_string()));
}
return Err(err(
Category::Timeout,
format!(
"durable: could not commit {key} within {:?} ({attempt} attempts, each \
definitively refused); nothing was committed",
self.tuning.commit_deadline
),
));
}
}
unreachable!("the attempt loop returns from inside")
}
fn adopt(&self, head: Head, etag: String, raw: Option<Map<String, Value>>) {
let mut state = lock(&self.state);
let raw = raw.or_else(|| {
serialize_head(&head, Some(&state.raw))
.ok()
.and_then(|body| parse_head(&body).ok())
.map(|(_, parsed)| parsed)
});
if let Some(raw) = raw {
state.raw = raw;
}
if !self.read_only {
if let Some(deadline) = lease_deadline_from(&head.lease, self.tuning.lease_ttl) {
state.lease_deadline = Some(deadline);
}
}
state.head = head;
state.etag = etag;
}
fn renew_lease(&self) -> Result<()> {
{
let state = lock(&self.state);
if self.read_only || state.fenced || state.lifecycle == Lifecycle::Closed {
return Ok(());
}
}
let owner = self.owner.clone();
let instance = self.instance.clone();
let ttl = self.tuning.lease_ttl.as_secs_f64();
self.commit_head(
HEAD_KEY,
move |current| {
let mut next = current.clone();
next.lease.owner = Some(owner.clone());
next.lease.instance = Some(instance.clone());
next.lease.expires_at = Some(now_seconds() + ttl);
next
},
|observed| observed.lease.instance_is(&self.instance),
false,
)?;
let mut state = lock(&self.state);
state.lease_deadline = lease_deadline_from(&state.head.lease, self.tuning.lease_ttl);
Ok(())
}
fn release_lease_locked(&self) -> Result<()> {
self.commit_head(
HEAD_KEY,
|current| {
let mut next = current.clone();
next.lease = Lease::released(current.lease.generation);
next
},
|observed| observed.lease.instance.is_none(),
true,
)?;
let mut state = lock(&self.state);
state.fenced = false;
state.lease_deadline = None;
Ok(())
}
fn close(&self) -> Result<()> {
{
let mut state = lock(&self.state);
if state.lifecycle != Lifecycle::Open {
return Ok(());
}
state.lifecycle = Lifecycle::Closing;
}
self.stop_heartbeat();
let _ops = lock(&self.ops);
let mut failure = None;
if !self.read_only {
let (fenced, pending) = {
let state = lock(&self.state);
(state.fenced, state.wal_buffer.len())
};
if fenced {
if pending > 0 {
failure = Some(err(
Category::LeaseFenced,
format!(
"durable: object {} lost its lease; {pending} buffered statement(s) \
were not persisted",
self.id
),
));
}
} else {
failure = self
.flush_locked(true)
.and_then(|_| self.release_lease_locked())
.err();
}
}
if let Err(e) = self.reclaim() {
failure = failure.or(Some(e));
}
lock(&self.state).lifecycle = Lifecycle::Closed;
match failure {
Some(e) => Err(e),
None => Ok(()),
}
}
fn reclaim(&self) -> Result<()> {
if self.reclaimed.swap(true, AtomicOrdering::SeqCst) {
return Ok(());
}
let closed = lock(&self.engine).close();
let removed = fs::remove_dir_all(&self.scratch.root).map_err(|e| {
backend_err(
e,
"durable: cannot remove the scratch directory".to_string(),
)
});
closed.and(removed)
}
fn stop_heartbeat(&self) {
self.signal_stop();
if let Some(handle) = lock(&self.heartbeat).take() {
let _ = handle.join();
}
}
fn database(&self) -> String {
lock(&self.state).head.manifest.db.clone()
}
fn stats(&self) -> Stats {
let state = lock(&self.state);
Stats {
id: self.id.clone(),
database: state.head.manifest.db.clone(),
read_only: self.read_only,
state: state.lifecycle.as_str(),
fenced: state.fenced,
generation: state.head.lease.generation,
owner: self.owner.clone(),
instance: self.instance.clone(),
committed_seq: state.head.manifest.seq,
base_key: state.head.manifest.base.as_ref().map(|b| b.key.clone()),
wal_segments: state.head.manifest.wal.len(),
executed_statements: state.statement_counter,
committed_statements: state.committed_statements,
pending_statements: state.wal_buffer.len(),
pending_bytes: state.wal_buffer_bytes,
last_flush_at: state.last_flush_at,
last_checkpoint_at: state.last_checkpoint_at,
}
}
}
fn lease_deadline_from(lease: &Lease, ttl: Duration) -> Option<Instant> {
let expires_at = lease.expires_at?;
let local = Instant::now() + ttl;
let remaining = expires_at - now_seconds();
if remaining <= 0.0 {
return Some(Instant::now());
}
let stored = Instant::now() + Duration::from_secs_f64(remaining);
Some(stored.min(local))
}
pub struct DurableObject {
core: Arc<Core>,
}
impl std::fmt::Debug for DurableObject {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.debug_struct("DurableObject")
.field("id", &self.core.id)
.field("read_only", &self.core.read_only)
.finish_non_exhaustive()
}
}
impl DurableObject {
pub fn id(&self) -> &str {
&self.core.id
}
pub fn read_only(&self) -> bool {
self.core.read_only
}
pub fn database(&self) -> String {
self.core.database()
}
pub fn generation(&self) -> u64 {
lock(&self.core.state).head.lease.generation
}
pub fn manifest(&self) -> Manifest {
lock(&self.core.state).head.manifest.clone()
}
pub fn scratch_path(&self) -> &Path {
&self.core.scratch.root
}
pub fn is_fenced(&self) -> bool {
lock(&self.core.state).fenced
}
pub fn stats(&self) -> Stats {
self.core.stats()
}
pub fn query(&self, sql: &str, format: OutputFormat) -> Result<Vec<u8>> {
self.core.query(sql, format)
}
pub fn execute(&self, sql: &str) -> Result<WriteTicket> {
self.core.execute(sql)
}
pub fn flush(&self) -> Result<Option<ObjectRef>> {
self.core.flush()
}
pub fn flush_through(&self, ticket: WriteTicket) -> Result<()> {
self.core.flush_through(ticket)
}
pub fn checkpoint(&self) -> Result<ObjectRef> {
self.core.checkpoint()
}
pub fn close(self) -> Result<()> {
self.core.close()
}
}
impl Drop for DurableObject {
fn drop(&mut self) {
if lock(&self.core.state).lifecycle == Lifecycle::Closed {
return;
}
self.core.stop_heartbeat();
let _ = self.core.reclaim();
lock(&self.core.state).lifecycle = Lifecycle::Closed;
}
}
pub(crate) fn open_object(
id: &str,
backend: Arc<dyn Backend>,
factory: &EngineFactory,
options: OpenOptions,
) -> Result<(DurableObject, bool)> {
let tuning = options.tuning.clone();
tuning.validate()?;
let owner = options
.owner
.clone()
.unwrap_or_else(|| format!("chdb-rust-{}", std::process::id()));
let instance = new_instance_id();
let mut engine = factory()?;
let probed = (|| -> Result<(RunningEngine, Option<HeadSnapshot>)> {
let running = RunningEngine {
version: engine.version()?,
backup_format: engine.backup_format()?,
};
let existing = read_head(&*backend)?;
match &existing {
Some(snapshot) => {
assert_readable(&snapshot.head)?;
assert_engine_compatible(&snapshot.head, &running)?;
if !options.read_only {
assert_writable(&snapshot.head)?;
}
}
None if options.read_only || options.existing_only => {
return Err(err(
Category::NotFound,
format!(
"durable: object {id} does not exist at {}",
backend.describe()
),
))
}
None => {}
}
Ok((running, existing))
})();
let (running, existing) = match probed {
Ok(value) => value,
Err(e) => {
let _ = engine.close();
return Err(e);
}
};
let existed = existing.is_some();
let snapshot = if options.read_only {
existing.expect("a read-only open of a missing object was refused above")
} else {
let database = options.database.clone().unwrap_or_else(|| "default".into());
let taken = match existing {
Some(existing) => acquire_lease(
&*backend,
existing,
LeaseParams {
instance: &instance,
owner: &owner,
tuning: &tuning,
force: options.force,
},
),
None => create_cold(
&*backend,
ColdParams {
id,
database: &database,
running: &running,
instance: &instance,
owner: &owner,
tuning: &tuning,
},
),
};
match taken {
Ok(snapshot) => snapshot,
Err(e) => {
let _ = engine.close();
return Err(e);
}
}
};
let scratch = match make_scratch(options.scratch_root.as_deref()) {
Ok(scratch) => scratch,
Err(e) => {
let _ = engine.close();
if !options.read_only {
let _ = release_lease(&*backend, &instance);
}
return Err(backend_err(
e,
"durable: cannot create a scratch directory".to_string(),
));
}
};
if let Err(e) = engine.start(EngineStartOptions {
data_path: scratch.data.clone(),
backups_allowed_path: scratch.backups.clone(),
}) {
let _ = engine.close();
let _ = fs::remove_dir_all(&scratch.root);
if !options.read_only {
let _ = release_lease(&*backend, &instance);
}
return Err(e);
}
let lease_deadline = if options.read_only {
None
} else {
lease_deadline_from(&snapshot.head.lease, tuning.lease_ttl)
};
let core = Arc::new(Core {
id: id.to_string(),
read_only: options.read_only,
backend,
engine: Mutex::new(engine),
scratch,
tuning,
instance,
owner,
running,
ops: Mutex::new(()),
head_gate: Mutex::new(()),
state: Mutex::new(State {
head: snapshot.head,
etag: snapshot.etag,
raw: snapshot.raw,
wal_buffer: Vec::new(),
wal_buffer_bytes: 0,
statement_counter: 0,
committed_statements: 0,
last_flush_at: None,
last_checkpoint_at: None,
lifecycle: Lifecycle::Open,
fenced: false,
lease_deadline,
}),
stop: (Mutex::new(false), Condvar::new()),
heartbeat: Mutex::new(None),
reclaimed: AtomicBool::new(false),
});
let opened = core.restore().and_then(|()| {
if core.read_only {
return Ok(());
}
core.renew_lease()
});
if let Err(e) = opened {
let _ = core.reclaim();
lock(&core.state).lifecycle = Lifecycle::Closed;
if !core.read_only {
let _ = release_lease(&*core.backend, &core.instance);
}
return Err(e);
}
if !core.read_only {
start_heartbeat(&core);
}
Ok((DurableObject { core }, existed))
}
fn start_heartbeat(core: &Arc<Core>) {
let worker = Arc::clone(core);
let interval = core.tuning.heartbeat_interval;
let handle = std::thread::Builder::new()
.name(format!("chdb-durable-lease-{}", core.id))
.spawn(move || loop {
let (flag, condvar) = &worker.stop;
let stopped = {
let guard = lock(flag);
let (guard, _timeout) = condvar
.wait_timeout(guard, interval)
.unwrap_or_else(|poisoned| poisoned.into_inner());
*guard
};
if stopped {
return;
}
{
let state = lock(&worker.state);
if state.lifecycle != Lifecycle::Open || state.fenced {
return;
}
}
if let Err(e) = worker.renew_lease() {
if e.category() == Category::LeaseFenced {
return; }
}
});
match handle {
Ok(handle) => *lock(&core.heartbeat) = Some(handle),
Err(_) => core.fence(),
}
}
#[cfg(test)]
mod test {
use super::*;
#[test]
fn a_heartbeat_slower_than_a_third_of_the_ttl_is_refused() {
let tuning = Tuning {
lease_ttl: Duration::from_secs(30),
heartbeat_interval: Duration::from_secs(11),
..Tuning::default()
};
let error = tuning.validate().expect_err("no room for a lost renewal");
assert!(error.to_string().contains("at most a third"), "{error}");
assert!(Tuning::default().validate().is_ok());
}
#[test]
fn a_zero_duration_would_put_an_expiry_in_the_past() {
for tuning in [
Tuning {
lease_ttl: Duration::ZERO,
..Tuning::default()
},
Tuning {
heartbeat_interval: Duration::ZERO,
..Tuning::default()
},
Tuning {
commit_deadline: Duration::ZERO,
..Tuning::default()
},
Tuning {
max_commit_attempts: 0,
..Tuning::default()
},
] {
assert!(tuning.validate().is_err(), "{tuning:?}");
}
}
#[test]
fn a_lease_deadline_is_the_earlier_of_the_clock_and_the_document() {
let ttl = Duration::from_secs(30);
let generous = Lease {
generation: 1,
owner: Some("w".into()),
instance: Some("i".into()),
expires_at: Some(now_seconds() + 3600.0),
};
let deadline = lease_deadline_from(&generous, ttl).expect("a held lease has a deadline");
assert!(deadline <= Instant::now() + ttl + Duration::from_secs(1));
let lapsed = Lease {
expires_at: Some(now_seconds() - 1.0),
..generous
};
assert!(lease_deadline_from(&lapsed, ttl).expect("still a deadline") <= Instant::now());
assert!(lease_deadline_from(&Lease::released(1), ttl).is_none());
}
}