use std::collections::{BTreeSet, HashMap};
use std::ops::Bound;
use crate::bucket::BucketName;
use crate::db::Db;
use crate::error::{Error, Result};
use crate::snapshot::Snapshot;
use crate::types::{KeyRange, KeyValue, ReadVersion, Value};
const RESERVED: &str = "\u{1}trine-branch\u{1}";
const SEP: char = '\u{1}';
fn registry_bucket() -> String {
format!("{RESERVED}registry")
}
fn data_bucket(branch: &str, user_bucket: &str) -> String {
format!("{RESERVED}{branch}{SEP}{user_bucket}")
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct BranchInfo {
fork: ReadVersion,
parent: Option<String>,
}
impl BranchInfo {
#[must_use]
pub const fn fork(&self) -> ReadVersion {
self.fork
}
#[must_use]
pub fn parent(&self) -> Option<&str> {
self.parent.as_deref()
}
}
struct RegistryEntry {
fork: ReadVersion,
parent: Option<String>,
written_buckets: BTreeSet<String>,
}
fn put_str(out: &mut Vec<u8>, value: &str) {
let len = u32::try_from(value.len()).unwrap_or(u32::MAX);
out.extend_from_slice(&len.to_le_bytes());
out.extend_from_slice(value.as_bytes());
}
impl RegistryEntry {
fn encode(&self) -> Vec<u8> {
let mut out = Vec::new();
out.extend_from_slice(&self.fork.as_u64().to_le_bytes());
let count = u32::try_from(self.written_buckets.len()).unwrap_or(u32::MAX);
out.extend_from_slice(&count.to_le_bytes());
for bucket in &self.written_buckets {
put_str(&mut out, bucket);
}
match &self.parent {
Some(parent) => {
out.push(1);
put_str(&mut out, parent);
}
None => out.push(0),
}
out
}
fn decode(bytes: &[u8]) -> Result<Self> {
let corrupt = || Error::Corruption {
message: "malformed branch registry entry".to_owned(),
};
let mut pos = 0usize;
let take_u32 = |pos: &mut usize| -> Result<u32> {
let raw: [u8; 4] = bytes
.get(*pos..*pos + 4)
.ok_or_else(corrupt)?
.try_into()
.expect("4");
*pos += 4;
Ok(u32::from_le_bytes(raw))
};
let fork_bytes: [u8; 8] = bytes.get(0..8).ok_or_else(corrupt)?.try_into().expect("8");
let fork = ReadVersion::from_u64(u64::from_le_bytes(fork_bytes));
pos += 8;
let count = take_u32(&mut pos)?;
let mut written_buckets = BTreeSet::new();
for _ in 0..count {
let len = take_u32(&mut pos)? as usize;
let name = bytes.get(pos..pos + len).ok_or_else(corrupt)?;
pos += len;
written_buckets.insert(String::from_utf8(name.to_vec()).map_err(|_| corrupt())?);
}
let parent = match bytes.get(pos) {
None | Some(&0) => None,
Some(&1) => {
pos += 1;
let len = take_u32(&mut pos)? as usize;
let name = bytes.get(pos..pos + len).ok_or_else(corrupt)?;
Some(String::from_utf8(name.to_vec()).map_err(|_| corrupt())?)
}
Some(_) => return Err(corrupt()),
};
Ok(Self {
fork,
parent,
written_buckets,
})
}
}
const TAG_PRESENT: u8 = 0;
const TAG_TOMBSTONE: u8 = 1;
fn encode_present(value: &[u8]) -> Vec<u8> {
let mut out = Vec::with_capacity(value.len() + 1);
out.push(TAG_PRESENT);
out.extend_from_slice(value);
out
}
enum OverlayWrite {
Put(Value),
Delete,
}
struct DurableLayer {
name: String,
written: BTreeSet<String>,
at: Option<Snapshot>,
}
struct DurableState {
chain: Vec<DurableLayer>,
leaf_fork: ReadVersion,
leaf_parent: Option<String>,
}
enum Backing {
Ephemeral(HashMap<(BucketName, Vec<u8>), OverlayWrite>),
Durable(DurableState),
}
pub struct Branch<'db> {
db: &'db Db,
fork: Snapshot,
backing: Backing,
}
impl<'db> Branch<'db> {
fn ephemeral(db: &'db Db, fork: Snapshot) -> Self {
Self {
db,
fork,
backing: Backing::Ephemeral(HashMap::new()),
}
}
fn durable(db: &'db Db, fork: Snapshot, state: DurableState) -> Self {
Self {
db,
fork,
backing: Backing::Durable(state),
}
}
#[must_use]
pub const fn fork_version(&self) -> ReadVersion {
self.fork.read_version()
}
#[must_use]
pub const fn is_durable(&self) -> bool {
matches!(self.backing, Backing::Durable(_))
}
pub fn get(&self, bucket: impl Into<BucketName>, key: &[u8]) -> Result<Option<Value>> {
let bucket = bucket.into();
match &self.backing {
Backing::Ephemeral(overlay) => match overlay.get(&(bucket.clone(), key.to_vec())) {
Some(OverlayWrite::Put(value)) => return Ok(Some(value.clone())),
Some(OverlayWrite::Delete) => return Ok(None),
None => {}
},
Backing::Durable(state) => {
for layer in &state.chain {
if !layer.written.contains(bucket.as_str()) {
continue;
}
let data = self
.db
.bucket_sync(data_bucket(&layer.name, bucket.as_str()))?;
let raw = match &layer.at {
None => data.get_sync(key)?,
Some(at) => data.get_at_sync(at, key)?,
};
if let Some(raw) = raw {
return Ok(decode_branch_value(&raw));
}
}
}
}
self.parent_get(&bucket, key)
}
fn parent_get(&self, bucket: &BucketName, key: &[u8]) -> Result<Option<Value>> {
self.db
.bucket_sync(bucket.clone())?
.get_at_sync(&self.fork, key)
}
pub fn put(
&mut self,
bucket: impl Into<BucketName>,
key: impl Into<Vec<u8>>,
value: impl Into<Value>,
) -> Result<()> {
self.write(bucket.into(), key.into(), OverlayWrite::Put(value.into()))
}
pub fn delete(&mut self, bucket: impl Into<BucketName>, key: impl Into<Vec<u8>>) -> Result<()> {
self.write(bucket.into(), key.into(), OverlayWrite::Delete)
}
fn write(&mut self, bucket: BucketName, key: Vec<u8>, write: OverlayWrite) -> Result<()> {
let db = self.db;
match &mut self.backing {
Backing::Ephemeral(overlay) => {
overlay.insert((bucket, key), write);
Ok(())
}
Backing::Durable(state) => {
let leaf_fork = state.leaf_fork;
let leaf_parent = state.leaf_parent.clone();
let leaf = &mut state.chain[0];
let data = db.bucket_sync(data_bucket(&leaf.name, bucket.as_str()))?;
match write {
OverlayWrite::Put(value) => data.put_sync(key, encode_present(&value))?,
OverlayWrite::Delete => data.put_sync(key, vec![TAG_TOMBSTONE])?,
}
if leaf.written.insert(bucket.as_str().to_owned()) {
persist_registry(
db,
&leaf.name,
&RegistryEntry {
fork: leaf_fork,
parent: leaf_parent,
written_buckets: leaf.written.clone(),
},
)?;
}
Ok(())
}
}
}
pub fn range(&self, bucket: impl Into<BucketName>, range: &KeyRange) -> Result<BranchRange> {
let bucket = bucket.into();
let mut sources: Vec<MergeSource> = Vec::new();
match &self.backing {
Backing::Ephemeral(overlay) => {
let mut entries: Vec<(Vec<u8>, Option<Value>)> = overlay
.iter()
.filter(|((overlay_bucket, key), _)| {
overlay_bucket == &bucket && range_contains(range, key)
})
.map(|((_, key), write)| {
let value = match write {
OverlayWrite::Put(value) => Some(value.clone()),
OverlayWrite::Delete => None,
};
(key.clone(), value)
})
.collect();
entries.sort_by(|a, b| a.0.cmp(&b.0));
sources.push(MergeSource::new(Box::new(entries.into_iter().map(Ok))));
}
Backing::Durable(state) => {
for layer in &state.chain {
if !layer.written.contains(bucket.as_str()) {
continue;
}
let data = self
.db
.bucket_sync(data_bucket(&layer.name, bucket.as_str()))?;
let rows = match &layer.at {
None => data.range_sync(range)?,
Some(at) => data.range_at_sync(at, range)?,
};
sources.push(MergeSource::new(Box::new(rows.map(|row| {
row.map(|kv| {
let value = decode_branch_value(&kv.value);
(kv.key, value)
})
}))));
}
}
}
let root = self
.db
.bucket_sync(bucket.clone())?
.range_at_sync(&self.fork, range)?;
sources.push(MergeSource::new(Box::new(
root.map(|row| row.map(|kv| (kv.key, Some(kv.value)))),
)));
Ok(BranchRange { sources })
}
}
type MergeRow = Result<(Vec<u8>, Option<Value>)>;
struct MergeSource {
iter: Box<dyn Iterator<Item = MergeRow>>,
head: Option<MergeRow>,
}
impl MergeSource {
fn new(mut iter: Box<dyn Iterator<Item = MergeRow>>) -> Self {
let head = iter.next();
Self { iter, head }
}
fn key(&self) -> Option<&[u8]> {
match &self.head {
Some(Ok((key, _))) => Some(key),
_ => None,
}
}
fn is_err(&self) -> bool {
matches!(&self.head, Some(Err(_)))
}
fn take(&mut self) -> Option<MergeRow> {
let row = self.head.take();
self.head = self.iter.next();
row
}
}
pub struct BranchRange {
sources: Vec<MergeSource>,
}
impl Iterator for BranchRange {
type Item = Result<KeyValue>;
fn next(&mut self) -> Option<Self::Item> {
loop {
for source in &mut self.sources {
if source.is_err() {
if let Some(Err(error)) = source.take() {
return Some(Err(error));
}
unreachable!("is_err guarantees an error head");
}
}
let mut smallest: Option<&[u8]> = None;
for source in &self.sources {
if let Some(key) = source.key() {
let replace = match smallest {
None => true,
Some(current) => key < current,
};
if replace {
smallest = Some(key);
}
}
}
let key = smallest?.to_vec();
let mut chosen: Option<Option<Value>> = None;
for source in &mut self.sources {
if source.key() == Some(key.as_slice()) {
if let Some(Ok((_, value))) = source.take() {
if chosen.is_none() {
chosen = Some(value);
}
}
}
}
if let Some(Some(value)) = chosen {
return Some(Ok(KeyValue::new(key, value)));
}
}
}
}
fn range_contains(range: &KeyRange, key: &[u8]) -> bool {
let after_start = match &range.start {
Bound::Unbounded => true,
Bound::Included(start) => key >= start.as_slice(),
Bound::Excluded(start) => key > start.as_slice(),
};
let before_end = match &range.end {
Bound::Unbounded => true,
Bound::Included(end) => key <= end.as_slice(),
Bound::Excluded(end) => key < end.as_slice(),
};
after_start && before_end
}
fn decode_branch_value(raw: &[u8]) -> Option<Value> {
match raw.first() {
Some(&TAG_PRESENT) => Some(raw[1..].to_vec()),
_ => None,
}
}
fn persist_registry(db: &Db, name: &str, entry: &RegistryEntry) -> Result<()> {
db.bucket_sync(registry_bucket())?
.put_sync(name.as_bytes().to_vec(), entry.encode())
}
async fn persist_registry_async(db: &Db, name: &str, entry: &RegistryEntry) -> Result<()> {
db.bucket(registry_bucket())
.await?
.put(name.as_bytes().to_vec(), entry.encode())
.await
}
async fn list_branches_async(db: &Db) -> Result<Vec<String>> {
let registry = db.bucket(registry_bucket()).await?;
let mut names = Vec::new();
for row in registry.range(&KeyRange::all()).await? {
let row = row?;
names.push(String::from_utf8(row.key).map_err(|_| Error::Corruption {
message: "branch registry holds a non-utf8 name".to_owned(),
})?);
}
Ok(names)
}
fn fork_checkpoint(branch: &str) -> String {
format!("{RESERVED}fork{SEP}{branch}")
}
impl Db {
pub fn branch_at(&self, version: ReadVersion) -> Result<Branch<'_>> {
Ok(Branch::ephemeral(self, self.snapshot_at(version)?))
}
pub fn branch_from_latest(&self) -> Result<Branch<'_>> {
self.branch_at(self.latest_read_version())
}
pub fn create_branch(&self, name: &str, from: ReadVersion) -> Result<()> {
if let Some(existing) = self.read_registry(name)? {
if existing.fork == from {
return Ok(());
}
return Err(Error::invalid_options(
"branch already exists with a different fork version",
));
}
match self.create_checkpoint_at_sync(&fork_checkpoint(name), from) {
Ok(()) | Err(Error::CheckpointAlreadyExists { .. }) => {}
Err(error) => return Err(error),
}
persist_registry(
self,
name,
&RegistryEntry {
fork: from,
parent: None,
written_buckets: BTreeSet::new(),
},
)
}
pub async fn create_branch_at(&self, name: &str, from: ReadVersion) -> Result<()> {
if let Some(existing) = self.read_registry_async(name).await? {
if existing.fork == from {
return Ok(());
}
return Err(Error::invalid_options(
"branch already exists with a different fork version",
));
}
match self
.create_checkpoint_at(&fork_checkpoint(name), from)
.await
{
Ok(()) | Err(Error::CheckpointAlreadyExists { .. }) => {}
Err(error) => return Err(error),
}
persist_registry_async(
self,
name,
&RegistryEntry {
fork: from,
parent: None,
written_buckets: BTreeSet::new(),
},
)
.await
}
pub fn create_branch_from(&self, name: &str, parent: &str) -> Result<()> {
if self.read_registry(parent)?.is_none() {
return Err(Error::invalid_options("parent branch does not exist"));
}
if self.read_registry(name)?.is_some() {
return Err(Error::invalid_options("branch already exists"));
}
let from = self.latest_read_version();
match self.create_checkpoint_at_sync(&fork_checkpoint(name), from) {
Ok(()) | Err(Error::CheckpointAlreadyExists { .. }) => {}
Err(error) => return Err(error),
}
persist_registry(
self,
name,
&RegistryEntry {
fork: from,
parent: Some(parent.to_owned()),
written_buckets: BTreeSet::new(),
},
)
}
pub fn open_branch(&self, name: &str) -> Result<Branch<'_>> {
let leaf = self
.read_registry(name)?
.ok_or_else(|| Error::invalid_options("no such branch"))?;
let leaf_fork = leaf.fork;
let leaf_parent = leaf.parent.clone();
let mut chain = vec![DurableLayer {
name: name.to_owned(),
written: leaf.written_buckets,
at: None,
}];
let mut child_fork = leaf.fork;
let mut parent = leaf.parent;
while let Some(parent_name) = parent {
let entry = self
.read_registry(&parent_name)?
.ok_or_else(|| Error::Corruption {
message: format!("branch {parent_name} is missing (an ancestor of {name})"),
})?;
chain.push(DurableLayer {
name: parent_name,
written: entry.written_buckets,
at: Some(self.snapshot_at(child_fork)?),
});
child_fork = entry.fork;
parent = entry.parent;
}
let root_fork = self.snapshot_at(child_fork)?;
Ok(Branch::durable(
self,
root_fork,
DurableState {
chain,
leaf_fork,
leaf_parent,
},
))
}
pub fn list_branches(&self) -> Result<Vec<String>> {
let registry = self.bucket_sync(registry_bucket())?;
let mut names = Vec::new();
for row in registry.range_sync(&KeyRange::all())? {
let row = row?;
names.push(String::from_utf8(row.key).map_err(|_| Error::Corruption {
message: "branch registry holds a non-utf8 name".to_owned(),
})?);
}
Ok(names)
}
pub fn delete_branch(&self, name: &str) -> Result<()> {
let entry = self
.read_registry(name)?
.ok_or_else(|| Error::invalid_options("no such branch"))?;
for other in self.list_branches()? {
if other == name {
continue;
}
if let Some(other_entry) = self.read_registry(&other)? {
if other_entry.parent.as_deref() == Some(name) {
return Err(Error::invalid_options(
"cannot delete a branch that still has child branches",
));
}
}
}
match self.delete_checkpoint_sync(&fork_checkpoint(name)) {
Ok(()) | Err(Error::CheckpointNotFound { .. }) => {}
Err(error) => return Err(error),
}
for user_bucket in &entry.written_buckets {
let data = data_bucket(name, user_bucket);
match self.drop_bucket_sync(data.clone()) {
Ok(()) => {}
Err(Error::UnsupportedBackend { .. }) => {
self.bucket_sync(data)?.delete_range_sync(KeyRange::all())?;
}
Err(error) => return Err(error),
}
}
self.bucket_sync(registry_bucket())?
.delete_sync(name.as_bytes().to_vec())
}
pub async fn delete_branch_async(&self, name: &str) -> Result<()> {
let entry = self
.read_registry_async(name)
.await?
.ok_or_else(|| Error::invalid_options("no such branch"))?;
for other in list_branches_async(self).await? {
if other == name {
continue;
}
if let Some(other_entry) = self.read_registry_async(&other).await? {
if other_entry.parent.as_deref() == Some(name) {
return Err(Error::invalid_options(
"cannot delete a branch that still has child branches",
));
}
}
}
match self.delete_checkpoint(&fork_checkpoint(name)).await {
Ok(()) | Err(Error::CheckpointNotFound { .. }) => {}
Err(error) => return Err(error),
}
for user_bucket in &entry.written_buckets {
let data = data_bucket(name, user_bucket);
match self.drop_bucket(data.clone()).await {
Ok(()) => {}
Err(Error::UnsupportedBackend { .. }) => {
self.bucket(data)
.await?
.delete_range(KeyRange::all())
.await?;
}
Err(error) => return Err(error),
}
}
self.bucket(registry_bucket())
.await?
.delete(name.as_bytes().to_vec())
.await
}
pub fn branch_info(&self, name: &str) -> Result<Option<BranchInfo>> {
Ok(self.read_registry(name)?.map(|entry| BranchInfo {
fork: entry.fork,
parent: entry.parent,
}))
}
fn read_registry(&self, name: &str) -> Result<Option<RegistryEntry>> {
match self
.bucket_sync(registry_bucket())?
.get_sync(name.as_bytes())?
{
Some(bytes) => Ok(Some(RegistryEntry::decode(&bytes)?)),
None => Ok(None),
}
}
async fn read_registry_async(&self, name: &str) -> Result<Option<RegistryEntry>> {
match self
.bucket(registry_bucket())
.await?
.get(name.as_bytes())
.await?
{
Some(bytes) => Ok(Some(RegistryEntry::decode(&bytes)?)),
None => Ok(None),
}
}
}
#[cfg(test)]
mod tests;