armdb 0.15.0

sharded bitcask key-value storage optimized for NVMe
Documentation
//! Active-generation commit locks and replay-debt cleanup.
use super::*;

pub(super) type Event<K> = (K, Option<Vec<u8>>, Option<Vec<u8>>);
pub(super) type EventCallback<K> = Box<dyn Fn(&Event<K>)>;
pub(super) struct SegDeferred<K> {
    pub(super) events: Vec<Event<K>>,
    pub(super) callback: Option<EventCallback<K>>,
}
impl<K> DeferredReplay for SegDeferred<K> {
    fn event_count(&self) -> usize {
        self.events.len()
    }
    fn replay_one(&mut self, local: usize) {
        if let (Some(callback), Some(event)) = (&self.callback, self.events.get(local)) {
            callback(event);
        }
    }
}

pub(super) struct Reservation {
    shard: usize,
    gsn: u64,
    applied: bool,
}

/// Drop executes cleanup while the owned generation guards still exist.
pub(super) struct SegCommitLocked<K: Key + Hash + Eq + Send + Sync + 'static> {
    pub(super) locks: GenerationLocks<K>,
    pub(super) core: Arc<SegCore<K>>,
    pub(super) view: Arc<View<K>>,
    pub(super) since_ms: Option<u64>,
    pub(super) reservations: Vec<Reservation>,
    pub(super) logged: bool,
    pub(super) cleaned: bool,
    pub(super) events: Vec<Event<K>>,
    pub(super) callback: Option<EventCallback<K>>,
    /// Stored values evaluation observed under these locks, kept only for
    /// `on_write`. Apply reuses them instead of reading older layers again
    /// after the log append, where a read error would fence the collection.
    pub(super) observed: HashMap<Vec<u8>, Option<Vec<u8>>>,
}
impl<K: Key + Hash + Eq + Send + Sync + 'static> SegCommitLocked<K> {
    fn lookup(&self, bytes: &[u8]) -> DbResult<Option<Vec<u8>>> {
        let key = checked_key::<K>(bytes)?;
        let h = key_hash(bytes, self.core.seed);
        match self.locks.lookup(h, &key)? {
            GenLookup::Live(v) => Ok(Some(v)),
            GenLookup::Tombstone => Ok(None),
            GenLookup::Absent => self.core.read_below(&self.view, h, &key, self.since_ms),
        }
    }
    /// The `on_write` old value. `None` skips the event: a best-effort
    /// callback must never turn into a post-log apply failure.
    fn event_old(&mut self, bytes: &[u8]) -> Option<Option<Vec<u8>>> {
        if let Some(old) = self.observed.remove(bytes) {
            return Some(old);
        }
        match self.lookup(bytes) {
            Ok(old) => Some(old),
            Err(e) => {
                tracing::warn!(
                    collection = self.core.meta.name,
                    error = %e,
                    "segmented on_write old value unreadable; event skipped"
                );
                None
            }
        }
    }
    fn cleanup(&mut self, fenced: &mut Vec<(usize, u64)>) {
        if self.cleaned {
            return;
        }
        let gen_ = self.locks.generation();
        let mut debt = false;
        for r in &self.reservations {
            let mut shard = gen_.engine().shards()[r.shard].lock();
            if self.logged && !r.applied {
                debt = true;
                let effective = shard.fence(r.gsn);
                if let Some(found) = fenced.iter_mut().find(|(sid, _)| *sid == r.shard) {
                    found.1 = found.1.min(effective);
                } else {
                    fenced.push((r.shard, effective));
                }
            }
            shard.clear_armed_gsn();
        }
        if debt {
            self.core
                .fence("logged segmented transaction has unapplied reservations".to_owned());
        }
        self.cleaned = true;
    }
    fn apply(&mut self, bytes: &[u8], value: Option<&[u8]>, tag: VersionTag) -> DbResult<usize> {
        let key = checked_key::<K>(bytes)?;
        let h = key_hash(bytes, self.core.seed);
        let gsn = bitcask(tag)?;
        let shard = shard_for_hash(h, self.locks.generation().shard_count());
        let pos = self
            .reservations
            .iter()
            .position(|r| r.shard == shard && r.gsn == gsn && !r.applied)
            .ok_or(DbError::Internal(
                "unreserved segmented transaction version",
            ))?;
        let old = if self.callback.is_some() {
            self.event_old(bytes)
        } else {
            None
        };
        self.locks.generation().engine().shards()[shard]
            .lock()
            .arm_gsn(gsn);
        let result = self.locks.append(h, &key, value, self.core.config.limits());
        self.locks.generation().engine().shards()[shard]
            .lock()
            .clear_armed_gsn();
        result?;
        self.reservations[pos].applied = true;
        self.core.signal.wake();
        match old {
            Some(old) if old.is_some() || value.is_some() => {
                self.events.push((key, old, value.map(<[u8]>::to_vec)));
                Ok(1)
            }
            _ => Ok(0),
        }
    }
}
impl<K: Key + Hash + Eq + Send + Sync + 'static> Drop for SegCommitLocked<K> {
    fn drop(&mut self) {
        self.cleanup(&mut Vec::new());
    }
}
impl<K: Key + Hash + Eq + Send + Sync + 'static> TxLocked for SegCommitLocked<K> {
    fn apply_domain(&self, bytes: &[u8]) -> DbResult<usize> {
        checked_key::<K>(bytes)?;
        Ok(shard_for_hash(
            key_hash(bytes, self.core.seed),
            self.locks.generation().shard_count(),
        ))
    }
    fn read(&mut self, bytes: &[u8]) -> DbResult<Option<Vec<u8>>> {
        let value = self.lookup(bytes)?;
        if self.callback.is_some() {
            // Evaluation reads storage only before the key has a pending
            // value, so the first observation is the pre-transaction state.
            self.observed
                .entry(bytes.to_vec())
                .or_insert_with(|| value.clone());
        }
        Ok(value)
    }
    fn contains(&mut self, bytes: &[u8]) -> DbResult<bool> {
        let key = checked_key::<K>(bytes)?;
        let h = key_hash(bytes, self.core.seed);
        let live = match self.locks.probe(h, &key)? {
            GenProbe::Live => true,
            GenProbe::Tombstone => false,
            GenProbe::Absent => self
                .core
                .contains_below(&self.view, h, &key, self.since_ms)?,
        };
        if self.callback.is_some() && !live {
            self.observed.entry(bytes.to_vec()).or_insert(None);
        }
        Ok(live)
    }
    fn reserve(&mut self, ops: &[PlannedOp<'_>]) -> DbResult<Vec<VersionTag>> {
        let domains = ops
            .iter()
            .map(|op| self.apply_domain(op.key))
            .collect::<DbResult<Vec<_>>>()?;
        let count = self.locks.generation().shard_count();
        let mut remaining = vec![0usize; count];
        for &s in &domains {
            remaining[s] += 1;
        }
        let mut next = vec![0u64; count];
        for (s, &n) in remaining.iter().enumerate() {
            if n > 0 {
                next[s] = self
                    .locks
                    .generation()
                    .engine()
                    .gsn()
                    .fetch_add(n as u64, Ordering::Relaxed);
            }
        }
        self.reservations.reserve(ops.len());
        Ok(domains
            .into_iter()
            .map(|shard| {
                let gsn = next[shard];
                next[shard] += 1;
                self.reservations.push(Reservation {
                    shard,
                    gsn,
                    applied: false,
                });
                VersionTag::Bitcask { gsn }
            })
            .collect())
    }
    fn apply_put(&mut self, k: &[u8], v: &[u8], t: VersionTag) -> DbResult<usize> {
        self.apply(k, Some(v), t)
    }
    fn apply_delete(&mut self, k: &[u8], t: VersionTag) -> DbResult<usize> {
        self.apply(k, None, t)
    }
    fn needs_replay(&mut self, _: &[u8], _: OpKind, _: VersionTag) -> DbResult<bool> {
        Err(DbError::Internal("commit context used for replay"))
    }
    fn mark_logged(&mut self) {
        self.logged = true;
    }
    fn abort(mut self: Box<Self>) {
        self.cleanup(&mut Vec::new());
    }
    #[cfg(test)]
    fn unapplied_reservations(&self) -> usize {
        self.reservations.iter().filter(|r| !r.applied).count()
    }
    #[cfg(test)]
    fn tx_debug_arm(&mut self, k: &[u8], tag: VersionTag) -> DbResult<()> {
        let s = self.apply_domain(k)?;
        self.locks.generation().engine().shards()[s]
            .lock()
            .arm_gsn(bitcask(tag)?);
        Ok(())
    }
    fn finish(
        mut self: Box<Self>,
        fenced: &mut Vec<(usize, u64)>,
        _: &mut crate::watch::WatchWakeBatch,
    ) -> DbResult<Box<dyn DeferredReplay>> {
        #[cfg(any(test, feature = "test-faults"))]
        if crate::seg::failpoints::finish_failed() {
            return Err(DbError::Internal("injected segmented finish failure"));
        }
        let deferred = Box::new(SegDeferred {
            events: std::mem::take(&mut self.events),
            callback: self.callback.take(),
        });
        self.cleanup(fenced);
        Ok(deferred)
    }
}