use std::collections::VecDeque;
use std::sync::atomic::{AtomicBool, AtomicU8, AtomicUsize, Ordering};
use std::sync::{Arc, Condvar, Mutex, MutexGuard, PoisonError, Weak};
use std::thread::JoinHandle;
use store_io_core::error::{Error, FirstCause, Op};
use store_io_platform::{Platform, Queue, Wake};
use crate::store::{Inner, Store};
pub(crate) type Flushed = Result<u64, Error>;
const NOWHERE: u8 = 0;
const IN_QUEUE: u8 = 1;
const ON_INBOX: u8 = 2;
pub(crate) struct Inbox<W> {
posted: Mutex<VecDeque<(u64, Flushed)>>,
count: AtomicUsize,
ready: Condvar,
blocked: AtomicU8,
due: AtomicBool,
woken: AtomicBool,
queue: W,
}
fn lock<T>(m: &Mutex<T>) -> MutexGuard<'_, T> {
m.lock().unwrap_or_else(PoisonError::into_inner)
}
impl<W: Wake> Inbox<W> {
pub(crate) fn new(depth: usize, queue: W) -> Self {
Self {
posted: Mutex::new(VecDeque::with_capacity(depth)),
count: AtomicUsize::new(0),
ready: Condvar::new(),
blocked: AtomicU8::new(NOWHERE),
due: AtomicBool::new(false),
woken: AtomicBool::new(false),
queue,
}
}
fn post(&self, qtag: u64, outcome: Flushed) {
lock(&self.posted).push_back((qtag, outcome));
let _was = self.count.fetch_add(1, Ordering::SeqCst);
self.due.store(true, Ordering::Relaxed);
}
fn signal(&self) {
if !self.due.swap(false, Ordering::Relaxed) {
return;
}
match self.blocked.load(Ordering::SeqCst) {
IN_QUEUE => self.queue.wake(),
ON_INBOX => {
let _sync = lock(&self.posted);
self.ready.notify_one();
}
_ => {}
}
}
pub(crate) fn pop(&self) -> Option<(u64, Flushed)> {
if self.count.load(Ordering::SeqCst) == 0 {
return None;
}
let got = lock(&self.posted).pop_front();
if got.is_some() {
let _was = self.count.fetch_sub(1, Ordering::SeqCst);
}
got
}
pub(crate) fn block_in_queue(&self) -> bool {
self.blocked.store(IN_QUEUE, Ordering::SeqCst);
if self.count.load(Ordering::SeqCst) > 0 || self.woken.load(Ordering::SeqCst) {
self.unblock();
return false;
}
true
}
pub(crate) fn unblock(&self) {
self.blocked.store(NOWHERE, Ordering::SeqCst);
}
pub(crate) fn wait(&self, wakes: bool) {
let mut posted = lock(&self.posted);
while posted.is_empty() && !(wakes && self.woken.load(Ordering::SeqCst)) {
self.blocked.store(ON_INBOX, Ordering::SeqCst);
posted = self
.ready
.wait(posted)
.unwrap_or_else(PoisonError::into_inner);
}
self.blocked.store(NOWHERE, Ordering::SeqCst);
}
pub(crate) fn take_woken(&self) -> bool {
self.woken.swap(false, Ordering::SeqCst)
}
fn wake(&self) {
self.woken.store(true, Ordering::SeqCst);
self.queue.wake();
let _sync = lock(&self.posted);
self.ready.notify_one();
}
}
pub struct SubmitterWaker<W>(Arc<Inbox<W>>);
impl<W> SubmitterWaker<W> {
pub(crate) fn new(inbox: &Arc<Inbox<W>>) -> Self {
Self(Arc::clone(inbox))
}
}
impl<W> Clone for SubmitterWaker<W> {
fn clone(&self) -> Self {
Self(Arc::clone(&self.0))
}
}
impl<W> core::fmt::Debug for SubmitterWaker<W> {
fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result {
f.write_str("SubmitterWaker")
}
}
impl<W: Wake> Wake for SubmitterWaker<W> {
fn wake(&self) {
self.0.wake();
}
}
impl<P: Platform> Store<P>
where
P::Queue: Send,
{
pub(crate) fn flusher(&self) -> Option<&Flusher<P>> {
self.inner
.flusher
.get_or_init(|| Flusher::start(Arc::downgrade(&self.inner)))
.as_ref()
}
}
struct Request<W> {
need: u64,
qtag: u64,
inbox: Arc<Inbox<W>>,
outcome: Option<Flushed>,
}
struct State<W> {
requests: Vec<Request<W>>,
stop: bool,
idle: bool,
}
struct Shared<W> {
state: Mutex<State<W>>,
wake: Condvar,
}
pub(crate) type WakerOf<P> = <<P as Platform>::Queue as Queue>::Waker;
pub(crate) struct Flusher<P: Platform> {
shared: Arc<Shared<WakerOf<P>>>,
thread: Option<JoinHandle<()>>,
}
impl<P: Platform> Flusher<P>
where
P::Queue: Send,
{
pub(crate) fn start(store: Weak<Inner<P>>) -> Option<Self> {
let shared = Arc::new(Shared {
state: Mutex::new(State {
requests: Vec::new(),
stop: false,
idle: false,
}),
wake: Condvar::new(),
});
let worker = Arc::clone(&shared);
let thread = std::thread::Builder::new()
.name("store-io-flush".into())
.spawn(move || run(&store, &worker))
.ok()?;
Some(Self {
shared,
thread: Some(thread),
})
}
pub(crate) fn request(&self, need: u64, qtag: u64, inbox: &Arc<Inbox<WakerOf<P>>>) -> bool {
let mut st = lock(&self.shared.state);
if st.stop {
return false;
}
st.requests.push(Request {
need,
qtag,
inbox: Arc::clone(inbox),
outcome: None,
});
let idle = st.idle;
drop(st);
if idle {
self.shared.wake.notify_one();
}
true
}
}
impl<P: Platform> Drop for Flusher<P> {
fn drop(&mut self) {
lock(&self.shared.state).stop = true;
self.shared.wake.notify_one();
if let Some(t) = self.thread.take() {
if t.thread().id() != std::thread::current().id() {
let _ended = t.join();
}
}
}
}
fn run<P: Platform>(store: &Weak<Inner<P>>, shared: &Shared<WakerOf<P>>)
where
P::Queue: Send,
{
let mut batch: Vec<Request<WakerOf<P>>> = Vec::new();
let mut sent: Vec<Request<WakerOf<P>>> = Vec::new();
loop {
{
let mut st = lock(&shared.state);
while st.requests.is_empty() && !st.stop {
st.idle = true;
st = shared.wake.wait(st).unwrap_or_else(PoisonError::into_inner);
}
st.idle = false;
if st.requests.is_empty() {
return;
}
std::mem::swap(&mut st.requests, &mut batch);
}
let mut guard = Unwind {
store,
shared,
batch: &mut batch,
sent: &mut sent,
answered: false,
};
let need = guard.batch.iter().map(|r| r.need).max().unwrap_or(0);
let Some(inner) = store.upgrade() else {
return;
};
let held = Store { inner };
let outcome = held.barrier_to(need).map(|()| need);
guard.decide(&held, &outcome);
drop(held);
guard.answer();
}
}
struct Unwind<'a, P: Platform>
where
P::Queue: Send,
{
store: &'a Weak<Inner<P>>,
shared: &'a Shared<WakerOf<P>>,
batch: &'a mut Vec<Request<WakerOf<P>>>,
sent: &'a mut Vec<Request<WakerOf<P>>>,
answered: bool,
}
impl<P: Platform> Unwind<'_, P>
where
P::Queue: Send,
{
fn decide(&mut self, store: &Store<P>, outcome: &Flushed) {
let domain = &store.inner.domain;
for r in self.batch.iter_mut() {
r.outcome = Some(match outcome {
Err(_) if domain.join_if_done(r.need) => Ok(r.need),
other => other.clone(),
});
}
}
fn answer(&mut self) {
while let Some(mut r) = self.batch.pop() {
let outcome = r.outcome.take().unwrap_or(Err(Error::Poisoned {
first: FirstCause {
op: Some(Op::FlushData),
raw: None,
},
}));
r.inbox.post(r.qtag, outcome);
self.sent.push(r);
}
for r in self.sent.drain(..) {
r.inbox.signal();
}
self.answered = true;
}
}
impl<P: Platform> Drop for Unwind<'_, P>
where
P::Queue: Send,
{
fn drop(&mut self) {
if self.answered {
return;
}
let first = FirstCause {
op: Some(Op::FlushData),
raw: None,
};
if let Some(inner) = self.store.upgrade() {
let store = Store { inner };
store.inner.domain.poison(first);
store.wake_frontiers();
}
let late = {
let mut st = lock(&self.shared.state);
st.stop = true;
std::mem::take(&mut st.requests)
};
for r in self.batch.drain(..).chain(late) {
r.inbox.post(r.qtag, Err(Error::Poisoned { first }));
r.inbox.signal();
}
for r in self.sent.drain(..) {
r.inbox.signal();
}
}
}
#[cfg(test)]
mod tests {
#![allow(clippy::unwrap_used, unused_results, unused_must_use)]
use std::path::Path;
use std::sync::Arc;
use store_io_platform::{Platform, Queue, QueueConfig};
use store_io_sim::{SimConfig, SimPlatform};
use std::sync::atomic::{AtomicUsize, Ordering};
use store_io_platform::Wake;
use super::{Inbox, Request, lock};
use crate::{Store, StoreOptions};
#[derive(Clone, Default)]
struct Count(Arc<AtomicUsize>);
impl Wake for Count {
fn wake(&self) {
let _was = self.0.fetch_add(1, Ordering::SeqCst);
}
}
impl Count {
fn get(&self) -> usize {
self.0.load(Ordering::SeqCst)
}
}
#[test]
fn test_the_inbox_wakes_only_a_blocked_queue_once() {
let n = Count::default();
let inbox = Inbox::new(4, n.clone());
inbox.post(1, Ok(1));
inbox.signal();
assert_eq!(n.get(), 0);
assert!(!inbox.block_in_queue());
assert!(inbox.pop().is_some());
assert!(inbox.block_in_queue());
inbox.post(2, Ok(2));
inbox.post(3, Ok(3));
inbox.signal();
inbox.signal();
assert_eq!(n.get(), 1);
inbox.unblock();
while inbox.pop().is_some() {}
inbox.wake();
assert_eq!(n.get(), 2);
assert!(!inbox.block_in_queue());
assert!(inbox.take_woken());
assert!(inbox.block_in_queue());
inbox.unblock();
}
#[test]
fn test_a_batch_flushes_for_its_largest_need() {
let p = SimPlatform::new(SimConfig::volatile(1200));
let s = Store::create(p.clone(), Path::new("/db"), StoreOptions::default()).unwrap();
let wal = s.provision_append_region("wal", 1 << 20).unwrap();
let t1 = wal.append(b"a").unwrap();
wal.sync_through(&t1).unwrap();
let t2 = wal.append(b"b").unwrap();
assert!(t2.need > t1.need);
let inbox = Arc::new(Inbox::new(
4,
p.queue(QueueConfig::default()).unwrap().waker(),
));
let f = s.flusher().unwrap();
{
let mut st = lock(&f.shared.state);
for (need, qtag) in [(t1.need, 1), (t2.need, 2)] {
st.requests.push(Request {
need,
qtag,
inbox: Arc::clone(&inbox),
outcome: None,
});
}
}
f.shared.wake.notify_one();
let mut got = Vec::new();
while got.len() < 2 {
inbox.wait(false);
while let Some(x) = inbox.pop() {
got.push(x);
}
}
got.sort_by_key(|g| g.0);
assert_eq!(
got[1],
(2, Ok(t2.need)),
"the batch flushed for a smaller need"
);
assert!(got[0].1.is_ok());
}
}