mod durable;
use std::marker::PhantomData;
use std::sync::Arc;
use store_io_core::error::{Error, ErrorContext, FirstCause, NotWrittenCause, Op, OsError};
use store_io_core::observe::Counter;
use store_io_platform::{IoBuf, IoOp, Owner, Platform, Queue, QueueConfig};
use crate::completion::{Completions, Rejected, SubmitterConfig, WriteMode};
use crate::exec::LaneMark;
use crate::flusher::{Inbox, SubmitterWaker, WakerOf};
use crate::gate::{Gate, Gated, OwnedPass};
use crate::io::{ctx, ctx_range, not_written};
use crate::lifecycle::retired_error;
use crate::optable::{Finish, Kind, Split};
use crate::queue_core::Core;
use crate::receipt::RegionPos;
use crate::region::{AppendRegion, PageRegion};
use crate::store::{LiveRegion, Store};
pub const MAX_DEPTH: u32 = 4096;
impl Gated for LiveRegion {
fn gate(&self) -> &Gate {
&self.gate
}
}
pub struct Submitter<P: Platform>
where
P::Queue: Send,
{
core: Core<P::Queue>,
inbox: Arc<Inbox<<P::Queue as Queue>::Waker>>,
id: u64,
store: Store<P>,
_thread: PhantomData<*const ()>,
}
impl<P: Platform> core::fmt::Debug for Submitter<P>
where
P::Queue: Send,
{
fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result {
f.debug_struct("Submitter")
.field("depth", &self.core.table.depth())
.field("in_flight", &self.core.table.live)
.field("ready", &self.core.table.ready.len())
.finish()
}
}
enum Req {
Write { offset: u64, buf: IoBuf },
Read { offset: u64, buf: IoBuf },
}
fn first_piece(state: &LiveRegion, start: u64, buf: &mut IoBuf) -> (u64, Option<Split>) {
let total = buf.len();
let piece = (state.map.run_left(start) as usize).min(total);
let split = (piece < total).then(|| {
let fits = buf.set_len(piece);
debug_assert!(fits, "a piece longer than its buffer");
Split { total, done: 0 }
});
(state.map.locate(start, piece as u64), split)
}
fn rejected(error: Error, buf: IoBuf) -> Rejected {
Rejected {
error,
buf: Some(buf),
}
}
macro_rules! admit {
($e:expr, $buf:ident) => {
match $e {
Ok(v) => v,
Err(error) => return Err(rejected(error, $buf)),
}
};
}
impl<P: Platform> Store<P>
where
P::Queue: Send,
{
pub fn submitter(&self, cfg: SubmitterConfig) -> Result<Submitter<P>, Error> {
let depth = cfg.depth.clamp(1, MAX_DEPTH);
let qcfg = QueueConfig {
depth,
parallel: true,
pool_threads: self.inner.opts.t3_pool_threads,
owner: Owner::Thread,
};
let inner = &*self.inner;
let queue = if inner.opts.register_file {
inner.platform.queue_registered(qcfg, &inner.file)
} else {
inner.platform.queue(qcfg)
};
let queue = queue.map_err(|raw| Error::Io {
op: Op::Queue,
raw,
ctx: ErrorContext::default(),
})?;
let n = depth as usize;
let mark = Arc::new(LaneMark::new());
let inbox = Arc::new(Inbox::new(n, queue.waker()));
let id = self.register_submitter(&mark);
Ok(Submitter {
core: Core::new(queue, n, mark),
inbox,
id,
store: self.clone(),
_thread: PhantomData,
})
}
}
impl<P: Platform> Submitter<P>
where
P::Queue: Send,
{
pub fn take_buf(&self, len: usize) -> Result<IoBuf, Error> {
if len > self.max_buf() {
return Err(not_written(
NotWrittenCause::TooLarge,
ErrorContext::default(),
));
}
self.store.take_any(len, ErrorContext::default())
}
#[must_use]
pub fn max_buf(&self) -> usize {
self.store.max_op()
}
#[must_use]
pub fn in_flight(&self) -> usize {
self.core
.table
.live
.min(self.core.queue.in_flight() + self.core.table.asked)
}
#[must_use]
pub fn waker(&self) -> SubmitterWaker<WakerOf<P>> {
SubmitterWaker::new(&self.inbox)
}
fn admit(
&self,
store: &Store<P>,
state: &Arc<LiveRegion>,
buf: &IoBuf,
) -> Result<OwnedPass<LiveRegion>, Error> {
let c = ctx(state);
if !Arc::ptr_eq(&store.inner, &self.store.inner) {
return Err(not_written(NotWrittenCause::ForeignPosition, c));
}
if !self.store.inner.pool.owns(buf) {
return Err(not_written(NotWrittenCause::ForeignBuffer, c));
}
if self.core.table.full() {
return Err(not_written(NotWrittenCause::QueueFull, c));
}
OwnedPass::enter(state).map_err(|why| retired_error(why, state))
}
pub fn append(
&mut self,
region: &AppendRegion<P>,
mut buf: IoBuf,
mode: WriteMode,
tag: u64,
) -> Result<RegionPos, Rejected> {
let state = region.state();
let c = ctx(state);
let pass = admit!(self.admit(region.store(), state, &buf), buf);
admit!(self.store.ready_for(state), buf);
let Some(f) = state.frontier.as_ref() else {
return Err(rejected(not_written(NotWrittenCause::NotReady, c), buf));
};
if !state.positioned.load(std::sync::atomic::Ordering::Acquire) {
return Err(rejected(
not_written(NotWrittenCause::NotPositioned, c),
buf,
));
}
let len = buf.len();
if len == 0 {
return Err(rejected(not_written(NotWrittenCause::Empty, c), buf));
}
let block = self.store.block() as usize;
if len.div_ceil(block) * block > buf.capacity() {
return Err(rejected(not_written(NotWrittenCause::TooLarge, c), buf));
}
let domain = &self.store.inner.domain;
let reserved = f
.reserve(len as u64)
.map_err(|why| match domain.poisoned() {
Some(first) => Error::Poisoned { first },
None => not_written(why, c),
});
let r = admit!(reserved, buf);
let held = region.store().hold_reservation(r);
let padded = (r.end - r.start) as usize;
buf.as_mut_capacity()[len..padded].fill(0);
let _fits = buf.set_len(padded);
let pos = RegionPos {
volume: self.store.inner.volume,
region: state.id,
generation: state.generation,
offset: r.start,
};
let (offset, split) = first_piece(state, r.start, &mut buf);
let req = Req::Write { offset, buf };
let durable = mode == WriteMode::Durable;
let issued = self.issue(req, tag, Kind::Append(r), pass, durable, split);
let _r = held.disarm();
match issued {
Ok(()) => Ok(pos),
Err((raw, buf)) => Err(Rejected {
error: self
.store
.unknown(Op::Write, raw, ctx_range(state, r.start, r.data_end)),
buf,
}),
}
}
pub fn write(
&mut self,
region: &PageRegion<P>,
pos: RegionPos,
mut buf: IoBuf,
mode: WriteMode,
tag: u64,
) -> Result<(), Rejected> {
let state = region.state();
let pass = admit!(self.admit(region.store(), state, &buf), buf);
admit!(self.store.ready_for(state), buf);
let end = admit!(region.check(&pos, buf.len()), buf);
let c = ctx_range(state, pos.offset, end);
let (offset, split) = first_piece(state, pos.offset, &mut buf);
let req = Req::Write { offset, buf };
let kind = Kind::Page {
start: pos.offset,
end,
};
self.issue(req, tag, kind, pass, mode == WriteMode::Durable, split)
.map_err(|(raw, buf)| Rejected {
error: Store::<P>::refused(raw, c),
buf,
})
}
pub fn read_append(
&mut self,
region: &AppendRegion<P>,
offset: u64,
buf: IoBuf,
tag: u64,
) -> Result<(), Rejected> {
self.read(region.store(), region.state(), offset, buf, tag)
}
pub fn read_page(
&mut self,
region: &PageRegion<P>,
pos: RegionPos,
buf: IoBuf,
tag: u64,
) -> Result<(), Rejected> {
let state = region.state();
let c = ctx(state);
if pos.volume != self.store.inner.volume || pos.region != state.id {
return Err(rejected(
not_written(NotWrittenCause::ForeignPosition, c),
buf,
));
}
if pos.generation != state.generation {
return Err(rejected(
not_written(NotWrittenCause::StaleGeneration, c),
buf,
));
}
self.read(region.store(), state, pos.offset, buf, tag)
}
fn read(
&mut self,
store: &Store<P>,
state: &Arc<LiveRegion>,
offset: u64,
mut buf: IoBuf,
tag: u64,
) -> Result<(), Rejected> {
let pass = admit!(self.admit(store, state, &buf), buf);
let len = buf.len() as u64;
let block = self.store.block();
let c = ctx_range(state, offset, offset.saturating_add(len));
if len == 0 {
return Err(rejected(not_written(NotWrittenCause::Empty, c), buf));
}
if offset % block != 0 || len % block != 0 {
return Err(rejected(not_written(NotWrittenCause::Misaligned, c), buf));
}
if offset.checked_add(len).is_none_or(|e| e > state.map.len()) {
return Err(rejected(not_written(NotWrittenCause::OutOfBounds, c), buf));
}
let (at, split) = first_piece(state, offset, &mut buf);
let req = Req::Read { offset: at, buf };
let kind = Kind::Read {
start: offset,
end: offset + len,
};
self.issue(req, tag, kind, pass, false, split)
.map_err(|(raw, buf)| Rejected {
error: match raw {
Some(raw) => Error::Io {
op: Op::Read,
raw,
ctx: c,
},
None => not_written(NotWrittenCause::QueueFull, c),
},
buf,
})
}
fn issue(
&mut self,
req: Req,
tag: u64,
kind: Kind,
pass: OwnedPass<LiveRegion>,
durable: bool,
split: Option<Split>,
) -> Result<(), (Option<OsError>, Option<IoBuf>)> {
let whole = |mut buf: IoBuf| {
if let Some(s) = split {
let fits = buf.set_len(s.total);
debug_assert!(fits, "a split buffer shorter than its operation");
}
buf
};
let Some((index, qtag)) = self.core.table.claim() else {
let buf = match req {
Req::Write { buf, .. } | Req::Read { buf, .. } => buf,
};
return Err((None, Some(whole(buf))));
};
let inner = &*self.store.inner;
let file = &inner.file;
let (op, write) = match req {
Req::Write { offset, buf } => {
let bytes = buf.len() as u64;
let op = IoOp::Write {
file,
offset,
buf,
dsync: inner.policy.dsync,
};
(op, Some(bytes))
}
Req::Read { offset, buf } => (IoOp::Read { file, offset, buf }, None),
};
self.core.table.start(index, tag, kind, Some(pass), durable);
if let Some(s) = split {
self.core.table.set_split(index, s);
}
if let Err(r) = self.core.queue.submit(op, qtag) {
self.core.table.abort(index);
if r.raw.is_none() {
inner.observer.add(Counter::Refused, 1);
}
return Err((r.raw, r.buf.map(whole)));
}
if let Some(bytes) = write {
inner.observer.add(Counter::Writes, 1);
inner.observer.add(Counter::WriteBytes, bytes);
}
Ok(())
}
#[must_use = "completions not taken stay with the submitter and count against its depth"]
pub fn reap(&mut self) -> Completions<'_> {
let _added = self.core.queue.reap(&mut self.core.raw);
self.process();
Completions {
ready: &mut self.core.table.ready,
}
}
#[must_use = "completions not taken stay with the submitter and count against its depth"]
pub fn wait(&mut self, min: usize) -> Result<Completions<'_>, Error> {
let want = min.min(self.core.table.live + self.core.table.ready.len());
if self.core.table.ready.len() >= want {
let _added = self.core.queue.reap(&mut self.core.raw);
self.process();
}
while self.core.table.ready.len() < want {
if self.inbox.take_woken() {
break;
}
let in_queue = self.core.queue.in_flight();
if in_queue > 0 {
let need = (want - self.core.table.ready.len()).min(in_queue);
let waited = if self.inbox.block_in_queue() {
let waited = self.core.queue.wait(need, &mut self.core.raw);
self.inbox.unblock();
waited
} else {
Ok(self.core.queue.reap(&mut self.core.raw))
};
self.process();
if let Err(raw) = waited {
return Err(Error::Io {
op: if self.core.table.writes_live() {
Op::Write
} else {
Op::Read
},
raw,
ctx: ErrorContext::default(),
});
}
} else if self.core.table.asked > 0 {
self.inbox.wait(true);
self.process();
} else {
break;
}
}
Ok(Completions {
ready: &mut self.core.table.ready,
})
}
fn process(&mut self) {
let _unwind = PoisonOnUnwind::new(&self.store);
for c in self.core.raw.drain() {
let mut finished = self.core.table.finish(&self.store, c.tag, c.buf, c.result);
while let Finish::Next {
qtag,
offset,
buf,
write,
} = finished
{
let inner = &*self.store.inner;
let file = &inner.file;
let bytes = buf.len() as u64;
let op = if write {
IoOp::Write {
file,
offset,
buf,
dsync: inner.policy.dsync,
}
} else {
IoOp::Read { file, offset, buf }
};
finished = match self.core.queue.submit(op, qtag) {
Ok(()) => {
if write {
inner.observer.add(Counter::Writes, 1);
inner.observer.add(Counter::WriteBytes, bytes);
}
Finish::Done
}
Err(r) => {
if r.raw.is_none() {
inner.observer.add(Counter::Refused, 1);
}
self.core
.table
.refused_next(&self.store, qtag, r.buf, r.raw)
}
};
}
if let Finish::Flush(qtag, need) = finished {
self.core.table.asks.push((qtag, need));
}
}
while let Some((qtag, need)) = self.core.table.asks.pop() {
durable::ask(
&mut self.core.table,
&self.store,
&self.inbox,
qtag,
need,
true,
);
}
while self.core.table.asked > 0 {
let Some((qtag, outcome)) = self.inbox.pop() else {
break;
};
self.core.table.asked -= 1;
self.core.table.flushed(&self.store, qtag, outcome);
}
}
}
struct PoisonOnUnwind<'a, P: Platform>
where
P::Queue: Send,
{
store: &'a Store<P>,
was: bool,
}
impl<'a, P: Platform> PoisonOnUnwind<'a, P>
where
P::Queue: Send,
{
fn new(store: &'a Store<P>) -> Self {
Self {
store,
was: std::thread::panicking(),
}
}
}
impl<P: Platform> Drop for PoisonOnUnwind<'_, P>
where
P::Queue: Send,
{
fn drop(&mut self) {
if !self.was && std::thread::panicking() {
self.store.inner.domain.poison(FirstCause {
op: Some(Op::Write),
raw: None,
});
self.store.wake_frontiers();
}
}
}
impl<P: Platform> Drop for Submitter<P>
where
P::Queue: Send,
{
fn drop(&mut self) {
let mut failed = None;
while self.core.table.live > 0 && self.core.queue.in_flight() > 0 {
match self
.core
.queue
.wait(self.core.queue.in_flight(), &mut self.core.raw)
{
Ok(_) => self.process(),
Err(raw) => {
failed = Some(raw);
break;
}
}
self.core.table.ready.clear();
}
let _added = self.core.queue.reap(&mut self.core.raw);
self.process();
while self.core.table.asked > 0 {
self.inbox.wait(false);
self.process();
}
if self.core.table.live > 0
&& (self.core.table.writes_live() || !self.core.table.consistent())
{
self.store.inner.domain.poison(FirstCause {
op: Some(Op::Write),
raw: failed,
});
self.store.wake_frontiers();
}
self.store.unregister_submitter(self.id);
}
}