use std::cell::{Cell, RefCell};
use std::marker::PhantomData;
use std::rc::{Rc, Weak};
use std::sync::Arc;
use store_io_core::error::{Error, ErrorContext, NotWrittenCause};
use store_io_platform::Clock;
use crate::qos::{Charge, Collect, Issuer, Qos};
pub(crate) struct Joined {
pub(crate) qos: Arc<Qos>,
pub(crate) pace: Option<Pace>,
}
#[derive(Clone)]
pub(crate) struct Pace {
issuer: Arc<Issuer>,
clock: Clock,
piece: usize,
leave: u64,
}
impl Pace {
pub(crate) fn new(qos: &Arc<Qos>, clock: Clock) -> Self {
Self {
issuer: Issuer::new(qos),
clock,
piece: qos.background_io() as usize,
leave: 0,
}
}
pub(crate) fn second(&self) -> Self {
Self {
leave: 1,
..self.clone()
}
}
pub(crate) fn piece(&self) -> usize {
self.piece
}
pub(crate) fn lone_piece(&self, max: usize) -> usize {
let (paced, cap) = self.issuer.pacing((self.clock)());
if paced {
return self.piece;
}
let quiet = self
.piece
.saturating_mul(usize::try_from(cap).unwrap_or(usize::MAX));
(quiet.min(max) / self.piece).max(1) * self.piece
}
fn key(&self) -> usize {
Arc::as_ptr(&self.issuer).cast::<()>() as usize
}
fn held(&self, charge: Charge) -> Held {
let key = self.key();
let _c = HELD.try_with(|h| {
let mut h = h.borrow_mut();
match h.iter_mut().find(|(k, _)| *k == key) {
Some((_, n)) => *n += 1,
None => h.push((key, 1)),
}
});
Held {
_charge: charge,
key,
_thread: PhantomData,
}
}
pub(crate) fn try_charge(&self) -> Option<Held> {
self.issuer
.admit_leaving((self.clock)(), self.leave)
.map(|c| self.held(c))
}
pub(crate) fn wait_charge(&self) -> Option<Held> {
self.obtain().ok()
}
pub(crate) fn admit(&self) -> Result<Held, Error> {
self.obtain().map_err(|()| {
let _n = ENDED.try_with(|n| n.set(n.get().wrapping_add(1)));
Error::NotWritten {
cause: NotWrittenCause::Interrupted,
ctx: ErrorContext::default(),
}
})
}
fn obtain(&self) -> Result<Held, ()> {
if let Some(h) = self.try_charge() {
return Ok(h);
}
let parts = parts_of(self.key());
if !parts.is_empty() {
if !self.issuer.pacing((self.clock)()).0 {
'one: loop {
let mut any = false;
for p in &parts {
any |= p.collect_one().map_err(|_| ())?;
if self.issuer.would_admit((self.clock)(), self.leave) {
break 'one;
}
}
if !any {
break;
}
}
crate::test_hook::step(crate::test_hook::Step::CollectRetry);
if let Some(h) = self.try_charge() {
return Ok(h);
}
}
for p in &parts {
p.collect_all().map_err(|_| ())?;
}
}
if held_of(self.key()) != 0 {
return Err(());
}
self.issuer
.wait_admit(&*self.clock, self.leave)
.map(|c| self.held(c))
.map_err(|_| ())
}
pub(crate) fn register(&self, part: Weak<dyn Collect>) -> Registered {
let id = NEXT_PART
.try_with(|n| {
let id = n.get();
n.set(id + 1);
id
})
.unwrap_or(u64::MAX);
let key = self.key();
let _c = PARTS.try_with(|p| p.borrow_mut().push((key, id, part)));
Registered {
id,
_issuer: Arc::clone(&self.issuer),
_thread: PhantomData,
}
}
#[cfg(all(test, not(loom)))]
pub(crate) fn is_second(&self) -> bool {
self.leave > 0
}
#[cfg(all(test, not(loom)))]
pub(crate) fn same_call(&self, other: &Self) -> bool {
Arc::ptr_eq(&self.issuer, &other.issuer)
}
#[cfg(all(test, not(loom)))]
pub(crate) fn held_here(&self) -> u32 {
held_of(self.key())
}
}
fn parts_of(key: usize) -> Vec<Rc<dyn Collect>> {
PARTS
.try_with(|p| {
p.borrow()
.iter()
.filter(|(k, _, _)| *k == key)
.filter_map(|(_, _, w)| w.upgrade())
.collect()
})
.unwrap_or_default()
}
fn held_of(key: usize) -> u32 {
HELD.try_with(|h| {
h.borrow()
.iter()
.find(|(k, _)| *k == key)
.map_or(0, |&(_, n)| n)
})
.unwrap_or(u32::MAX)
}
type PartEntry = (usize, u64, Weak<dyn Collect>);
thread_local! {
static HELD: RefCell<Vec<(usize, u32)>> = const { RefCell::new(Vec::new()) };
static PARTS: RefCell<Vec<PartEntry>> = const { RefCell::new(Vec::new()) };
static NEXT_PART: Cell<u64> = const { Cell::new(0) };
static ENDED: Cell<u64> = const { Cell::new(0) };
}
pub(crate) fn admissions_ended() -> u64 {
ENDED.try_with(Cell::get).unwrap_or(0)
}
pub(crate) struct Held {
_charge: Charge,
key: usize,
_thread: PhantomData<*const ()>,
}
impl Drop for Held {
fn drop(&mut self) {
let key = self.key;
let _c = HELD.try_with(|h| {
let mut h = h.borrow_mut();
if let Some(i) = h.iter().position(|(k, _)| *k == key) {
h[i].1 = h[i].1.saturating_sub(1);
if h[i].1 == 0 {
let _gone = h.swap_remove(i);
}
}
});
}
}
pub(crate) struct Registered {
id: u64,
_issuer: Arc<Issuer>,
_thread: PhantomData<*const ()>,
}
impl Drop for Registered {
fn drop(&mut self) {
let id = self.id;
let _c = PARTS.try_with(|p| p.borrow_mut().retain(|(_, i, _)| *i != id));
}
}
#[cfg(all(test, not(loom)))]
mod tests {
#![allow(clippy::unwrap_used, clippy::expect_used, unused_results)]
use std::path::Path;
use std::sync::{Arc, Mutex};
use store_io_core::IoClass;
use store_io_core::error::{Error, NotWrittenCause};
use store_io_format::slot::{FORMAT_1, FORMAT_2};
use store_io_sim::{SimConfig, SimPlatform};
use crate::store::container_name;
use crate::{Store, StoreOptions};
fn sampled(
p: &SimPlatform,
sample: impl Fn() -> u64 + Send + Sync + 'static,
f: impl FnOnce(),
) -> (Vec<u64>, u64) {
let seen = Arc::new(Mutex::new(Vec::new()));
let out = Arc::clone(&seen);
p.with_world(|w| {
w.set_submit_probe(Some(Arc::new(move || {
out.lock().unwrap().push(sample());
})))
});
let before = p.with_world(|w| w.write_submits());
f();
p.with_world(|w| w.set_submit_probe(None));
let writes = p.with_world(|w| w.write_submits()) - before;
let seen = seen.lock().unwrap().clone();
(seen, writes)
}
fn background(p: &SimPlatform, dir: &str, format: u16) -> Store<SimPlatform> {
Store::create_named(
p.clone(),
Path::new(dir),
StoreOptions {
container_format: format,
..StoreOptions::default()
},
&container_name().unwrap(),
IoClass::Background,
)
.unwrap()
}
fn work(s: &Store<SimPlatform>) {
let _log = s.provision_append_region("log", 1 << 20).unwrap();
let slot = s.provision_slot("obj").unwrap();
let _receipt = slot.commit(b"object").unwrap();
}
#[test]
fn test_every_write_of_a_background_store_is_charged() {
for (seed, format) in [(45, FORMAT_1), (46, FORMAT_2)] {
let p = SimPlatform::new(SimConfig::volatile(seed));
let w = Store::create(p.clone(), Path::new("/w"), StoreOptions::default()).unwrap();
let w2 = w.clone();
let (seen, writes) = sampled(
&p,
move || w2.qos().in_flight,
|| {
work(&background(&p, "/bg", format));
},
);
let charged = seen.iter().filter(|&&n| n >= 1).count() as u64;
assert_eq!(charged, writes, "format {format}: {seen:?}");
assert!(writes > 0);
drop(w);
}
}
#[test]
fn test_a_scan_takes_capped_buffers() {
let p = SimPlatform::new(SimConfig::volatile(52));
let s = Store::create(p.clone(), Path::new("/s"), StoreOptions::default()).unwrap();
let log = s.provision_append_region("log", 1 << 20).unwrap();
let t = log.append(&[9u8; 1 << 20]).unwrap();
let _receipt = log.sync_through(&t).unwrap();
let s2 = s.clone();
let holdings = move || {
s2.inner
.pool
.background_holdings()
.iter()
.map(|c| u64::from(c.1))
.sum()
};
let (held, writes) = sampled(&p, holdings, || {
let _sum = log
.scan(0, |_| std::ops::ControlFlow::Continue(()))
.unwrap();
});
assert_eq!(writes, 0);
assert!(!held.is_empty());
assert!(held.iter().all(|&n| n >= 1), "{held:?}");
}
#[test]
fn test_a_clone_is_the_same_call() {
let p = SimPlatform::new(SimConfig::volatile(53));
let s = Store::create(p.clone(), Path::new("/s"), StoreOptions::default()).unwrap();
let a = s.inner.domain.new_pace().unwrap();
assert!(a.same_call(&a.clone()));
assert!(!a.same_call(&s.inner.domain.new_pace().unwrap()));
}
#[test]
fn test_a_second_part_leaves_the_first_a_piece() {
let p = SimPlatform::new(SimConfig::volatile(57));
let s = Store::create(p.clone(), Path::new("/s"), StoreOptions::default()).unwrap();
p.with_world(|w| w.advance_clock_ns(10_000_000_000));
assert!(!s.qos().active);
let first = s.inner.domain.new_pace().unwrap();
let second = first.second();
let mut held: Vec<_> = (0..7).map(|_| second.try_charge().unwrap()).collect();
assert!(second.try_charge().is_none(), "one left");
held.push(first.try_charge().unwrap());
assert!(first.try_charge().is_none(), "the cap");
drop(held);
}
#[derive(Default)]
struct FakeState {
held: Vec<crate::pace::Held>,
fail: bool,
ones: u32,
alls: u32,
}
struct Fake(std::cell::RefCell<FakeState>);
impl Fake {
fn holding(
pace: &crate::pace::Pace,
n: usize,
) -> (std::rc::Rc<Self>, crate::pace::Registered) {
let held = (0..n).map(|_| pace.try_charge().unwrap()).collect();
let f = std::rc::Rc::new(Fake(std::cell::RefCell::new(FakeState {
held,
..FakeState::default()
})));
let weak: std::rc::Weak<Fake> = std::rc::Rc::downgrade(&f);
let reg = pace.register(weak);
(f, reg)
}
fn state(&self) -> std::cell::Ref<'_, FakeState> {
self.0.borrow()
}
}
impl crate::qos::Collect for Fake {
fn collect_one(&self) -> Result<bool, store_io_core::error::OsError> {
let Ok(mut s) = self.0.try_borrow_mut() else {
return Ok(false);
};
s.ones += 1;
if s.fail {
return Err(EIO);
}
Ok(s.held.pop().is_some())
}
fn collect_all(&self) -> Result<(), store_io_core::error::OsError> {
let Ok(mut s) = self.0.try_borrow_mut() else {
return Ok(());
};
s.alls += 1;
if s.fail {
return Err(EIO);
}
s.held.clear();
Ok(())
}
fn failed(&self) -> Option<(store_io_core::error::OsError, u64)> {
None
}
}
const EIO: store_io_core::error::OsError = store_io_core::error::OsError {
code: 5,
source: store_io_core::error::OsErrorSource::Errno,
};
fn within(f: impl FnOnce() + Send + 'static) {
let (tx, rx) = std::sync::mpsc::channel();
let _t = std::thread::spawn(move || {
f();
let _sent = tx.send(());
});
rx.recv_timeout(std::time::Duration::from_secs(20))
.expect("the admission never returned (or panicked)");
}
fn quiet(seed: u64) -> (SimPlatform, Store<SimPlatform>, crate::pace::Pace) {
let p = SimPlatform::new(SimConfig::volatile(seed));
let s = Store::create(p.clone(), Path::new("/s"), StoreOptions::default()).unwrap();
p.with_world(|w| w.advance_clock_ns(10_000_000_000));
assert!(!s.qos().active);
let pace = s.inner.domain.new_pace().unwrap();
(p, s, pace)
}
#[test]
fn test_a_refused_piece_collects_another_part_one_at_a_time() {
within(|| {
let (_p, _s, pace) = quiet(60);
let (fake, _reg) = Fake::holding(&pace, 8);
assert_eq!(pace.held_here(), 8);
assert!(pace.try_charge().is_none(), "the call's cap");
let piece = pace.admit().unwrap();
let st = fake.state();
assert_eq!((st.ones, st.alls, st.held.len()), (1, 0, 7));
assert_eq!(pace.held_here(), 8, "seven of the part's and the piece");
drop(st);
drop(piece);
});
}
#[test]
fn test_a_paced_refusal_collects_every_piece_of_the_call() {
within(|| {
let (_p, s, pace) = quiet(61);
let (fake, _reg) = Fake::holding(&pace, 2);
let _hold = s.commit_hold();
assert!(s.qos().active);
assert!(pace.try_charge().is_none(), "paced: the call holds two");
let _piece = pace.admit().unwrap();
let st = fake.state();
assert_eq!((st.ones, st.alls, st.held.len()), (0, 1, 0));
});
}
static HOLDS: Mutex<Vec<crate::CommitHold>> = Mutex::new(Vec::new());
#[test]
fn test_a_commit_between_the_collection_and_the_retry_collects_the_rest() {
within(|| {
let (_p, s, pace) = quiet(62);
let (fake, _reg) = Fake::holding(&pace, 8);
let s2 = s.clone();
crate::test_hook::lib_tests::at(
crate::test_hook::Step::CollectRetry,
Box::new(move || HOLDS.lock().unwrap().push(s2.commit_hold())),
);
let piece = pace.admit();
crate::test_hook::lib_tests::clear();
assert!(s.qos().active, "the commit came");
let _piece = piece.unwrap();
let st = fake.state();
assert_eq!((st.ones, st.alls, st.held.len()), (1, 1, 0));
HOLDS.lock().unwrap().clear();
});
}
#[test]
fn test_pieces_no_collector_holds_refuse_the_wait() {
within(|| {
let (_p, _s, pace) = quiet(67);
let held: Vec<_> = (0..8).map(|_| pace.try_charge().unwrap()).collect();
assert!(pace.admit().is_err(), "never registered");
drop(held);
let (fake, _reg) = Fake::holding(&pace, 8);
{
let _asking = fake.0.borrow_mut();
assert!(pace.admit().is_err(), "registered, borrowed");
}
assert_eq!(fake.state().ones, 0, "borrowed: skipped");
});
within(|| {
let (_p, _s, pace) = quiet(63);
let (fake, reg) = Fake::holding(&pace, 8);
drop(reg);
let e = pace.admit().err().unwrap();
assert!(matches!(
e,
Error::NotWritten {
cause: NotWrittenCause::Interrupted,
..
}
));
assert!(pace.wait_charge().is_none());
assert_eq!(fake.state().held.len(), 8);
assert_eq!(fake.state().ones, 0, "unregistered: never asked");
});
}
#[test]
fn test_a_failed_collection_ends_the_admission() {
within(|| {
let (_p, _s, pace) = quiet(64);
let (fake, _reg) = Fake::holding(&pace, 8);
fake.0.borrow_mut().fail = true;
assert!(pace.admit().is_err());
let st = fake.state();
assert_eq!((st.ones, st.alls), (1, 0));
});
}
#[test]
fn test_a_second_part_collects_until_it_would_be_admitted_leaving_one() {
within(|| {
let (_p, _s, pace) = quiet(68);
let second = pace.second();
let (fake, _reg) = Fake::holding(&pace, 8);
let _piece = second.admit().unwrap();
let st = fake.state();
assert_eq!((st.ones, st.alls, st.held.len()), (2, 0, 6));
});
}
#[test]
fn test_counts_and_collectors_are_per_record() {
within(|| {
let (_p, s, a) = quiet(66);
let b = s.inner.domain.new_pace().unwrap();
let (fa, _ra) = Fake::holding(&a, 8);
let (fb, _rb) = Fake::holding(&b, 8);
assert_eq!((a.held_here(), b.held_here()), (8, 8));
let _piece = b.admit().unwrap();
assert_eq!(fa.state().ones, 0, "another call's part untouched");
assert_eq!(fa.state().held.len(), 8);
assert_eq!(fb.state().ones, 1);
assert_eq!((a.held_here(), b.held_here()), (8, 8));
});
}
#[test]
fn test_would_admit_answers_as_an_admission() {
let (_p, s, pace) = quiet(65);
let second = pace.second();
let now = (pace.clock)();
let mut held = Vec::new();
for n in 0..8 {
assert!(pace.issuer.would_admit(now, 0), "{n} held");
assert_eq!(second.issuer.would_admit(now, 1), n < 7, "{n} held");
held.push(pace.try_charge().unwrap());
}
assert!(!pace.issuer.would_admit(now, 0), "the cap");
assert!(pace.try_charge().is_none());
assert_eq!(pace.issuer.in_flight(), 8, "no charge taken");
held.truncate(1);
let _hold = s.commit_hold();
let now = (pace.clock)();
assert!(!pace.issuer.would_admit(now, 0), "paced, one in flight");
held.clear();
assert!(pace.issuer.would_admit(now, 0), "paced, none in flight");
}
#[test]
fn test_a_background_store_s_writes_take_capped_buffers() {
let p = SimPlatform::new(SimConfig::volatile(47));
let s = background(&p, "/bg", FORMAT_2);
let s2 = s.clone();
let holdings = move || {
s2.inner
.pool
.background_holdings()
.iter()
.map(|c| u64::from(c.1))
.sum()
};
let (held, writes) = sampled(&p, holdings, || work(&s));
let capped = held.iter().filter(|&&n| n >= 1).count() as u64;
assert_eq!(capped, writes, "{held:?}");
assert!(writes > 0);
}
}