use std::cell::Cell;
use std::fmt;
use std::mem::size_of;
use std::sync::Arc;
use std::sync::atomic::{AtomicU64, Ordering};
use super::{ChargeError, Limits, heap_cost};
use crate::budget::{AllocError, BudgetExceeded, MemoryBudget, Permit, Reservation, Unbounded};
use crate::error::{LimitError, LimitKind};
const fn arc_bytes(size: usize) -> u64 {
size as u64 + 2 * size_of::<usize>() as u64
}
struct Account {
used: AtomicU64,
limit: u64,
budget: Arc<dyn MemoryBudget>,
what: Reservation,
}
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub struct WorkPast {
done: u64,
allowance: u64,
}
const _: () = crate::send_sync::<WorkPast>();
impl WorkPast {
#[must_use]
pub const fn done(&self) -> u64 {
self.done
}
#[must_use]
pub const fn allowance(&self) -> u64 {
self.allowance
}
}
pub struct Meter {
account: Arc<Account>,
work: AtomicU64,
work_limit: u64,
work_refusal: AtomicU64,
granted: AtomicU64,
}
const _: () = crate::send_sync::<Meter>();
impl fmt::Debug for Meter {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
f.debug_struct("Meter")
.field("used", &self.used())
.field("limit", &self.limit())
.field("decoded", &self.decoded())
.field("what", &self.reservation())
.finish_non_exhaustive()
}
}
impl Meter {
#[must_use]
pub fn allocator_refused(&self, requested: u64) -> AllocError {
let error = AllocError {
requested,
what: self.reservation(),
};
self.account.budget.allocator_refused(error);
error
}
pub const OWN_HEAP: u64 = Self::REOPEN_HEAP + heap_cost(arc_bytes(size_of::<Account>()));
pub const REOPEN_HEAP: u64 = heap_cost(arc_bytes(size_of::<Self>()));
#[must_use]
pub fn new(limit: u64, budget: Arc<dyn MemoryBudget>, what: Reservation) -> Arc<Self> {
Self::with_decode_work(limit, u64::MAX, budget, what)
}
#[must_use]
pub fn with_decode_work(
limit: u64,
max_decode_work: u64,
budget: Arc<dyn MemoryBudget>,
what: Reservation,
) -> Arc<Self> {
Arc::new(Self {
account: Arc::new(Account {
used: AtomicU64::new(0),
limit,
budget,
what,
}),
work: AtomicU64::new(0),
work_limit: max_decode_work,
work_refusal: AtomicU64::new(0),
granted: AtomicU64::new(0),
})
}
#[must_use]
pub fn for_input(
limits: &Limits,
budget: Arc<dyn MemoryBudget>,
what: Reservation,
) -> Arc<Self> {
Self::with_decode_work(
limits.max_input_memory,
limits.max_decode_work,
budget,
what,
)
}
#[must_use]
pub fn reopen(&self) -> Arc<Self> {
Arc::new(Self {
account: self.account.clone(),
work: AtomicU64::new(0),
work_limit: self.work_limit,
work_refusal: AtomicU64::new(0),
granted: AtomicU64::new(0),
})
}
#[must_use]
pub fn work_scope(&self, limit: u64) -> Arc<Self> {
Arc::new(Self {
account: self.account.clone(),
work: AtomicU64::new(0),
work_limit: limit,
work_refusal: AtomicU64::new(0),
granted: AtomicU64::new(0),
})
}
#[must_use]
pub fn attempted_work(&self) -> u64 {
self.decoded()
.max(self.work_refusal.load(Ordering::Acquire))
}
#[must_use]
pub fn same_input(&self, other: &Self) -> bool {
Arc::ptr_eq(&self.account, &other.account)
}
pub(crate) fn input_address(&self) -> usize {
Arc::as_ptr(&self.account).addr()
}
pub fn count_decode(&self, bytes: u64) -> Result<(), ChargeError> {
self.count_work(bytes).map_err(ChargeError::Limit)
}
pub(crate) fn count_work(&self, bytes: u64) -> Result<(), LimitError> {
let limit = self.work_limit;
self.work
.try_update(Ordering::AcqRel, Ordering::Acquire, |done| {
done.checked_add(bytes).filter(|t| *t <= limit)
})
.map(|_| ())
.map_err(|done| self.decode_refused(done, bytes))
}
#[must_use]
pub fn try_count_work(&self, bytes: u64) -> bool {
let limit = self.work_limit;
self.work
.try_update(Ordering::AcqRel, Ordering::Acquire, |done| {
done.checked_add(bytes).filter(|t| *t <= limit)
})
.is_ok()
}
pub(crate) fn uncount_decode(&self, bytes: u64) {
let _ = self
.work
.try_update(Ordering::AcqRel, Ordering::Acquire, |done| {
Some(done.saturating_sub(bytes))
});
}
pub(crate) fn add_work(&self, bytes: u64) {
let _ = self
.work
.try_update(Ordering::AcqRel, Ordering::Acquire, |done| {
Some(done.saturating_add(bytes))
});
}
#[must_use]
pub fn work_used_up(&self) -> Option<WorkPast> {
let done = self.decoded();
(done > self.work_limit).then_some(WorkPast {
done,
allowance: self.work_limit,
})
}
#[inline]
pub(crate) fn work_refused(&self) -> Option<LimitError> {
let done = self.decoded();
if done > self.work_limit {
Some(LimitError::refused(
LimitKind::DecodeWork,
done,
self.work_limit,
))
} else {
None
}
}
fn decode_refused(&self, done: u64, bytes: u64) -> LimitError {
let attempted = done.saturating_add(bytes);
self.work_refusal.fetch_max(attempted, Ordering::AcqRel);
LimitError::refused(LimitKind::DecodeWork, attempted, self.work_limit)
}
#[must_use]
pub fn work_left(&self) -> u64 {
self.work_limit.saturating_sub(self.decoded())
}
#[must_use]
pub fn decoded(&self) -> u64 {
self.work.load(Ordering::Acquire)
}
#[must_use]
pub fn granted(&self) -> u64 {
self.granted.load(Ordering::Acquire)
}
#[must_use]
pub fn unlimited() -> Arc<Self> {
Self::new(u64::MAX, Arc::new(Unbounded), Reservation::Input)
}
#[must_use]
pub fn used(&self) -> u64 {
self.account.used.load(Ordering::Acquire)
}
#[must_use]
pub fn limit(&self) -> u64 {
self.account.limit
}
#[must_use]
pub fn reservation(&self) -> Reservation {
self.account.what
}
pub fn charge(self: &Arc<Self>, bytes: u64) -> Result<Charge, ChargeError> {
let permit = self.reserve(bytes)?;
Ok(Charge {
meter: self.clone(),
permit,
more: Vec::new(),
})
}
fn reserve(&self, bytes: u64) -> Result<Permit, ChargeError> {
let account = &self.account;
let limit = account.limit;
account
.used
.try_update(Ordering::AcqRel, Ordering::Acquire, |used| {
used.checked_add(bytes).filter(|t| *t <= limit)
})
.map_err(|used| {
LimitError::refused(LimitKind::InputMemory, used.saturating_add(bytes), limit)
})?;
let permit = account
.budget
.try_reserve(bytes, account.what)
.map_err(|e| {
self.refund(bytes);
crate::counters::budget(e.what);
ChargeError::Budget(counted(e))
})?;
let _ = self
.granted
.try_update(Ordering::AcqRel, Ordering::Acquire, |g| {
Some(g.saturating_add(bytes))
});
GrantScope::credit(self, bytes);
Ok(permit)
}
fn refund(&self, bytes: u64) {
let _ = self
.account
.used
.try_update(Ordering::AcqRel, Ordering::Acquire, |used| {
Some(used.saturating_sub(bytes))
});
}
}
thread_local! {
static POOL_REFUSALS: Cell<u64> = const { Cell::new(0) };
}
pub(crate) fn pool_refusals() -> u64 {
POOL_REFUSALS.with(Cell::get)
}
fn counted(e: BudgetExceeded) -> BudgetExceeded {
if e.what == Reservation::Dependency {
POOL_REFUSALS.with(|n| n.set(n.get().wrapping_add(1)));
}
e
}
pub(crate) fn refusal(requested: u64, what: Reservation) -> BudgetExceeded {
counted(BudgetExceeded { requested, what })
}
const GRANT_SCOPES: usize = 64;
struct GrantFrames {
frames: [(usize, u64); GRANT_SCOPES],
len: usize,
}
thread_local! {
static GRANTS: std::cell::RefCell<GrantFrames> = const {
std::cell::RefCell::new(GrantFrames {
frames: [(0, 0); GRANT_SCOPES],
len: 0,
})
};
}
pub(crate) struct GrantScope {
depth: Option<usize>,
fallback: u64,
}
impl GrantScope {
pub(crate) fn enter(meter: &Meter) -> Self {
let addr = std::ptr::from_ref(meter).addr();
let depth = GRANTS
.try_with(|g| {
let mut g = g.borrow_mut();
let at = g.len;
let frame = g.frames.get_mut(at)?;
*frame = (addr, 0);
g.len = at + 1;
Some(at)
})
.ok()
.flatten();
Self {
depth,
fallback: meter.granted(),
}
}
pub(crate) fn finish(self, meter: &Meter) -> u64 {
match self.depth {
Some(at) => GRANTS
.try_with(|g| g.borrow().frames.get(at).map_or(0, |f| f.1))
.unwrap_or(0),
None => meter.granted().saturating_sub(self.fallback),
}
}
fn credit(meter: &Meter, bytes: u64) {
let addr = std::ptr::from_ref(meter).addr();
let _ = GRANTS.try_with(|g| {
let mut g = g.borrow_mut();
let len = g.len;
for frame in g.frames.iter_mut().take(len) {
if frame.0 == addr {
frame.1 = frame.1.saturating_add(bytes);
}
}
});
}
}
impl Drop for GrantScope {
fn drop(&mut self) {
if let Some(at) = self.depth {
let _ = GRANTS.try_with(|g| g.borrow_mut().len = at);
}
}
}
#[must_use = "a charge returns its bytes when dropped"]
pub struct Charge {
meter: Arc<Meter>,
permit: Permit,
more: Vec<Charge>,
}
const _: () = crate::send_sync::<Charge>();
impl fmt::Debug for Charge {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
f.debug_struct("Charge")
.field("bytes", &self.bytes())
.field("more", &self.more.len())
.finish_non_exhaustive()
}
}
impl Charge {
pub fn zero(meter: &Arc<Meter>) -> Self {
Self {
meter: meter.clone(),
permit: Permit::detached(0),
more: Vec::new(),
}
}
#[must_use]
pub fn bytes(&self) -> u64 {
self.more
.iter()
.fold(self.permit.bytes(), |n, c| n.saturating_add(c.bytes()))
}
#[must_use]
pub fn meter(&self) -> &Arc<Meter> {
&self.meter
}
pub fn grow(&mut self, bytes: u64) -> Result<(), ChargeError> {
let permit = self.meter.reserve(bytes)?;
self.permit.absorb(permit);
Ok(())
}
pub fn release(&mut self, bytes: u64) {
let held = self.permit.bytes();
let n = bytes.min(held);
self.meter.refund(n);
self.permit.shrink_to(held - n);
}
pub fn absorb(&mut self, mut other: Charge) {
for c in std::mem::take(&mut other.more) {
self.absorb(c);
}
if Arc::ptr_eq(&self.meter, &other.meter) {
self.permit
.absorb(std::mem::replace(&mut other.permit, Permit::detached(0)));
} else if other.permit.bytes() > 0 {
match self
.more
.iter_mut()
.find(|c| Arc::ptr_eq(&c.meter, &other.meter))
{
Some(kept) => kept.absorb(other),
None => self.more.push(other),
}
}
}
#[must_use]
pub fn records(&self) -> usize {
self.more
.iter()
.fold(self.permit.records(), |n, c| n + 1 + c.records())
}
}
impl Drop for Charge {
fn drop(&mut self) {
self.meter.refund(self.permit.bytes());
}
}