use core::sync::atomic::Ordering::Relaxed;
use crate::class::{self, NCLASSES};
use crate::heap::Heap;
use crate::segment::SEGMENT_BYTES;
use crate::stats::Stats;
use crate::tally::SPAN;
#[cfg(test)]
use crate::{class::SPAN_BYTES, segment::NO_CLASS};
impl Heap {
#[must_use]
pub fn snapshot(&self) -> Stats {
let t = &self.tally;
let (parked, parked_live) = if self.parked.is_null() {
(0, 0)
} else {
let p = unsafe { &*self.parked };
(p.bytes.load(Relaxed) as u64, p.live.load(Relaxed) as u64)
};
let held = self.held_bytes();
let assigned = t.assigned_spans();
Stats {
mapped: t.segments * SEGMENT_BYTES as u64,
live: self.live_bytes - parked_live,
rounding: self.rounding_total(held) - (parked - parked_live),
cache: parked,
span_free: t.touched - held - t.returned + self.claims_unused_bytes(),
returned: t.returned_spans * SPAN + t.returned,
virgin: t.virgin_spans * SPAN + (assigned - t.empty_spans) * SPAN - t.touched,
hysteresis: (t.held_spans + t.empty_spans) * SPAN,
segment_overhead: t.segments * SPAN,
large_count: 0,
spans_assigned: assigned,
}
}
fn held_bytes(&self) -> u64 {
let mut held = 0u64;
for c in 0..NCLASSES {
held += u64::from(self.class_live[c]) * class::size_of(c) as u64;
}
held
}
fn rounding_total(&self, held: u64) -> u64 {
held - self.claims_unused_bytes() - self.live_bytes
}
#[cfg(test)]
pub(crate) fn snapshot_walked(&self) -> Stats {
use crate::segment::{FIRST_DATA_SPAN, SPANS_PER_SEGMENT};
let rounding = self.rounding_total(self.held_bytes());
let mut st = Stats { live: self.live_bytes, rounding, ..Stats::default() };
let mut seg = self.segments;
while !seg.is_null() {
let s = unsafe { &*seg };
st.mapped += SEGMENT_BYTES as u64;
st.segment_overhead += SPAN_BYTES as u64;
let mut node = s.foreign.load(core::sync::atomic::Ordering::Acquire);
while !node.is_null() {
let p = unsafe { core::ptr::NonNull::new_unchecked(node) };
let requested = unsafe { crate::segment::foreign_requested(p) } as u64;
let cls = s.spans[crate::segment::span_index_of(p)].class as usize;
let slot = class::size_of(cls) as u64;
st.cache += slot;
st.live -= requested;
st.rounding -= slot - requested;
node = unsafe { node.cast::<*mut u8>().read() };
}
for ix in FIRST_DATA_SPAN..SPANS_PER_SEGMENT {
add_span(&mut st, &s.spans[ix]);
if s.spans[ix].class != NO_CLASS {
st.spans_assigned += 1;
}
}
seg = s.next;
}
st.span_free += self.claims_unused_bytes();
st
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub(crate) enum Unassigned {
Virgin,
Returned,
Held,
}
pub(crate) fn unassigned_bucket(retired: bool, discarded: u16) -> Unassigned {
if !retired {
Unassigned::Virgin
} else if discarded == crate::pagemap::ALL_PAGES_DISCARDED {
Unassigned::Returned
} else {
Unassigned::Held
}
}
#[cfg(test)]
fn add_span(st: &mut Stats, meta: &crate::segment::SpanMeta) {
if meta.class == NO_CLASS {
match unassigned_bucket(meta.retired, meta.discarded) {
Unassigned::Virgin => st.virgin += SPAN_BYTES as u64,
Unassigned::Returned => st.returned += SPAN_BYTES as u64,
Unassigned::Held => st.hysteresis += SPAN_BYTES as u64,
}
return;
}
if meta.live == 0 {
st.hysteresis += SPAN_BYTES as u64;
return;
}
let slot = class::size_of(meta.class as usize) as u64;
for i in 0..u32::from(meta.high_water) {
if meta.is_live(i) {
continue;
}
let (pa, pb) = crate::pagemap::pages_of_slot(i, slot as usize);
let all_gone = (pa..=pb).all(|p| meta.discarded & (1u16 << p) != 0);
if all_gone {
st.returned += slot;
} else {
st.span_free += slot;
}
}
st.virgin += SPAN_BYTES as u64 - u64::from(meta.high_water) * slot;
}
#[cfg(test)]
mod unassigned_tests {
use super::{Unassigned, unassigned_bucket};
use crate::pagemap::ALL_PAGES_DISCARDED;
#[test]
fn an_unassigned_span_is_one_of_three_things() {
assert_eq!(unassigned_bucket(false, 0), Unassigned::Virgin);
assert_eq!(unassigned_bucket(false, ALL_PAGES_DISCARDED), Unassigned::Virgin);
assert_eq!(unassigned_bucket(true, ALL_PAGES_DISCARDED), Unassigned::Returned);
assert_eq!(unassigned_bucket(true, 0), Unassigned::Held);
assert_eq!(unassigned_bucket(true, ALL_PAGES_DISCARDED >> 1), Unassigned::Held);
}
}