use std::cell::RefCell;
use tf_tree_core::{FrameId, LookupError, Plan};
use crate::tree::Tree;
const SLOTS: usize = 16;
const MIX: u64 = 0x9E37_79B9_7F4A_7C15;
#[derive(Clone, Copy, PartialEq, Eq)]
struct Key {
scope: u64,
target: u32,
source: u32,
generation: u64,
}
#[derive(Clone, Copy)]
struct Entry {
key: Key,
plan: Plan,
}
thread_local! {
static CACHE: RefCell<[Option<Entry>; SLOTS]> = const { RefCell::new([None; SLOTS]) };
}
fn index(key: Key) -> usize {
let mut h = key.scope;
h = h.wrapping_mul(MIX) ^ u64::from(key.target);
h = h.wrapping_mul(MIX) ^ u64::from(key.source);
h = h.wrapping_mul(MIX) ^ key.generation;
(h as usize) & (SLOTS - 1)
}
pub(crate) fn with_plan<R>(
tree: &Tree,
target: FrameId,
source: FrameId,
generation: u64,
f: impl FnOnce(&Plan) -> R,
) -> Result<(R, bool), LookupError> {
let key = Key {
scope: tree.cache_scope(),
target: target.get(),
source: source.get(),
generation,
};
let idx = index(key);
CACHE.with(|c| {
{
let slots = c.borrow();
if let Some(entry) = &slots[idx] {
if entry.key == key {
return Ok((f(&entry.plan), true));
}
}
}
let plan = tree.plan(target, source)?;
c.borrow_mut()[idx] = Some(Entry { key, plan });
Ok((f(&plan), false))
})
}
#[cfg(test)]
mod tests {
#![allow(clippy::unwrap_used, clippy::expect_used)]
use crate::{Iso3, TreeBuilder};
#[test]
fn the_low_bit_mask_wins_on_a_small_tree_and_ties_on_a_large_one() {
fn hashed(key: super::Key) -> usize {
let mut h = key.scope;
h = h.wrapping_mul(super::MIX) ^ u64::from(key.target);
h = h.wrapping_mul(super::MIX) ^ u64::from(key.source);
h = h.wrapping_mul(super::MIX) ^ key.generation;
(h.wrapping_mul(super::MIX) >> (u64::BITS - super::SLOTS.trailing_zeros())) as usize
}
let residency = |frames: u32, pairs: usize| {
let mut state = 0x1234_5678_9ABC_DEF1u64;
let mut next = move || {
state ^= state << 13;
state ^= state >> 7;
state ^= state << 17;
state
};
let resident = |slots: &[usize]| {
let mut counts = [0usize; super::SLOTS];
for &s in slots {
counts[s] += 1;
}
slots.iter().filter(|&&s| counts[s] == 1).count() as f64 / slots.len() as f64
};
let (mut mask_total, mut hash_total) = (0.0, 0.0);
let trials = 2000;
for _ in 0..trials {
let (mut mask_slots, mut hash_slots) = (Vec::new(), Vec::new());
for _ in 0..pairs {
let key = super::Key {
scope: 1,
target: (next() % u64::from(frames)) as u32,
source: (next() % u64::from(frames)) as u32,
generation: 0,
};
mask_slots.push(super::index(key));
hash_slots.push(hashed(key));
}
mask_total += resident(&mask_slots);
hash_total += resident(&hash_slots);
}
(
mask_total / f64::from(trials),
hash_total / f64::from(trials),
)
};
let (mask, hash) = residency(8, 6);
assert!(
mask > hash + 0.15,
"on an 8-frame tree the mask {mask} should beat the alternative {hash} \
by the margin the choice was made on"
);
let (mask, hash) = residency(40, 6);
assert!(
(mask - hash).abs() < 0.05,
"on a 40-frame tree the mask {mask} and the alternative {hash} tie; \
a gap either way means the index changed shape, not tuning"
);
}
#[test]
fn two_trees_keep_separate_entries_and_still_hit() {
let build = || {
TreeBuilder::new()
.static_edge("a", "b", &Iso3::IDENTITY)
.build()
.unwrap()
};
let first = build();
let second = build();
assert_ne!(
first.cache_scope(),
second.cache_scope(),
"two heap trees are two arenas"
);
let key_of = |t: &crate::Tree| {
let a = t.frame("a").unwrap();
let b = t.frame("b").unwrap();
(a, b, t.guard().generation())
};
let (a1, b1, g1) = key_of(&first);
let (a2, b2, g2) = key_of(&second);
assert_eq!((a1.get(), b1.get(), g1), (a2.get(), b2.get(), g2));
let probe = |t: &crate::Tree, target, source, g| {
super::with_plan(t, target, source, g, |_| ()).unwrap().1
};
assert!(!probe(&first, b1, a1, g1));
assert!(probe(&first, b1, a1, g1), "the first tree's repeat hits");
assert!(
!probe(&second, b2, a2, g2),
"the second tree must not be served the first tree's plan"
);
assert!(probe(&second, b2, a2, g2), "the second tree's repeat hits");
}
#[test]
fn cache_hits_and_invalidates_on_generation() {
let tree = TreeBuilder::new()
.static_edge("a", "b", &Iso3::IDENTITY)
.static_edge("b", "c", &Iso3::IDENTITY)
.build()
.unwrap();
let a = tree.frame("a").unwrap();
let b = tree.frame("b").unwrap();
let c = tree.frame("c").unwrap();
let gen1 = tree.guard().generation();
let (g1_stamped, hit1) =
super::with_plan(&tree, b, a, gen1, tf_tree_core::Plan::generation).unwrap();
assert!(!hit1, "first compile is a miss");
assert_eq!(g1_stamped, gen1);
let (_, hit2) = super::with_plan(&tree, b, a, gen1, |_| ()).unwrap();
assert!(hit2, "repeat lookup hits the cache");
tree.reparent(c, a).unwrap();
let gen2 = tree.guard().generation();
assert_ne!(gen1, gen2, "re-parent must change the generation");
let (g3_stamped, _hit3) =
super::with_plan(&tree, b, a, gen2, tf_tree_core::Plan::generation).unwrap();
assert_eq!(
g3_stamped, gen2,
"post-change plan is stamped with the new generation"
);
}
#[test]
fn the_cache_hits_exactly_where_its_index_predicts() {
let build = || {
TreeBuilder::new()
.static_edge("a", "b", &Iso3::IDENTITY)
.static_edge("b", "c", &Iso3::IDENTITY)
.build()
.unwrap()
};
let resident = |slots: &[usize]| {
slots
.iter()
.filter(|&&s| slots.iter().filter(|&&o| o == s).count() == 1)
.count()
};
const ROUNDS: usize = 3;
for n in [2usize, 16, 17] {
let trees: Vec<crate::Tree> = (0..n).map(|_| build()).collect();
let a = trees[0].frame("a").unwrap();
let c = trees[0].frame("c").unwrap();
let g = trees[0].guard().generation();
for t in &trees {
assert_eq!(
(
t.frame("a").unwrap().get(),
t.frame("c").unwrap().get(),
t.guard().generation()
),
(a.get(), c.get(), g),
"the trees must agree on everything but their arena id"
);
}
let slots: Vec<usize> = trees
.iter()
.map(|t| {
super::index(super::Key {
scope: t.cache_scope(),
target: a.get(),
source: c.get(),
generation: g,
})
})
.collect();
let residues: Vec<usize> = trees
.iter()
.map(|t| (t.cache_scope() as usize) & (super::SLOTS - 1))
.collect();
let (mut hits, mut total) = (0usize, 0usize);
for round in 0..ROUNDS {
for tree in &trees {
let (_, hit) = super::with_plan(tree, a, c, g, |_| ()).unwrap();
if round > 0 {
total += 1;
hits += usize::from(hit);
}
}
}
assert_eq!(
hits,
resident(&slots) * (ROUNDS - 1),
"{n} trees: the cache hit {hits} times in {total} steady-state \
lookups, but its own index puts {} of them in a slot no other \
tree shares",
resident(&slots)
);
assert_eq!(
resident(&slots),
resident(&residues),
"{n} trees: `index` must separate arena ids exactly as their low \
{} bits do — it is a permutation of them, and that is what \
keeps consecutive ids from thrashing",
super::SLOTS.trailing_zeros()
);
if n > super::SLOTS {
assert!(
hits < total,
"{n} trees cannot all be resident in {} slots — {hits} of \
{total} means the arena component stopped separating them",
super::SLOTS
);
}
}
}
#[cfg(all(feature = "shm", target_os = "linux"))]
#[test]
fn two_handles_on_one_shared_arena_share_their_plans() {
let owner = TreeBuilder::new()
.static_edge("a", "b", &Iso3::IDENTITY)
.build_shared("tf_tree-cache-identity-test")
.unwrap();
let fd = owner
.shared_fd()
.expect("a build_shared tree has a segment fd")
.try_clone_to_owned()
.unwrap();
let peer = crate::Tree::attach_shared(fd, crate::AttachMode::ReadOnly).unwrap();
assert_eq!(
owner.cache_scope(),
peer.cache_scope(),
"one segment is one arena; both handles must key the same"
);
assert_eq!(
owner.cache_scope() >> 63,
1,
"a shared scope carries the tag bit that keeps it out of the counter's space"
);
let a = owner.frame("a").unwrap();
let b = owner.frame("b").unwrap();
let g = owner.guard().generation();
assert!(
!super::with_plan(&owner, b, a, g, |_| ()).unwrap().1,
"cold cache"
);
assert!(
super::with_plan(&peer, b, a, g, |_| ()).unwrap().1,
"the peer's FIRST lookup must reuse the owner's plan, not recompile"
);
}
}