use vyre::ir::model::expr::GeneratorRef;
use vyre::ir::{BufferDecl, DataType, Expr, Node, Program};
use vyre_driver::DispatchConfig;
use vyre_libs::region::{wrap_anonymous, wrap_child};
use vyre_primitives::hash::fnv1a::{fnv1a32_initial_expr, fnv1a32_update_byte_expr};
use crate::reference::{MergeEntry, MergeTable};
use crate::{PairKey, Rank, TokenId, ExatokError};
pub const LOOKUP_OP_ID: &str = "exatok::gpu_rank_table::lookup";
pub const PROBE_OP_ID: &str = "exatok::gpu_rank_table::probe";
pub const SLOT_WORDS: u32 = 4;
pub const SLOT_KEY_LO: u32 = 0;
pub const SLOT_KEY_HI: u32 = 1;
pub const SLOT_RANK: u32 = 2;
pub const SLOT_ID: u32 = 3;
pub const MAX_PROBE_LENGTH: u32 = 32;
pub const MAX_CAPACITY: u32 = 1 << 24;
pub const PAIR_LO_BUFFER: &str = "pair_lo";
pub const PAIR_HI_BUFFER: &str = "pair_hi";
pub const TABLE_SLOTS_BUFFER: &str = "table_slots";
pub const OUT_BUFFER: &str = "out_probe";
pub const LOOKUP_WORKGROUP: u32 = 256;
const SLOTS_PER_ROW: u64 = 4;
const TABLE_ARTIFACT: &str = "merge rank table";
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
enum Placement {
Placed,
Duplicate(Rank),
WindowExhausted,
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct RankTable {
capacity: u32,
limit: u32,
len: u32,
max_probe: u32,
slots: Vec<u32>,
}
impl RankTable {
pub fn with_capacity(capacity: u32) -> Result<Self, ExatokError> {
if capacity == 0 || !capacity.is_power_of_two() {
return Err(ExatokError::Artifact {
artifact: TABLE_ARTIFACT.to_owned(),
location: format!("capacity {capacity}"),
detail: "table capacity must be a non-zero power of two so the home slot is a bit mask".to_owned(),
fix: "round the capacity up with u32::next_power_of_two, or call RankTable::from_entries which sizes it".to_owned(),
});
}
if capacity > MAX_CAPACITY {
return Err(ExatokError::Artifact {
artifact: TABLE_ARTIFACT.to_owned(),
location: format!("capacity {capacity}"),
detail: format!("capacity exceeds the {MAX_CAPACITY} slot ceiling"),
fix: "shard the merge table across several uploads, or raise gpu_rank_table::MAX_CAPACITY after checking device memory".to_owned(),
});
}
let mut slots = vec![0_u32; capacity as usize * SLOT_WORDS as usize];
for slot in 0..capacity as usize {
slots[slot * SLOT_WORDS as usize + SLOT_RANK as usize] = Rank::ABSENT.0;
}
Ok(Self {
capacity,
limit: capacity.min(MAX_PROBE_LENGTH),
len: 0,
max_probe: 0,
slots,
})
}
pub fn from_entries(
artifact: &str,
entries: impl IntoIterator<Item = (PairKey, TokenId, Rank)>,
) -> Result<Self, ExatokError> {
let rows: Vec<(PairKey, TokenId, Rank)> = entries.into_iter().collect();
for &(pair, _, rank) in &rows {
if !rank.is_present() {
return Err(absent_rank_error(artifact, pair));
}
}
let mut capacity = Self::capacity_for(rows.len())?;
loop {
let mut table = Self::with_capacity(capacity)?;
let mut exhausted = false;
for &(pair, id, rank) in &rows {
match table.place(pair, id, rank) {
Placement::Placed => {}
Placement::Duplicate(existing) => {
return Err(duplicate_error(artifact, pair, existing, rank));
}
Placement::WindowExhausted => {
exhausted = true;
break;
}
}
}
if !exhausted {
return Ok(table);
}
capacity = capacity
.checked_mul(2)
.filter(|grown| *grown <= MAX_CAPACITY)
.ok_or_else(|| ExatokError::Artifact {
artifact: artifact.to_owned(),
location: format!("{} rows at capacity {capacity}", rows.len()),
detail: format!(
"a key still exceeded the {MAX_PROBE_LENGTH} slot probe cap at the {MAX_CAPACITY} slot ceiling"
),
fix: "shard the merge table, or widen gpu_rank_table::MAX_PROBE_LENGTH".to_owned(),
})?;
}
}
pub fn insert(
&mut self,
artifact: &str,
pair: PairKey,
id: TokenId,
rank: Rank,
) -> Result<(), ExatokError> {
if !rank.is_present() {
return Err(absent_rank_error(artifact, pair));
}
match self.place(pair, id, rank) {
Placement::Placed => Ok(()),
Placement::Duplicate(existing) => Err(duplicate_error(artifact, pair, existing, rank)),
Placement::WindowExhausted => Err(ExatokError::Artifact {
artifact: artifact.to_owned(),
location: format!(
"{} at home slot {}",
pair_location(pair),
self.home_slot(pair)
),
detail: format!(
"all {} slots within {MAX_PROBE_LENGTH} of the home slot are occupied ({} of {} slots used)",
self.limit, self.len, self.capacity
),
fix: "rebuild with RankTable::from_entries, which sizes capacity for the row count and grows on window exhaustion".to_owned(),
}),
}
}
#[must_use]
pub fn home_slot(&self, pair: PairKey) -> u32 {
pair.probe_hash() & (self.capacity - 1)
}
#[must_use]
pub const fn capacity(&self) -> u32 {
self.capacity
}
#[must_use]
pub const fn window(&self) -> u32 {
if self.max_probe == 0 {
1
} else {
self.max_probe
}
}
#[must_use]
pub const fn len(&self) -> u32 {
self.len
}
#[must_use]
pub const fn is_empty(&self) -> bool {
self.len == 0
}
#[must_use]
pub fn slot_words(&self) -> &[u32] {
&self.slots
}
#[must_use]
pub fn dispatch_inputs(&self, pairs: &[PairKey]) -> Vec<Vec<u8>> {
let (pair_lo, pair_hi) = pack_pair_columns(pairs);
let pack = vyre_primitives::wire::pack_u32_slice;
vec![pack(&pair_lo), pack(&pair_hi), pack(&self.slots)]
}
pub fn lookup_program(&self, lanes: u32) -> Result<Program, ExatokError> {
lookup_program(self.capacity, self.window(), lanes)
}
fn capacity_for(rows: usize) -> Result<u32, ExatokError> {
let rows = rows as u64;
let target = rows
.saturating_mul(SLOTS_PER_ROW)
.max(1)
.next_power_of_two();
u32::try_from(target)
.ok()
.filter(|capacity| *capacity <= MAX_CAPACITY)
.ok_or_else(|| ExatokError::Artifact {
artifact: TABLE_ARTIFACT.to_owned(),
location: format!("{rows} rows"),
detail: format!(
"a quarter-load table for {rows} rows needs {target} slots, above the {MAX_CAPACITY} slot ceiling"
),
fix: "shard the merge table across several uploads".to_owned(),
})
}
const fn base(slot: u32) -> usize {
slot as usize * SLOT_WORDS as usize
}
fn place(&mut self, pair: PairKey, id: TokenId, rank: Rank) -> Placement {
let (lo, hi) = split_key(pair);
let home = self.home_slot(pair);
let mask = self.capacity - 1;
for step in 0..self.limit {
let base = Self::base((home + step) & mask);
if self.slots[base + SLOT_RANK as usize] == Rank::ABSENT.0 {
self.slots[base + SLOT_KEY_LO as usize] = lo;
self.slots[base + SLOT_KEY_HI as usize] = hi;
self.slots[base + SLOT_RANK as usize] = rank.0;
self.slots[base + SLOT_ID as usize] = id.0;
self.len += 1;
self.max_probe = self.max_probe.max(step + 1);
return Placement::Placed;
}
if self.slots[base + SLOT_KEY_LO as usize] == lo
&& self.slots[base + SLOT_KEY_HI as usize] == hi
{
return Placement::Duplicate(Rank(self.slots[base + SLOT_RANK as usize]));
}
}
Placement::WindowExhausted
}
}
impl MergeTable for RankTable {
fn lookup(&self, pair: PairKey) -> Option<MergeEntry> {
let (lo, hi) = split_key(pair);
let home = self.home_slot(pair);
let mask = self.capacity - 1;
for step in 0..self.window() {
let base = Self::base((home + step) & mask);
let rank = self.slots[base + SLOT_RANK as usize];
if rank == Rank::ABSENT.0 {
return None;
}
if self.slots[base + SLOT_KEY_LO as usize] == lo
&& self.slots[base + SLOT_KEY_HI as usize] == hi
{
return Some(MergeEntry {
id: TokenId(self.slots[base + SLOT_ID as usize]),
rank: Rank(rank),
});
}
}
None
}
}
#[must_use]
pub const fn split_key(pair: PairKey) -> (u32, u32) {
(pair.right().0, pair.left().0)
}
#[must_use]
pub fn pack_pair_columns(pairs: &[PairKey]) -> (Vec<u32>, Vec<u32>) {
let mut lo = Vec::with_capacity(pairs.len());
let mut hi = Vec::with_capacity(pairs.len());
for pair in pairs {
let (pair_lo, pair_hi) = split_key(*pair);
lo.push(pair_lo);
hi.push(pair_hi);
}
(lo, hi)
}
#[must_use]
pub fn probe_rank_var(prefix: &str) -> String {
format!("{prefix}_rank")
}
#[must_use]
pub fn probe_id_var(prefix: &str) -> String {
format!("{prefix}_id")
}
#[must_use]
pub fn probe_nodes(
prefix: &str,
slots: &str,
capacity: u32,
window: u32,
key_lo: Expr,
key_hi: Expr,
) -> Vec<Node> {
let mask = capacity.wrapping_sub(1);
let lo_var = format!("{prefix}_query_lo");
let hi_var = format!("{prefix}_query_hi");
let home_var = format!("{prefix}_home");
let done_var = format!("{prefix}_done");
let step_var = format!("{prefix}_step");
let base_var = format!("{prefix}_base");
let slot_rank_var = format!("{prefix}_slot_rank");
let rank_var = probe_rank_var(prefix);
let id_var = probe_id_var(prefix);
let slot_rank = Expr::var(slot_rank_var.as_str());
let base = Expr::var(base_var.as_str());
let word = |offset: u32| Expr::load(slots, Expr::add(base.clone(), Expr::u32(offset)));
let occupied = Expr::ne(slot_rank.clone(), Expr::u32(Rank::ABSENT.0));
let key_matches = Expr::and(
Expr::eq(word(SLOT_KEY_LO), Expr::var(lo_var.as_str())),
Expr::eq(word(SLOT_KEY_HI), Expr::var(hi_var.as_str())),
);
vec![
Node::let_bind(lo_var.as_str(), key_lo),
Node::let_bind(hi_var.as_str(), key_hi),
Node::let_bind(
home_var.as_str(),
Expr::bitand(
probe_hash_expr(Expr::var(lo_var.as_str()), Expr::var(hi_var.as_str())),
Expr::u32(mask),
),
),
Node::let_bind(rank_var.as_str(), Expr::u32(Rank::ABSENT.0)),
Node::let_bind(id_var.as_str(), Expr::u32(0)),
Node::let_bind(done_var.as_str(), Expr::u32(0)),
Node::loop_for(
step_var.as_str(),
Expr::u32(0),
Expr::u32(window),
vec![Node::if_then(
Expr::eq(Expr::var(done_var.as_str()), Expr::u32(0)),
vec![
Node::let_bind(
base_var.as_str(),
Expr::mul(
Expr::bitand(
Expr::add(
Expr::var(home_var.as_str()),
Expr::var(step_var.as_str()),
),
Expr::u32(mask),
),
Expr::u32(SLOT_WORDS),
),
),
Node::let_bind(slot_rank_var.as_str(), word(SLOT_RANK)),
Node::if_then(
Expr::eq(slot_rank.clone(), Expr::u32(Rank::ABSENT.0)),
vec![Node::assign(done_var.as_str(), Expr::u32(1))],
),
Node::if_then(
Expr::and(occupied, key_matches),
vec![
Node::assign(rank_var.as_str(), slot_rank),
Node::assign(id_var.as_str(), word(SLOT_ID)),
Node::assign(done_var.as_str(), Expr::u32(1)),
],
),
],
)],
),
]
}
#[must_use]
pub fn probe_child(
parent_op_id: &str,
prefix: &str,
slots: &str,
capacity: u32,
window: u32,
key_lo: Expr,
key_hi: Expr,
then: Vec<Node>,
) -> Node {
let mut body = probe_nodes(prefix, slots, capacity, window, key_lo, key_hi);
body.extend(then);
wrap_child(
PROBE_OP_ID,
GeneratorRef {
name: parent_op_id.to_owned(),
},
body,
)
}
fn probe_hash_expr(key_lo: Expr, key_hi: Expr) -> Expr {
let mut hash = fnv1a32_initial_expr();
for word in [key_lo, key_hi] {
hash = fnv1a32_update_byte_expr(hash, word.clone());
for shift in [8_u32, 16, 24] {
hash = fnv1a32_update_byte_expr(hash, Expr::shr(word.clone(), Expr::u32(shift)));
}
}
hash
}
pub fn lookup_program(capacity: u32, window: u32, lanes: u32) -> Result<Program, ExatokError> {
if capacity == 0 || !capacity.is_power_of_two() || capacity > MAX_CAPACITY {
return Err(ExatokError::Artifact {
artifact: TABLE_ARTIFACT.to_owned(),
location: format!("lookup program capacity {capacity}"),
detail: format!(
"capacity must be a non-zero power of two no greater than {MAX_CAPACITY}"
),
fix: "pass RankTable::capacity() of the table being uploaded".to_owned(),
});
}
let max_window = capacity.min(MAX_PROBE_LENGTH);
if window == 0 || window > max_window {
return Err(ExatokError::Artifact {
artifact: TABLE_ARTIFACT.to_owned(),
location: format!("lookup program probe window {window}"),
detail: format!(
"probe window must be between 1 and {max_window} for a {capacity} slot table"
),
fix: "pass RankTable::window() of the table being uploaded".to_owned(),
});
}
if lanes == 0 {
return Err(ExatokError::Artifact {
artifact: TABLE_ARTIFACT.to_owned(),
location: "lookup program lane count 0".to_owned(),
detail: "a zero-length dispatch declares zero-element buffers, which no backend can bind".to_owned(),
fix: "skip the dispatch when there are no pairs to look up".to_owned(),
});
}
let words = capacity
.checked_mul(SLOT_WORDS)
.ok_or_else(|| ExatokError::Artifact {
artifact: TABLE_ARTIFACT.to_owned(),
location: format!("lookup program capacity {capacity}"),
detail: "capacity times SLOT_WORDS overflows a u32 buffer element count".to_owned(),
fix: "shard the merge table across several uploads".to_owned(),
})?;
let out_words = lanes.checked_mul(2).ok_or_else(|| ExatokError::Artifact {
artifact: TABLE_ARTIFACT.to_owned(),
location: format!("lookup program lane count {lanes}"),
detail: "two output words per lane overflows a u32 buffer element count".to_owned(),
fix: "split the query batch across several dispatches".to_owned(),
})?;
let lane = Expr::var("lane");
let body = vec![
Node::let_bind("lane", Expr::gid_x()),
Node::if_then(
Expr::lt(lane.clone(), Expr::u32(lanes)),
vec![probe_child(
LOOKUP_OP_ID,
"probe",
TABLE_SLOTS_BUFFER,
capacity,
window,
Expr::load(PAIR_LO_BUFFER, lane.clone()),
Expr::load(PAIR_HI_BUFFER, lane.clone()),
vec![
Node::store(
OUT_BUFFER,
lane.clone(),
Expr::var(probe_rank_var("probe").as_str()),
),
Node::store(
OUT_BUFFER,
Expr::add(lane, Expr::u32(lanes)),
Expr::var(probe_id_var("probe").as_str()),
),
],
)],
),
];
Ok(Program::wrapped(
vec![
BufferDecl::read(PAIR_LO_BUFFER, 0, DataType::U32).with_count(lanes),
BufferDecl::read(PAIR_HI_BUFFER, 1, DataType::U32).with_count(lanes),
BufferDecl::read(TABLE_SLOTS_BUFFER, 2, DataType::U32).with_count(words),
BufferDecl::output(OUT_BUFFER, 3, DataType::U32).with_count(out_words),
],
[LOOKUP_WORKGROUP, 1, 1],
vec![wrap_anonymous(LOOKUP_OP_ID, body)],
)
.with_entry_op_id(LOOKUP_OP_ID))
}
#[must_use]
pub fn lookup_dispatch_config(lanes: u32) -> DispatchConfig {
let mut config = DispatchConfig::default();
config.dispatch_elements = Some(lanes);
config
}
fn pair_location(pair: PairKey) -> String {
format!("pair ({}, {})", pair.left().0, pair.right().0)
}
fn absent_rank_error(artifact: &str, pair: PairKey) -> ExatokError {
ExatokError::Artifact {
artifact: artifact.to_owned(),
location: pair_location(pair),
detail: "merge rank equals the ABSENT sentinel u32::MAX, which marks an empty slot in the rank word".to_owned(),
fix: "renumber merges so every real rank stays below u32::MAX, keeping the sentinel unused so a min reduction skips unmergeable pairs".to_owned(),
}
}
fn duplicate_error(
artifact: &str,
pair: PairKey,
existing: Rank,
repeated: Rank,
) -> ExatokError {
ExatokError::Artifact {
artifact: artifact.to_owned(),
location: pair_location(pair),
detail: format!(
"duplicate merge: rank {} already recorded, rank {} seen again",
existing.0, repeated.0
),
fix: "remove the repeated pair from the merge list so exactly one rank governs it"
.to_owned(),
}
}
#[cfg(test)]
#[allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)]
mod tests {
use super::*;
#[test]
fn host_and_device_fnv1a_constants_are_the_same_values() {
assert_eq!(crate::FNV1A32_OFFSET_BASIS, 0x811c_9dc5);
assert_eq!(
crate::FNV1A32_OFFSET_BASIS,
vyre_primitives::hash::fnv1a::FNV1A32_OFFSET
);
assert_eq!(crate::FNV1A32_PRIME, 0x0100_0193);
assert_eq!(
crate::FNV1A32_PRIME,
vyre_primitives::hash::fnv1a::FNV1A32_PRIME
);
}
#[test]
fn empty_slot_marker_equals_the_primitive_miss_value() {
assert_eq!(Rank::ABSENT.0, vyre_primitives::hash::table::MISS_VALUE);
}
#[test]
fn split_key_puts_right_token_in_the_low_word() {
let pair = PairKey::new(TokenId(0xAAAA_BBBB), TokenId(0x1111_2222));
assert_eq!(split_key(pair), (0x1111_2222, 0xAAAA_BBBB));
assert_eq!(
pair.bits().to_le_bytes(),
[0x22, 0x22, 0x11, 0x11, 0xBB, 0xBB, 0xAA, 0xAA]
);
}
#[test]
fn a_fresh_table_stamps_absent_into_every_rank_word() {
let table = RankTable::with_capacity(8).unwrap();
assert_eq!(table.slot_words().len(), 8 * SLOT_WORDS as usize);
for slot in 0..8_u32 {
let base = RankTable::base(slot);
assert_eq!(table.slot_words()[base + SLOT_KEY_LO as usize], 0);
assert_eq!(table.slot_words()[base + SLOT_KEY_HI as usize], 0);
assert_eq!(
table.slot_words()[base + SLOT_RANK as usize],
Rank::ABSENT.0,
"slot {slot} rank word must start ABSENT"
);
assert_eq!(table.slot_words()[base + SLOT_ID as usize], 0);
}
assert_eq!(table.len(), 0);
assert!(
table.is_empty(),
"is_empty() must agree with the exact len()==0 pinned above"
);
}
#[test]
fn an_inserted_row_lands_in_the_documented_word_order() {
let mut table = RankTable::with_capacity(16).unwrap();
let pair = PairKey::new(TokenId(0x0001_0203), TokenId(0x0405_0607));
table
.insert("unit", pair, TokenId(9_999), Rank(1_234))
.unwrap();
let base = RankTable::base(table.home_slot(pair));
assert_eq!(table.slot_words()[base + SLOT_KEY_LO as usize], 0x0405_0607);
assert_eq!(table.slot_words()[base + SLOT_KEY_HI as usize], 0x0001_0203);
assert_eq!(table.slot_words()[base + SLOT_RANK as usize], 1_234);
assert_eq!(table.slot_words()[base + SLOT_ID as usize], 9_999);
assert_eq!(table.len(), 1);
}
#[test]
fn build_capacity_stays_a_power_of_two_at_or_below_quarter_load() {
assert_eq!(RankTable::capacity_for(0).unwrap(), 1);
assert_eq!(RankTable::capacity_for(1).unwrap(), 4);
assert_eq!(RankTable::capacity_for(3).unwrap(), 16);
assert_eq!(RankTable::capacity_for(50_000).unwrap(), 262_144);
assert_eq!(RankTable::capacity_for(446_189).unwrap(), 2_097_152);
for rows in [1_usize, 7, 100, 4_096, 50_000, 233_378, 446_189] {
let capacity = RankTable::capacity_for(rows).unwrap();
assert!(capacity.is_power_of_two(), "rows={rows} capacity={capacity}");
assert!(
u64::from(capacity) >= (rows as u64) * SLOTS_PER_ROW,
"rows={rows} capacity={capacity} is above quarter load"
);
}
}
#[test]
fn capacity_above_the_ceiling_is_refused() {
let err = RankTable::with_capacity(MAX_CAPACITY * 2).unwrap_err();
assert!(
err.to_string().contains("slot ceiling"),
"wrong diagnostic: {err}"
);
let err = RankTable::capacity_for(1 << 30).unwrap_err();
assert!(
err.to_string().contains("slot ceiling"),
"wrong diagnostic: {err}"
);
}
#[test]
fn non_power_of_two_capacity_is_refused() {
for capacity in [0_u32, 3, 100, 1000] {
let err = RankTable::with_capacity(capacity).unwrap_err();
assert!(
err.to_string().contains("power of two"),
"capacity {capacity} gave the wrong diagnostic: {err}"
);
}
}
#[test]
fn probe_bound_tracks_the_longest_probe_actually_needed() {
let mut table = RankTable::with_capacity(1024).unwrap();
assert_eq!(table.window(), 1);
table
.insert("unit", PairKey::new(TokenId(1), TokenId(2)), TokenId(3), Rank(0))
.unwrap();
assert_eq!(table.window(), 1, "one key sits in its own home slot");
let victim = PairKey::new(TokenId(700), TokenId(800));
let home = table.home_slot(victim);
let squatter = (0_u32..)
.map(|n| PairKey::new(TokenId(900_000 + n), TokenId(7)))
.find(|candidate| table.home_slot(*candidate) == home)
.unwrap();
table
.insert("unit", squatter, TokenId(11), Rank(11))
.unwrap();
table.insert("unit", victim, TokenId(22), Rank(22)).unwrap();
assert_eq!(table.window(), 2);
assert_eq!(table.rank(victim), Rank(22));
assert_eq!(table.rank(squatter), Rank(11));
}
#[test]
fn lookup_program_refuses_a_window_outside_the_table_bound() {
assert!(lookup_program(1024, 0, 8).is_err());
assert!(lookup_program(1024, MAX_PROBE_LENGTH + 1, 8).is_err());
assert!(lookup_program(4, 5, 8).is_err(), "window must clamp to capacity");
assert!(lookup_program(1024, MAX_PROBE_LENGTH, 8).is_ok());
assert!(lookup_program(4, 4, 8).is_ok());
}
#[test]
fn probe_child_nests_its_consumer_inside_a_region_naming_its_parent() {
let node = probe_child(
LOOKUP_OP_ID,
"probe",
TABLE_SLOTS_BUFFER,
1024,
8,
Expr::u32(1),
Expr::u32(2),
vec![Node::store(
OUT_BUFFER,
Expr::u32(0),
Expr::var(probe_rank_var("probe").as_str()),
)],
);
let bare = probe_nodes("probe", TABLE_SLOTS_BUFFER, 1024, 8, Expr::u32(1), Expr::u32(2));
match node {
Node::Region {
generator,
source_region,
body,
..
} => {
assert_eq!(generator.as_str(), PROBE_OP_ID);
assert_eq!(
source_region.map(|parent| parent.name),
Some(LOOKUP_OP_ID.to_owned())
);
assert_eq!(
body.len(),
bare.len() + 1,
"the region must hold the probe fragment plus the one consumer node"
);
assert!(
matches!(body.last(), Some(Node::Store { buffer, .. }) if buffer == OUT_BUFFER),
"the consumer must be the region's last node, not its sibling"
);
}
other => panic!("probe_child must emit a Region, got {other:?}"),
}
}
#[test]
fn op_ids_use_the_tier_four_exatok_prefix() {
for id in [LOOKUP_OP_ID, PROBE_OP_ID] {
assert!(id.starts_with("exatok::gpu_rank_table::"), "bad op id: {id}");
assert!(!id.contains("vyre-libs::"), "bad op id: {id}");
}
}
}