use crate::sched::io_pool::global_io_pool;
const PREFETCH_LINE_BYTES: usize = 64;
#[cfg(target_arch = "x86_64")]
#[inline(always)]
unsafe fn prefetch_hint_l3(ptr: *const u8) {
use core::arch::x86_64::{_mm_prefetch, _MM_HINT_T2};
unsafe {
_mm_prefetch(ptr as *const i8, _MM_HINT_T2);
}
}
#[cfg(target_arch = "x86_64")]
#[inline(always)]
unsafe fn prefetch_hint_l2(ptr: *const u8) {
use core::arch::x86_64::{_mm_prefetch, _MM_HINT_T1};
unsafe {
_mm_prefetch(ptr as *const i8, _MM_HINT_T1);
}
}
#[cfg(not(target_arch = "x86_64"))]
#[inline(always)]
unsafe fn prefetch_hint_l3(_ptr: *const u8) {
}
#[cfg(not(target_arch = "x86_64"))]
#[inline(always)]
unsafe fn prefetch_hint_l2(_ptr: *const u8) {
}
pub fn prefetch_into_l3<T>(slice: &[T]) {
let Some(pool) = global_io_pool() else { return };
let addr = slice.as_ptr() as usize;
let bytes = std::mem::size_of_val(slice);
pool.submit(move || {
let base = addr as *const u8;
unsafe {
let mut off = 0usize;
while off < bytes {
prefetch_hint_l3(base.add(off));
off = off.saturating_add(PREFETCH_LINE_BYTES);
}
}
});
}
pub fn prefetch_into_l2<T>(slice: &[T]) {
let Some(pool) = global_io_pool() else { return };
let addr = slice.as_ptr() as usize;
let bytes = std::mem::size_of_val(slice);
pool.submit(move || {
let base = addr as *const u8;
unsafe {
let mut off = 0usize;
while off < bytes {
prefetch_hint_l2(base.add(off));
off = off.saturating_add(PREFETCH_LINE_BYTES);
}
}
});
}
#[inline]
pub fn prefetch_into_l3_inline<T>(slice: &[T]) {
let base = slice.as_ptr() as *const u8;
let bytes = std::mem::size_of_val(slice);
unsafe {
let mut off = 0usize;
while off < bytes {
prefetch_hint_l3(base.add(off));
off = off.saturating_add(PREFETCH_LINE_BYTES);
}
}
}
#[inline]
pub fn prefetch_into_l2_inline<T>(slice: &[T]) {
let base = slice.as_ptr() as *const u8;
let bytes = std::mem::size_of_val(slice);
unsafe {
let mut off = 0usize;
while off < bytes {
prefetch_hint_l2(base.add(off));
off = off.saturating_add(PREFETCH_LINE_BYTES);
}
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn prefetch_inline_does_not_crash_on_small_slice() {
let v: Vec<u32> = (0..16).collect();
prefetch_into_l3_inline(&v);
prefetch_into_l2_inline(&v);
}
#[test]
fn prefetch_inline_does_not_crash_on_large_slice() {
let v: Vec<u32> = (0..256 * 1024).collect();
prefetch_into_l3_inline(&v);
}
#[test]
fn prefetch_inline_does_not_crash_on_empty_slice() {
let v: Vec<u32> = Vec::new();
prefetch_into_l3_inline(&v);
prefetch_into_l2_inline(&v);
}
#[test]
fn prefetch_async_noop_when_pool_disabled() {
let v: Vec<u32> = (0..1024).collect();
let t0 = std::time::Instant::now();
prefetch_into_l3(&v);
let elapsed = t0.elapsed();
assert!(
elapsed < std::time::Duration::from_millis(10),
"async prefetch should be ~immediate when pool disabled"
);
}
}