yo_common/
prefetch.rs

1//! Telling the cache what the next walk is going to want.
2//!
3//! `04` section 3 walks a drained batch twice. The first walk works out which
4//! index bucket each command will land in and asks for that line; the second
5//! walk executes, and by then the line is on its way or already there. The
6//! whole batch is the prefetch distance, which is 64 rather than Valkey's or
7//! Redis 8.4's 16, because Y1 means there is no lock held across the window and
8//! no other thread that can invalidate a bucket between the ask and the use.
9//!
10//! There is no stable portable intrinsic for this, so there are three
11//! implementations here and they are all one instruction. x86_64 gets
12//! `prefetcht0`, aarch64 gets `prfm pldl1keep`, and anything else gets nothing
13//! at all, because a hint that has to be emulated is not a hint. Miri also gets
14//! nothing, since it does not run inline assembly and there is no correctness
15//! in here for it to check.
16
17/// Ask for the cache line at `p`, for reading, into every level of cache.
18///
19/// A hint and only a hint. It cannot fault, it cannot fail, and it does not
20/// change what any later load returns. The only thing it can do wrong is be
21/// pointed at a line nobody wants, which costs bandwidth and nothing else.
22///
23/// The pointer is not dereferenced, so it does not have to be aligned and it
24/// does not have to be readable, but it should point at something real or the
25/// prefetch is just noise.
26#[inline(always)]
27pub fn prefetch_read(p: *const u8) {
28    // Miri does not do inline assembly and has nothing to say about a hint, so
29    // it gets the version where the hint is not there.
30    #[cfg(miri)]
31    let _ = p;
32
33    #[cfg(all(not(miri), target_arch = "x86_64"))]
34    // SAFETY: `_mm_prefetch` is a hint. It does not read through the pointer,
35    // it cannot fault whatever the pointer holds, and it is available on every
36    // x86_64 because SSE is part of the baseline.
37    unsafe {
38        core::arch::x86_64::_mm_prefetch(p.cast::<i8>(), core::arch::x86_64::_MM_HINT_T0);
39    }
40
41    #[cfg(all(not(miri), target_arch = "aarch64"))]
42    // SAFETY: `prfm` is a hint. The architecture defines it as having no effect
43    // other than on performance, it does not fault on an address the load would
44    // have faulted on, and the operand is only read.
45    unsafe {
46        core::arch::asm!(
47            "prfm pldl1keep, [{p}]",
48            p = in(reg) p,
49            options(nostack, readonly, preserves_flags),
50        );
51    }
52
53    #[cfg(all(not(miri), not(any(target_arch = "x86_64", target_arch = "aarch64"))))]
54    let _ = p;
55}
56
57/// The same hint for a reference, which is the shape the call sites have.
58#[inline(always)]
59pub fn prefetch<T>(r: &T) {
60    prefetch_read(core::ptr::from_ref(r).cast::<u8>());
61}
62
63#[cfg(test)]
64mod tests {
65    use super::*;
66
67    /// There is nothing to assert about a hint except that it happened and the
68    /// program carried on, which is exactly what this checks. It is here so
69    /// that a build for an architecture without one of the two instructions
70    /// still compiles and runs this path.
71    #[test]
72    fn a_hint_changes_nothing_it_can_be_asked_about() {
73        let v: Vec<u64> = (0..1024).collect();
74        for i in (0..v.len()).step_by(8) {
75            prefetch(&v[i]);
76        }
77        assert_eq!(v[1023], 1023);
78        prefetch_read(v.as_ptr().cast());
79        assert_eq!(v.iter().sum::<u64>(), (0..1024u64).sum());
80    }
81
82    /// A pointer past the end is still only a hint. This is the case that would
83    /// be a segfault if the implementation ever became a load.
84    #[test]
85    fn a_line_nobody_owns_is_still_only_a_hint() {
86        let v = [1u8, 2, 3, 4];
87        // SAFETY: `add` on the one past the end pointer is in bounds for the
88        // pointer arithmetic rules, and nothing dereferences it here.
89        let past = unsafe { v.as_ptr().add(v.len()) };
90        prefetch_read(past);
91        assert_eq!(v[0], 1);
92    }
93}
yo_common/prefetch.rs

yo_common/
prefetch.rs
