subetha_cxc/
cache_ops.rs

1//! Cache-line stewardship: instruction-level hints the compiler
2//! never emits on its own.
3//!
4//! LLVM treats every store identically; it has no model of WHICH
5//! core reads a line next. These wrappers encode that knowledge at
6//! the three points the substrate has it:
7//!
8//! | op | when | effect |
9//! |---|---|---|
10//! | [`prefetchw`] | before a CAS/RMW on a contended line | requests the line in Modified state, collapsing the read-for-ownership round trip the CAS pays otherwise |
11//! | [`cldemote`] | after the producer publishes a slot + sequence | pushes the just-written line toward the shared LLC so the consumer's first read is an LLC hit, not a cross-core snoop-and-forward |
12//! | [`sfence`] | after non-temporal stores, before the `Release` publish | orders weakly-ordered streaming stores ahead of the sequence store other cores acquire on |
13//!
14//! Safety/portability posture (per the project's fallback mandate):
15//! every op compiles to a plain no-op on non-x86_64 targets, and on
16//! x86_64 each is either architecturally NOP-safe on silicon that
17//! lacks it or baseline-guaranteed:
18//!
19//! - `PREFETCHW` (`0F 0D /1`): prefetch hints never fault; AMD
20//!   since K6, Intel since Broadwell, and pre-Broadwell Intel
21//!   executes the encoding as a NOP (it sits in the NOP-reserved
22//!   hint space).
23//! - `CLDEMOTE` (`NP 0F 1C /0`): per the Intel ISA reference,
24//!   "on processors which do not support the CLDEMOTE instruction
25//!   (including legacy hardware) the instruction will be treated
26//!   as a NOP". CPUID `7.0` ECX bit 25 reports real support
27//!   ([`has_cldemote`]) for diagnostics; execution needs no gate.
28//! - `SFENCE` is baseline x86-64 (SSE2).
29//!
30//! Both hint wrappers are emitted as explicit byte sequences, not
31//! mnemonics, so the assembler never gates them behind target
32//! features the baseline build does not enable.
33
34/// Write-intent prefetch of the cache line containing `addr`.
35/// Call immediately before a `compare_exchange` / `fetch_*` on a
36/// line another core probably owns: the line arrives in Modified
37/// state instead of being upgraded mid-RMW.
38#[inline(always)]
39pub fn prefetchw(addr: *const u8) {
40    #[cfg(target_arch = "x86_64")]
41    unsafe {
42        // PREFETCHW m8 = 0F 0D /1; modrm 0x08 = [rax] with reg=/1.
43        core::arch::asm!(
44            ".byte 0x0f, 0x0d, 0x08",
45            in("rax") addr,
46            options(nostack, preserves_flags, readonly),
47        );
48    }
49    #[cfg(not(target_arch = "x86_64"))]
50    {
51        _ = addr;
52    }
53}
54
55/// Demote the cache line containing `addr` toward the shared LLC.
56/// Call after the producer's final store to a line whose next
57/// reader is another core (slot payload, published sequence).
58/// Architecturally a NOP wherever unsupported; the hint may also
59/// be ignored by hardware - it is never load-bearing.
60#[inline(always)]
61pub fn cldemote(addr: *const u8) {
62    #[cfg(target_arch = "x86_64")]
63    unsafe {
64        // CLDEMOTE m8 = NP 0F 1C /0; modrm 0x00 = [rax] with reg=/0.
65        core::arch::asm!(
66            ".byte 0x0f, 0x1c, 0x00",
67            in("rax") addr,
68            options(nostack, preserves_flags, readonly),
69        );
70    }
71    #[cfg(not(target_arch = "x86_64"))]
72    {
73        _ = addr;
74    }
75}
76
77/// Store fence: orders all prior stores (including non-temporal
78/// ones, which `Release` ordering alone does NOT cover) before any
79/// later store. Required between a streaming copy and the
80/// sequence-publish store that makes it visible.
81#[inline(always)]
82pub fn sfence() {
83    #[cfg(target_arch = "x86_64")]
84    unsafe {
85        core::arch::x86_64::_mm_sfence();
86    }
87    #[cfg(not(target_arch = "x86_64"))]
88    core::sync::atomic::fence(core::sync::atomic::Ordering::Release);
89}
90
91/// Whether this CPU actually implements CLDEMOTE (CPUID `7.0` ECX
92/// bit 25). Diagnostic only - [`cldemote`] is NOP-safe regardless.
93pub fn has_cldemote() -> bool {
94    #[cfg(target_arch = "x86_64")]
95    {
96        use std::sync::OnceLock;
97        static PROBE: OnceLock<bool> = OnceLock::new();
98        *PROBE.get_or_init(|| {
99            let max_basic = core::arch::x86_64::__cpuid_count(0, 0).eax;
100            max_basic >= 7
101                && core::arch::x86_64::__cpuid_count(7, 0).ecx & (1 << 25) != 0
102        })
103    }
104    #[cfg(not(target_arch = "x86_64"))]
105    {
106        false
107    }
108}
109
110#[cfg(test)]
111mod tests {
112    use super::*;
113
114    #[test]
115    fn hints_execute_without_faulting() {
116        // The whole point: safe to execute blind on any x86_64 (and
117        // no-ops elsewhere). Run each against stack and heap lines.
118        let stack_val: u64 = 42;
119        let heap_val = Box::new([0u8; 256]);
120        for addr in [
121            &stack_val as *const u64 as *const u8,
122            heap_val.as_ptr(),
123            unsafe { heap_val.as_ptr().add(128) },
124        ] {
125            prefetchw(addr);
126            cldemote(addr);
127        }
128        sfence();
129        assert_eq!(stack_val, 42);
130    }
131
132    #[test]
133    fn cldemote_probe_is_stable() {
134        assert_eq!(has_cldemote(), has_cldemote());
135        println!("cldemote supported: {}", has_cldemote());
136    }
137}
subetha_cxc/cache_ops.rs

subetha_cxc/
cache_ops.rs
