et_abi/lib.rs
1//! Shared host/device ABI for the ET-SoC-1: the kernel-launch argument structs,
2//! defined **once** and used by both the host launcher and the device kernel.
3//!
4//! Kernel arguments are passed by pointer: the host stages an argument struct in
5//! device memory and the firmware delivers its address to the kernel (in `a0`).
6//! Because both the host (x86-64) and the device (RV64) are little-endian, the
7//! in-memory `#[repr(C)]` layout *is* the wire layout, so no explicit
8//! serialisation is needed -- the host takes the struct's bytes and the kernel
9//! reinterprets the pointer. Defining each struct here keeps the two sides from
10//! drifting (mismatched field order, sizes, or padding).
11
12#![no_std]
13
14/// ET-SoC-1 cache-line size, in bytes.
15///
16/// Per-hart outputs are laid out at this stride on both sides: the host strides
17/// its padded arrays by it and the device writes each hart's cell at
18/// `base + hart * CACHE_LINE`. Defining it once here keeps the two from drifting,
19/// which on this software-coherent part would cause silent false-sharing
20/// corruption.
21pub const CACHE_LINE: usize = 64;
22
23/// A wrapper that aligns `T` to a cache-line boundary.
24///
25/// On the ET-SoC-1 (a software-coherent architecture), two values sharing a
26/// cache line that are written by distinct harts without explicit cache
27/// operations cause false-sharing corruption. Wrapping per-hart output data
28/// in `CachePadded` ensures each instance occupies a distinct 64-byte line,
29/// making cross-hart false sharing structurally impossible regardless of the
30/// surrounding allocation layout.
31///
32/// The inner value is accessed directly via the public tuple field `0`.
33///
34/// # Example
35///
36/// ```
37/// use et_abi::CachePadded;
38/// let cell: CachePadded<u64> = CachePadded(0);
39/// assert_eq!(core::mem::align_of::<CachePadded<u64>>(), 64);
40/// ```
41#[repr(align(64))]
42#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
43pub struct CachePadded<T>(pub T);
44
45/// Harts per compute shire on the ET-SoC-1 (architectural constant).
46pub const HARTS_PER_SHIRE: u32 = 64;
47
48/// Harts per neighbourhood on the ET-SoC-1 (architectural constant).
49pub const HARTS_PER_NEIGHBOURHOOD: u32 = 16;
50
51/// A plain-old-data kernel-argument struct exchanged between host and device.
52///
53/// # Safety
54/// Implementors must be `#[repr(C)]`, contain only integer fields with no
55/// padding, and be valid for any bit pattern. Then [`DeviceArgs::as_bytes`] and
56/// [`DeviceArgs::from_ptr`] are a faithful round-trip on little-endian hosts and
57/// devices.
58pub unsafe trait DeviceArgs: Sized + Copy {
59 /// Borrow the struct as its on-wire bytes (host side: stage these in device
60 /// memory as the launch arguments).
61 fn as_bytes(&self) -> &[u8] {
62 // SAFETY: `Self` is repr(C) POD (trait contract), so its bytes are a
63 // valid representation of length `size_of::<Self>()`.
64 unsafe {
65 core::slice::from_raw_parts(
66 self as *const Self as *const u8,
67 core::mem::size_of::<Self>(),
68 )
69 }
70 }
71
72 /// Reinterpret a device-memory pointer as these arguments (device side).
73 ///
74 /// # Safety
75 /// `ptr` must point to at least `size_of::<Self>()` bytes of a valid,
76 /// suitably aligned instance -- e.g. the launch-args region the firmware
77 /// passed in `a0`.
78 unsafe fn from_ptr<'a>(ptr: *const u8) -> &'a Self {
79 // SAFETY: forwarded to the caller's contract on `ptr`.
80 unsafe { &*(ptr as *const Self) }
81 }
82}
83
84// ---------------------------------------------------------------------------
85// DevicePod
86// ---------------------------------------------------------------------------
87
88/// Types that may be copied verbatim between host and device memory.
89///
90/// # Safety
91/// An implementor must be "plain old data": a scalar, or a `#[repr(C)]` struct
92/// composed entirely of such types, with no padding bytes and valid for every
93/// bit pattern. These invariants make it safe to reinterpret a value (or slice)
94/// as its raw bytes and back again, which the upload/download DMA paths rely on.
95///
96/// The provided implementations cover the standard integer and floating-point
97/// scalars. Implement this trait for your own `#[repr(C)]` POD structs to store
98/// them in a `DeviceBuffer` or pass them via `upload_slice`.
99///
100/// Because the trait is defined in `et-abi`, external crates can implement it
101/// for their own types without running into the orphan rule.
102pub unsafe trait DevicePod: Copy + 'static {}
103
104unsafe impl DevicePod for u8 {}
105unsafe impl DevicePod for u16 {}
106unsafe impl DevicePod for u32 {}
107unsafe impl DevicePod for u64 {}
108unsafe impl DevicePod for u128 {}
109unsafe impl DevicePod for i8 {}
110unsafe impl DevicePod for i16 {}
111unsafe impl DevicePod for i32 {}
112unsafe impl DevicePod for i64 {}
113unsafe impl DevicePod for i128 {}
114unsafe impl DevicePod for f32 {}
115unsafe impl DevicePod for f64 {}
116unsafe impl DevicePod for usize {}
117unsafe impl DevicePod for isize {}
118
119// ---------------------------------------------------------------------------
120// Tensor-extension constants
121// ---------------------------------------------------------------------------
122
123/// Required alignment for all matrix pointers and row strides used with the
124/// ET-SoC-1 tensor-load/store instructions. TensorLoad and TensorStore each
125/// require the source or destination address to be 64-byte aligned.
126pub const TENSOR_ALIGN: usize = 64;
127
128/// Number of addressable cache lines in each Minion's L1 scratchpad.
129/// TensorLoad START field is 6 bits, spanning lines 0..47 inclusive.
130pub const SCP_LINES: usize = 48;
131
132/// Bytes per L1 scratchpad line (one cache line).
133pub const SCP_LINE_BYTES: usize = 64;
134
135/// Minion cores per compute shire on the ET-SoC-1.
136/// Each shire has 32 dual-threaded Minion cores (64 harts total).
137pub const MINIONS_PER_SHIRE: u32 = 32;
138
139// ---------------------------------------------------------------------------
140// GEMM tile dimensions
141// ---------------------------------------------------------------------------
142
143/// Number of C output rows computed per tile by TensorFMA32.
144/// Equals the maximum AROWS+1 value (4-bit field, max 15 -> 16 rows).
145pub const GEMM_TILE_M: usize = 16;
146
147/// Inner-dimension (K) slice processed per TensorFMA32 call.
148/// Limited to 16 f32 values per A-matrix row fitting in one 64-byte
149/// scratchpad line (ACOLS field is 4-bit, max 15 -> 16 columns).
150pub const GEMM_TILE_K: usize = 16;
151
152/// Number of f32 output columns produced per TensorFMA32 call (BCOLS=3 gives
153/// 4*(3+1) = 16 columns). Each tile row occupies exactly 64 bytes in the FP
154/// register file. N need not be a multiple of this value; the last tile column
155/// may be partial, with the hardware writing 64 bytes per row regardless --
156/// the caller reads only the N valid columns from the 64-byte-aligned allocation.
157pub const GEMM_TILE_N: usize = 16;
158
159// ---------------------------------------------------------------------------
160// GemmArgs
161// ---------------------------------------------------------------------------
162
163/// Arguments for the single-precision general matrix multiplication (sGEMM)
164/// kernel (`sgemm-rs`), implementing C = alpha*A*B + beta*C.
165///
166/// # Layout invariants (v0.1 restrictions)
167/// - `alpha` must be `1.0` and `beta` must be `0.0`.
168/// - `n` may be any positive integer; partial last-column tiles are handled
169/// transparently via 64-byte-aligned row padding.
170/// - `a`, `b`, `c` must be [`TENSOR_ALIGN`]-byte aligned device addresses.
171/// - `lda`, `ldb`, `ldc` must be multiples of [`TENSOR_ALIGN`] (64 bytes).
172///
173/// All dimensions are in elements; leading dimensions are in bytes.
174///
175/// # ABI layout
176/// The four 8-byte fields (`a`, `b`, `c`, `n_shires`) are grouped first to
177/// give the struct 8-byte alignment with no internal or trailing padding:
178/// `4*8 + 8*4 = 64 bytes` total.
179#[repr(C)]
180#[derive(Clone, Copy, Debug, PartialEq)]
181pub struct GemmArgs {
182 /// Device address of A [M x K], row-major, 64-byte aligned.
183 pub a: u64,
184 /// Device address of B [K x N], row-major, 64-byte aligned.
185 pub b: u64,
186 /// Device address of C [M x N], row-major, 64-byte aligned.
187 pub c: u64,
188 /// Number of participating compute shires. Stored as `u64` to keep
189 /// all 8-byte fields contiguous and the total struct size a multiple
190 /// of the struct's 8-byte alignment. Effective range: 1..=34.
191 pub n_shires: u64,
192 /// Number of rows of A and C (M dimension).
193 pub m: u32,
194 /// Number of columns of B and C (N dimension). May be any positive integer;
195 /// the last output tile column is partial when N is not a multiple of 16.
196 pub n: u32,
197 /// Shared inner dimension (K): columns of A and rows of B.
198 pub k: u32,
199 /// Row stride of A in bytes (multiple of 64).
200 pub lda: u32,
201 /// Row stride of B in bytes (multiple of 64).
202 pub ldb: u32,
203 /// Row stride of C in bytes (multiple of 64).
204 pub ldc: u32,
205 /// A*B scaling factor. Must be `1.0` in v0.1.
206 pub alpha: f32,
207 /// C scaling factor. Must be `0.0` in v0.1.
208 pub beta: f32,
209}
210
211// SAFETY: repr(C); 4 u64 fields followed by 8 u32/f32 fields, ordered by
212// decreasing size -> no padding. 4*8 + 8*4 = 64 bytes, a multiple of the
213// struct's 8-byte alignment.
214unsafe impl DeviceArgs for GemmArgs {}
215const _: () = assert!(core::mem::size_of::<GemmArgs>() == 64);
216
217/// Arguments for the cache-coherence test kernel (`cache-test-rs`).
218///
219/// Each primary Minion hart writes its global Minion index to
220/// `output[minion_idx]` (stride = 64 bytes, one u32 per cache line), then
221/// calls `cache_writeback` and `fence`. The host downloads the padded array
222/// and verifies `output[i] == i as u32`.
223#[repr(C)]
224#[derive(Clone, Copy, Debug, PartialEq, Eq)]
225pub struct CacheTestArgs {
226 /// Device address of the output buffer:
227 /// `n_shires * MINIONS_PER_SHIRE` entries of `u32`, each at stride 64.
228 pub output: u64,
229 /// Number of participating compute shires (1..=32).
230 pub n_shires: u64,
231 /// Writeback destination level (`CacheDest` discriminant: 1 = L2, 2 = L3,
232 /// 3 = Mem/DDR). Selects how far the writeback propagates so the fault can be
233 /// narrowed to the DDR path; only `Mem` is host-DMA visible.
234 pub dest: u64,
235}
236
237// SAFETY: repr(C), three u64 fields, no padding.
238unsafe impl DeviceArgs for CacheTestArgs {}
239const _: () = assert!(core::mem::size_of::<CacheTestArgs>() == 24);
240
241/// Arguments for the data-parallel reduction kernel (`reduce-rs`).
242#[repr(C)]
243#[derive(Clone, Copy, Debug, PartialEq, Eq)]
244pub struct ReduceArgs {
245 /// Device address of the input array (`n` x `u32`).
246 pub input: u64,
247 /// Device address of the output array (`n_harts` x one `u64` per cache line).
248 pub out: u64,
249 /// Number of input elements.
250 pub n: u32,
251 /// Number of participating harts.
252 pub n_harts: u32,
253}
254
255// SAFETY: repr(C), only u64/u32 fields ordered by decreasing size -> no padding.
256unsafe impl DeviceArgs for ReduceArgs {}
257const _: () = assert!(core::mem::size_of::<ReduceArgs>() == 24);
258
259/// Bytes of output per Minion written by the tensor-extension test kernel: one
260/// cache line for each of its three subtests. Shared by the kernel and the host
261/// so the two cannot disagree on the layout.
262pub const TENSOR_EXT_TEST_OUT_STRIDE: usize = 3 * CACHE_LINE;
263
264/// Arguments for the tensor-extension instruction test kernel (`tensor-ext-test`).
265///
266/// All Minions read the same shared input buffers and write their results to
267/// per-Minion sections of `output` for independent host verification.
268///
269/// # Output buffer layout (per Minion, stride [`TENSOR_EXT_TEST_OUT_STRIDE`])
270/// Minion `shire * 32 + m` writes at byte offset
271/// `(shire * 32 + m) * TENSOR_EXT_TEST_OUT_STRIDE`:
272/// - `[0..64)`: TensorFMA16A32 result: 4 x f32, expected `[5.0, 0.0, 0.0, 0.0]`.
273/// - `[64..128)`: TensorIMA8A32 result: 4 x i32 stored as f32 bit patterns,
274/// expected `[4, 0, 0, 0]`.
275/// - `[128..192)`: TensorStoreFromScp passthrough: 64 bytes copied verbatim from
276/// L1 scratchpad line 0, expected to equal the `a_fp16` buffer contents.
277///
278/// # Input data
279/// Inputs encode the simplest non-trivial tile (AROWS=0, ACOLS=0, BCOLS=0):
280/// - `a_fp16`: `[2.0_f16, 3.0_f16, 0..0]` (64 bytes; first 4 bytes used).
281/// - `b_fp16`: TenB-interleaved fp16 pairs: `[1.0, 1.0, 0.0, ...]`
282/// (64 bytes; first 4 bytes = `b[0,0]=1.0` and `b[1,0]=1.0` for output col 0).
283/// - `a_int8`: `[1, 1, 1, 1, 0..0]` (64 bytes; first 4 bytes used).
284/// - `b_int8`: IMA8A32-interleaved int8 groups: col 0 = `[1,1,1,1]`, cols 1-3 = 0
285/// (64 bytes; first 16 bytes used).
286#[repr(C)]
287#[derive(Clone, Copy, Debug, PartialEq)]
288pub struct TensorExtTestArgs {
289 /// Base address of the output buffer: `n_shires * 32 *
290 /// TENSOR_EXT_TEST_OUT_STRIDE` bytes, 64-byte aligned.
291 pub output: u64,
292 /// 64-byte-aligned device address of the fp16 A input (64 bytes).
293 pub a_fp16: u64,
294 /// 64-byte-aligned device address of the fp16 B input in TenB interleaved
295 /// format (64 bytes). (PRM TensorFMA16A32: `TenB[k].h[j*2+0] = b[2k,j]`.)
296 pub b_fp16: u64,
297 /// 64-byte-aligned device address of the int8 A input (64 bytes).
298 pub a_int8: u64,
299 /// 64-byte-aligned device address of the int8 B input in IMA8A32 interleaved
300 /// format (64 bytes). (PRM TensorIMA8A32: word j = `[b[0,j]|b[1,j]|b[2,j]|b[3,j]]`.)
301 pub b_int8: u64,
302 /// Extent of the output array in shires (1..=32). Cells are indexed by
303 /// physical shire number, so this is one more than the highest launched
304 /// shire, not the number of launched shires; Minions in shires at or
305 /// beyond it do not write.
306 pub n_shires: u64,
307}
308
309// SAFETY: repr(C), six u64 fields, no padding.
310unsafe impl DeviceArgs for TensorExtTestArgs {}
311const _: () = assert!(core::mem::size_of::<TensorExtTestArgs>() == 48);
312
313/// Arguments for the PS SIMD instruction verification kernel (`simd-test-rs`).
314///
315/// Each primary Minion computes `(minion_idx + 1) as f32 * 3.0` in all 16
316/// lanes of one C-tile row (FP registers f0 and f1) using the `FBCX.PS`
317/// (broadcast) and `FMUL.PS` (element-wise multiply) instructions, and stores
318/// the row to its output cell with TensorStore. The host verifies all 16
319/// values of every cell.
320///
321/// # Output buffer layout
322/// `n_shires * MINIONS_PER_SHIRE` cells of 16 `f32` (one 64-byte cache line
323/// each). Minion `i = shire * 32 + m`, indexed by physical shire, writes the
324/// cell at byte offset `i * 64`.
325#[repr(C)]
326#[derive(Clone, Copy, Debug, PartialEq)]
327pub struct SimdTestArgs {
328 /// Device address of the output buffer.
329 pub output: u64,
330 /// Extent of the output array in shires (1..=32): one more than the highest
331 /// launched shire. Minions in shires at or beyond it do not write.
332 pub n_shires: u64,
333}
334
335// SAFETY: repr(C), two u64 fields, no padding.
336unsafe impl DeviceArgs for SimdTestArgs {}
337const _: () = assert!(core::mem::size_of::<SimdTestArgs>() == 16);
338
339#[cfg(test)]
340mod tests {
341 use super::*;
342
343 #[test]
344 fn gemm_args_size() {
345 // 4 u64 + 8 u32/f32 = 32 + 32 = 64 bytes, a multiple of 8.
346 assert_eq!(core::mem::size_of::<GemmArgs>(), 64);
347 }
348
349 #[test]
350 fn gemm_args_roundtrip() {
351 let a = GemmArgs {
352 a: 0x0080_0100_0000,
353 b: 0x0080_0200_0000,
354 c: 0x0080_0300_0000,
355 n_shires: 4,
356 m: 128,
357 n: 64,
358 k: 256,
359 lda: 1024, // 256 * 4 bytes, 64-byte aligned
360 ldb: 256, // 64 * 4 bytes, 64-byte aligned
361 ldc: 256, // 64 * 4 bytes, 64-byte aligned
362 alpha: 1.0,
363 beta: 0.0,
364 };
365 let bytes = a.as_bytes();
366 assert_eq!(bytes.len(), 64);
367 let b = unsafe { GemmArgs::from_ptr(bytes.as_ptr()) };
368 assert_eq!(*b, a);
369 // Verify leading dimensions are 64-byte aligned as the kernel requires.
370 assert_eq!(a.lda as usize % TENSOR_ALIGN, 0);
371 assert_eq!(a.ldb as usize % TENSOR_ALIGN, 0);
372 assert_eq!(a.ldc as usize % TENSOR_ALIGN, 0);
373 // Verify N is a multiple of GEMM_TILE_N.
374 assert_eq!(a.n as usize % GEMM_TILE_N, 0);
375 }
376
377 #[test]
378 fn reduce_args_roundtrip() {
379 let a = ReduceArgs {
380 input: 0x0080_0580_1000,
381 out: 0x0080_0590_0000,
382 n: 262_144,
383 n_harts: 64,
384 };
385 let bytes = a.as_bytes();
386 assert_eq!(bytes.len(), 24);
387 // The device would do exactly this from the args pointer.
388 let b = unsafe { ReduceArgs::from_ptr(bytes.as_ptr()) };
389 assert_eq!(*b, a);
390 }
391}