et_soc1/buffer.rs
1//! Typed device-memory buffers layered over the byte-oriented DMA primitives.
2//!
3//! [`Device::alloc`] plus [`Device::memcpy_h2d`]/[`Device::memcpy_d2h`] are the
4//! low-level interface: they deal in raw addresses and `&[u8]`, so call sites
5//! reinterpret their data with `unsafe` slice casts and track sizes by hand.
6//! The types here restore that type information. A [`DeviceBuffer<T>`] is a
7//! lightweight handle (device address, element count, element type); the owning
8//! [`Device`] uploads and downloads whole typed slices, so no byte arithmetic or
9//! `unsafe` reinterpretation appears at the call site.
10//!
11//! [`PaddedArray<T>`] additionally lays out one element per cache line, which the
12//! ET-SoC-1's software-managed coherence requires when distinct harts write
13//! distinct elements: without the padding, two harts sharing a cache line
14//! corrupt each other's writes silently (false sharing). It pairs with the
15//! device-side `Grid::output_cell`, which writes each hart's result at
16//! `base + hart * CACHE_LINE`.
17
18use std::marker::PhantomData;
19use std::mem::size_of;
20
21use et_abi::CACHE_LINE;
22pub use et_abi::DevicePod;
23
24use crate::device::{Device, DeviceRegion, DmaOptions};
25use crate::error::{Error, Result};
26use crate::transport::Transport;
27
28/// Reinterpret a POD slice as its byte representation.
29fn as_bytes<E: DevicePod>(data: &[E]) -> &[u8] {
30 // SAFETY: `E: DevicePod` is POD, so the `size_of_val` bytes backing `data`
31 // are a valid byte representation of it.
32 unsafe { std::slice::from_raw_parts(data.as_ptr() as *const u8, std::mem::size_of_val(data)) }
33}
34
35/// Mutable byte view of a slice of device POD values.
36fn as_bytes_mut<E: DevicePod>(data: &mut [E]) -> &mut [u8] {
37 // SAFETY: as in `as_bytes`; additionally, any byte pattern written through
38 // the view is a valid `E`, because `E: DevicePod` admits every bit pattern.
39 unsafe {
40 std::slice::from_raw_parts_mut(data.as_mut_ptr() as *mut u8, std::mem::size_of_val(data))
41 }
42}
43
44/// A host vector of `n` zero-initialised values of `E`, ready to receive a DMA.
45fn zeroed_vec<E: DevicePod>(n: usize) -> Result<Vec<E>> {
46 byte_len_of::<E>(n)?;
47 let mut out: Vec<E> = Vec::with_capacity(n);
48 // SAFETY: capacity for `n` values is reserved, the bytes are initialised to
49 // zero before the length is set, and all-zero is a valid `E: DevicePod`.
50 unsafe {
51 std::ptr::write_bytes(out.as_mut_ptr(), 0, n);
52 out.set_len(n);
53 }
54 Ok(out)
55}
56
57/// Size in bytes of `n` values of `E`, or [`Error::Limit`] on overflow.
58fn byte_len_of<E>(n: usize) -> Result<usize> {
59 n.checked_mul(size_of::<E>()).ok_or_else(|| {
60 Error::Limit(format!(
61 "{n} elements of {} bytes overflow usize",
62 size_of::<E>()
63 ))
64 })
65}
66
67/// A typed handle to a contiguous device array of `len` values of `T`.
68///
69/// Created by [`Device::alloc_array`] or [`Device::upload`]. The handle is
70/// `Copy` and does not own the DRAM (there is no device-side free); it simply
71/// records where the array lives and how to interpret it.
72#[derive(Clone, Copy, Debug)]
73pub struct DeviceBuffer<T> {
74 region: DeviceRegion,
75 len: usize,
76 _marker: PhantomData<T>,
77}
78
79impl<T: DevicePod> DeviceBuffer<T> {
80 /// Device address of the first element (use as a kernel argument).
81 pub fn addr(&self) -> u64 {
82 self.region.addr
83 }
84
85 /// Number of elements.
86 pub fn len(&self) -> usize {
87 self.len
88 }
89
90 /// Whether the buffer holds no elements.
91 pub fn is_empty(&self) -> bool {
92 self.len == 0
93 }
94
95 /// Size of the buffer in bytes.
96 pub fn byte_len(&self) -> usize {
97 self.len * size_of::<T>()
98 }
99
100 /// The underlying untyped region.
101 pub fn region(&self) -> DeviceRegion {
102 self.region
103 }
104}
105
106/// A device array in which each element occupies a full cache line.
107///
108/// Use this for per-hart outputs so distinct harts never share a cache line
109/// (see the module docs). `T` must be no larger than one cache line.
110#[derive(Clone, Copy, Debug)]
111pub struct PaddedArray<T> {
112 region: DeviceRegion,
113 len: usize,
114 _marker: PhantomData<T>,
115}
116
117impl<T: DevicePod> PaddedArray<T> {
118 /// Device address of element 0 (element `i` lives at `addr + i * CACHE_LINE`).
119 pub fn addr(&self) -> u64 {
120 self.region.addr
121 }
122
123 /// Number of elements (cache lines).
124 pub fn len(&self) -> usize {
125 self.len
126 }
127
128 /// Whether the array holds no elements.
129 pub fn is_empty(&self) -> bool {
130 self.len == 0
131 }
132
133 /// The stride between consecutive elements, in bytes (one cache line).
134 pub fn stride(&self) -> usize {
135 CACHE_LINE
136 }
137
138 /// The underlying untyped region.
139 pub fn region(&self) -> DeviceRegion {
140 self.region
141 }
142}
143
144impl<Tr: Transport> Device<Tr> {
145 /// Allocate an uninitialised device array of `n` values of `E`.
146 pub fn alloc_array<E: DevicePod>(&self, n: usize) -> Result<DeviceBuffer<E>> {
147 let region = self.alloc(byte_len_of::<E>(n)? as u64)?;
148 Ok(DeviceBuffer {
149 region,
150 len: n,
151 _marker: PhantomData,
152 })
153 }
154
155 /// Allocate a device array and upload `data` into it (host -> device).
156 pub fn upload<E: DevicePod>(&self, data: &[E]) -> Result<DeviceBuffer<E>> {
157 let buf = self.alloc_array::<E>(data.len())?;
158 self.write_buffer(&buf, data)?;
159 Ok(buf)
160 }
161
162 /// Copy `data` into an existing device buffer (host -> device).
163 pub fn write_buffer<E: DevicePod>(&self, buf: &DeviceBuffer<E>, data: &[E]) -> Result<()> {
164 if data.len() > buf.len() {
165 return Err(Error::Limit(format!(
166 "write of {} elements exceeds buffer capacity {}",
167 data.len(),
168 buf.len()
169 )));
170 }
171 self.memcpy_h2d(as_bytes(data), buf.addr())
172 }
173
174 /// Download a device buffer into a host `Vec` (device -> host).
175 pub fn download<E: DevicePod>(&self, buf: &DeviceBuffer<E>) -> Result<Vec<E>> {
176 let mut out = zeroed_vec::<E>(buf.len())?;
177 self.memcpy_d2h(buf.addr(), as_bytes_mut(&mut out))?;
178 Ok(out)
179 }
180
181 /// Allocate a cache-line-padded device array of `n` values of `E`.
182 ///
183 /// # Panics
184 /// If `E` is larger than one cache line.
185 pub fn alloc_padded<E: DevicePod>(&self, n: usize) -> Result<PaddedArray<E>> {
186 assert!(
187 size_of::<E>() <= CACHE_LINE,
188 "padded element ({} bytes) exceeds one cache line ({CACHE_LINE} bytes)",
189 size_of::<E>()
190 );
191 let region = self.alloc(byte_len_of::<[u8; CACHE_LINE]>(n)? as u64)?;
192 Ok(PaddedArray {
193 region,
194 len: n,
195 _marker: PhantomData,
196 })
197 }
198
199 /// Download a cache-line-padded array, extracting one `E` from each line.
200 pub fn download_padded<E: DevicePod>(&self, arr: &PaddedArray<E>) -> Result<Vec<E>> {
201 let mut raw = vec![0u8; arr.len() * CACHE_LINE];
202 self.memcpy_d2h(arr.addr(), &mut raw)?;
203 let mut out = Vec::with_capacity(arr.len());
204 for i in 0..arr.len() {
205 let off = i * CACHE_LINE;
206 // SAFETY: `E: DevicePod` is valid for any bit pattern; `off .. off +
207 // size_of::<E>()` lies within `raw` (guaranteed by the allocation of
208 // `len * CACHE_LINE` and `size_of::<E>() <= CACHE_LINE`). The offset
209 // need not be aligned, hence `read_unaligned`.
210 out.push(unsafe { std::ptr::read_unaligned(raw.as_ptr().add(off) as *const E) });
211 }
212 Ok(out)
213 }
214
215 /// Fill every byte of `region` with `value` (host -> device).
216 ///
217 /// Test programs use this to prefill an output buffer with a sentinel (for
218 /// example `0xFF`, a NaN pattern for `f32`) before a launch, so that a kernel
219 /// which fails to write a result cannot pass on data left in DRAM by an
220 /// earlier run: allocation restarts at the same addresses and does not
221 /// clear memory.
222 pub fn fill(&self, region: DeviceRegion, value: u8) -> Result<()> {
223 let len = usize::try_from(region.size).map_err(|_| {
224 Error::Limit(format!(
225 "fill of {} bytes exceeds the host address space",
226 region.size
227 ))
228 })?;
229 self.memcpy_h2d(&vec![value; len], region.addr)
230 }
231
232 /// Upload a typed slice to device address `dst`, without allocating a new
233 /// [`DeviceBuffer`].
234 ///
235 /// Use when `dst` is an address already held externally, or when writing
236 /// into a sub-region of a larger device allocation. Equivalent to
237 /// [`Device::memcpy_h2d`] but accepts a typed slice rather than `&[u8]`.
238 ///
239 /// For concurrent DMA via a specific submission queue, use
240 /// [`Device::upload_slice_opts`].
241 pub fn upload_slice<E: DevicePod>(&self, data: &[E], dst: u64) -> Result<()> {
242 self.memcpy_h2d(as_bytes(data), dst)
243 }
244
245 /// Upload a typed slice to device address `dst` using explicit [`DmaOptions`].
246 ///
247 /// Equivalent to [`Device::upload_slice`] but routes the DMA command
248 /// through `opts`, enabling SQ selection and optional timeout override.
249 pub fn upload_slice_opts<E: DevicePod>(
250 &self,
251 data: &[E],
252 dst: u64,
253 opts: &DmaOptions,
254 ) -> Result<()> {
255 self.memcpy_h2d_opts(as_bytes(data), dst, opts)
256 }
257
258 /// Download `n` values of `E` from device address `src` into a host `Vec`.
259 ///
260 /// The untracked counterpart to [`Device::upload_slice`]: downloads from a
261 /// raw device address without a [`DeviceBuffer`] handle. Callers are
262 /// responsible for ensuring `src` is valid and `n` does not exceed the
263 /// allocated region.
264 ///
265 /// For concurrent DMA, use [`Device::download_slice_opts`].
266 pub fn download_slice<E: DevicePod>(&self, src: u64, n: usize) -> Result<Vec<E>> {
267 let mut out = zeroed_vec::<E>(n)?;
268 self.memcpy_d2h(src, as_bytes_mut(&mut out))?;
269 Ok(out)
270 }
271
272 /// Download `n` values of `E` from device address `src` using explicit
273 /// [`DmaOptions`].
274 ///
275 /// Equivalent to [`Device::download_slice`] but routes the DMA command
276 /// through `opts`.
277 pub fn download_slice_opts<E: DevicePod>(
278 &self,
279 src: u64,
280 n: usize,
281 opts: &DmaOptions,
282 ) -> Result<Vec<E>> {
283 let mut out = zeroed_vec::<E>(n)?;
284 self.memcpy_d2h_opts(src, as_bytes_mut(&mut out), opts)?;
285 Ok(out)
286 }
287}
288
289#[cfg(test)]
290mod tests {
291 use super::*;
292
293 #[test]
294 fn pod_bytes_are_little_endian() {
295 let v = [1u32, 0x0203_0405u32];
296 let b = as_bytes(&v);
297 assert_eq!(b.len(), 8);
298 assert_eq!(&b[0..4], &1u32.to_le_bytes());
299 assert_eq!(&b[4..8], &0x0203_0405u32.to_le_bytes());
300 }
301}