et-rs 0.6.3

Host-side Rust interface to the Esperanto ET-SoC-1 RISC-V accelerator
Documentation
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
//! [`Transport`] backed by the PCIe kernel driver's operations character device.
//!
//! This backend targets real ET-SoC-1 hardware exposed at `/dev/etN_ops`. Every
//! method is a thin wrapper over one `ETSOC1_IOCTL_*` call. Where the driver's
//! precise return convention is not pinned down by the uapi header (the exact
//! encoding of "queue full" and "queue empty", and whether byte counts arrive
//! via the ioctl return value or the descriptor), the interpretation is called
//! out in comments and kept in one place so it can be reconciled against the
//! driver on the hardware host.

use super::{DeviceConfig, DeviceProperties, DmaHostBuffer, DramInfo, PoppedResponse, Transport};
use crate::error::{Error, Result};
use crate::ffi::ops;
use crate::ioctl;
use std::cell::Cell;
use std::os::fd::{AsRawFd, FromRawFd, OwnedFd, RawFd};
use std::path::{Path, PathBuf};
use std::time::Duration;

/// A [`Transport`] over one device's operations node (`/dev/etN_ops`).
pub struct IoctlTransport {
    fd: OwnedFd,
    /// Cached `GET_SQ_MAX_MSG_SIZE`, used to size completion-queue read buffers.
    max_msg: Cell<Option<usize>>,
    /// Filesystem path of the device node, stored so the device can be
    /// re-opened after a full reset. `None` when constructed via
    /// [`IoctlTransport::from_owned_fd`] (path is unknown in that case).
    path: Option<PathBuf>,
}

impl IoctlTransport {
    /// Open device `index`, i.e. `/dev/et{index}_ops`.
    pub fn open(index: u32) -> Result<Self> {
        Self::open_path(format!("/dev/et{index}_ops"))
    }

    /// Open an operations node at an explicit filesystem path.
    pub fn open_path<P: AsRef<Path>>(path: P) -> Result<Self> {
        let path_buf = path.as_ref().to_path_buf();
        let fd = open_node(&path_buf, "open(/dev/etN_ops)")?;
        Ok(Self {
            fd,
            max_msg: Cell::new(None),
            path: Some(path_buf),
        })
    }

    /// Wrap an already-open operations node file descriptor.
    ///
    /// Ownership of `fd` is transferred to the returned transport. The device
    /// path is unknown, so [`Device::reset_device`](crate::Device::reset_device) is not available on a
    /// transport constructed this way.
    pub fn from_owned_fd(fd: OwnedFd) -> Self {
        Self {
            fd,
            max_msg: Cell::new(None),
            path: None,
        }
    }

    /// Filesystem path of the device node, if known.
    ///
    /// Returns `None` when the transport was constructed via
    /// [`IoctlTransport::from_owned_fd`].
    pub fn device_path(&self) -> Option<&Path> {
        self.path.as_deref()
    }

    /// Open the device node at `path`, submit `cmd` via `PUSH_SQ` with
    /// `flags`, then close the node.
    ///
    /// Returns `Ok(())` once the command is queued. Does not wait for a CQ
    /// response; intended for fire-and-forget Service Processor commands (e.g.
    /// ETSOC reset) where the device may reset before producing a reply.
    pub fn push_one_cmd(path: &Path, cmd: &[u8], flags: u8) -> Result<()> {
        let transport = Self::open_path(path)?;
        // A freshly opened device node has a non-full SQ; treat queue-full
        // (push_sq returns false) as a protocol error rather than retrying.
        let queued = transport.push_sq(0, cmd, flags)?;
        if !queued {
            return Err(Error::Protocol(
                "management SQ full on first submit; command not sent".into(),
            ));
        }
        Ok(())
    }

    /// Path of the management node paired with this operations node
    /// (`/dev/etN_ops` -> `/dev/etN_mgmt`), if the operations path is known and
    /// follows the driver's naming convention.
    pub fn mgmt_path(&self) -> Option<PathBuf> {
        self.path.as_deref().and_then(mgmt_path_for)
    }

    /// Open the management node for a single management-only ioctl. The node
    /// is single-open in the driver, so it is closed again when the returned
    /// descriptor drops.
    fn open_mgmt(&self) -> Result<OwnedFd> {
        let path = self.mgmt_path().ok_or_else(|| {
            Error::Protocol(
                "management node path unknown: the transport was built from a raw \
                 descriptor or its path does not end in `_ops`"
                    .into(),
            )
        })?;
        open_node(&path, "open(/dev/etN_mgmt)")
    }

    fn raw(&self) -> RawFd {
        self.fd.as_raw_fd()
    }

    fn cached_max_msg(&self) -> Result<usize> {
        if let Some(v) = self.max_msg.get() {
            return Ok(v);
        }
        let v = self.sq_max_msg_size()? as usize;
        self.max_msg.set(Some(v));
        Ok(v)
    }

    /// Wait for `events` (a `poll(2)` event mask) on the device fd.
    fn poll(&self, events: libc::c_short, timeout: Duration) -> Result<bool> {
        let mut pfd = libc::pollfd {
            fd: self.raw(),
            events,
            revents: 0,
        };
        let millis = timeout.as_millis().min(libc::c_int::MAX as u128) as libc::c_int;
        // SAFETY: single valid pollfd, count 1.
        let rc = unsafe { libc::poll(&mut pfd, 1, millis) };
        if rc < 0 {
            // An interrupted wait is reported as "not ready"; the caller's
            // deadline loop re-polls.
            if ioctl::interrupted() {
                return Ok(false);
            }
            return Err(Error::last_os("poll"));
        }
        Ok(rc > 0 && (pfd.revents & events) != 0)
    }
}

impl Transport for IoctlTransport {
    fn dram_info(&self) -> Result<DramInfo> {
        let info: ops::dram_info =
            ioctl::read_scalar(self.raw(), ioctl::GET_USER_DRAM_INFO, "GET_USER_DRAM_INFO")?;
        Ok(DramInfo {
            base: info.base,
            size: info.size,
            dma_max_elem_size: info.dma_max_elem_size,
            dma_max_elem_count: info.dma_max_elem_count,
            // The driver reports `align_in_bits` as 8, 32 or 64 bits
            // (MEM_REGION_DMA_ALIGNMENT_*BIT). It is carried through unchanged
            // and consumed as a byte quantum, which over-aligns to 64 bytes;
            // the kernels rely on that cache-line alignment, so it is retained
            // deliberately rather than converted to bytes.
            dma_alignment: info.align_in_bits,
        })
    }

    fn device_config(&self) -> Result<DeviceConfig> {
        let cfg: ops::dev_config = ioctl::read_scalar(
            self.raw(),
            ioctl::GET_DEVICE_CONFIGURATION,
            "GET_DEVICE_CONFIGURATION",
        )?;
        Ok(DeviceConfig {
            shire_mask: u64::from(cfg.cm_shire_mask),
            cache_line: u32::from(cfg.cache_line_size),
        })
    }

    fn device_properties(&self) -> Result<DeviceProperties> {
        let cfg: ops::dev_config = ioctl::read_scalar(
            self.raw(),
            ioctl::GET_DEVICE_CONFIGURATION,
            "GET_DEVICE_CONFIGURATION",
        )?;
        Ok(DeviceProperties {
            total_l3_size: cfg.total_l3_size,
            total_l2_size: cfg.total_l2_size,
            total_scp_size: cfg.total_scp_size,
            ddr_bandwidth: cfg.ddr_bandwidth,
            minion_boot_freq: if cfg.minion_boot_freq > 0 {
                Some(cfg.minion_boot_freq)
            } else {
                None
            },
            shire_mask: cfg.cm_shire_mask,
            form_factor: cfg.form_factor,
            tdp: cfg.tdp,
            cache_line_size: cfg.cache_line_size,
            num_l2_cache_banks: cfg.num_l2_cache_banks,
            sync_min_shire_id: cfg.sync_min_shire_id,
            arch_rev: cfg.arch_rev,
            devnum: cfg.devnum,
        })
    }

    fn fw_update(&self, image: &[u8]) -> Result<()> {
        // The driver reads `size` bytes from `ubuf`; the buffer must outlive the
        // call, which it does for the borrow's duration.
        let mut desc = ops::fw_update_desc {
            ubuf: image.as_ptr() as *mut libc::c_void,
            size: image.len() as u64,
        };
        // FW_UPDATE is served only by the management node; the ops node
        // rejects it with EINVAL (et-soc1-pcie.c, esperanto_pcie_mgmt_ioctl).
        let mgmt = self.open_mgmt()?;
        // SAFETY: descriptor points at a live, correctly sized image buffer.
        unsafe {
            ioctl::ioctl(
                mgmt.as_raw_fd(),
                ioctl::FW_UPDATE,
                (&raw mut desc).cast(),
                "FW_UPDATE",
            )?;
        }
        Ok(())
    }

    fn sq_count(&self) -> Result<u16> {
        ioctl::read_scalar::<u16>(self.raw(), ioctl::GET_SQ_COUNT, "GET_SQ_COUNT")
    }

    fn sq_max_msg_size(&self) -> Result<u16> {
        ioctl::read_scalar::<u16>(
            self.raw(),
            ioctl::GET_SQ_MAX_MSG_SIZE,
            "GET_SQ_MAX_MSG_SIZE",
        )
    }

    fn push_sq(&self, sq_index: u16, cmd: &[u8], flags: u8) -> Result<bool> {
        // Zero-initialise so any tail padding the driver copies is defined.
        let mut desc: ops::cmd_desc = unsafe { std::mem::zeroed() };
        desc.cmd = cmd.as_ptr() as *mut libc::c_void;
        desc.size = cmd.len() as u16;
        desc.sq_index = sq_index;
        desc.flags = flags;
        let rc = loop {
            // SAFETY: descriptor points at the live `cmd` buffer for the call.
            let rc = unsafe {
                libc::ioctl(
                    self.raw(),
                    ioctl::PUSH_SQ,
                    (&raw mut desc).cast::<libc::c_void>(),
                )
            };
            if rc >= 0 || !ioctl::interrupted() {
                break rc;
            }
        };
        if rc < 0 {
            let err = std::io::Error::last_os_error();
            // A full submission queue is reported as a transient errno rather
            // than a hard failure; the caller should retry after `wait_sq`.
            return match err.raw_os_error() {
                Some(libc::EAGAIN) | Some(libc::ENOSPC) | Some(libc::ENOBUFS) => Ok(false),
                _ => Err(Error::Io {
                    op: "PUSH_SQ",
                    source: err,
                }),
            };
        }
        Ok(true)
    }

    fn pop_cq(&self) -> Result<Option<PoppedResponse>> {
        // Determine which CQ has data before attempting to pop. The
        // `GET_CQ_AVAIL_BITMAP` ioctl returns a bitmask where bit N indicates
        // that CQ N holds at least one response. `POP_CQ` takes `cq_index` as
        // an input specifying which CQ to dequeue from, so reading from the
        // wrong CQ (always index 0) would miss responses that land on CQ 1+
        // (e.g. the response to a CM reset command, which the firmware places
        // on the high-priority CQ).
        let bitmap = ioctl::read_scalar::<u64>(
            self.raw(),
            ioctl::GET_CQ_AVAIL_BITMAP,
            "GET_CQ_AVAIL_BITMAP",
        )?;
        if bitmap == 0 {
            return Ok(None);
        }
        let cq_index = bitmap.trailing_zeros() as u16;

        let cap = self.cached_max_msg()?.max(8);
        let mut buf = vec![0u8; cap];
        let mut desc = ops::rsp_desc {
            rsp: buf.as_mut_ptr() as *mut libc::c_void,
            size: cap as u16,
            cq_index,
        };
        let rc = loop {
            // SAFETY: descriptor points at the live, `cap`-byte `buf`.
            let rc = unsafe {
                libc::ioctl(
                    self.raw(),
                    ioctl::POP_CQ,
                    (&raw mut desc).cast::<libc::c_void>(),
                )
            };
            if rc >= 0 || !ioctl::interrupted() {
                break rc;
            }
        };
        if rc < 0 {
            let err = std::io::Error::last_os_error();
            // An empty completion queue is reported as EAGAIN; treat as "no
            // response yet" rather than an error.
            return match err.raw_os_error() {
                Some(libc::EAGAIN) | Some(libc::ENOMSG) => Ok(None),
                _ => Err(Error::Io {
                    op: "POP_CQ",
                    source: err,
                }),
            };
        }
        // The number of bytes popped is reported via the ioctl return value when
        // positive, otherwise via the descriptor's updated `size`. A zero-length
        // result means the queue was empty.
        let len = if rc > 0 {
            rc as usize
        } else {
            desc.size as usize
        };
        if len == 0 {
            return Ok(None);
        }
        buf.truncate(len.min(cap));
        Ok(Some(PoppedResponse {
            bytes: buf,
            cq_index: desc.cq_index,
        }))
    }

    fn extract_trace(&self, trace_type: u8) -> Result<Vec<u8>> {
        // EXTRACT_TRACE_BUFFER is served only by the management node; the ops
        // node rejects it with EINVAL. The size query is issued on the same node
        // so that both calls resolve the same trace region.
        let mgmt = self.open_mgmt()?;
        // Query the buffer size first; the trace descriptor itself carries no
        // length, so the host must allocate exactly what the driver expects.
        let mut ty = trace_type;
        // SAFETY: `ty` is a live u8; the request encodes sizeof(u8).
        let size = unsafe {
            ioctl::ioctl(
                mgmt.as_raw_fd(),
                ioctl::GET_TRACE_BUFFER_SIZE,
                (&raw mut ty).cast(),
                "GET_TRACE_BUFFER_SIZE",
            )?
        } as usize;

        let mut buf = vec![0u8; size];
        // Zero-initialise so any tail padding the driver copies is defined.
        let mut desc: ops::trace_desc = unsafe { std::mem::zeroed() };
        desc.trace_type = trace_type;
        desc.buf = buf.as_mut_ptr() as *mut libc::c_void;
        // SAFETY: descriptor points at the live, `size`-byte `buf`.
        unsafe {
            ioctl::ioctl(
                mgmt.as_raw_fd(),
                ioctl::EXTRACT_TRACE_BUFFER,
                (&raw mut desc).cast(),
                "EXTRACT_TRACE_BUFFER",
            )?;
        }
        Ok(buf)
    }

    fn wait_cq(&self, timeout: Duration) -> Result<bool> {
        self.poll(libc::POLLIN, timeout)
    }

    fn wait_sq(&self, timeout: Duration) -> Result<bool> {
        self.poll(libc::POLLOUT, timeout)
    }

    fn dma_host_buffer(&self, size: usize) -> Result<Box<dyn DmaHostBuffer>> {
        // The driver DMAs only to/from its own CMA region, which is obtained by
        // mmap-ing the operations node (each mapping is a fresh CMA allocation).
        // It resolves the bus address from the mapped virtual address itself, so
        // the DMA node's physical field is left 0. This mirrors DevicePcie's
        // allocDmaBuffer: mmap(NULL, size, RW, MAP_SHARED, fd, 0).
        let len = size.max(1);
        // SAFETY: standard mmap of `len` bytes from the device fd.
        let ptr = unsafe {
            libc::mmap(
                std::ptr::null_mut(),
                len,
                libc::PROT_READ | libc::PROT_WRITE,
                libc::MAP_SHARED,
                self.raw(),
                0,
            )
        };
        if ptr == libc::MAP_FAILED {
            return Err(Error::last_os("mmap(DMA buffer)"));
        }
        Ok(Box::new(MmapDmaBuffer {
            ptr: ptr.cast(),
            len,
        }))
    }
}

/// Open a device node read-write with close-on-exec, retrying on `EINTR`.
fn open_node(path: &Path, op: &'static str) -> Result<OwnedFd> {
    use std::os::unix::ffi::OsStrExt;
    let mut bytes = path.as_os_str().as_bytes().to_vec();
    bytes.push(0);
    loop {
        // SAFETY: `bytes` is a valid NUL-terminated C string for the call.
        let raw = unsafe { libc::open(bytes.as_ptr().cast(), libc::O_RDWR | libc::O_CLOEXEC) };
        if raw >= 0 {
            // SAFETY: `raw` is a freshly opened, owned file descriptor.
            return Ok(unsafe { OwnedFd::from_raw_fd(raw) });
        }
        if !ioctl::interrupted() {
            return Err(Error::last_os(op));
        }
    }
}

/// Management-node path paired with the operations node at `ops`: the final
/// path component's `_ops` suffix is replaced by `_mgmt`, leaving the directory
/// untouched (so `/srv/my_ops/et0_ops` maps to `/srv/my_ops/et0_mgmt`).
/// Returns `None` if the file name does not end in `_ops`.
pub(crate) fn mgmt_path_for(ops: &Path) -> Option<PathBuf> {
    let name = ops.file_name()?.to_str()?;
    let stem = name.strip_suffix("_ops")?;
    Some(ops.with_file_name(format!("{stem}_mgmt")))
}

/// A DMA host buffer mapped from the driver's CMA pool. Its mapped virtual
/// address is DMA-capable; the driver resolves the bus address, so the physical
/// field is reported as 0.
struct MmapDmaBuffer {
    ptr: *mut u8,
    len: usize,
}

impl DmaHostBuffer for MmapDmaBuffer {
    fn as_mut_slice(&mut self) -> &mut [u8] {
        // SAFETY: `ptr` maps `len` writable bytes for the buffer's lifetime.
        unsafe { std::slice::from_raw_parts_mut(self.ptr, self.len) }
    }
    fn as_slice(&self) -> &[u8] {
        // SAFETY: `ptr` maps `len` readable bytes for the buffer's lifetime.
        unsafe { std::slice::from_raw_parts(self.ptr, self.len) }
    }
    fn virt_addr(&self) -> u64 {
        self.ptr as u64
    }
    fn phys_addr(&self) -> u64 {
        0
    }
}

impl Drop for MmapDmaBuffer {
    fn drop(&mut self) {
        // SAFETY: `ptr`/`len` came from the mmap above, unmapped once.
        unsafe { libc::munmap(self.ptr.cast(), self.len) };
    }
}