1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
//! The high-level [`Device`] handle: DRAM allocation, kernel loading, launch,
//! device-to-host DMA and trace extraction.
//!
//! [`Device`] is generic over a [`Transport`]; the default,
//! [`IoctlTransport`], drives real hardware through `/dev/etN_ops`. The device
//! command model is single-threaded, so state that mutates during otherwise
//! read-only operations (the DRAM bump pointer and the tag counter) is held in
//! `Cell`s and the command methods take `&self`.
use crate::elf;
use crate::error::{Error, Result};
use crate::ffi::ops;
use crate::proto::{self, cmd_flags, desc_flags};
use crate::transport::{DeviceProperties, DramInfo, IoctlTransport, PoppedResponse, Transport};
use std::cell::{Cell, RefCell};
use std::collections::HashMap;
use std::thread;
use std::time::{Duration, Instant};
/// A device-resident kernel, ready to be launched.
#[derive(Clone, Copy, Debug)]
pub struct LoadedKernel {
/// Device address at which execution begins (the ELF entry point).
pub code_start_address: u64,
}
/// A handle to a region of device DRAM returned by [`Device::alloc`].
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub struct DeviceRegion {
/// Device physical base address of the region.
pub addr: u64,
/// Size of the region in bytes.
pub size: u64,
}
impl DeviceRegion {
/// The half-open address range `[addr, addr + size)`.
pub fn end(&self) -> u64 {
self.addr + self.size
}
}
/// U-mode trace capture configuration for a kernel launch, mirroring
/// [`proto::TraceInitInfo`] with a device-resident trace buffer.
#[derive(Clone, Copy, Debug)]
pub struct TraceConfig {
/// Device address of the trace buffer (typically an [`Device::alloc`] region).
pub buffer: u64,
/// Size of the trace buffer in bytes.
pub buffer_size: u32,
/// Per-hart free-space threshold at which the device raises a full event.
pub threshold: u32,
/// Bitmask of shires for which trace capture is enabled.
pub shire_mask: u64,
/// Bitmask of threads within a shire for which trace capture is enabled.
pub thread_mask: u64,
/// Bitmask selecting which events to trace.
pub event_mask: u32,
/// Bitmask selecting which filters apply to the traced events.
pub filter_mask: u32,
}
impl TraceConfig {
/// Enable full user tracing of every thread, event and filter for `shire_mask`,
/// dumping into the whole of `buffer`. Mirrors the configuration used by the
/// SDK "hello world" test drive.
pub fn full(buffer: DeviceRegion, shire_mask: u64) -> Self {
TraceConfig {
buffer: buffer.addr,
buffer_size: buffer.size as u32,
threshold: 0,
shire_mask,
thread_mask: u64::MAX,
event_mask: u32::MAX,
filter_mask: u32::MAX,
}
}
fn to_init_info(self) -> proto::TraceInitInfo {
proto::TraceInitInfo {
buffer: self.buffer,
buffer_size: self.buffer_size,
threshold: self.threshold,
shire_mask: self.shire_mask,
thread_mask: self.thread_mask,
event_mask: self.event_mask,
filter_mask: self.filter_mask,
}
}
}
/// Options controlling a DMA transfer ([`Device::memcpy_h2d_opts`] /
/// [`Device::memcpy_d2h_opts`]).
///
/// ```
/// use et_soc1::DmaOptions;
/// use std::time::Duration;
/// // Route DMA to SQ 1, with a 60 s timeout for a large transfer.
/// let opts = DmaOptions::new().on_sq(1).with_timeout(Duration::from_secs(60));
/// ```
///
/// `#[non_exhaustive]`: additional fields may be added in future patch releases.
#[derive(Clone, Copy, Debug, Default)]
#[non_exhaustive]
pub struct DmaOptions {
/// Submission queue to push DMA commands onto (default: 0).
pub sq_index: u16,
/// Maximum duration to wait for DMA completion.
///
/// `None` (the default) means wait indefinitely. Use
/// [`DmaOptions::with_timeout`] to set an upper bound.
timeout: Option<Duration>,
}
impl DmaOptions {
/// Default DMA options: submission queue 0, unlimited completion wait.
pub fn new() -> Self {
Self::default()
}
/// Route DMA commands to submission queue `idx`.
///
/// Using a different queue from the kernel launch (which defaults to SQ 0)
/// allows the firmware to process DMA and compute concurrently, enabling
/// double-buffering patterns. Verify with [`Device::topology`] that the
/// device has more than one queue before selecting `idx > 0`.
pub fn on_sq(mut self, idx: u16) -> Self {
self.sq_index = idx;
self
}
/// Set the maximum duration to wait for a DMA completion response.
///
/// By default, DMA waits are unlimited: the call returns as soon as the
/// firmware acknowledges the transfer. Use this method to impose a bound for
/// defensive error detection on DMA paths that should be fast.
pub fn with_timeout(mut self, t: Duration) -> Self {
self.timeout = Some(t);
self
}
}
/// Options controlling a single kernel launch.
///
/// `#[non_exhaustive]`: additional fields may be added in future patch releases.
/// Construct via [`LaunchOptions::new`] and the provided builder methods.
#[derive(Clone, Debug)]
#[non_exhaustive]
pub struct LaunchOptions {
/// Bitmask of compute shires the kernel executes on.
pub shire_mask: u64,
/// Whether to drain outstanding commands before launching (barrier).
pub barrier: bool,
/// Whether to flush the L3 cache before launching.
pub flush_l3: bool,
/// Optional U-mode trace configuration.
pub trace: Option<TraceConfig>,
/// Optional embedded kernel arguments blob.
pub args: Vec<u8>,
/// Optional U-mode stack configuration.
pub stack: Option<proto::UserStackCfg>,
/// Device address of a U-mode exception buffer (0 if unused).
pub exception_buffer: u64,
/// Submission queue to push the launch command onto.
pub sq_index: u16,
/// Maximum duration to wait for the kernel completion response.
/// `None` inherits the device-level default from
/// [`Transport::default_launch_timeout`], resolved at launch time.
timeout: Option<Duration>,
}
impl LaunchOptions {
/// Launch on `shire_mask` with a barrier and no tracing or arguments.
pub fn new(shire_mask: u64) -> Self {
LaunchOptions {
shire_mask,
barrier: true,
flush_l3: false,
trace: None,
args: Vec::new(),
stack: None,
exception_buffer: 0,
sq_index: 0,
timeout: None,
}
}
/// Enable U-mode tracing with the given configuration.
pub fn with_trace(mut self, trace: TraceConfig) -> Self {
self.trace = Some(trace);
self
}
/// Attach an embedded kernel-arguments blob.
pub fn with_args(mut self, args: Vec<u8>) -> Self {
self.args = args;
self
}
/// Clear the `BARRIER` flag so the firmware may start this kernel before
/// all prior commands on the same SQ have completed.
///
/// Use this when you have ensured ordering by other means -- for example,
/// data was uploaded on a separate SQ (via [`DmaOptions::on_sq`]) and the
/// kernel issues a `BARRIER` on its own queue to drain only the compute
/// stream, not the DMA stream.
///
/// The default constructor sets `barrier = true`; call `without_barrier()`
/// last so it is not overridden by a later builder call.
pub fn without_barrier(mut self) -> Self {
self.barrier = false;
self
}
/// Route this launch command to submission queue `idx`.
///
/// The default is SQ 0. Routing a kernel launch to a dedicated compute
/// queue while DMA goes to SQ 1 (via [`DmaOptions::on_sq`]) allows the
/// firmware to schedule both streams concurrently.
pub fn on_sq(mut self, idx: u16) -> Self {
self.sq_index = idx;
self
}
/// Set the maximum duration to wait for the kernel completion response.
///
/// The default is 10 s, which is adequate for short test kernels but
/// insufficient for production workloads (large matrix multiplications,
/// multi-frame radiosity solves, etc.). Pass a duration at least as large
/// as the worst-case kernel execution time; erring on the side of a longer
/// timeout is safe -- the call returns as soon as the response arrives.
///
/// ```
/// use et_soc1::LaunchOptions;
/// use std::time::Duration;
///
/// let opts = LaunchOptions::new(0x1)
/// .with_timeout(Duration::from_secs(300)); // 5 min for a long kernel
/// ```
pub fn with_timeout(mut self, t: Duration) -> Self {
self.timeout = Some(t);
self
}
}
/// Timing counters reported alongside a kernel-launch response, in device cycles.
#[derive(Clone, Copy, Debug, Default)]
pub struct LaunchTiming {
/// Timestamp at which the command was dispatched.
pub start_ts: u64,
/// Cycles between dispatch and completion.
pub execute_dur: u64,
/// Cycles between arrival and dispatch.
pub wait_dur: u64,
}
/// Outcome of a successful [`Device::launch`] or [`Device::wait_launch`].
#[derive(Clone, Copy, Debug, Default)]
pub struct LaunchResult {
/// Device timing counters for the launch.
pub timing: LaunchTiming,
}
/// Handle to an in-flight kernel launch, returned by [`Device::launch_async`].
///
/// Pass to [`Device::wait_launch`] to block until the kernel completes and
/// retrieve its [`LaunchResult`]. The handle carries the CQ tag allocated for
/// the launch; dropping it without calling `wait_launch` leaves the response
/// in the device's completion stash until the next `wait_launch` or `launch`
/// clears it.
#[derive(Debug)]
pub struct PendingLaunch {
tag: u16,
/// Timeout for [`Device::wait_launch`], copied from [`LaunchOptions::timeout`].
timeout: Duration,
}
/// A connected ET-SoC-1 device.
pub struct Device<T: Transport = IoctlTransport> {
transport: T,
dram: DramInfo,
/// Bump-allocation cursor within the user DRAM region.
next: Cell<u64>,
/// Reused device region for launch arguments, grown on demand so repeated
/// launches do not leak a fresh region each time.
args_scratch: Cell<Option<DeviceRegion>>,
/// Monotonic command correlation tag.
tag: Cell<u16>,
/// Responses that arrived from the CQ for tags other than the one currently
/// being waited for. Keyed by tag ID. Used by `collect_response` to park
/// out-of-order responses so concurrent in-flight commands do not lose each
/// other's completions (e.g. a kernel launch response arriving while a DMA
/// command is being collected).
stash: RefCell<HashMap<u16, PoppedResponse>>,
/// Default timeout for kernel completion waits (not DMA, which is unlimited
/// unless overridden via [`DmaOptions::with_timeout`]).
/// Initialised from [`Transport::default_launch_timeout`]; may be overridden
/// at runtime via [`Device::set_default_launch_timeout`].
default_timeout: Cell<Duration>,
}
/// A saved position of the DRAM bump allocator, taken by [`Device::alloc_mark`]
/// and passed to [`Device::reset_to`] to reclaim everything allocated since.
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub struct AllocMark(u64);
impl Device<IoctlTransport> {
/// Open device `index` (`/dev/et{index}_ops`) and query its DRAM geometry.
pub fn open(index: u32) -> Result<Self> {
Self::with_transport(IoctlTransport::open(index)?)
}
/// Trigger a full ETSOC device reset and return a freshly opened device.
///
/// Perform a full ETSOC device reset and return a fresh handle.
///
/// Consumes `self` so the ops device node is closed before the firmware
/// completes the reset (the driver requires this: it will not finish while
/// any file descriptor to the ops node remains open).
///
/// The sequence is:
/// 1. Derive the management device path from the ops path
/// (`/dev/etN_ops` -> `/dev/etN_mgmt`).
/// 2. Drop `self`, closing the ops fd.
/// 3. Open `/dev/etN_mgmt` and submit a `device_mgmt_etsoc_reset_cmd_t`
/// via `PUSH_SQ` with `CMD_DESC_FLAG_ETSOC_RESET`. The firmware
/// initiates the reset on receipt; no response arrives.
/// 4. Poll `/dev/etN_ops` every 250 ms until the device is accessible
/// again, up to a 30 s deadline.
/// 5. Re-open and return a new `Device<IoctlTransport>`.
///
/// Note: `CMD_DESC_FLAG_ETSOC_RESET` is rejected (EINVAL) on the ops
/// node; it is only valid on the Service Processor management node.
///
/// Returns [`Error::Protocol`] if the transport was constructed via
/// [`IoctlTransport::from_owned_fd`] (path is unknown in that case).
/// Returns [`Error::Timeout`] if the device does not come back within 30 s.
///
/// Use [`Device::reset_shires`] instead when only the compute minions need
/// recovery and the firmware is still responsive.
pub fn reset_device(self) -> Result<Self> {
// Capture the ops path before consuming self.
let path = self
.transport
.device_path()
.ok_or_else(|| {
Error::Protocol(
"reset_device requires a path-based transport; \
use Device::open or IoctlTransport::open_path"
.into(),
)
})?
.to_path_buf();
// Derive the management node path: /dev/et0_ops -> /dev/et0_mgmt.
// The driver creates one mgmt node per ops node under the same /dev.
let ops_str = path.to_string_lossy();
let mgmt_str = ops_str.replace("_ops", "_mgmt");
drop(ops_str); // release the borrow of `path`
// Build the ETSOC reset command (device_mgmt_etsoc_reset_cmd_t, 16 B).
let tag = self.next_tag();
let cmd = proto::build_etsoc_reset(tag);
// Close the ops fd. The driver will not complete the reset while any
// ops fd remains open; this drop satisfies that requirement.
drop(self);
// Submit the reset via the management node with ETSOC_RESET flag.
// The management node accepts this flag; the ops node returns EINVAL.
IoctlTransport::push_one_cmd(
std::path::Path::new(&mgmt_str),
&cmd,
desc_flags::ETSOC_RESET,
)?;
// Poll until the ops node is accessible again (reset complete) or
// the 30 s deadline elapses.
let deadline = Instant::now() + Duration::from_secs(30);
loop {
thread::sleep(Duration::from_millis(250));
match IoctlTransport::open_path(&path) {
Ok(transport) => return Device::with_transport(transport),
Err(_) if Instant::now() < deadline => continue,
Err(_) => {
return Err(Error::Timeout {
operation: "device reset",
limit: Duration::from_secs(30),
});
}
}
}
}
}
#[cfg(feature = "emu")]
impl Device<crate::transport::FfiTransport> {
/// Boot the SDK software emulator and open it as a device.
///
/// `sdk_prefix` is the SDK install root (e.g. `/opt/et`) and `run_dir` a
/// writable directory for emulator logs. This blocks while the emulator
/// boots firmware. Requires the `emu` feature.
pub fn open_emulator<P: AsRef<std::path::Path>, Q: AsRef<std::path::Path>>(
sdk_prefix: P,
run_dir: Q,
) -> Result<Self> {
Self::with_transport(crate::transport::FfiTransport::new(sdk_prefix, run_dir)?)
}
}
impl<T: Transport> Device<T> {
/// Build a device over an explicit transport (an alternative backend, or a
/// test double). The DRAM geometry is queried immediately.
pub fn with_transport(transport: T) -> Result<Self> {
let dram = transport.dram_info()?;
let default_timeout = transport.default_launch_timeout();
Ok(Device {
transport,
dram,
next: Cell::new(dram.base),
args_scratch: Cell::new(None),
tag: Cell::new(0),
stash: RefCell::new(HashMap::new()),
default_timeout: Cell::new(default_timeout),
})
}
/// Override the default kernel completion timeout for this device.
///
/// The initial value comes from [`crate::transport::Transport::default_launch_timeout`]:
/// 10 s for [`IoctlTransport`] (real hardware) and 1 h for the FFI emulator.
/// Per-launch timeouts set via [`LaunchOptions::with_timeout`] take
/// precedence over this device-level default.
///
/// DMA timeouts are not affected by this setting; use
/// [`DmaOptions::with_timeout`] to bound an individual DMA transfer.
///
/// Use this when all (or most) kernels run longer than the transport default
/// and adding `.with_timeout` to every `LaunchOptions` would be repetitive.
pub fn set_default_launch_timeout(&self, timeout: Duration) {
self.default_timeout.set(timeout);
}
/// The device's user DRAM region geometry and DMA limits.
pub fn dram_info(&self) -> DramInfo {
self.dram
}
/// The device's compute topology: the present compute-shire mask plus the
/// architectural per-shire geometry. Query this to size work to the device
/// instead of hard-coding shire masks and hart counts.
pub fn topology(&self) -> Result<crate::Topology> {
let cfg = self.transport.device_config()?;
Ok(crate::Topology {
shire_mask: cfg.shire_mask,
harts_per_shire: crate::topology::HARTS_PER_SHIRE,
harts_per_neighbourhood: crate::topology::HARTS_PER_NEIGHBOURHOOD,
cache_line: cfg.cache_line,
})
}
/// All device properties reported by `ETSOC1_IOCTL_GET_DEVICE_CONFIGURATION`.
///
/// Includes cache sizes, DDR bandwidth, and `minion_boot_freq` (MHz), which
/// can be used to convert a PMU cycle-count delta to wall time:
///
/// ```text
/// if let Some(freq_mhz) = props.minion_boot_freq {
/// elapsed_us = cycles as f64 / (freq_mhz as f64);
/// }
/// ```
///
/// `minion_boot_freq` is `None` when the transport cannot provide a clock
/// value (e.g. the default or emulator transport).
pub fn properties(&self) -> Result<DeviceProperties> {
self.transport.device_properties()
}
/// Borrow the underlying transport.
pub fn transport(&self) -> &T {
&self.transport
}
/// Bytes still available in the DRAM bump allocator.
pub fn dram_available(&self) -> u64 {
(self.dram.base + self.dram.size).saturating_sub(self.next.get())
}
/// Allocate a region of device DRAM aligned to at least a cache line.
///
/// This is a monotonic bump allocator: individual regions are not freed, but
/// a span can be reclaimed as a group with [`Device::alloc_mark`] and
/// [`Device::reset_to`]. Alignment is `max(dma_alignment, CACHE_LINE)`:
/// the cache-line floor prevents two caller-allocated regions from sharing a
/// line, which would cause silent false-sharing corruption on this
/// software-coherent device even when neither region is accessed from the
/// device concurrently.
pub fn alloc(&self, size: u64) -> Result<DeviceRegion> {
let align = self.dram.alignment().max(et_abi::CACHE_LINE as u64);
let start = align_up(self.next.get(), align);
let end = self.dram.base + self.dram.size;
if start > end || size > end - start {
return Err(Error::OutOfMemory {
requested: size,
available: self.dram_available(),
});
}
self.next.set(start + size);
Ok(DeviceRegion { addr: start, size })
}
/// Record the current allocator position for a later [`Device::reset_to`].
///
/// This is the arena/scratch pattern: mark a point, allocate freely, then
/// reset to reclaim it all at once.
pub fn alloc_mark(&self) -> AllocMark {
AllocMark(self.next.get())
}
/// Reclaim every allocation made since `mark`, rewinding the bump allocator.
///
/// Any [`DeviceRegion`] or [`DeviceBuffer`](crate::DeviceBuffer) obtained
/// after `mark` must not be used afterwards: the DRAM it names may be handed
/// out again. This cannot cause host-side undefined behaviour, but using a
/// reclaimed region will read or overwrite unrelated device data. `mark` must
/// come from this device; a mark ahead of the current position is ignored.
pub fn reset_to(&self, mark: AllocMark) {
if mark.0 < self.next.get() {
self.next.set(mark.0);
}
// Drop the launch-args scratch if it now lies in the reclaimed span, so
// the next launch re-allocates rather than reusing freed DRAM.
if let Some(r) = self.args_scratch.get()
&& r.addr >= mark.0
{
self.args_scratch.set(None);
}
}
/// Device region for a launch-argument payload of `len` bytes, reused across
/// launches and grown on demand so repeated launches do not each leak a
/// region. Reuse is safe because a launch runs to completion (the default
/// barrier) before its scratch could be handed out again.
fn args_region(&self, len: u64) -> Result<DeviceRegion> {
if let Some(r) = self.args_scratch.get()
&& r.size >= len
{
return Ok(r);
}
let region = self.alloc(len)?;
self.args_scratch.set(Some(region));
Ok(region)
}
/// Load a RISC-V ELF device kernel into device DRAM.
///
/// Compute kernels are position-dependent and linked at a fixed U-mode
/// address that coincides with the base of the user DRAM region; there is no
/// firmware-side ELF loader for them (that is what `FW_UPDATE` is for, and it
/// rejects kernel ELFs). Loading therefore DMA-writes each `PT_LOAD` segment
/// to its virtual address, zero-filling any `.bss` tail, and reserves the
/// occupied DRAM so subsequent [`Device::alloc`] calls do not overlap the
/// code. The returned [`LoadedKernel`] carries the ELF entry point for use as
/// the launch `code_start_address`.
///
/// Call this before any [`Device::alloc`] or [`Device::alloc_padded`] calls.
/// The kernel ELF is linked at the DRAM base address and this function DMA-
/// writes each `PT_LOAD` segment unconditionally to its `p_vaddr`. Any prior
/// allocation at those addresses is silently overwritten with kernel code.
pub fn load_kernel(&self, elf_image: &[u8]) -> Result<LoadedKernel> {
let image = elf::parse(elf_image)?;
let region_end = self.dram.base + self.dram.size;
let mut occupied_end = self.next.get();
for seg in &image.segments {
if seg.vaddr < self.dram.base || seg.vaddr + seg.mem_size > region_end {
return Err(Error::Limit(format!(
"kernel segment [{:#x}, {:#x}) lies outside the DRAM region [{:#x}, {:#x})",
seg.vaddr,
seg.vaddr + seg.mem_size,
self.dram.base,
region_end
)));
}
if seg.file_size > 0 {
let start = seg.file_offset as usize;
let end = start + seg.file_size as usize;
self.memcpy_h2d(&elf_image[start..end], seg.vaddr)?;
}
// Zero-initialise the `.bss` tail present in memory but not in the file.
if seg.mem_size > seg.file_size {
let zeros = vec![0u8; (seg.mem_size - seg.file_size) as usize];
self.memcpy_h2d(&zeros, seg.vaddr + seg.file_size)?;
}
occupied_end = occupied_end.max(seg.vaddr + seg.mem_size);
}
// Reserve the DRAM the kernel occupies against future allocations.
let align = self.dram.alignment().max(1);
self.next.set(align_up(occupied_end, align).min(region_end));
Ok(LoadedKernel {
code_start_address: image.entry,
})
}
/// Update device firmware via `FW_UPDATE`.
///
/// This is for signed firmware images, not compute kernels; use
/// [`Device::load_kernel`] for the latter.
pub fn update_firmware(&self, image: &[u8]) -> Result<()> {
self.transport.fw_update(image)
}
/// Submit a kernel launch and return immediately without blocking.
///
/// The kernel is pushed onto the device submission queue; it begins
/// executing as soon as the firmware dequeues it. Pass the returned
/// [`PendingLaunch`] to [`Device::wait_launch`] to block until completion.
///
/// Between `launch_async` and `wait_launch` the caller may issue other
/// device operations (DMA, a second launch, etc.); their responses are
/// stashed and returned correctly when each is collected. This enables
/// double-buffering patterns: while the kernel processes buffer A, the
/// host can DMA-fill buffer B concurrently.
///
/// If `opts.barrier` is `true` (the default), the firmware will drain all
/// prior commands before starting this kernel. Set `barrier` to `false`
/// only when the caller has ensured the prior operations (typically the
/// DMA filling the kernel's input buffer) have completed.
pub fn launch_async(
&self,
kernel: &LoadedKernel,
opts: &LaunchOptions,
) -> Result<PendingLaunch> {
let mut flags: u16 = 0;
if opts.barrier {
flags |= cmd_flags::BARRIER;
}
if opts.flush_l3 {
flags |= cmd_flags::FLUSH_L3;
}
// Optional argument payload: trace configuration (40 B) then stack
// configuration (8 B), each present only when the corresponding flag is set.
let mut payload = Vec::new();
if let Some(trace) = opts.trace {
flags |= cmd_flags::COMPUTE_KERNEL_TRACE;
payload.extend_from_slice(&trace.to_init_info().to_bytes());
}
if let Some(stack) = opts.stack {
flags |= cmd_flags::USER_STACK_CFG;
payload.extend_from_slice(&stack.to_bytes());
}
// Kernel arguments are delivered by pointer: the firmware passes the
// device address of the args blob in `a0` at kernel entry. Stage the
// bytes into device DRAM first, then encode the pointer in the command.
// (Verified on device: `a0` carries the pointer; `ra` is 0 at entry
// despite the SDK docs, and an embedded payload leaves neither populated.)
let mut pointer_to_args: u64 = 0;
if !opts.args.is_empty() {
let region = self.args_region(opts.args.len() as u64)?;
self.memcpy_h2d(&opts.args, region.addr)?;
pointer_to_args = region.addr;
}
let tag = self.next_tag();
let cmd = proto::build_kernel_launch(
tag,
flags,
kernel.code_start_address,
pointer_to_args,
opts.exception_buffer,
opts.shire_mask,
&payload,
);
self.push_cmd(opts.sq_index, &cmd, 0)?;
Ok(PendingLaunch {
tag,
timeout: opts.timeout.unwrap_or_else(|| self.default_timeout.get()),
})
}
/// Block until a pending kernel launch completes and return its result.
///
/// Any completion responses for other in-flight commands that arrive while
/// waiting are stashed automatically and returned by their own `wait_launch`
/// or subsequent `launch` call.
///
/// The timeout applied is the one set via [`LaunchOptions::with_timeout`]
/// when the launch was submitted (default 10 s).
pub fn wait_launch(&self, pending: PendingLaunch) -> Result<LaunchResult> {
let rsp = self.collect_response(pending.tag, Some(pending.timeout), "kernel completion")?;
let status = proto::response_status(&rsp.bytes)
.ok_or_else(|| Error::Protocol("kernel-launch response truncated".into()))?;
if status != ops::DEV_OPS_API_KERNEL_LAUNCH_RESPONSE::DEV_OPS_API_KERNEL_LAUNCH_RESPONSE_KERNEL_COMPLETED {
return Err(Error::KernelLaunch {
status,
status_name: proto::kernel_launch_status_name(status),
detail: proto::parse_kernel_error_ptr(&rsp.bytes),
});
}
Ok(LaunchResult {
timing: parse_launch_timing(&rsp.bytes),
})
}
/// Launch a loaded kernel and wait for its completion response.
///
/// Equivalent to [`Device::launch_async`] followed immediately by
/// [`Device::wait_launch`]. Use `launch_async` + `wait_launch` directly
/// when overlapping computation with DMA or issuing multiple concurrent
/// launches.
pub fn launch(&self, kernel: &LoadedKernel, opts: &LaunchOptions) -> Result<LaunchResult> {
let pending = self.launch_async(kernel, opts)?;
self.wait_launch(pending)
}
/// Launch `kernel` across `shire_mask` (SPMD) with typed arguments.
///
/// This bundles the argument staging that [`Device::launch`] otherwise does by
/// hand: `args` is the shared [`et_abi::DeviceArgs`] struct the kernel reads,
/// serialised and delivered by pointer (the firmware passes its address in
/// `a0`). Equivalent to `launch(kernel, &LaunchOptions::new(shire_mask)
/// .with_args(args.as_bytes().to_vec()))`.
pub fn launch_spmd<A: et_abi::DeviceArgs>(
&self,
kernel: &LoadedKernel,
shire_mask: u64,
args: &A,
) -> Result<LaunchResult> {
let opts = LaunchOptions::new(shire_mask).with_args(args.as_bytes().to_vec());
self.launch(kernel, &opts)
}
/// As [`Device::launch_spmd`], additionally capturing a U-mode trace.
pub fn launch_spmd_traced<A: et_abi::DeviceArgs>(
&self,
kernel: &LoadedKernel,
shire_mask: u64,
args: &A,
trace: TraceConfig,
) -> Result<LaunchResult> {
let opts = LaunchOptions::new(shire_mask)
.with_args(args.as_bytes().to_vec())
.with_trace(trace);
self.launch(kernel, &opts)
}
/// As [`Device::launch_spmd`] with explicit launch options.
///
/// The `shire_mask` is taken from `opts`; `args` is serialised and appended
/// via [`LaunchOptions::with_args`], overwriting any args already in `opts`.
/// Use this to set a per-launch timeout or a non-default submission queue:
///
/// ```
/// use et_soc1::{LaunchOptions, LoadedKernel};
/// use std::time::Duration;
///
/// # fn example(device: &et_soc1::Device, kernel: &LoadedKernel, args: &impl et_abi::DeviceArgs) -> et_soc1::Result<et_soc1::LaunchResult> {
/// device.launch_spmd_opts(
/// kernel,
/// args,
/// LaunchOptions::new(0xFFFF_FFFF).with_timeout(Duration::from_secs(120)),
/// )
/// # }
/// ```
pub fn launch_spmd_opts<A: et_abi::DeviceArgs>(
&self,
kernel: &LoadedKernel,
args: &A,
opts: LaunchOptions,
) -> Result<LaunchResult> {
self.launch(kernel, &opts.with_args(args.as_bytes().to_vec()))
}
/// As [`Device::launch_spmd_traced`] with explicit launch options.
///
/// The `shire_mask` is taken from `opts`; `args` is appended via
/// [`LaunchOptions::with_args`] and `trace` via [`LaunchOptions::with_trace`],
/// overwriting any previously set values in `opts`.
pub fn launch_spmd_traced_opts<A: et_abi::DeviceArgs>(
&self,
kernel: &LoadedKernel,
args: &A,
trace: TraceConfig,
opts: LaunchOptions,
) -> Result<LaunchResult> {
let opts = opts.with_args(args.as_bytes().to_vec()).with_trace(trace);
self.launch(kernel, &opts)
}
/// Copy `dst.len()` bytes from device address `src` into host memory via a
/// DMA read-list command, splitting the transfer to honour the device's DMA
/// element-size and element-count limits.
///
/// The transfer is staged through a transport-provided DMA host buffer (see
/// [`crate::transport::DmaHostBuffer`]) and copied out afterwards, so it works
/// whether the backend pins arbitrary host memory or requires registered DMA
/// memory.
///
/// Intermediate DMA batches (when the transfer spans more than
/// `dma_max_elem_count` nodes) are issued without the `BARRIER` flag so the
/// firmware's DMA engine can pipeline them; only the final batch uses
/// `BARRIER` to ensure completion before the function returns.
///
/// Uses the default [`DmaOptions`] (SQ 0). Use [`Device::memcpy_d2h_opts`]
/// to select a different submission queue.
pub fn memcpy_d2h(&self, src: u64, dst: &mut [u8]) -> Result<()> {
self.memcpy_d2h_opts(src, dst, &DmaOptions::default())
}
/// Copy `dst.len()` bytes from device address `src` into host memory,
/// routing DMA commands according to `opts`.
///
/// Equivalent to [`Device::memcpy_d2h`] but with explicit [`DmaOptions`].
/// Use `opts.on_sq(1)` to send DMA to a separate submission queue, enabling
/// concurrent DMA and compute when paired with [`Device::launch_async`].
pub fn memcpy_d2h_opts(&self, src: u64, dst: &mut [u8], opts: &DmaOptions) -> Result<()> {
let total = dst.len();
if total == 0 {
return Ok(());
}
let max_elem = (self.dram.dma_max_elem_size as usize).max(1);
let max_nodes = (self.dram.dma_max_elem_count as usize).max(1);
let host = self.transport.dma_host_buffer(total)?;
let hvirt = host.virt_addr();
let hphys = host.phys_addr();
let mut offset = 0usize;
let mut nodes: Vec<proto::DmaReadNode> = Vec::with_capacity(max_nodes);
while offset < total {
let len = (total - offset).min(max_elem);
nodes.push(proto::DmaReadNode {
dst_host_virt_addr: hvirt + offset as u64,
dst_host_phy_addr: node_phys(hphys, offset),
src_device_phy_addr: src + offset as u64,
size: len as u32,
_pad: [0; 4],
});
offset += len;
if nodes.len() == max_nodes || offset >= total {
// Barrier only on the final batch: earlier batches may overlap
// with subsequent firmware DMA scheduling for better throughput.
let is_last = offset >= total;
let flags = if is_last { cmd_flags::BARRIER } else { 0 };
self.dma_read_command(&nodes, flags, opts.sq_index, opts.timeout)?;
nodes.clear();
}
}
dst.copy_from_slice(host.as_slice());
Ok(())
}
/// Copy `src.len()` bytes from host memory to device address `dst` via a DMA
/// write-list command, splitting the transfer to honour the device's DMA
/// element-size and element-count limits. The data is staged through a
/// transport-provided DMA host buffer.
///
/// Intermediate batches are issued without `BARRIER` (see [`memcpy_d2h`]);
/// only the final batch uses `BARRIER` to guarantee completion on return.
///
/// Uses the default [`DmaOptions`] (SQ 0). Use [`Device::memcpy_h2d_opts`]
/// to select a different submission queue.
pub fn memcpy_h2d(&self, src: &[u8], dst: u64) -> Result<()> {
self.memcpy_h2d_opts(src, dst, &DmaOptions::default())
}
/// Copy `src.len()` bytes from host memory to device address `dst`,
/// routing DMA commands according to `opts`.
///
/// Equivalent to [`Device::memcpy_h2d`] but with explicit [`DmaOptions`].
/// Use `opts.on_sq(1)` to send DMA to a separate submission queue, enabling
/// concurrent DMA and compute when paired with [`Device::launch_async`].
pub fn memcpy_h2d_opts(&self, src: &[u8], dst: u64, opts: &DmaOptions) -> Result<()> {
let total = src.len();
if total == 0 {
return Ok(());
}
let max_elem = (self.dram.dma_max_elem_size as usize).max(1);
let max_nodes = (self.dram.dma_max_elem_count as usize).max(1);
let mut host = self.transport.dma_host_buffer(total)?;
host.as_mut_slice().copy_from_slice(src);
let hvirt = host.virt_addr();
let hphys = host.phys_addr();
let mut offset = 0usize;
let mut nodes: Vec<proto::DmaWriteNode> = Vec::with_capacity(max_nodes);
while offset < total {
let len = (total - offset).min(max_elem);
nodes.push(proto::DmaWriteNode {
src_host_virt_addr: hvirt + offset as u64,
src_host_phy_addr: node_phys(hphys, offset),
dst_device_phy_addr: dst + offset as u64,
size: len as u32,
_pad: [0; 4],
});
offset += len;
if nodes.len() == max_nodes || offset >= total {
let is_last = offset >= total;
let flags = if is_last { cmd_flags::BARRIER } else { 0 };
self.dma_write_command(&nodes, flags, opts.sq_index, opts.timeout)?;
nodes.clear();
}
}
Ok(())
}
/// Extract the compute-minion trace buffer (`TRACE_BUFFER_CM`).
pub fn extract_cm_trace(&self) -> Result<Vec<u8>> {
self.extract_trace(ops::trace_buffer_type::TRACE_BUFFER_CM as u8)
}
/// Extract a device trace buffer of the given `trace_buffer_type`.
pub fn extract_trace(&self, trace_type: u8) -> Result<Vec<u8>> {
self.transport.extract_trace(trace_type)
}
/// Reset the compute minions on the shires in `shire_mask`.
///
/// Sends a `CM_RESET_CMD` and blocks until the firmware acknowledges it.
/// The device node remains open; DMA and host state are unaffected. This
/// is the appropriate response to a kernel that has hung inside the RISC-V
/// compute shires without corrupting firmware state.
///
/// On success the reset shires are ready for a new kernel launch. Any
/// previously loaded kernels remain mapped (the DRAM allocator state is
/// not modified); loaded kernels can be launched again immediately.
///
/// Returns [`Error::Device`] with `command = "cm-reset"` if the firmware
/// rejects the request (e.g. an invalid `shire_mask`).
pub fn reset_shires(&self, shire_mask: u64) -> Result<()> {
let tag = self.next_tag();
let cmd = proto::build_cm_reset(tag, shire_mask);
// CM reset goes through the high-priority SQ (HPSQ), which is the MM
// firmware's management path. The standard SQ (flags=0) is not
// monitored for management commands on this firmware version; submitting
// there produces no response. CMD_DESC_FLAG_HIGH_PRIORITY routes the
// command to the HPSQ. The `collect_response` caller uses
// `GET_CQ_AVAIL_BITMAP` to find whichever CQ the response lands on.
let timeout = Some(self.default_timeout.get());
let rsp = self.submit(0, &cmd, desc_flags::HIGH_PRIORITY, tag, timeout)?;
let status = proto::cm_reset_response_status(&rsp.bytes)
.ok_or_else(|| Error::Protocol("CM reset response truncated".into()))?;
if status != ops::DEV_OPS_API_CM_RESET_RESPONSE::DEV_OPS_API_CM_RESET_RESPONSE_SUCCESS {
return Err(Error::Device {
command: "cm-reset",
code: status,
});
}
Ok(())
}
// --- internals ---
fn dma_read_command(
&self,
nodes: &[proto::DmaReadNode],
flags: u16,
sq_index: u16,
timeout: Option<Duration>,
) -> Result<()> {
let tag = self.next_tag();
let cmd = proto::build_dma_readlist(tag, flags, nodes);
let rsp = self.submit(sq_index, &cmd, desc_flags::DMA, tag, timeout)?;
let status = proto::response_status(&rsp.bytes)
.ok_or_else(|| Error::Protocol("DMA read-list response truncated".into()))?;
if status != ops::DEV_OPS_API_DMA_RESPONSE::DEV_OPS_API_DMA_RESPONSE_COMPLETE {
return Err(Error::Device {
command: "dma-readlist",
code: status,
});
}
Ok(())
}
fn dma_write_command(
&self,
nodes: &[proto::DmaWriteNode],
flags: u16,
sq_index: u16,
timeout: Option<Duration>,
) -> Result<()> {
let tag = self.next_tag();
let cmd = proto::build_dma_writelist(tag, flags, nodes);
let rsp = self.submit(sq_index, &cmd, desc_flags::DMA, tag, timeout)?;
let status = proto::response_status(&rsp.bytes)
.ok_or_else(|| Error::Protocol("DMA write-list response truncated".into()))?;
if status != ops::DEV_OPS_API_DMA_RESPONSE::DEV_OPS_API_DMA_RESPONSE_COMPLETE {
return Err(Error::Device {
command: "dma-writelist",
code: status,
});
}
Ok(())
}
/// Push a command onto the submission queue, retrying until space is available.
fn push_cmd(&self, sq_index: u16, cmd: &[u8], desc_flags: u8) -> Result<()> {
// Longest a single `wait_sq` blocks before re-polling. A backend whose
// wait returns immediately (the emulator does) must not be mistaken for
// a genuine timeout; the deadline is the sole authority on giving up.
let limit = self.default_timeout.get();
let slice = Duration::from_millis(250);
let deadline = Instant::now() + limit;
loop {
if self.transport.push_sq(sq_index, cmd, desc_flags)? {
return Ok(());
}
if Instant::now() >= deadline {
return Err(Error::Timeout {
operation: "submission-queue space",
limit,
});
}
if !self.transport.wait_sq(remaining(deadline).min(slice))? {
std::thread::sleep(Duration::from_millis(1));
}
}
}
/// Block until the CQ response bearing `expected_tag` arrives.
///
/// `operation` names the blocking operation for diagnostic purposes (e.g.
/// `"kernel completion"`, `"DMA completion"`). Responses for other in-flight
/// tags are placed in `self.stash` so that concurrent `launch_async` /
/// `wait_launch` sequences do not discard each other's completions.
///
/// `timeout` is `None` for unlimited waits (DMA default) or `Some(d)` to
/// surface an [`Error::Timeout`] after `d` elapses.
fn collect_response(
&self,
expected_tag: u16,
timeout: Option<Duration>,
operation: &'static str,
) -> Result<PoppedResponse> {
let slice = Duration::from_millis(250);
let deadline = timeout.map(|t| Instant::now() + t);
// Check the stash before polling the CQ: if a prior `collect_response`
// parked this tag, return it immediately without touching the hardware.
if let Some(rsp) = self.stash.borrow_mut().remove(&expected_tag) {
return Ok(rsp);
}
loop {
if let Some(rsp) = self.transport.pop_cq()? {
match proto::ResponseHeader::parse(&rsp.bytes) {
Some(hdr) if hdr.tag_id == expected_tag => return Ok(rsp),
Some(hdr) => {
// Response for a different in-flight command; park it.
self.stash.borrow_mut().insert(hdr.tag_id, rsp);
}
None => {} // Malformed response; discard silently.
}
continue;
}
if let Some(dl) = deadline {
if Instant::now() >= dl {
return Err(Error::Timeout {
operation,
limit: timeout.unwrap(),
});
}
// A false return means no completion arrived in this slice; keep
// polling until the deadline.
if !self.transport.wait_cq(remaining(dl).min(slice))? {
std::thread::sleep(Duration::from_millis(1));
}
} else {
// No deadline: block in wait_cq for up to `slice` then retry.
if !self.transport.wait_cq(slice)? {
std::thread::sleep(Duration::from_millis(1));
}
}
}
}
/// Push a command and block for the response bearing `expected_tag`.
///
/// A thin wrapper around [`push_cmd`] + [`collect_response`]; used for
/// operations that are inherently synchronous (DMA commands). Kernel launches
/// use [`Device::launch_async`] + [`Device::wait_launch`] so they can carry
/// a per-launch timeout.
fn submit(
&self,
sq_index: u16,
cmd: &[u8],
desc_flags: u8,
expected_tag: u16,
timeout: Option<Duration>,
) -> Result<PoppedResponse> {
self.push_cmd(sq_index, cmd, desc_flags)?;
self.collect_response(expected_tag, timeout, "DMA completion")
}
fn next_tag(&self) -> u16 {
let t = self.tag.get();
self.tag.set(t.wrapping_add(1));
t
}
}
/// Physical address for a DMA node at `offset` into a staging buffer whose base
/// physical address is `base`. A zero base means the backend resolves the
/// physical address itself, so it stays zero.
fn node_phys(base: u64, offset: usize) -> u64 {
if base == 0 { 0 } else { base + offset as u64 }
}
fn align_up(value: u64, align: u64) -> u64 {
debug_assert!(align.is_power_of_two());
(value + align - 1) & !(align - 1)
}
fn remaining(deadline: Instant) -> Duration {
deadline.saturating_duration_since(Instant::now())
}
fn parse_launch_timing(buf: &[u8]) -> LaunchTiming {
let rd = |off: usize| -> u64 {
buf.get(off..off + 8)
.map(|s| u64::from_le_bytes(s.try_into().unwrap()))
.unwrap_or(0)
};
// Following the 8-byte response header: start_ts, execute_dur, wait_dur.
LaunchTiming {
start_ts: rd(8),
execute_dur: rd(16),
wait_dur: rd(24),
}
}