cubecl_runtime/server/base.rs
1use super::Handle;
2use crate::{
3 client::ComputeClient,
4 compiler::CompilationError,
5 config::{CubeClRuntimeConfig, RuntimeConfig, compilation::BoundsCheckMode},
6 kernel::KernelMetadata,
7 logging::ServerLogger,
8 memory_management::{ArenaStats, ManagedMemoryHandle, MemoryAllocationMode, MemoryUsage},
9 runtime::Runtime,
10 server::Binding,
11 storage::{ComputeStorage, ManagedResource},
12 tma::{OobFill, TensorMapFormat, TensorMapInterleave, TensorMapPrefetch, TensorMapSwizzle},
13};
14use ahash::AHasher;
15use alloc::boxed::Box;
16#[cfg(feature = "profile-tracy")]
17use alloc::format;
18use alloc::string::String;
19use alloc::sync::Arc;
20use alloc::vec::Vec;
21use core::{
22 fmt::Debug,
23 hash::{Hash, Hasher},
24};
25use cubecl_common::{
26 backtrace::BackTrace,
27 bytes::Bytes,
28 device::{self, DeviceId},
29 future::DynFut,
30 profile::ProfileDuration,
31 stream_id::StreamId,
32 stub::RwLock,
33};
34use cubecl_ir::{DeviceProperties, ElemType, StorageType};
35use cubecl_zspace::{Shape, Strides, metadata::Metadata};
36use hashbrown::HashSet;
37use thiserror::Error;
38
39#[derive(Error, Clone)]
40#[cfg_attr(std_io, derive(serde::Serialize, serde::Deserialize))]
41/// An error during profiling.
42pub enum ProfileError {
43 /// An unknown error happened during profiling
44 #[error(
45 "An unknown error happened during profiling\nCaused by:\n {reason}\nBacktrace:\n{backtrace}"
46 )]
47 Unknown {
48 /// The caused of the error
49 reason: String,
50 /// The captured backtrace.
51 #[cfg_attr(std_io, serde(skip))]
52 backtrace: BackTrace,
53 },
54
55 /// No profiling was registered
56 #[error("No profiling registered\nBacktrace:\n{backtrace}")]
57 NotRegistered {
58 /// The captured backtrace.
59 #[cfg_attr(std_io, serde(skip))]
60 backtrace: BackTrace,
61 },
62
63 /// A launch error happened during profiling
64 #[error("A launch error happened during profiling\nCaused by:\n {0}")]
65 Launch(#[from] LaunchError),
66
67 /// An execution error happened during profiling
68 #[error("An execution error happened during profiling\nCaused by:\n {0}")]
69 Server(#[from] Box<ServerError>),
70}
71
72impl core::fmt::Debug for ProfileError {
73 fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result {
74 f.write_fmt(format_args!("{self}"))
75 }
76}
77
78/// Contains many different types that are useful for server implementations and compute clients.
79pub struct ServerUtilities<Server: ComputeServer> {
80 /// The time when `profile-tracy` is activated.
81 #[cfg(feature = "profile-tracy")]
82 pub epoch_time: web_time::Instant,
83 /// The GPU client when `profile-tracy` is activated.
84 #[cfg(feature = "profile-tracy")]
85 pub gpu_client: tracy_client::GpuContext,
86 /// Information shared between all servers.
87 pub properties: DeviceProperties,
88 /// Stable hash of the device properties
89 pub properties_hash: u64,
90 /// Information specific to the current server.
91 pub info: Server::Info,
92 /// The logger based on global cubecl configs.
93 pub logger: Arc<ServerLogger>,
94 /// How to create the allocation.
95 pub layout_policy: Server::MemoryLayoutPolicy,
96 /// How to enforce bounds checking on kernels.
97 pub check_mode: BoundsCheckMode,
98 /// A set containing the ids for which the inter-device communication has already been initialized.
99 pub initialized_comms: RwLock<HashSet<CommunicationId>>,
100}
101
102/// Defines how the memory layout is determined.
103pub trait MemoryLayoutPolicy: Send + Sync + 'static {
104 /// Applies the memory layout policy to a list of descriptors.
105 ///
106 /// Returns a vector of `MemoryLayout`, one per descriptor, with layouts that share a
107 /// single `Binding`.
108 fn apply(
109 &self,
110 stream_id: StreamId,
111 descriptors: &[MemoryLayoutDescriptor],
112 ) -> (Handle, Vec<MemoryLayout>);
113}
114
115impl<Server: core::fmt::Debug> core::fmt::Debug for ServerUtilities<Server>
116where
117 Server: ComputeServer,
118 Server::Info: core::fmt::Debug,
119{
120 fn fmt(&self, f: &mut core::fmt::Formatter) -> core::fmt::Result {
121 f.debug_struct("ServerUtilities")
122 .field("properties", &self.properties)
123 .field("info", &self.info)
124 .field("logger", &self.logger)
125 .finish()
126 }
127}
128
129impl<S: ComputeServer> ServerUtilities<S> {
130 /// Creates a new server utilities.
131 pub fn new(
132 properties: DeviceProperties,
133 logger: Arc<ServerLogger>,
134 info: S::Info,
135 allocator: S::MemoryLayoutPolicy,
136 ) -> Self {
137 // Start a tracy client if needed.
138 #[cfg(feature = "profile-tracy")]
139 let client = tracy_client::Client::start();
140
141 Self {
142 properties_hash: properties.checksum(),
143 properties,
144 logger,
145 // Create the GPU client if needed.
146 #[cfg(feature = "profile-tracy")]
147 gpu_client: client
148 .clone()
149 .new_gpu_context(
150 Some(&format!("{info:?}")),
151 // In the future should ask the server what makes sense here. 'Invalid' atm is a generic stand-in (Tracy doesn't have CUDA/RocM atm anyway).
152 tracy_client::GpuContextType::Invalid,
153 0, // Timestamps are manually aligned to this epoch so start at 0.
154 1.0, // Timestamps are manually converted to be nanoseconds so period is 1.
155 )
156 .unwrap(),
157 #[cfg(feature = "profile-tracy")]
158 epoch_time: web_time::Instant::now(),
159 info,
160 layout_policy: allocator,
161 check_mode: CubeClRuntimeConfig::get().compilation.check_mode,
162 initialized_comms: RwLock::new(HashSet::default()),
163 }
164 }
165}
166
167/// Kernel Launch Errors.
168#[derive(Error, Clone)]
169#[cfg_attr(std_io, derive(serde::Serialize, serde::Deserialize))]
170pub enum LaunchError {
171 /// The given kernel can't be compiled.
172 #[error("A compilation error happened during launch\nCaused by:\n {0}")]
173 CompilationError(#[from] CompilationError),
174
175 /// The server is out of memory.
176 #[error(
177 "An out-of-memory error happened during launch\nCaused by:\n {reason}\nBacktrace\n{backtrace}"
178 )]
179 OutOfMemory {
180 /// The caused of the memory error.
181 reason: String,
182 /// The backtrace for this error.
183 #[cfg_attr(std_io, serde(skip))]
184 backtrace: BackTrace,
185 },
186
187 /// Too many resources were requested
188 #[error("Too many resources were requested during launch\n{0}")]
189 TooManyResources(#[from] ResourceLimitError),
190
191 /// Unknown launch error.
192 #[error(
193 "An unknown error happened during launch\nCaused by:\n {reason}\nBacktrace\n{backtrace}"
194 )]
195 Unknown {
196 /// The caused of the unknown error.
197 reason: String,
198 /// The backtrace for this error.
199 #[cfg_attr(std_io, serde(skip))]
200 backtrace: BackTrace,
201 },
202
203 /// Can't launch because of an IO Error.
204 #[error("An io error happened during launch\nCaused by:\n {0}")]
205 IoError(#[from] IoError),
206}
207
208/// Resource limit errors.
209#[derive(Error, Clone)]
210#[cfg_attr(std_io, derive(serde::Serialize, serde::Deserialize))]
211pub enum ResourceLimitError {
212 /// Shared memory exceeds maximum
213 #[error(
214 "Too much shared memory requested.\nRequested {requested} bytes, maximum {max} bytes available.\nBacktrace\n{backtrace}"
215 )]
216 SharedMemory {
217 /// Value requested
218 requested: usize,
219 /// Maximum value
220 max: usize,
221 /// The backtrace for this error.
222 #[cfg_attr(std_io, serde(skip))]
223 backtrace: BackTrace,
224 },
225 /// Total units exceeds maximum
226 #[error(
227 "Total unit count exceeds maximum.\nRequested {requested} units, max units is {max}.\nBacktrace\n{backtrace}"
228 )]
229 Units {
230 /// Requested value
231 requested: u32,
232 /// Maximum value
233 max: u32,
234 /// The backtrace for this error.
235 #[cfg_attr(std_io, serde(skip))]
236 backtrace: BackTrace,
237 },
238 /// `CubeDim` exceeds maximum
239 #[error(
240 "Cube dim exceeds maximum bounds.\nRequested {requested:?}, max is {max:?}.\nBacktrace\n{backtrace}"
241 )]
242 CubeDim {
243 /// Requested value
244 requested: (u32, u32, u32),
245 /// Maximum value
246 max: (u32, u32, u32),
247 /// The backtrace for this error.
248 #[cfg_attr(std_io, serde(skip))]
249 backtrace: BackTrace,
250 },
251}
252
253impl core::fmt::Debug for LaunchError {
254 fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result {
255 f.write_fmt(format_args!("{self}"))
256 }
257}
258
259impl core::fmt::Debug for ResourceLimitError {
260 fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result {
261 f.write_fmt(format_args!("{self}"))
262 }
263}
264
265/// Error that can happen asynchronously while executing registered kernels.
266#[derive(Error, Debug, Clone)]
267#[cfg_attr(std_io, derive(serde::Serialize, serde::Deserialize))]
268pub enum ServerError {
269 /// A generic runtime error.
270 #[error("An error happened during execution\nCaused by:\n {reason}\nBacktrace:\n{backtrace}")]
271 Generic {
272 /// The details of the generic error.
273 reason: String,
274 /// The backtrace for this error.
275 #[cfg_attr(std_io, serde(skip))]
276 backtrace: BackTrace,
277 },
278
279 /// A launch error happened during profiling
280 #[error("A launch error happened during profiling\nCaused by:\n {0}")]
281 Launch(#[from] LaunchError),
282
283 /// An execution error happened during profiling
284 #[error("An execution error happened during profiling\nCaused by:\n {0}")]
285 Profile(#[from] ProfileError),
286
287 /// An execution error happened during profiling
288 #[error("An execution error happened during profiling\nCaused by:\n {0}")]
289 Io(#[from] IoError),
290
291 /// The server is an invalid state.
292 #[error("The server is in an invalid state\nCaused by:\n {errors:?}")]
293 ServerUnhealthy {
294 /// The details of the generic error.
295 errors: Vec<Self>,
296 /// The backtrace for this error.
297 #[cfg_attr(std_io, serde(skip))]
298 backtrace: BackTrace,
299 },
300}
301
302/// How errors are handled in a stream when executing a task.
303#[derive(Clone, Copy)]
304pub struct StreamErrorMode {
305 /// Whether the task still executes even if the stream is in error.
306 pub ignore: bool,
307 /// Whether the errors are flushed by the current task.
308 pub flush: bool,
309}
310
311/// The compute server is responsible for handling resources and computations over resources.
312///
313/// Everything in the server is mutable, therefore it should be solely accessed through the
314/// [`ComputeClient`] for thread safety.
315pub trait ComputeServer:
316 Send + core::fmt::Debug + ServerCommunication + device::DeviceService + 'static
317where
318 Self: Sized,
319{
320 /// The kernel type defines the computation algorithms.
321 type Kernel: KernelMetadata;
322 /// Information that can be retrieved for the runtime.
323 type Info: Debug + Send + Sync;
324 /// Manages how allocations are performed for a server.
325 type MemoryLayoutPolicy: MemoryLayoutPolicy;
326 /// The [storage](ComputeStorage) type defines how data is stored and accessed.
327 type Storage: ComputeStorage;
328
329 /// Initializes [memory](ManagedMemoryHandle) on the given [stream](StreamId) with the given size.
330 fn initialize_memory(&mut self, memory: ManagedMemoryHandle, size: u64, stream_id: StreamId);
331
332 /// Whether this backend can capture a region of enqueued work into a
333 /// replayable object at all. Callers must check this before capturing:
334 /// the capture methods below panic on a backend that cannot.
335 fn graph_capture_supported(&self) -> bool {
336 false
337 }
338
339 /// Defer the stream's pending-drop flush without opening a capture.
340 ///
341 /// This is the PRE-WARM for a capture, and it is needed because a capture
342 /// changes what the allocator has to hold at once. Normally a staging
343 /// buffer is handed back a few launches after it is used, so a region of
344 /// N launches only ever needs a handful of pages live. Inside a capture
345 /// nothing is handed back -- the flush waits on a fence, which a capture
346 /// cannot contain, and the buffers must stay alive anyway because the
347 /// graph re-reads them on every replay. So the region's requirement jumps
348 /// from its per-launch working set to its SIMULTANEOUS HIGH-WATER MARK,
349 /// and the pool discovers that by allocating, which is the one thing that
350 /// breaks a capture.
351 ///
352 /// Running the region once with frees deferred and no capture open lets
353 /// the pool reach that high-water mark the legal way. Flushing afterwards
354 /// returns the slices but keeps the pages, so the capture that follows
355 /// finds every page it needs already there.
356 fn graph_defer_frees(&mut self, _defer: bool, _stream_id: StreamId) {
357 panic!("graph capture is not supported by this backend");
358 }
359
360 /// Route every allocation on `stream_id` into the CAPTURE ARENA, and
361 /// return the generation this opening is.
362 ///
363 /// Call this once to WARM (arena open, no capture: the region runs eagerly
364 /// and the arena learns exactly which slices it needs), then again around
365 /// the capture itself. See
366 /// [`MemoryManagement::arena_begin`](crate::memory_management::MemoryManagement::arena_begin)
367 /// for why an arena and not a hold.
368 ///
369 /// Reopening the arena and capturing again DELIBERATELY hands the second
370 /// graph the same slices: that is what makes two captures of one region
371 /// comparable, and what lets one graph stand in for another step. The two
372 /// then share scratch, which is safe exactly when they are captures of the
373 /// same region -- checked by the request sequence each capture recorded --
374 /// and replayed serially, which the single stream gives for free.
375 fn graph_arena_begin(&mut self, _stream_id: StreamId) -> u64 {
376 panic!("graph capture is not supported by this backend");
377 }
378
379 /// Stop routing allocations into the arena. The arena keeps every page it
380 /// owns -- closing it ends allocation from it, never its reservation.
381 fn graph_arena_end(&mut self, _stream_id: StreamId) {
382 panic!("graph capture is not supported by this backend");
383 }
384
385 /// What the arena holds, and how many requests it had to take from the
386 /// driver. A miss taken while a capture is open is a graph memory node.
387 fn graph_arena_stats(&mut self, _stream_id: StreamId) -> ArenaStats {
388 ArenaStats::default()
389 }
390
391 /// Zero the arena's served/miss counters so the capture pass can be
392 /// counted apart from the warm pass.
393 fn graph_arena_reset_counters(&mut self, _stream_id: StreamId) {}
394
395 /// What the allocator's RESERVE half costs, as `(calls, nanoseconds)`.
396 ///
397 /// A decode step makes roughly one reservation per kernel launch, and all
398 /// of it is host work inside the term a graph replay removes. The count has
399 /// been reported for a while; this is the time, because a count is not a
400 /// time and the two lead to opposite conclusions at 0.5 us and at 5 us a
401 /// request. Zero unless `CUBECL_TIME_RESERVE=1`. It does NOT include the
402 /// free half, which happens on handle drop.
403 fn reserve_timing(&mut self, _stream_id: StreamId) -> (u64, u64) {
404 (0, 0)
405 }
406
407 /// Zero the reservation timer.
408 fn reserve_timing_reset(&mut self, _stream_id: StreamId) {}
409
410 /// Begin capturing work enqueued on `stream_id` instead of running it.
411 /// Between this and [`ComputeServer::graph_capture_end`] nothing executes;
412 /// the enqueued work is recorded as a graph. Anything that BLOCKS THE HOST
413 /// on device progress inside the captured region (a sync, a fence wait, a
414 /// blocking read) invalidates the capture, because there is no device
415 /// progress to wait for.
416 fn graph_capture_begin(&mut self, _stream_id: StreamId) {
417 panic!("graph capture is not supported by this backend");
418 }
419
420 /// Close the capture opened by [`ComputeServer::graph_capture_begin`],
421 /// instantiate the result, and return an id naming it for replay. The
422 /// captured graph holds the EXACT device pointers seen during capture, so
423 /// every buffer it touches must live at a stable address across replays.
424 fn graph_capture_end(&mut self, _stream_id: StreamId) -> u64 {
425 panic!("graph capture is not supported by this backend");
426 }
427
428 /// Enqueue one replay of the graph named by `id` on `stream_id`.
429 fn graph_replay(&mut self, _id: u64, _stream_id: StreamId) {
430 panic!("graph capture is not supported by this backend");
431 }
432
433 /// 0 = not capturing, 1 = capturing, 2 = INVALIDATED, 3 = query failed.
434 /// A capture that has been invalidated keeps accepting work silently and
435 /// only reports it at `end`, so this is the only way to find WHICH call
436 /// broke one.
437 fn graph_capture_status(&mut self, _stream_id: StreamId) -> u32 {
438 3
439 }
440
441 /// How many nodes the captured graph holds. This is the unit the replay
442 /// cost is quoted per, so it belongs next to the graph, not in a log line.
443 fn graph_node_count(&mut self, _id: u64) -> usize {
444 panic!("graph capture is not supported by this backend");
445 }
446
447 /// How many kernel launches the OPEN capture has recorded so far.
448 ///
449 /// This is the index a caller brackets its own calls with, so that after
450 /// the capture closes it can name the nodes a particular call produced
451 /// without knowing anything about how many kernels that call fans out to.
452 /// It counts launches, not nodes: a launch is one node, but a graph also
453 /// holds memcpy and memset nodes that no launch made.
454 fn graph_capture_launch_count(&mut self) -> usize {
455 panic!("graph capture is not supported by this backend");
456 }
457
458 /// How many launches the closed graph `id` recorded.
459 fn graph_launch_count(&mut self, _id: u64) -> usize {
460 panic!("graph capture is not supported by this backend");
461 }
462
463 /// What launch `idx` of graph `id` was recorded WITH: its grid, its packed
464 /// scalar+metadata blob, and the device address of every buffer it bound.
465 ///
466 /// This is the half of patching that makes it checkable. A caller that
467 /// wants to move a scalar has to know which slot of the blob holds it, and
468 /// the honest way to learn that is to look at what the blob actually
469 /// contained when the value was known -- not to re-derive cubecl's packing
470 /// order by hand and hope.
471 fn graph_launch_params(&mut self, _id: u64, _idx: usize) -> GraphLaunchParams {
472 panic!("graph capture is not supported by this backend");
473 }
474
475 /// Rewrite launch `idx` of graph `id`'s EXECUTABLE in place.
476 ///
477 /// The graph is untouched; only the instantiated exec changes, which is
478 /// what makes this cheap enough to do per step. Everything not named in
479 /// `patch` keeps the value it was captured with.
480 fn graph_patch_launch(&mut self, _id: u64, _idx: usize, _patch: GraphLaunchPatch) {
481 panic!("graph capture is not supported by this backend");
482 }
483
484 /// A census of the graph's nodes by KIND, as `(CUgraphNodeType, count)`.
485 ///
486 /// A graph is not only its kernels. Everything else a capture recorded --
487 /// host-to-device copies, memsets, child graphs -- is a node too, and a
488 /// node no kernel-parameter rewrite can reach. Whether a region is
489 /// replayable on a later step is therefore first of all a question about
490 /// how much of it is NOT a kernel launch.
491 fn graph_node_kinds(&mut self, _id: u64) -> Vec<(u32, usize)> {
492 panic!("graph capture is not supported by this backend");
493 }
494
495 /// Rewrite several launches of one graph in a single round trip.
496 ///
497 /// Per-launch [`ComputeServer::graph_patch_launch`] is one submit each, and
498 /// on a region with a hundred moving launches the submits cost far more
499 /// than the rewrites do. The batch exists because that overhead is the
500 /// caller's to avoid, not something to measure and report as the price of
501 /// patching.
502 fn graph_patch_launches(&mut self, _id: u64, _patches: Vec<(usize, GraphLaunchPatch)>) {
503 panic!("graph capture is not supported by this backend");
504 }
505
506 /// Every `(address, bytes)` the graph ALLOCATES for itself.
507 ///
508 /// A capture that could not avoid allocating turns each allocation into a
509 /// memory node, and the memory a graph owns has a virtual address the graph
510 /// fixes -- the same one on every launch of that exec, and NOT the same one
511 /// as any other graph's. So these addresses are already pinned for as long
512 /// as one exec is being replayed, and rewriting them to another graph's is
513 /// wrong by construction rather than merely unnecessary. A caller
514 /// re-pointing a region's buffers has to be able to tell them apart from
515 /// the ones that came from the pool.
516 fn graph_alloc_regions(&mut self, _id: u64) -> Vec<(u64, u64)> {
517 panic!("graph capture is not supported by this backend");
518 }
519
520 /// Every memcpy node's `(src, dst, bytes, kind)`, sorted.
521 ///
522 /// A capture records copies as well as launches, and a copy is not a
523 /// kernel: no parameter rewrite reaches it. When the data a copy carries is
524 /// per-step, the region is not replayable on a later step no matter how
525 /// well its kernels are patched, and this is how that is checked rather
526 /// than assumed.
527 fn graph_memcpy_specs(&mut self, _id: u64) -> Vec<(u64, u64, u64, u32)> {
528 panic!("graph capture is not supported by this backend");
529 }
530
531 /// Release a captured graph. An unknown id is ignored.
532 fn graph_destroy(&mut self, _id: u64) {}
533
534 /// Register an externally-owned GPU buffer that ALIASES host memory (e.g.
535 /// mmap'd pile pages) as a tensor [`Handle`], with ZERO copy. `ptr`/`page_len`
536 /// describe a page-aligned superset region (required by Metal's
537 /// `newBufferWithBytesNoCopy:`); `offset`/`size` locate the actual tensor
538 /// bytes inside it. The caller guarantees the host memory outlives every
539 /// handle derived from it. Default impl panics — only backends that can
540 /// import host buffers (wgpu/Metal) override this.
541 fn register_external_aliased(
542 &mut self,
543 _ptr: *mut core::ffi::c_void,
544 _page_len: u64,
545 _offset: u64,
546 _size: u64,
547 _keepalive: alloc::sync::Arc<dyn core::any::Any + Send + Sync>,
548 _stream_id: StreamId,
549 ) -> Handle {
550 panic!("register_external_aliased is not supported by this backend");
551 }
552
553 /// Reserves N [Bytes] of the provided sizes to be used as staging to load data.
554 fn staging(
555 &mut self,
556 _sizes: &[usize],
557 _stream_id: StreamId,
558 ) -> Result<Vec<Bytes>, ServerError> {
559 Err(IoError::UnsupportedIoOperation {
560 backtrace: BackTrace::capture(),
561 }
562 .into())
563 }
564
565 /// Retrieve the server logger.
566 fn logger(&self) -> Arc<ServerLogger>;
567
568 /// Retrieve the server utilities.
569 fn utilities(&self) -> Arc<ServerUtilities<Self>>;
570
571 /// Given bindings, returns the owned resources as bytes.
572 fn read(
573 &mut self,
574 descriptors: Vec<CopyDescriptor>,
575 stream_id: StreamId,
576 ) -> DynFut<Result<Vec<Bytes>, ServerError>>;
577
578 /// Writes the specified bytes into the buffers given
579 fn write(&mut self, descriptors: Vec<(CopyDescriptor, Bytes)>, stream_id: StreamId);
580
581 /// Wait for the completion of every task in the server.
582 fn sync(&mut self, stream_id: StreamId) -> DynFut<Result<(), ServerError>>;
583
584 /// Given a resource handle, returns the storage resource.
585 fn get_resource(
586 &mut self,
587 binding: Binding,
588 stream_id: StreamId,
589 ) -> Result<ManagedResource<<Self::Storage as ComputeStorage>::Resource>, ServerError>;
590
591 /// Executes the `kernel` over the given memory `handles`.
592 ///
593 /// Kernels have mutable access to every resource they are given
594 /// and are responsible of determining which should be read or written.
595 ///
596 /// # Safety
597 ///
598 /// When executing with mode [`ExecutionMode::Unchecked`], out-of-bound reads and writes can happen.
599 unsafe fn launch(
600 &mut self,
601 kernel: Self::Kernel,
602 count: CubeCount,
603 bindings: KernelArguments,
604 kind: ExecutionMode,
605 stream_id: StreamId,
606 );
607
608 /// Flush all outstanding tasks in the server.
609 fn flush(&mut self, stream_id: StreamId) -> Result<(), ServerError>;
610
611 /// The current memory usage of the server.
612 fn memory_usage(&mut self, stream_id: StreamId) -> Result<MemoryUsage, ServerError>;
613
614 /// Ask the server to release memory that it can release.
615 fn memory_cleanup(&mut self, stream_id: StreamId);
616
617 /// Enable collecting timestamps.
618 fn start_profile(&mut self, stream_id: StreamId) -> Result<ProfilingToken, ServerError>;
619
620 /// Disable collecting timestamps.
621 fn end_profile(
622 &mut self,
623 stream_id: StreamId,
624 token: ProfilingToken,
625 ) -> Result<ProfileDuration, ProfileError>;
626
627 /// Update the memory mode of allocation in the server.
628 fn allocation_mode(&mut self, mode: MemoryAllocationMode, stream_id: StreamId);
629}
630
631/// An ID unique to any unordered combination of devices.
632#[derive(Clone, Debug, Hash, Eq, PartialEq)]
633pub struct CommunicationId {
634 /// The ID as a `String`.
635 pub id: u64,
636}
637
638impl From<Vec<DeviceId>> for CommunicationId {
639 fn from(mut value: Vec<DeviceId>) -> Self {
640 // Make sure that device ids are sorted so that any combination of the same devices uses the same communicator.
641 value.sort();
642 let mut hasher = AHasher::default();
643 value.hash(&mut hasher);
644 CommunicationId {
645 id: hasher.finish(),
646 }
647 }
648}
649
650/// Different reduce operations.
651pub enum ReduceOperation {
652 /// Sum.
653 Sum,
654 /// Mean.
655 Mean,
656}
657
658/// Defines functions for optimized data transfer between servers, supporting custom communication
659/// mechanisms such as peer-to-peer communication or specialized implementations.
660pub trait ServerCommunication {
661 /// Indicates whether server-to-server communication is enabled for this implementation.
662 const SERVER_COMM_ENABLED: bool;
663
664 /// Ensure that all queued collective operations have been executed.
665 ///
666 /// # Arguments
667 ///
668 /// * `stream_id` - The [`StreamId`] of the stream waiting for the sync.
669 ///
670 /// # Returns
671 ///
672 /// Returns a `Result` containing an `ServerError` if the operation fails.
673 #[allow(unused_variables)]
674 fn sync_collective(&mut self, stream_id: StreamId) -> Result<(), ServerError> {
675 todo!() // For backends other than cuda.
676 }
677
678 /// Initialize the communication between the devices in `device_ids`.
679 ///
680 /// # Arguments
681 ///
682 /// * `device_ids` - The IDs of the devices that need communication.
683 ///
684 /// # Returns
685 ///
686 /// Returns a `Result` containing an `ServerError` if the operation fails.
687 #[allow(unused_variables)]
688 fn comm_init(&mut self, device_ids: Vec<DeviceId>) -> Result<(), ServerError> {
689 unimplemented!()
690 }
691
692 /// Performs an `all_reduce` operation on the input data and writes it to the output buffer.
693 /// see <https://docs.nvidia.com/deeplearning/nccl/user-guide/docs/usage/collectives.html#allreduce>
694 ///
695 /// # Arguments
696 ///
697 /// * `src` - The data to be reduced.
698 /// * `dst` - Where to write the result.
699 /// * `dtype` - The element type of the data being reduced
700 /// * `stream_id` - The data's stream id.
701 /// * `op` - The reduce's aggregation operation e.g. mean, sum, etc.
702 /// * `device_ids` - The list of device ids from which to `all_reduce`.
703 ///
704 /// # Returns
705 ///
706 /// Returns a `Result` containing an `ServerError` if the operation fails.
707 #[allow(unused_variables)]
708 fn all_reduce(
709 &mut self,
710 src: Binding,
711 dst: Binding,
712 dtype: ElemType,
713 stream_id: StreamId,
714 op: ReduceOperation,
715 device_ids: Vec<DeviceId>,
716 ) -> Result<(), ServerError> {
717 unimplemented!()
718 }
719
720 /// Sends data from this server to a destination server.
721 ///
722 /// # Arguments
723 ///
724 /// * `desc` - A descriptor specifying the data to be sent, including shape, strides, and binding.
725 /// * `dtype` - The element type of the data being sent.
726 /// * `stream_id` - The stream ID associated with the server's operation.
727 /// * `device_id_dst` - ID of the device receiving the data.
728 ///
729 /// # Returns
730 ///
731 /// Returns a `Result` containing an `ServerError` if the operation fails.
732 #[allow(unused_variables)]
733 fn send(
734 &mut self,
735 desc: CopyDescriptor,
736 dtype: ElemType,
737 stream_id: StreamId,
738 device_id_dst: DeviceId,
739 ) -> Result<(), ServerError> {
740 unimplemented!()
741 }
742
743 /// Receive data from another server.
744 ///
745 /// # Arguments
746 ///
747 /// * `handle` - The handle in which the received data is written.
748 /// * `dtype` - The element type of the data being sent.
749 /// * `stream_id` - The stream ID associated with the server's operation.
750 /// * `device_id_src` - ID of the device sending the data.
751 ///
752 /// # Returns
753 ///
754 /// Returns a `Result` containing an `ServerError` if the operation fails.
755 #[allow(unused_variables)]
756 fn recv(
757 &mut self,
758 handle: Handle,
759 dtype: ElemType,
760 stream_id: StreamId,
761 device_id_src: DeviceId,
762 ) -> Result<(), ServerError> {
763 unimplemented!()
764 }
765}
766
767#[derive(Debug, Clone, Copy, Hash, PartialEq, Eq)]
768/// Profiling identification so that the server can support recursive and overlapping profilings.
769pub struct ProfilingToken {
770 /// The token value.
771 pub id: u64,
772}
773
774/// Type of allocation, either contiguous or optimized (row-aligned when possible)
775#[derive(Debug, Clone, Copy, Hash, PartialEq, Eq)]
776pub enum MemoryLayoutStrategy {
777 /// Contiguous layout, with no padding
778 Contiguous,
779 /// Optimized for access speed. In practice this means row-aligned with padding for runtimes
780 /// that support it.
781 Optimized,
782}
783
784/// Descriptor for a new tensor allocation
785#[derive(new, Debug, Clone)]
786pub struct MemoryLayoutDescriptor {
787 /// Strategy used to create the memory layout.
788 pub strategy: MemoryLayoutStrategy,
789 /// Shape of the tensor
790 pub shape: Shape,
791 /// Size of each element in the tensor (used for conversion of shape to bytes)
792 pub elem_size: usize,
793}
794
795impl MemoryLayoutDescriptor {
796 /// Create an optimized allocation descriptor
797 pub fn optimized(shape: Shape, elem_size: usize) -> Self {
798 MemoryLayoutDescriptor::new(MemoryLayoutStrategy::Optimized, shape, elem_size)
799 }
800
801 /// Create a contiguous allocation descriptor
802 pub fn contiguous(shape: Shape, elem_size: usize) -> Self {
803 MemoryLayoutDescriptor::new(MemoryLayoutStrategy::Contiguous, shape, elem_size)
804 }
805}
806
807/// An allocation with associated strides. Strides depend on tensor layout.
808#[derive(Debug, Clone)]
809pub struct MemoryLayout {
810 /// The handle for the memory resource
811 pub memory: Handle,
812 /// TODO: `Strides` should become `Layout`.
813 ///
814 /// The strides of the tensor
815 pub strides: Strides,
816}
817
818impl MemoryLayout {
819 /// Create a new memory layout.
820 pub fn new(handle: Handle, strides: impl Into<Strides>) -> Self {
821 MemoryLayout {
822 memory: handle,
823 strides: strides.into(),
824 }
825 }
826}
827
828/// A reason for an error.
829#[derive(Default, Clone)]
830pub struct Reason {
831 inner: ReasonInner,
832}
833
834#[cfg(std_io)]
835mod _reason_serde {
836 use super::*;
837
838 use alloc::string::ToString;
839 use serde::{Deserialize, Deserializer, Serialize, Serializer};
840
841 impl Serialize for Reason {
842 fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
843 where
844 S: Serializer,
845 {
846 // Use the Display implementation (via to_string) to flatten the enum
847 serializer.serialize_str(&self.to_string())
848 }
849 }
850
851 impl<'de> Deserialize<'de> for Reason {
852 fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
853 where
854 D: Deserializer<'de>,
855 {
856 // Deserialize into a standard String first
857 let s = String::deserialize(deserializer)?;
858
859 // Wrap it in the Dynamic variant since we can't safely
860 // reconstruct a 'static str from a runtime string.
861 Ok(Reason {
862 inner: ReasonInner::Dynamic(Arc::new(s)),
863 })
864 }
865 }
866}
867
868#[derive(Default, Clone)]
869enum ReasonInner {
870 Static(&'static str),
871 Dynamic(Arc<String>),
872 #[default]
873 NotProvided,
874}
875
876impl core::fmt::Display for Reason {
877 fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result {
878 match &self.inner {
879 ReasonInner::Static(content) => f.write_str(content),
880 ReasonInner::Dynamic(content) => f.write_str(content),
881 ReasonInner::NotProvided => f.write_str("No reason provided for the error"),
882 }
883 }
884}
885
886impl core::fmt::Debug for Reason {
887 fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result {
888 core::fmt::Display::fmt(&self, f)
889 }
890}
891
892impl From<&'static str> for Reason {
893 fn from(value: &'static str) -> Self {
894 Self {
895 inner: ReasonInner::Static(value),
896 }
897 }
898}
899
900impl From<String> for Reason {
901 fn from(value: String) -> Self {
902 Self {
903 inner: ReasonInner::Dynamic(Arc::new(value)),
904 }
905 }
906}
907
908/// Error returned from `create`/`read`/`write` functions. Due to async execution not all errors
909/// are able to be caught, so some IO errors will still panic.
910#[derive(Error, Clone)]
911#[cfg_attr(std_io, derive(serde::Serialize, serde::Deserialize))]
912pub enum IoError {
913 /// Buffer size exceeds the max available
914 #[error("can't allocate buffer of size: {size}\n{backtrace}")]
915 BufferTooBig {
916 /// The size of the buffer in bytes.
917 size: u64,
918 /// The captured backtrace.
919 #[cfg_attr(std_io, serde(skip))]
920 backtrace: BackTrace,
921 },
922
923 /// Strides aren't supported for this copy operation on this runtime
924 #[error("the provided strides are not supported for this operation\n{backtrace}")]
925 UnsupportedStrides {
926 /// The backtrace.
927 #[cfg_attr(std_io, serde(skip))]
928 backtrace: BackTrace,
929 },
930
931 /// Memory wasn't found in the memory pool
932 #[error("couldn't find resource for that handle: {reason}\n{backtrace}")]
933 NotFound {
934 /// The backtrace.
935 #[cfg_attr(std_io, serde(skip))]
936 backtrace: BackTrace,
937 /// The reason the handle is invalid.
938 reason: Reason,
939 },
940
941 /// Handle wasn't found in the memory pool
942 #[error("couldn't free the handle, since it is currently in used. \n{backtrace}")]
943 FreeError {
944 /// The backtrace.
945 #[cfg_attr(std_io, serde(skip))]
946 backtrace: BackTrace,
947 },
948
949 /// Unknown error happened during execution
950 #[error("Unknown error happened during execution\n{backtrace}")]
951 Unknown {
952 /// Details of the error
953 description: String,
954 /// The backtrace.
955 #[cfg_attr(std_io, serde(skip))]
956 backtrace: BackTrace,
957 },
958
959 /// The current IO operation is not supported
960 #[error("The current IO operation is not supported\n{backtrace}")]
961 UnsupportedIoOperation {
962 /// The backtrace.
963 #[cfg_attr(std_io, serde(skip))]
964 backtrace: BackTrace,
965 },
966
967 /// Can't perform the IO operation because of a runtime error.
968 #[error("Can't perform the IO operation because of a runtime error: {0}")]
969 Execution(#[from] Box<ServerError>),
970}
971
972impl core::fmt::Debug for IoError {
973 fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result {
974 f.write_fmt(format_args!("{self}"))
975 }
976}
977
978/// Arguments to execute a kernel.
979#[derive(Debug, Default)]
980pub struct KernelArguments {
981 /// Buffer bindings
982 pub buffers: Vec<Binding>,
983 /// Packed scalars and metadata. First scalars sorted by type, then static metadata,
984 /// then dynamic metadata.
985 pub info: MetadataBindingInfo,
986 /// Tensor map bindings
987 pub tensor_maps: Vec<TensorMapBinding>,
988}
989
990impl core::fmt::Display for KernelArguments {
991 fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result {
992 f.write_str("KernelArguments")?;
993 for b in self.buffers.iter() {
994 f.write_fmt(format_args!("\n - buffer: {b:?}\n"))?;
995 }
996
997 Ok(())
998 }
999}
1000
1001impl KernelArguments {
1002 /// Create a new bindings struct
1003 pub fn new() -> Self {
1004 Self::default()
1005 }
1006
1007 /// Add a buffer binding
1008 pub fn with_buffer(mut self, binding: Binding) -> Self {
1009 self.buffers.push(binding);
1010 self
1011 }
1012
1013 /// Extend the buffers with `bindings`
1014 pub fn with_buffers(mut self, bindings: Vec<Binding>) -> Self {
1015 self.buffers.extend(bindings);
1016 self
1017 }
1018
1019 /// Set the info to `info`
1020 pub fn with_info(mut self, info: MetadataBindingInfo) -> Self {
1021 self.info = info;
1022 self
1023 }
1024
1025 /// Extend the tensor maps with `bindings`
1026 pub fn with_tensor_maps(mut self, bindings: Vec<TensorMapBinding>) -> Self {
1027 self.tensor_maps.extend(bindings);
1028 self
1029 }
1030}
1031
1032/// Binding of a set of scalars of the same type to execute a kernel.
1033///
1034/// The [`ComputeServer`] is responsible to convert those info into actual [`Binding`] when launching
1035/// kernels.
1036#[derive(new, Debug, Default)]
1037pub struct MetadataBindingInfo {
1038 /// Scalar and metadata values
1039 pub data: Vec<u64>,
1040 /// Start of the dynamically sized portion of the metadata, relative to the entire info buffer
1041 pub dynamic_metadata_offset: usize,
1042}
1043
1044impl MetadataBindingInfo {
1045 /// Create a new binding info for custom data, for externally compiled kernels.
1046 pub fn custom(data: Vec<u64>) -> Self {
1047 Self::new(data, 0)
1048 }
1049}
1050
1051/// A binding with shape and stride info for non-contiguous reading
1052#[derive(new, Debug)]
1053pub struct CopyDescriptor {
1054 /// Binding for the memory resource
1055 pub handle: Binding,
1056 /// Shape of the resource
1057 pub shape: Shape,
1058 /// Strides of the resource
1059 pub strides: Strides,
1060 /// Size of each element in the resource
1061 pub elem_size: usize,
1062}
1063
1064/// A tensor map used with TMA ops
1065#[derive(new, Debug)]
1066pub struct TensorMapBinding {
1067 /// The binding for the backing tensor
1068 pub binding: Binding,
1069 /// The tensormap metadata
1070 pub map: TensorMapMeta,
1071}
1072
1073/// `TensorMap` metadata for the opaque proxy used in TMA copies
1074#[derive(Debug, Clone)]
1075pub struct TensorMapMeta {
1076 /// Tensormap format (tiled or im2col)
1077 pub format: TensorMapFormat,
1078 /// Metadata of the backing tensor
1079 pub metadata: Metadata,
1080 /// Element stride, usually 1 but may be 2 for complex tensors
1081 /// For im2col, this is equivalent to the kernel stride
1082 pub elem_stride: Strides,
1083 /// Interleave mode
1084 pub interleave: TensorMapInterleave,
1085 /// Swizzle mode
1086 pub swizzle: TensorMapSwizzle,
1087 /// Prefetch settings
1088 pub prefetch: TensorMapPrefetch,
1089 /// OOB fill value
1090 pub oob_fill: OobFill,
1091 /// Storage type
1092 pub storage_ty: StorageType,
1093}
1094
1095/// Specifieds the number of cubes to be dispatched for a kernel.
1096///
1097/// This translates to eg. a grid for CUDA, or to `num_workgroups` for wgsl.
1098#[allow(clippy::large_enum_variant)]
1099pub enum CubeCount {
1100 /// Dispatch a known count of x, y, z cubes.
1101 Static(u32, u32, u32),
1102 /// Dispatch an amount based on the values in this buffer. The buffer should contain a u32 array [x, y, z].
1103 Dynamic(Binding),
1104}
1105
1106/// Defines how to select cube count based on the number of cubes required.
1107pub enum CubeCountSelection {
1108 /// If the number of cubes is the same as required.
1109 Exact(CubeCount),
1110 /// If the number of cubes isn't the same as required.
1111 ///
1112 /// This can happen based on the hardware limit, requiring the kernel to perform OOB checks.
1113 Approx(CubeCount, u32),
1114}
1115
1116impl CubeCountSelection {
1117 /// Creates a [`CubeCount`] while respecting the hardware limits.
1118 pub fn new<R: Runtime>(client: &ComputeClient<R>, num_cubes: u32) -> Self {
1119 let cube_count = cube_count_spread(&client.properties().hardware.max_cube_count, num_cubes);
1120
1121 let num_cubes_actual = cube_count[0] * cube_count[1] * cube_count[2];
1122 let cube_count = CubeCount::Static(cube_count[0], cube_count[1], cube_count[2]);
1123
1124 match num_cubes_actual == num_cubes {
1125 true => CubeCountSelection::Exact(cube_count),
1126 false => CubeCountSelection::Approx(cube_count, num_cubes_actual),
1127 }
1128 }
1129
1130 /// If some cubes will be idle.
1131 pub fn has_idle(&self) -> bool {
1132 matches!(self, Self::Approx(..))
1133 }
1134
1135 /// Converts into [`CubeCount`].
1136 pub fn cube_count(self) -> CubeCount {
1137 match self {
1138 CubeCountSelection::Exact(cube_count) => cube_count,
1139 CubeCountSelection::Approx(cube_count, _) => cube_count,
1140 }
1141 }
1142}
1143
1144impl From<CubeCountSelection> for CubeCount {
1145 fn from(value: CubeCountSelection) -> Self {
1146 value.cube_count()
1147 }
1148}
1149
1150impl CubeCount {
1151 /// Create a new static cube count with the given x = y = z = 1.
1152 pub fn new_single() -> Self {
1153 CubeCount::Static(1, 1, 1)
1154 }
1155
1156 /// Create a new static cube count with the given x, and y = z = 1.
1157 pub fn new_1d(x: u32) -> Self {
1158 CubeCount::Static(x, 1, 1)
1159 }
1160
1161 /// Create a new static cube count with the given x and y, and z = 1.
1162 pub fn new_2d(x: u32, y: u32) -> Self {
1163 CubeCount::Static(x, y, 1)
1164 }
1165
1166 /// Create a new static cube count with the given x, y and z.
1167 pub fn new_3d(x: u32, y: u32, z: u32) -> Self {
1168 CubeCount::Static(x, y, z)
1169 }
1170
1171 /// Checks whether the cube count is definitely empty, i.e. has 0 dispatches.
1172 pub fn is_empty(&self) -> bool {
1173 match self {
1174 Self::Static(x, y, z) => *x == 0 || *y == 0 || *z == 0,
1175 Self::Dynamic(_) => false,
1176 }
1177 }
1178}
1179
1180impl Debug for CubeCount {
1181 fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result {
1182 match self {
1183 CubeCount::Static(x, y, z) => f.write_fmt(format_args!("({x}, {y}, {z})")),
1184 CubeCount::Dynamic(_) => f.write_str("binding"),
1185 }
1186 }
1187}
1188
1189impl Clone for CubeCount {
1190 fn clone(&self) -> Self {
1191 match self {
1192 Self::Static(x, y, z) => Self::Static(*x, *y, *z),
1193 Self::Dynamic(binding) => Self::Dynamic(binding.clone()),
1194 }
1195 }
1196}
1197
1198#[derive(Debug, PartialEq, Eq, Clone, Copy, Hash)]
1199#[cfg_attr(std_io, derive(serde::Serialize, serde::Deserialize))]
1200#[allow(missing_docs)]
1201/// The number of units across all 3 axis totalling to the number of working units in a cube.
1202pub struct CubeDim {
1203 /// The number of units in the x axis.
1204 pub x: u32,
1205 /// The number of units in the y axis.
1206 pub y: u32,
1207 /// The number of units in the z axis.
1208 pub z: u32,
1209}
1210
1211impl CubeDim {
1212 /// Creates a new [`CubeDim`] based on the maximum number of tasks that can be parellalized by units, in other words,
1213 /// by the maximum number of working units.
1214 ///
1215 /// # Notes
1216 ///
1217 /// For complex problems, you probably want to have your own logic function to create the
1218 /// [`CubeDim`], but for simpler problems such as elemwise-operation, this is a great default.
1219 pub fn new<R: Runtime>(client: &ComputeClient<R>, working_units: usize) -> Self {
1220 let properties = client.properties();
1221 let plane_size = properties.hardware.plane_size_max;
1222 let plane_count = Self::calculate_plane_count_per_cube(
1223 working_units as u32,
1224 plane_size,
1225 properties.hardware.num_cpu_cores,
1226 );
1227
1228 // Make sure it respects the max units per cube (especially on wasm)
1229 let limit = properties.hardware.max_units_per_cube / plane_size;
1230
1231 // Ensure at least 1 plane so CubeDim is always valid (num_elems() > 0).
1232 Self::new_2d(plane_size, u32::min(limit, plane_count).max(1))
1233 }
1234
1235 fn calculate_plane_count_per_cube(
1236 working_units: u32,
1237 plane_dim: u32,
1238 num_cpu_cores: Option<u32>,
1239 ) -> u32 {
1240 match num_cpu_cores {
1241 Some(num_cores) => core::cmp::min(num_cores, working_units),
1242 None => {
1243 let plane_count_max = core::cmp::max(1, working_units / plane_dim);
1244
1245 // Ensures `plane_count` is a power of 2.
1246 const NUM_PLANE_MAX: u32 = 8u32;
1247 const NUM_PLANE_MAX_LOG2: u32 = NUM_PLANE_MAX.ilog2();
1248 let plane_count_max_log2 =
1249 core::cmp::min(NUM_PLANE_MAX_LOG2, u32::ilog2(plane_count_max));
1250 2u32.pow(plane_count_max_log2)
1251 }
1252 }
1253 }
1254
1255 /// Create a new cube dim with x = y = z = 1.
1256 pub const fn new_single() -> Self {
1257 Self { x: 1, y: 1, z: 1 }
1258 }
1259
1260 /// Create a new cube dim with the given x, and y = z = 1.
1261 pub const fn new_1d(x: u32) -> Self {
1262 Self { x, y: 1, z: 1 }
1263 }
1264
1265 /// Create a new cube dim with the given x and y, and z = 1.
1266 pub const fn new_2d(x: u32, y: u32) -> Self {
1267 Self { x, y, z: 1 }
1268 }
1269
1270 /// Create a new cube dim with the given x, y and z.
1271 /// This is equivalent to the [new](CubeDim::new) function.
1272 pub const fn new_3d(x: u32, y: u32, z: u32) -> Self {
1273 Self { x, y, z }
1274 }
1275
1276 /// Total numbers of units per cube
1277 pub const fn num_elems(&self) -> u32 {
1278 self.x * self.y * self.z
1279 }
1280
1281 /// Whether this `CubeDim` can fully contain `other`
1282 pub const fn can_contain(&self, other: CubeDim) -> bool {
1283 self.x >= other.x && self.y >= other.y && self.z >= other.z
1284 }
1285}
1286
1287impl From<(u32, u32, u32)> for CubeDim {
1288 fn from(value: (u32, u32, u32)) -> Self {
1289 CubeDim::new_3d(value.0, value.1, value.2)
1290 }
1291}
1292
1293impl From<CubeDim> for (u32, u32, u32) {
1294 fn from(val: CubeDim) -> Self {
1295 (val.x, val.y, val.z)
1296 }
1297}
1298
1299/// The kind of execution to be performed.
1300#[derive(Default, Hash, PartialEq, Eq, Clone, Debug, Copy)]
1301#[cfg_attr(std_io, derive(serde::Serialize, serde::Deserialize))]
1302pub enum ExecutionMode {
1303 /// Checked kernels are safe.
1304 #[default]
1305 Checked,
1306 /// Validate OOB and alert if OOB access occurs
1307 Validate,
1308 /// Unchecked kernels are unsafe.
1309 Unchecked,
1310}
1311
1312fn cube_count_spread(max: &(u32, u32, u32), num_cubes: u32) -> [u32; 3] {
1313 let max_cube_counts = [max.0, max.1, max.2];
1314 let mut num_cubes = [num_cubes, 1, 1];
1315 let base = 2;
1316
1317 let mut reduce_count = |i: usize| {
1318 if num_cubes[i] <= max_cube_counts[i] {
1319 return true;
1320 }
1321
1322 loop {
1323 num_cubes[i] = num_cubes[i].div_ceil(base);
1324 num_cubes[i + 1] *= base;
1325
1326 if num_cubes[i] <= max_cube_counts[i] {
1327 return false;
1328 }
1329 }
1330 };
1331
1332 for i in 0..2 {
1333 if reduce_count(i) {
1334 break;
1335 }
1336 }
1337
1338 num_cubes
1339}
1340
1341#[cfg(test)]
1342mod tests {
1343 use super::*;
1344
1345 #[test_log::test]
1346 fn safe_num_cubes_even() {
1347 let max = (32, 32, 32);
1348 let required = 2048;
1349
1350 let actual = cube_count_spread(&max, required);
1351 let expected = [32, 32, 2];
1352 assert_eq!(actual, expected);
1353 }
1354
1355 #[test_log::test]
1356 fn safe_num_cubes_odd() {
1357 let max = (48, 32, 16);
1358 let required = 3177;
1359
1360 let actual = cube_count_spread(&max, required);
1361 let expected = [25, 32, 4];
1362 assert_eq!(actual, expected);
1363 }
1364}
1365
1366/// What one captured launch was recorded with.
1367///
1368/// `info` is cubecl's packed argument blob -- scalars first, sorted by type,
1369/// then static metadata, then dynamic metadata -- exactly as the launch
1370/// presented it. `ptrs` are the device addresses the launch bound, in binding
1371/// order. Both are the values baked into the graph node, which is why reading
1372/// them back is the only sound way to locate a value inside them.
1373#[derive(Debug, Clone, PartialEq, Eq)]
1374pub struct GraphLaunchParams {
1375 /// The cube count the launch was recorded with.
1376 pub grid: (u32, u32, u32),
1377 /// The cube dim the launch was recorded with.
1378 pub block: (u32, u32, u32),
1379 /// The packed scalar and metadata blob, as captured.
1380 pub info: Vec<u64>,
1381 /// Whether `info` reached the kernel as a by-value grid constant. When it
1382 /// did not, it lives in a device buffer instead and no parameter rewrite
1383 /// can move it.
1384 pub info_is_grid_constant: bool,
1385 /// The device address of every buffer the launch bound, in binding order.
1386 pub ptrs: Vec<u64>,
1387 /// Where the DYNAMIC half of `info` starts.
1388 ///
1389 /// `info[..dyn_offset]` is the scalars and the static metadata and reaches
1390 /// the kernel BY VALUE, so a parameter rewrite moves it.
1391 /// `info[dyn_offset..]` is every bound tensor's shape and stride list; it
1392 /// is variable-length, so it cannot be a fixed-size kernel parameter and
1393 /// is uploaded by a memcpy node instead. A word that moved between two
1394 /// captures is patchable or staged according to which side of this it
1395 /// falls on, and reporting the two as one number says neither.
1396 pub dyn_offset: usize,
1397 /// Whether the launch has a pinned staging buffer the graph owns, i.e.
1398 /// whether its dynamic half can be rewritten at all.
1399 pub has_staging: bool,
1400 /// The kernel, named. Populated only for launches recorded by a capture.
1401 pub name: String,
1402}
1403
1404/// What to change about one captured launch. Fields left `None`/empty keep the
1405/// captured value.
1406#[derive(Debug, Clone, Default, PartialEq, Eq)]
1407pub struct GraphLaunchPatch {
1408 /// A new cube count. This is what moves a launch whose GEOMETRY depends on
1409 /// a per-step quantity, such as an attention split count that grows with
1410 /// the context.
1411 pub grid: Option<(u32, u32, u32)>,
1412 /// A new packed argument blob. Must be the same length as the captured
1413 /// one: the kernel's parameter size is fixed at compile time and a shorter
1414 /// blob would have the driver read past it.
1415 pub info: Option<Vec<u64>>,
1416 /// `(binding index, new device address)` for buffers that must be
1417 /// re-pointed. Empty is the normal case and the one the design is for --
1418 /// a region whose buffers are pinned patches scalars only.
1419 pub ptrs: Vec<(usize, u64)>,
1420}