Skip to main content

cubecl_runtime/server/
base.rs

1use super::Handle;
2use crate::{
3    client::ComputeClient,
4    compiler::CompilationError,
5    config::{CubeClRuntimeConfig, RuntimeConfig, compilation::BoundsCheckMode},
6    kernel::KernelMetadata,
7    logging::ServerLogger,
8    memory_management::{ArenaStats, ManagedMemoryHandle, MemoryAllocationMode, MemoryUsage},
9    runtime::Runtime,
10    server::Binding,
11    storage::{ComputeStorage, ManagedResource},
12    tma::{OobFill, TensorMapFormat, TensorMapInterleave, TensorMapPrefetch, TensorMapSwizzle},
13};
14use ahash::AHasher;
15use alloc::boxed::Box;
16#[cfg(feature = "profile-tracy")]
17use alloc::format;
18use alloc::string::String;
19use alloc::sync::Arc;
20use alloc::vec::Vec;
21use core::{
22    fmt::Debug,
23    hash::{Hash, Hasher},
24};
25use cubecl_common::{
26    backtrace::BackTrace,
27    bytes::Bytes,
28    device::{self, DeviceId},
29    future::DynFut,
30    profile::ProfileDuration,
31    stream_id::StreamId,
32    stub::RwLock,
33};
34use cubecl_ir::{DeviceProperties, ElemType, StorageType};
35use cubecl_zspace::{Shape, Strides, metadata::Metadata};
36use hashbrown::HashSet;
37use thiserror::Error;
38
39#[derive(Error, Clone)]
40#[cfg_attr(std_io, derive(serde::Serialize, serde::Deserialize))]
41/// An error during profiling.
42pub enum ProfileError {
43    /// An unknown error happened during profiling
44    #[error(
45        "An unknown error happened during profiling\nCaused by:\n  {reason}\nBacktrace:\n{backtrace}"
46    )]
47    Unknown {
48        /// The caused of the error
49        reason: String,
50        /// The captured backtrace.
51        #[cfg_attr(std_io, serde(skip))]
52        backtrace: BackTrace,
53    },
54
55    /// No profiling was registered
56    #[error("No profiling registered\nBacktrace:\n{backtrace}")]
57    NotRegistered {
58        /// The captured backtrace.
59        #[cfg_attr(std_io, serde(skip))]
60        backtrace: BackTrace,
61    },
62
63    /// A launch error happened during profiling
64    #[error("A launch error happened during profiling\nCaused by:\n  {0}")]
65    Launch(#[from] LaunchError),
66
67    /// An execution error happened during profiling
68    #[error("An execution error happened during profiling\nCaused by:\n  {0}")]
69    Server(#[from] Box<ServerError>),
70}
71
72impl core::fmt::Debug for ProfileError {
73    fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result {
74        f.write_fmt(format_args!("{self}"))
75    }
76}
77
78/// Contains many different types that are useful for server implementations and compute clients.
79pub struct ServerUtilities<Server: ComputeServer> {
80    /// The time when `profile-tracy` is activated.
81    #[cfg(feature = "profile-tracy")]
82    pub epoch_time: web_time::Instant,
83    /// The GPU client when `profile-tracy` is activated.
84    #[cfg(feature = "profile-tracy")]
85    pub gpu_client: tracy_client::GpuContext,
86    /// Information shared between all servers.
87    pub properties: DeviceProperties,
88    /// Stable hash of the device properties
89    pub properties_hash: u64,
90    /// Information specific to the current server.
91    pub info: Server::Info,
92    /// The logger based on global cubecl configs.
93    pub logger: Arc<ServerLogger>,
94    /// How to create the allocation.
95    pub layout_policy: Server::MemoryLayoutPolicy,
96    /// How to enforce bounds checking on kernels.
97    pub check_mode: BoundsCheckMode,
98    /// A set containing the ids for which the inter-device communication has already been initialized.
99    pub initialized_comms: RwLock<HashSet<CommunicationId>>,
100}
101
102/// Defines how the memory layout is determined.
103pub trait MemoryLayoutPolicy: Send + Sync + 'static {
104    /// Applies the memory layout policy to a list of descriptors.
105    ///
106    /// Returns a vector of `MemoryLayout`, one per descriptor, with layouts that share a
107    /// single `Binding`.
108    fn apply(
109        &self,
110        stream_id: StreamId,
111        descriptors: &[MemoryLayoutDescriptor],
112    ) -> (Handle, Vec<MemoryLayout>);
113}
114
115impl<Server: core::fmt::Debug> core::fmt::Debug for ServerUtilities<Server>
116where
117    Server: ComputeServer,
118    Server::Info: core::fmt::Debug,
119{
120    fn fmt(&self, f: &mut core::fmt::Formatter) -> core::fmt::Result {
121        f.debug_struct("ServerUtilities")
122            .field("properties", &self.properties)
123            .field("info", &self.info)
124            .field("logger", &self.logger)
125            .finish()
126    }
127}
128
129impl<S: ComputeServer> ServerUtilities<S> {
130    /// Creates a new server utilities.
131    pub fn new(
132        properties: DeviceProperties,
133        logger: Arc<ServerLogger>,
134        info: S::Info,
135        allocator: S::MemoryLayoutPolicy,
136    ) -> Self {
137        // Start a tracy client if needed.
138        #[cfg(feature = "profile-tracy")]
139        let client = tracy_client::Client::start();
140
141        Self {
142            properties_hash: properties.checksum(),
143            properties,
144            logger,
145            // Create the GPU client if needed.
146            #[cfg(feature = "profile-tracy")]
147            gpu_client: client
148                .clone()
149                .new_gpu_context(
150                    Some(&format!("{info:?}")),
151                    // In the future should ask the server what makes sense here. 'Invalid' atm is a generic stand-in (Tracy doesn't have CUDA/RocM atm anyway).
152                    tracy_client::GpuContextType::Invalid,
153                    0,   // Timestamps are manually aligned to this epoch so start at 0.
154                    1.0, // Timestamps are manually converted to be nanoseconds so period is 1.
155                )
156                .unwrap(),
157            #[cfg(feature = "profile-tracy")]
158            epoch_time: web_time::Instant::now(),
159            info,
160            layout_policy: allocator,
161            check_mode: CubeClRuntimeConfig::get().compilation.check_mode,
162            initialized_comms: RwLock::new(HashSet::default()),
163        }
164    }
165}
166
167/// Kernel Launch Errors.
168#[derive(Error, Clone)]
169#[cfg_attr(std_io, derive(serde::Serialize, serde::Deserialize))]
170pub enum LaunchError {
171    /// The given kernel can't be compiled.
172    #[error("A compilation error happened during launch\nCaused by:\n  {0}")]
173    CompilationError(#[from] CompilationError),
174
175    /// The server is out of memory.
176    #[error(
177        "An out-of-memory error happened during launch\nCaused by:\n  {reason}\nBacktrace\n{backtrace}"
178    )]
179    OutOfMemory {
180        /// The caused of the memory error.
181        reason: String,
182        /// The backtrace for this error.
183        #[cfg_attr(std_io, serde(skip))]
184        backtrace: BackTrace,
185    },
186
187    /// Too many resources were requested
188    #[error("Too many resources were requested during launch\n{0}")]
189    TooManyResources(#[from] ResourceLimitError),
190
191    /// Unknown launch error.
192    #[error(
193        "An unknown error happened during launch\nCaused by:\n  {reason}\nBacktrace\n{backtrace}"
194    )]
195    Unknown {
196        /// The caused of the unknown error.
197        reason: String,
198        /// The backtrace for this error.
199        #[cfg_attr(std_io, serde(skip))]
200        backtrace: BackTrace,
201    },
202
203    /// Can't launch because of an IO Error.
204    #[error("An io error happened during launch\nCaused by:\n  {0}")]
205    IoError(#[from] IoError),
206}
207
208/// Resource limit errors.
209#[derive(Error, Clone)]
210#[cfg_attr(std_io, derive(serde::Serialize, serde::Deserialize))]
211pub enum ResourceLimitError {
212    /// Shared memory exceeds maximum
213    #[error(
214        "Too much shared memory requested.\nRequested {requested} bytes, maximum {max} bytes available.\nBacktrace\n{backtrace}"
215    )]
216    SharedMemory {
217        /// Value requested
218        requested: usize,
219        /// Maximum value
220        max: usize,
221        /// The backtrace for this error.
222        #[cfg_attr(std_io, serde(skip))]
223        backtrace: BackTrace,
224    },
225    /// Total units exceeds maximum
226    #[error(
227        "Total unit count exceeds maximum.\nRequested {requested} units, max units is {max}.\nBacktrace\n{backtrace}"
228    )]
229    Units {
230        /// Requested value
231        requested: u32,
232        /// Maximum value
233        max: u32,
234        /// The backtrace for this error.
235        #[cfg_attr(std_io, serde(skip))]
236        backtrace: BackTrace,
237    },
238    /// `CubeDim` exceeds maximum
239    #[error(
240        "Cube dim exceeds maximum bounds.\nRequested {requested:?}, max is {max:?}.\nBacktrace\n{backtrace}"
241    )]
242    CubeDim {
243        /// Requested value
244        requested: (u32, u32, u32),
245        /// Maximum value
246        max: (u32, u32, u32),
247        /// The backtrace for this error.
248        #[cfg_attr(std_io, serde(skip))]
249        backtrace: BackTrace,
250    },
251}
252
253impl core::fmt::Debug for LaunchError {
254    fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result {
255        f.write_fmt(format_args!("{self}"))
256    }
257}
258
259impl core::fmt::Debug for ResourceLimitError {
260    fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result {
261        f.write_fmt(format_args!("{self}"))
262    }
263}
264
265/// Error that can happen asynchronously while executing registered kernels.
266#[derive(Error, Debug, Clone)]
267#[cfg_attr(std_io, derive(serde::Serialize, serde::Deserialize))]
268pub enum ServerError {
269    /// A generic runtime error.
270    #[error("An error happened during execution\nCaused by:\n  {reason}\nBacktrace:\n{backtrace}")]
271    Generic {
272        /// The details of the generic error.
273        reason: String,
274        /// The backtrace for this error.
275        #[cfg_attr(std_io, serde(skip))]
276        backtrace: BackTrace,
277    },
278
279    /// A launch error happened during profiling
280    #[error("A launch error happened during profiling\nCaused by:\n  {0}")]
281    Launch(#[from] LaunchError),
282
283    /// An execution error happened during profiling
284    #[error("An execution error happened during profiling\nCaused by:\n  {0}")]
285    Profile(#[from] ProfileError),
286
287    /// An execution error happened during profiling
288    #[error("An execution error happened during profiling\nCaused by:\n  {0}")]
289    Io(#[from] IoError),
290
291    /// The server is an invalid state.
292    #[error("The server is in an invalid state\nCaused by:\n  {errors:?}")]
293    ServerUnhealthy {
294        /// The details of the generic error.
295        errors: Vec<Self>,
296        /// The backtrace for this error.
297        #[cfg_attr(std_io, serde(skip))]
298        backtrace: BackTrace,
299    },
300}
301
302/// How errors are handled in a stream when executing a task.
303#[derive(Clone, Copy)]
304pub struct StreamErrorMode {
305    /// Whether the task still executes even if the stream is in error.
306    pub ignore: bool,
307    /// Whether the errors are flushed by the current task.
308    pub flush: bool,
309}
310
311/// The compute server is responsible for handling resources and computations over resources.
312///
313/// Everything in the server is mutable, therefore it should be solely accessed through the
314/// [`ComputeClient`] for thread safety.
315pub trait ComputeServer:
316    Send + core::fmt::Debug + ServerCommunication + device::DeviceService + 'static
317where
318    Self: Sized,
319{
320    /// The kernel type defines the computation algorithms.
321    type Kernel: KernelMetadata;
322    /// Information that can be retrieved for the runtime.
323    type Info: Debug + Send + Sync;
324    /// Manages how allocations are performed for a server.
325    type MemoryLayoutPolicy: MemoryLayoutPolicy;
326    /// The [storage](ComputeStorage) type defines how data is stored and accessed.
327    type Storage: ComputeStorage;
328
329    /// Initializes [memory](ManagedMemoryHandle) on the given [stream](StreamId) with the given size.
330    fn initialize_memory(&mut self, memory: ManagedMemoryHandle, size: u64, stream_id: StreamId);
331
332    /// Whether this backend can capture a region of enqueued work into a
333    /// replayable object at all. Callers must check this before capturing:
334    /// the capture methods below panic on a backend that cannot.
335    fn graph_capture_supported(&self) -> bool {
336        false
337    }
338
339    /// Defer the stream's pending-drop flush without opening a capture.
340    ///
341    /// This is the PRE-WARM for a capture, and it is needed because a capture
342    /// changes what the allocator has to hold at once. Normally a staging
343    /// buffer is handed back a few launches after it is used, so a region of
344    /// N launches only ever needs a handful of pages live. Inside a capture
345    /// nothing is handed back -- the flush waits on a fence, which a capture
346    /// cannot contain, and the buffers must stay alive anyway because the
347    /// graph re-reads them on every replay. So the region's requirement jumps
348    /// from its per-launch working set to its SIMULTANEOUS HIGH-WATER MARK,
349    /// and the pool discovers that by allocating, which is the one thing that
350    /// breaks a capture.
351    ///
352    /// Running the region once with frees deferred and no capture open lets
353    /// the pool reach that high-water mark the legal way. Flushing afterwards
354    /// returns the slices but keeps the pages, so the capture that follows
355    /// finds every page it needs already there.
356    fn graph_defer_frees(&mut self, _defer: bool, _stream_id: StreamId) {
357        panic!("graph capture is not supported by this backend");
358    }
359
360    /// Route every allocation on `stream_id` into the CAPTURE ARENA, and
361    /// return the generation this opening is.
362    ///
363    /// Call this once to WARM (arena open, no capture: the region runs eagerly
364    /// and the arena learns exactly which slices it needs), then again around
365    /// the capture itself. See
366    /// [`MemoryManagement::arena_begin`](crate::memory_management::MemoryManagement::arena_begin)
367    /// for why an arena and not a hold.
368    ///
369    /// Reopening the arena and capturing again DELIBERATELY hands the second
370    /// graph the same slices: that is what makes two captures of one region
371    /// comparable, and what lets one graph stand in for another step. The two
372    /// then share scratch, which is safe exactly when they are captures of the
373    /// same region -- checked by the request sequence each capture recorded --
374    /// and replayed serially, which the single stream gives for free.
375    fn graph_arena_begin(&mut self, _stream_id: StreamId) -> u64 {
376        panic!("graph capture is not supported by this backend");
377    }
378
379    /// Stop routing allocations into the arena. The arena keeps every page it
380    /// owns -- closing it ends allocation from it, never its reservation.
381    fn graph_arena_end(&mut self, _stream_id: StreamId) {
382        panic!("graph capture is not supported by this backend");
383    }
384
385    /// What the arena holds, and how many requests it had to take from the
386    /// driver. A miss taken while a capture is open is a graph memory node.
387    fn graph_arena_stats(&mut self, _stream_id: StreamId) -> ArenaStats {
388        ArenaStats::default()
389    }
390
391    /// Zero the arena's served/miss counters so the capture pass can be
392    /// counted apart from the warm pass.
393    fn graph_arena_reset_counters(&mut self, _stream_id: StreamId) {}
394
395    /// What the allocator's RESERVE half costs, as `(calls, nanoseconds)`.
396    ///
397    /// A decode step makes roughly one reservation per kernel launch, and all
398    /// of it is host work inside the term a graph replay removes. The count has
399    /// been reported for a while; this is the time, because a count is not a
400    /// time and the two lead to opposite conclusions at 0.5 us and at 5 us a
401    /// request. Zero unless `CUBECL_TIME_RESERVE=1`. It does NOT include the
402    /// free half, which happens on handle drop.
403    fn reserve_timing(&mut self, _stream_id: StreamId) -> (u64, u64) {
404        (0, 0)
405    }
406
407    /// Zero the reservation timer.
408    fn reserve_timing_reset(&mut self, _stream_id: StreamId) {}
409
410    /// Begin capturing work enqueued on `stream_id` instead of running it.
411    /// Between this and [`ComputeServer::graph_capture_end`] nothing executes;
412    /// the enqueued work is recorded as a graph. Anything that BLOCKS THE HOST
413    /// on device progress inside the captured region (a sync, a fence wait, a
414    /// blocking read) invalidates the capture, because there is no device
415    /// progress to wait for.
416    fn graph_capture_begin(&mut self, _stream_id: StreamId) {
417        panic!("graph capture is not supported by this backend");
418    }
419
420    /// Close the capture opened by [`ComputeServer::graph_capture_begin`],
421    /// instantiate the result, and return an id naming it for replay. The
422    /// captured graph holds the EXACT device pointers seen during capture, so
423    /// every buffer it touches must live at a stable address across replays.
424    fn graph_capture_end(&mut self, _stream_id: StreamId) -> u64 {
425        panic!("graph capture is not supported by this backend");
426    }
427
428    /// Enqueue one replay of the graph named by `id` on `stream_id`.
429    fn graph_replay(&mut self, _id: u64, _stream_id: StreamId) {
430        panic!("graph capture is not supported by this backend");
431    }
432
433    /// 0 = not capturing, 1 = capturing, 2 = INVALIDATED, 3 = query failed.
434    /// A capture that has been invalidated keeps accepting work silently and
435    /// only reports it at `end`, so this is the only way to find WHICH call
436    /// broke one.
437    fn graph_capture_status(&mut self, _stream_id: StreamId) -> u32 {
438        3
439    }
440
441    /// How many nodes the captured graph holds. This is the unit the replay
442    /// cost is quoted per, so it belongs next to the graph, not in a log line.
443    fn graph_node_count(&mut self, _id: u64) -> usize {
444        panic!("graph capture is not supported by this backend");
445    }
446
447    /// How many kernel launches the OPEN capture has recorded so far.
448    ///
449    /// This is the index a caller brackets its own calls with, so that after
450    /// the capture closes it can name the nodes a particular call produced
451    /// without knowing anything about how many kernels that call fans out to.
452    /// It counts launches, not nodes: a launch is one node, but a graph also
453    /// holds memcpy and memset nodes that no launch made.
454    fn graph_capture_launch_count(&mut self) -> usize {
455        panic!("graph capture is not supported by this backend");
456    }
457
458    /// How many launches the closed graph `id` recorded.
459    fn graph_launch_count(&mut self, _id: u64) -> usize {
460        panic!("graph capture is not supported by this backend");
461    }
462
463    /// What launch `idx` of graph `id` was recorded WITH: its grid, its packed
464    /// scalar+metadata blob, and the device address of every buffer it bound.
465    ///
466    /// This is the half of patching that makes it checkable. A caller that
467    /// wants to move a scalar has to know which slot of the blob holds it, and
468    /// the honest way to learn that is to look at what the blob actually
469    /// contained when the value was known -- not to re-derive cubecl's packing
470    /// order by hand and hope.
471    fn graph_launch_params(&mut self, _id: u64, _idx: usize) -> GraphLaunchParams {
472        panic!("graph capture is not supported by this backend");
473    }
474
475    /// Rewrite launch `idx` of graph `id`'s EXECUTABLE in place.
476    ///
477    /// The graph is untouched; only the instantiated exec changes, which is
478    /// what makes this cheap enough to do per step. Everything not named in
479    /// `patch` keeps the value it was captured with.
480    fn graph_patch_launch(&mut self, _id: u64, _idx: usize, _patch: GraphLaunchPatch) {
481        panic!("graph capture is not supported by this backend");
482    }
483
484    /// A census of the graph's nodes by KIND, as `(CUgraphNodeType, count)`.
485    ///
486    /// A graph is not only its kernels. Everything else a capture recorded --
487    /// host-to-device copies, memsets, child graphs -- is a node too, and a
488    /// node no kernel-parameter rewrite can reach. Whether a region is
489    /// replayable on a later step is therefore first of all a question about
490    /// how much of it is NOT a kernel launch.
491    fn graph_node_kinds(&mut self, _id: u64) -> Vec<(u32, usize)> {
492        panic!("graph capture is not supported by this backend");
493    }
494
495    /// Rewrite several launches of one graph in a single round trip.
496    ///
497    /// Per-launch [`ComputeServer::graph_patch_launch`] is one submit each, and
498    /// on a region with a hundred moving launches the submits cost far more
499    /// than the rewrites do. The batch exists because that overhead is the
500    /// caller's to avoid, not something to measure and report as the price of
501    /// patching.
502    fn graph_patch_launches(&mut self, _id: u64, _patches: Vec<(usize, GraphLaunchPatch)>) {
503        panic!("graph capture is not supported by this backend");
504    }
505
506    /// Every `(address, bytes)` the graph ALLOCATES for itself.
507    ///
508    /// A capture that could not avoid allocating turns each allocation into a
509    /// memory node, and the memory a graph owns has a virtual address the graph
510    /// fixes -- the same one on every launch of that exec, and NOT the same one
511    /// as any other graph's. So these addresses are already pinned for as long
512    /// as one exec is being replayed, and rewriting them to another graph's is
513    /// wrong by construction rather than merely unnecessary. A caller
514    /// re-pointing a region's buffers has to be able to tell them apart from
515    /// the ones that came from the pool.
516    fn graph_alloc_regions(&mut self, _id: u64) -> Vec<(u64, u64)> {
517        panic!("graph capture is not supported by this backend");
518    }
519
520    /// Every memcpy node's `(src, dst, bytes, kind)`, sorted.
521    ///
522    /// A capture records copies as well as launches, and a copy is not a
523    /// kernel: no parameter rewrite reaches it. When the data a copy carries is
524    /// per-step, the region is not replayable on a later step no matter how
525    /// well its kernels are patched, and this is how that is checked rather
526    /// than assumed.
527    fn graph_memcpy_specs(&mut self, _id: u64) -> Vec<(u64, u64, u64, u32)> {
528        panic!("graph capture is not supported by this backend");
529    }
530
531    /// Release a captured graph. An unknown id is ignored.
532    fn graph_destroy(&mut self, _id: u64) {}
533
534    /// Register an externally-owned GPU buffer that ALIASES host memory (e.g.
535    /// mmap'd pile pages) as a tensor [`Handle`], with ZERO copy. `ptr`/`page_len`
536    /// describe a page-aligned superset region (required by Metal's
537    /// `newBufferWithBytesNoCopy:`); `offset`/`size` locate the actual tensor
538    /// bytes inside it. The caller guarantees the host memory outlives every
539    /// handle derived from it. Default impl panics — only backends that can
540    /// import host buffers (wgpu/Metal) override this.
541    fn register_external_aliased(
542        &mut self,
543        _ptr: *mut core::ffi::c_void,
544        _page_len: u64,
545        _offset: u64,
546        _size: u64,
547        _keepalive: alloc::sync::Arc<dyn core::any::Any + Send + Sync>,
548        _stream_id: StreamId,
549    ) -> Handle {
550        panic!("register_external_aliased is not supported by this backend");
551    }
552
553    /// Reserves N [Bytes] of the provided sizes to be used as staging to load data.
554    fn staging(
555        &mut self,
556        _sizes: &[usize],
557        _stream_id: StreamId,
558    ) -> Result<Vec<Bytes>, ServerError> {
559        Err(IoError::UnsupportedIoOperation {
560            backtrace: BackTrace::capture(),
561        }
562        .into())
563    }
564
565    /// Retrieve the server logger.
566    fn logger(&self) -> Arc<ServerLogger>;
567
568    /// Retrieve the server utilities.
569    fn utilities(&self) -> Arc<ServerUtilities<Self>>;
570
571    /// Given bindings, returns the owned resources as bytes.
572    fn read(
573        &mut self,
574        descriptors: Vec<CopyDescriptor>,
575        stream_id: StreamId,
576    ) -> DynFut<Result<Vec<Bytes>, ServerError>>;
577
578    /// Writes the specified bytes into the buffers given
579    fn write(&mut self, descriptors: Vec<(CopyDescriptor, Bytes)>, stream_id: StreamId);
580
581    /// Wait for the completion of every task in the server.
582    fn sync(&mut self, stream_id: StreamId) -> DynFut<Result<(), ServerError>>;
583
584    /// Given a resource handle, returns the storage resource.
585    fn get_resource(
586        &mut self,
587        binding: Binding,
588        stream_id: StreamId,
589    ) -> Result<ManagedResource<<Self::Storage as ComputeStorage>::Resource>, ServerError>;
590
591    /// Executes the `kernel` over the given memory `handles`.
592    ///
593    /// Kernels have mutable access to every resource they are given
594    /// and are responsible of determining which should be read or written.
595    ///
596    /// # Safety
597    ///
598    /// When executing with mode [`ExecutionMode::Unchecked`], out-of-bound reads and writes can happen.
599    unsafe fn launch(
600        &mut self,
601        kernel: Self::Kernel,
602        count: CubeCount,
603        bindings: KernelArguments,
604        kind: ExecutionMode,
605        stream_id: StreamId,
606    );
607
608    /// Flush all outstanding tasks in the server.
609    fn flush(&mut self, stream_id: StreamId) -> Result<(), ServerError>;
610
611    /// The current memory usage of the server.
612    fn memory_usage(&mut self, stream_id: StreamId) -> Result<MemoryUsage, ServerError>;
613
614    /// Ask the server to release memory that it can release.
615    fn memory_cleanup(&mut self, stream_id: StreamId);
616
617    /// Enable collecting timestamps.
618    fn start_profile(&mut self, stream_id: StreamId) -> Result<ProfilingToken, ServerError>;
619
620    /// Disable collecting timestamps.
621    fn end_profile(
622        &mut self,
623        stream_id: StreamId,
624        token: ProfilingToken,
625    ) -> Result<ProfileDuration, ProfileError>;
626
627    /// Update the memory mode of allocation in the server.
628    fn allocation_mode(&mut self, mode: MemoryAllocationMode, stream_id: StreamId);
629}
630
631/// An ID unique to any unordered combination of devices.
632#[derive(Clone, Debug, Hash, Eq, PartialEq)]
633pub struct CommunicationId {
634    /// The ID as a `String`.
635    pub id: u64,
636}
637
638impl From<Vec<DeviceId>> for CommunicationId {
639    fn from(mut value: Vec<DeviceId>) -> Self {
640        // Make sure that device ids are sorted so that any combination of the same devices uses the same communicator.
641        value.sort();
642        let mut hasher = AHasher::default();
643        value.hash(&mut hasher);
644        CommunicationId {
645            id: hasher.finish(),
646        }
647    }
648}
649
650/// Different reduce operations.
651pub enum ReduceOperation {
652    /// Sum.
653    Sum,
654    /// Mean.
655    Mean,
656}
657
658/// Defines functions for optimized data transfer between servers, supporting custom communication
659/// mechanisms such as peer-to-peer communication or specialized implementations.
660pub trait ServerCommunication {
661    /// Indicates whether server-to-server communication is enabled for this implementation.
662    const SERVER_COMM_ENABLED: bool;
663
664    /// Ensure that all queued collective operations have been executed.
665    ///
666    /// # Arguments
667    ///
668    /// * `stream_id` - The [`StreamId`] of the stream waiting for the sync.
669    ///
670    /// # Returns
671    ///
672    /// Returns a `Result` containing an `ServerError` if the operation fails.
673    #[allow(unused_variables)]
674    fn sync_collective(&mut self, stream_id: StreamId) -> Result<(), ServerError> {
675        todo!() // For backends other than cuda.
676    }
677
678    /// Initialize the communication between the devices in `device_ids`.
679    ///
680    /// # Arguments
681    ///
682    /// * `device_ids` - The IDs of the devices that need communication.
683    ///
684    /// # Returns
685    ///
686    /// Returns a `Result` containing an `ServerError` if the operation fails.
687    #[allow(unused_variables)]
688    fn comm_init(&mut self, device_ids: Vec<DeviceId>) -> Result<(), ServerError> {
689        unimplemented!()
690    }
691
692    /// Performs an `all_reduce` operation on the input data and writes it to the output buffer.
693    /// see <https://docs.nvidia.com/deeplearning/nccl/user-guide/docs/usage/collectives.html#allreduce>
694    ///
695    /// # Arguments
696    ///
697    /// * `src` - The data to be reduced.
698    /// * `dst` - Where to write the result.
699    /// * `dtype` - The element type of the data being reduced
700    /// * `stream_id` - The data's stream id.
701    /// * `op` - The reduce's aggregation operation e.g. mean, sum, etc.
702    /// * `device_ids` - The list of device ids from which to `all_reduce`.
703    ///
704    /// # Returns
705    ///
706    /// Returns a `Result` containing an `ServerError` if the operation fails.
707    #[allow(unused_variables)]
708    fn all_reduce(
709        &mut self,
710        src: Binding,
711        dst: Binding,
712        dtype: ElemType,
713        stream_id: StreamId,
714        op: ReduceOperation,
715        device_ids: Vec<DeviceId>,
716    ) -> Result<(), ServerError> {
717        unimplemented!()
718    }
719
720    /// Sends data from this server to a destination server.
721    ///
722    /// # Arguments
723    ///
724    /// * `desc` - A descriptor specifying the data to be sent, including shape, strides, and binding.
725    /// * `dtype` - The element type of the data being sent.
726    /// * `stream_id` - The stream ID associated with the server's operation.
727    /// * `device_id_dst` - ID of the device receiving the data.
728    ///
729    /// # Returns
730    ///
731    /// Returns a `Result` containing an `ServerError` if the operation fails.
732    #[allow(unused_variables)]
733    fn send(
734        &mut self,
735        desc: CopyDescriptor,
736        dtype: ElemType,
737        stream_id: StreamId,
738        device_id_dst: DeviceId,
739    ) -> Result<(), ServerError> {
740        unimplemented!()
741    }
742
743    /// Receive data from another server.
744    ///
745    /// # Arguments
746    ///
747    /// * `handle` - The handle in which the received data is written.
748    /// * `dtype` - The element type of the data being sent.
749    /// * `stream_id` - The stream ID associated with the server's operation.
750    /// * `device_id_src` - ID of the device sending the data.
751    ///
752    /// # Returns
753    ///
754    /// Returns a `Result` containing an `ServerError` if the operation fails.
755    #[allow(unused_variables)]
756    fn recv(
757        &mut self,
758        handle: Handle,
759        dtype: ElemType,
760        stream_id: StreamId,
761        device_id_src: DeviceId,
762    ) -> Result<(), ServerError> {
763        unimplemented!()
764    }
765}
766
767#[derive(Debug, Clone, Copy, Hash, PartialEq, Eq)]
768/// Profiling identification so that the server can support recursive and overlapping profilings.
769pub struct ProfilingToken {
770    /// The token value.
771    pub id: u64,
772}
773
774/// Type of allocation, either contiguous or optimized (row-aligned when possible)
775#[derive(Debug, Clone, Copy, Hash, PartialEq, Eq)]
776pub enum MemoryLayoutStrategy {
777    /// Contiguous layout, with no padding
778    Contiguous,
779    /// Optimized for access speed. In practice this means row-aligned with padding for runtimes
780    /// that support it.
781    Optimized,
782}
783
784/// Descriptor for a new tensor allocation
785#[derive(new, Debug, Clone)]
786pub struct MemoryLayoutDescriptor {
787    /// Strategy used to create the memory layout.
788    pub strategy: MemoryLayoutStrategy,
789    /// Shape of the tensor
790    pub shape: Shape,
791    /// Size of each element in the tensor (used for conversion of shape to bytes)
792    pub elem_size: usize,
793}
794
795impl MemoryLayoutDescriptor {
796    /// Create an optimized allocation descriptor
797    pub fn optimized(shape: Shape, elem_size: usize) -> Self {
798        MemoryLayoutDescriptor::new(MemoryLayoutStrategy::Optimized, shape, elem_size)
799    }
800
801    /// Create a contiguous allocation descriptor
802    pub fn contiguous(shape: Shape, elem_size: usize) -> Self {
803        MemoryLayoutDescriptor::new(MemoryLayoutStrategy::Contiguous, shape, elem_size)
804    }
805}
806
807/// An allocation with associated strides. Strides depend on tensor layout.
808#[derive(Debug, Clone)]
809pub struct MemoryLayout {
810    /// The handle for the memory resource
811    pub memory: Handle,
812    /// TODO: `Strides` should become `Layout`.
813    ///
814    /// The strides of the tensor
815    pub strides: Strides,
816}
817
818impl MemoryLayout {
819    /// Create a new memory layout.
820    pub fn new(handle: Handle, strides: impl Into<Strides>) -> Self {
821        MemoryLayout {
822            memory: handle,
823            strides: strides.into(),
824        }
825    }
826}
827
828/// A reason for an error.
829#[derive(Default, Clone)]
830pub struct Reason {
831    inner: ReasonInner,
832}
833
834#[cfg(std_io)]
835mod _reason_serde {
836    use super::*;
837
838    use alloc::string::ToString;
839    use serde::{Deserialize, Deserializer, Serialize, Serializer};
840
841    impl Serialize for Reason {
842        fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
843        where
844            S: Serializer,
845        {
846            // Use the Display implementation (via to_string) to flatten the enum
847            serializer.serialize_str(&self.to_string())
848        }
849    }
850
851    impl<'de> Deserialize<'de> for Reason {
852        fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
853        where
854            D: Deserializer<'de>,
855        {
856            // Deserialize into a standard String first
857            let s = String::deserialize(deserializer)?;
858
859            // Wrap it in the Dynamic variant since we can't safely
860            // reconstruct a 'static str from a runtime string.
861            Ok(Reason {
862                inner: ReasonInner::Dynamic(Arc::new(s)),
863            })
864        }
865    }
866}
867
868#[derive(Default, Clone)]
869enum ReasonInner {
870    Static(&'static str),
871    Dynamic(Arc<String>),
872    #[default]
873    NotProvided,
874}
875
876impl core::fmt::Display for Reason {
877    fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result {
878        match &self.inner {
879            ReasonInner::Static(content) => f.write_str(content),
880            ReasonInner::Dynamic(content) => f.write_str(content),
881            ReasonInner::NotProvided => f.write_str("No reason provided for the error"),
882        }
883    }
884}
885
886impl core::fmt::Debug for Reason {
887    fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result {
888        core::fmt::Display::fmt(&self, f)
889    }
890}
891
892impl From<&'static str> for Reason {
893    fn from(value: &'static str) -> Self {
894        Self {
895            inner: ReasonInner::Static(value),
896        }
897    }
898}
899
900impl From<String> for Reason {
901    fn from(value: String) -> Self {
902        Self {
903            inner: ReasonInner::Dynamic(Arc::new(value)),
904        }
905    }
906}
907
908/// Error returned from `create`/`read`/`write` functions. Due to async execution not all errors
909/// are able to be caught, so some IO errors will still panic.
910#[derive(Error, Clone)]
911#[cfg_attr(std_io, derive(serde::Serialize, serde::Deserialize))]
912pub enum IoError {
913    /// Buffer size exceeds the max available
914    #[error("can't allocate buffer of size: {size}\n{backtrace}")]
915    BufferTooBig {
916        /// The size of the buffer in bytes.
917        size: u64,
918        /// The captured backtrace.
919        #[cfg_attr(std_io, serde(skip))]
920        backtrace: BackTrace,
921    },
922
923    /// Strides aren't supported for this copy operation on this runtime
924    #[error("the provided strides are not supported for this operation\n{backtrace}")]
925    UnsupportedStrides {
926        /// The backtrace.
927        #[cfg_attr(std_io, serde(skip))]
928        backtrace: BackTrace,
929    },
930
931    /// Memory wasn't found in the memory pool
932    #[error("couldn't find resource for that handle: {reason}\n{backtrace}")]
933    NotFound {
934        /// The backtrace.
935        #[cfg_attr(std_io, serde(skip))]
936        backtrace: BackTrace,
937        /// The reason the handle is invalid.
938        reason: Reason,
939    },
940
941    /// Handle wasn't found in the memory pool
942    #[error("couldn't free the handle, since it is currently in used. \n{backtrace}")]
943    FreeError {
944        /// The backtrace.
945        #[cfg_attr(std_io, serde(skip))]
946        backtrace: BackTrace,
947    },
948
949    /// Unknown error happened during execution
950    #[error("Unknown error happened during execution\n{backtrace}")]
951    Unknown {
952        /// Details of the error
953        description: String,
954        /// The backtrace.
955        #[cfg_attr(std_io, serde(skip))]
956        backtrace: BackTrace,
957    },
958
959    /// The current IO operation is not supported
960    #[error("The current IO operation is not supported\n{backtrace}")]
961    UnsupportedIoOperation {
962        /// The backtrace.
963        #[cfg_attr(std_io, serde(skip))]
964        backtrace: BackTrace,
965    },
966
967    /// Can't perform the IO operation because of a runtime error.
968    #[error("Can't perform the IO operation because of a runtime error: {0}")]
969    Execution(#[from] Box<ServerError>),
970}
971
972impl core::fmt::Debug for IoError {
973    fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result {
974        f.write_fmt(format_args!("{self}"))
975    }
976}
977
978/// Arguments to execute a kernel.
979#[derive(Debug, Default)]
980pub struct KernelArguments {
981    /// Buffer bindings
982    pub buffers: Vec<Binding>,
983    /// Packed scalars and metadata. First scalars sorted by type, then static metadata,
984    /// then dynamic metadata.
985    pub info: MetadataBindingInfo,
986    /// Tensor map bindings
987    pub tensor_maps: Vec<TensorMapBinding>,
988}
989
990impl core::fmt::Display for KernelArguments {
991    fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result {
992        f.write_str("KernelArguments")?;
993        for b in self.buffers.iter() {
994            f.write_fmt(format_args!("\n - buffer: {b:?}\n"))?;
995        }
996
997        Ok(())
998    }
999}
1000
1001impl KernelArguments {
1002    /// Create a new bindings struct
1003    pub fn new() -> Self {
1004        Self::default()
1005    }
1006
1007    /// Add a buffer binding
1008    pub fn with_buffer(mut self, binding: Binding) -> Self {
1009        self.buffers.push(binding);
1010        self
1011    }
1012
1013    /// Extend the buffers with `bindings`
1014    pub fn with_buffers(mut self, bindings: Vec<Binding>) -> Self {
1015        self.buffers.extend(bindings);
1016        self
1017    }
1018
1019    /// Set the info to `info`
1020    pub fn with_info(mut self, info: MetadataBindingInfo) -> Self {
1021        self.info = info;
1022        self
1023    }
1024
1025    /// Extend the tensor maps with `bindings`
1026    pub fn with_tensor_maps(mut self, bindings: Vec<TensorMapBinding>) -> Self {
1027        self.tensor_maps.extend(bindings);
1028        self
1029    }
1030}
1031
1032/// Binding of a set of scalars of the same type to execute a kernel.
1033///
1034/// The [`ComputeServer`] is responsible to convert those info into actual [`Binding`] when launching
1035/// kernels.
1036#[derive(new, Debug, Default)]
1037pub struct MetadataBindingInfo {
1038    /// Scalar and metadata values
1039    pub data: Vec<u64>,
1040    /// Start of the dynamically sized portion of the metadata, relative to the entire info buffer
1041    pub dynamic_metadata_offset: usize,
1042}
1043
1044impl MetadataBindingInfo {
1045    /// Create a new binding info for custom data, for externally compiled kernels.
1046    pub fn custom(data: Vec<u64>) -> Self {
1047        Self::new(data, 0)
1048    }
1049}
1050
1051/// A binding with shape and stride info for non-contiguous reading
1052#[derive(new, Debug)]
1053pub struct CopyDescriptor {
1054    /// Binding for the memory resource
1055    pub handle: Binding,
1056    /// Shape of the resource
1057    pub shape: Shape,
1058    /// Strides of the resource
1059    pub strides: Strides,
1060    /// Size of each element in the resource
1061    pub elem_size: usize,
1062}
1063
1064/// A tensor map used with TMA ops
1065#[derive(new, Debug)]
1066pub struct TensorMapBinding {
1067    /// The binding for the backing tensor
1068    pub binding: Binding,
1069    /// The tensormap metadata
1070    pub map: TensorMapMeta,
1071}
1072
1073/// `TensorMap` metadata for the opaque proxy used in TMA copies
1074#[derive(Debug, Clone)]
1075pub struct TensorMapMeta {
1076    /// Tensormap format (tiled or im2col)
1077    pub format: TensorMapFormat,
1078    /// Metadata of the backing tensor
1079    pub metadata: Metadata,
1080    /// Element stride, usually 1 but may be 2 for complex tensors
1081    /// For im2col, this is equivalent to the kernel stride
1082    pub elem_stride: Strides,
1083    /// Interleave mode
1084    pub interleave: TensorMapInterleave,
1085    /// Swizzle mode
1086    pub swizzle: TensorMapSwizzle,
1087    /// Prefetch settings
1088    pub prefetch: TensorMapPrefetch,
1089    /// OOB fill value
1090    pub oob_fill: OobFill,
1091    /// Storage type
1092    pub storage_ty: StorageType,
1093}
1094
1095/// Specifieds the number of cubes to be dispatched for a kernel.
1096///
1097/// This translates to eg. a grid for CUDA, or to `num_workgroups` for wgsl.
1098#[allow(clippy::large_enum_variant)]
1099pub enum CubeCount {
1100    /// Dispatch a known count of x, y, z cubes.
1101    Static(u32, u32, u32),
1102    /// Dispatch an amount based on the values in this buffer. The buffer should contain a u32 array [x, y, z].
1103    Dynamic(Binding),
1104}
1105
1106/// Defines how to select cube count based on the number of cubes required.
1107pub enum CubeCountSelection {
1108    /// If the number of cubes is the same as required.
1109    Exact(CubeCount),
1110    /// If the number of cubes isn't the same as required.
1111    ///
1112    /// This can happen based on the hardware limit, requiring the kernel to perform OOB checks.
1113    Approx(CubeCount, u32),
1114}
1115
1116impl CubeCountSelection {
1117    /// Creates a [`CubeCount`] while respecting the hardware limits.
1118    pub fn new<R: Runtime>(client: &ComputeClient<R>, num_cubes: u32) -> Self {
1119        let cube_count = cube_count_spread(&client.properties().hardware.max_cube_count, num_cubes);
1120
1121        let num_cubes_actual = cube_count[0] * cube_count[1] * cube_count[2];
1122        let cube_count = CubeCount::Static(cube_count[0], cube_count[1], cube_count[2]);
1123
1124        match num_cubes_actual == num_cubes {
1125            true => CubeCountSelection::Exact(cube_count),
1126            false => CubeCountSelection::Approx(cube_count, num_cubes_actual),
1127        }
1128    }
1129
1130    /// If some cubes will be idle.
1131    pub fn has_idle(&self) -> bool {
1132        matches!(self, Self::Approx(..))
1133    }
1134
1135    /// Converts into [`CubeCount`].
1136    pub fn cube_count(self) -> CubeCount {
1137        match self {
1138            CubeCountSelection::Exact(cube_count) => cube_count,
1139            CubeCountSelection::Approx(cube_count, _) => cube_count,
1140        }
1141    }
1142}
1143
1144impl From<CubeCountSelection> for CubeCount {
1145    fn from(value: CubeCountSelection) -> Self {
1146        value.cube_count()
1147    }
1148}
1149
1150impl CubeCount {
1151    /// Create a new static cube count with the given x = y = z = 1.
1152    pub fn new_single() -> Self {
1153        CubeCount::Static(1, 1, 1)
1154    }
1155
1156    /// Create a new static cube count with the given x, and y = z = 1.
1157    pub fn new_1d(x: u32) -> Self {
1158        CubeCount::Static(x, 1, 1)
1159    }
1160
1161    /// Create a new static cube count with the given x and y, and z = 1.
1162    pub fn new_2d(x: u32, y: u32) -> Self {
1163        CubeCount::Static(x, y, 1)
1164    }
1165
1166    /// Create a new static cube count with the given x, y and z.
1167    pub fn new_3d(x: u32, y: u32, z: u32) -> Self {
1168        CubeCount::Static(x, y, z)
1169    }
1170
1171    /// Checks whether the cube count is definitely empty, i.e. has 0 dispatches.
1172    pub fn is_empty(&self) -> bool {
1173        match self {
1174            Self::Static(x, y, z) => *x == 0 || *y == 0 || *z == 0,
1175            Self::Dynamic(_) => false,
1176        }
1177    }
1178}
1179
1180impl Debug for CubeCount {
1181    fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result {
1182        match self {
1183            CubeCount::Static(x, y, z) => f.write_fmt(format_args!("({x}, {y}, {z})")),
1184            CubeCount::Dynamic(_) => f.write_str("binding"),
1185        }
1186    }
1187}
1188
1189impl Clone for CubeCount {
1190    fn clone(&self) -> Self {
1191        match self {
1192            Self::Static(x, y, z) => Self::Static(*x, *y, *z),
1193            Self::Dynamic(binding) => Self::Dynamic(binding.clone()),
1194        }
1195    }
1196}
1197
1198#[derive(Debug, PartialEq, Eq, Clone, Copy, Hash)]
1199#[cfg_attr(std_io, derive(serde::Serialize, serde::Deserialize))]
1200#[allow(missing_docs)]
1201/// The number of units across all 3 axis totalling to the number of working units in a cube.
1202pub struct CubeDim {
1203    /// The number of units in the x axis.
1204    pub x: u32,
1205    /// The number of units in the y axis.
1206    pub y: u32,
1207    /// The number of units in the z axis.
1208    pub z: u32,
1209}
1210
1211impl CubeDim {
1212    /// Creates a new [`CubeDim`] based on the maximum number of tasks that can be parellalized by units, in other words,
1213    /// by the maximum number of working units.
1214    ///
1215    /// # Notes
1216    ///
1217    /// For complex problems, you probably want to have your own logic function to create the
1218    /// [`CubeDim`], but for simpler problems such as elemwise-operation, this is a great default.
1219    pub fn new<R: Runtime>(client: &ComputeClient<R>, working_units: usize) -> Self {
1220        let properties = client.properties();
1221        let plane_size = properties.hardware.plane_size_max;
1222        let plane_count = Self::calculate_plane_count_per_cube(
1223            working_units as u32,
1224            plane_size,
1225            properties.hardware.num_cpu_cores,
1226        );
1227
1228        // Make sure it respects the max units per cube (especially on wasm)
1229        let limit = properties.hardware.max_units_per_cube / plane_size;
1230
1231        // Ensure at least 1 plane so CubeDim is always valid (num_elems() > 0).
1232        Self::new_2d(plane_size, u32::min(limit, plane_count).max(1))
1233    }
1234
1235    fn calculate_plane_count_per_cube(
1236        working_units: u32,
1237        plane_dim: u32,
1238        num_cpu_cores: Option<u32>,
1239    ) -> u32 {
1240        match num_cpu_cores {
1241            Some(num_cores) => core::cmp::min(num_cores, working_units),
1242            None => {
1243                let plane_count_max = core::cmp::max(1, working_units / plane_dim);
1244
1245                // Ensures `plane_count` is a power of 2.
1246                const NUM_PLANE_MAX: u32 = 8u32;
1247                const NUM_PLANE_MAX_LOG2: u32 = NUM_PLANE_MAX.ilog2();
1248                let plane_count_max_log2 =
1249                    core::cmp::min(NUM_PLANE_MAX_LOG2, u32::ilog2(plane_count_max));
1250                2u32.pow(plane_count_max_log2)
1251            }
1252        }
1253    }
1254
1255    /// Create a new cube dim with x = y = z = 1.
1256    pub const fn new_single() -> Self {
1257        Self { x: 1, y: 1, z: 1 }
1258    }
1259
1260    /// Create a new cube dim with the given x, and y = z = 1.
1261    pub const fn new_1d(x: u32) -> Self {
1262        Self { x, y: 1, z: 1 }
1263    }
1264
1265    /// Create a new cube dim with the given x and y, and z = 1.
1266    pub const fn new_2d(x: u32, y: u32) -> Self {
1267        Self { x, y, z: 1 }
1268    }
1269
1270    /// Create a new cube dim with the given x, y and z.
1271    /// This is equivalent to the [new](CubeDim::new) function.
1272    pub const fn new_3d(x: u32, y: u32, z: u32) -> Self {
1273        Self { x, y, z }
1274    }
1275
1276    /// Total numbers of units per cube
1277    pub const fn num_elems(&self) -> u32 {
1278        self.x * self.y * self.z
1279    }
1280
1281    /// Whether this `CubeDim` can fully contain `other`
1282    pub const fn can_contain(&self, other: CubeDim) -> bool {
1283        self.x >= other.x && self.y >= other.y && self.z >= other.z
1284    }
1285}
1286
1287impl From<(u32, u32, u32)> for CubeDim {
1288    fn from(value: (u32, u32, u32)) -> Self {
1289        CubeDim::new_3d(value.0, value.1, value.2)
1290    }
1291}
1292
1293impl From<CubeDim> for (u32, u32, u32) {
1294    fn from(val: CubeDim) -> Self {
1295        (val.x, val.y, val.z)
1296    }
1297}
1298
1299/// The kind of execution to be performed.
1300#[derive(Default, Hash, PartialEq, Eq, Clone, Debug, Copy)]
1301#[cfg_attr(std_io, derive(serde::Serialize, serde::Deserialize))]
1302pub enum ExecutionMode {
1303    /// Checked kernels are safe.
1304    #[default]
1305    Checked,
1306    /// Validate OOB and alert if OOB access occurs
1307    Validate,
1308    /// Unchecked kernels are unsafe.
1309    Unchecked,
1310}
1311
1312fn cube_count_spread(max: &(u32, u32, u32), num_cubes: u32) -> [u32; 3] {
1313    let max_cube_counts = [max.0, max.1, max.2];
1314    let mut num_cubes = [num_cubes, 1, 1];
1315    let base = 2;
1316
1317    let mut reduce_count = |i: usize| {
1318        if num_cubes[i] <= max_cube_counts[i] {
1319            return true;
1320        }
1321
1322        loop {
1323            num_cubes[i] = num_cubes[i].div_ceil(base);
1324            num_cubes[i + 1] *= base;
1325
1326            if num_cubes[i] <= max_cube_counts[i] {
1327                return false;
1328            }
1329        }
1330    };
1331
1332    for i in 0..2 {
1333        if reduce_count(i) {
1334            break;
1335        }
1336    }
1337
1338    num_cubes
1339}
1340
1341#[cfg(test)]
1342mod tests {
1343    use super::*;
1344
1345    #[test_log::test]
1346    fn safe_num_cubes_even() {
1347        let max = (32, 32, 32);
1348        let required = 2048;
1349
1350        let actual = cube_count_spread(&max, required);
1351        let expected = [32, 32, 2];
1352        assert_eq!(actual, expected);
1353    }
1354
1355    #[test_log::test]
1356    fn safe_num_cubes_odd() {
1357        let max = (48, 32, 16);
1358        let required = 3177;
1359
1360        let actual = cube_count_spread(&max, required);
1361        let expected = [25, 32, 4];
1362        assert_eq!(actual, expected);
1363    }
1364}
1365
1366/// What one captured launch was recorded with.
1367///
1368/// `info` is cubecl's packed argument blob -- scalars first, sorted by type,
1369/// then static metadata, then dynamic metadata -- exactly as the launch
1370/// presented it. `ptrs` are the device addresses the launch bound, in binding
1371/// order. Both are the values baked into the graph node, which is why reading
1372/// them back is the only sound way to locate a value inside them.
1373#[derive(Debug, Clone, PartialEq, Eq)]
1374pub struct GraphLaunchParams {
1375    /// The cube count the launch was recorded with.
1376    pub grid: (u32, u32, u32),
1377    /// The cube dim the launch was recorded with.
1378    pub block: (u32, u32, u32),
1379    /// The packed scalar and metadata blob, as captured.
1380    pub info: Vec<u64>,
1381    /// Whether `info` reached the kernel as a by-value grid constant. When it
1382    /// did not, it lives in a device buffer instead and no parameter rewrite
1383    /// can move it.
1384    pub info_is_grid_constant: bool,
1385    /// The device address of every buffer the launch bound, in binding order.
1386    pub ptrs: Vec<u64>,
1387    /// Where the DYNAMIC half of `info` starts.
1388    ///
1389    /// `info[..dyn_offset]` is the scalars and the static metadata and reaches
1390    /// the kernel BY VALUE, so a parameter rewrite moves it.
1391    /// `info[dyn_offset..]` is every bound tensor's shape and stride list; it
1392    /// is variable-length, so it cannot be a fixed-size kernel parameter and
1393    /// is uploaded by a memcpy node instead. A word that moved between two
1394    /// captures is patchable or staged according to which side of this it
1395    /// falls on, and reporting the two as one number says neither.
1396    pub dyn_offset: usize,
1397    /// Whether the launch has a pinned staging buffer the graph owns, i.e.
1398    /// whether its dynamic half can be rewritten at all.
1399    pub has_staging: bool,
1400    /// The kernel, named. Populated only for launches recorded by a capture.
1401    pub name: String,
1402}
1403
1404/// What to change about one captured launch. Fields left `None`/empty keep the
1405/// captured value.
1406#[derive(Debug, Clone, Default, PartialEq, Eq)]
1407pub struct GraphLaunchPatch {
1408    /// A new cube count. This is what moves a launch whose GEOMETRY depends on
1409    /// a per-step quantity, such as an attention split count that grows with
1410    /// the context.
1411    pub grid: Option<(u32, u32, u32)>,
1412    /// A new packed argument blob. Must be the same length as the captured
1413    /// one: the kernel's parameter size is fixed at compile time and a shorter
1414    /// blob would have the driver read past it.
1415    pub info: Option<Vec<u64>>,
1416    /// `(binding index, new device address)` for buffers that must be
1417    /// re-pointed. Empty is the normal case and the one the design is for --
1418    /// a region whose buffers are pinned patches scalars only.
1419    pub ptrs: Vec<(usize, u64)>,
1420}