Skip to main content

cubecl_runtime/
client.rs

1use crate::{
2    config::{TypeNameFormatLevel, type_name_format},
3    kernel::KernelMetadata,
4    logging::ProfileLevel,
5    memory_management::{ArenaStats, MemoryAllocationMode, MemoryUsage},
6    runtime::Runtime,
7    server::{
8        CommunicationId, ComputeServer, CopyDescriptor, CubeCount, ExecutionMode,
9        GraphLaunchParams, GraphLaunchPatch, Handle, IoError,
10        KernelArguments, MemoryLayout, MemoryLayoutDescriptor, MemoryLayoutPolicy,
11        MemoryLayoutStrategy, ProfileError, ReduceOperation, ServerCommunication, ServerError,
12        ServerUtilities,
13    },
14    storage::{ComputeStorage, ManagedResource},
15};
16use alloc::{format, sync::Arc, vec, vec::Vec};
17use cubecl_common::{
18    backtrace::BackTrace,
19    bytes::{AllocationProperty, Bytes},
20    device::{Device, DeviceId},
21    device_handle::DeviceHandle,
22    future::DynFut,
23    profile::ProfileDuration,
24};
25use cubecl_ir::{DeviceProperties, ElemType, VectorSize, features::Features};
26use cubecl_zspace::Shape;
27
28#[allow(unused)]
29use cubecl_common::profile::TimingMethod;
30use cubecl_common::stream_id::StreamId;
31
32/// The `ComputeClient` is the entry point to require tasks from the `ComputeServer`.
33/// It should be obtained for a specific device via the Compute struct.
34pub struct ComputeClient<R: Runtime> {
35    device: DeviceHandle<R::Server>,
36    utilities: Arc<ServerUtilities<R::Server>>,
37    stream_id: Option<StreamId>,
38}
39
40impl<R: Runtime> Clone for ComputeClient<R> {
41    fn clone(&self) -> Self {
42        Self {
43            device: self.device.clone(),
44            utilities: self.utilities.clone(),
45            stream_id: self.stream_id,
46        }
47    }
48}
49
50impl<R: Runtime> ComputeClient<R> {
51    /// Get the info of the current backend.
52    pub fn info(&self) -> &<R::Server as ComputeServer>::Info {
53        &self.utilities.info
54    }
55
56    /// Create a new client with a new server.
57    pub fn init<D: Device>(device: &D, server: R::Server) -> Self {
58        let utilities = server.utilities();
59        let context = DeviceHandle::<R::Server>::insert(device.to_id(), server)
60            .expect("Can't create a new client on an already registered server");
61
62        Self {
63            device: context,
64            utilities,
65            stream_id: None,
66        }
67    }
68
69    /// Load the client for the given device.
70    pub fn load<D: Device>(device: &D) -> Self {
71        let context = DeviceHandle::<R::Server>::new(device.to_id());
72
73        // This is safe because we now know the return type of [`DeviceHandle::utilities()`].
74        let utilities = context
75            .utilities()
76            .downcast::<ServerUtilities<R::Server>>()
77            .expect("Can downcast to `ServerUtilities`");
78
79        Self {
80            device: context,
81            utilities,
82            stream_id: None,
83        }
84    }
85
86    fn stream_id(&self) -> StreamId {
87        match self.stream_id {
88            Some(val) => val,
89            None => StreamId::current(),
90        }
91    }
92
93    /// Set the stream in which the current client is operating on.
94    ///
95    /// # Safety
96    ///
97    /// This is highly unsafe and should probably only be used by the CubeCL/Burn projects for now.
98    pub unsafe fn set_stream(&mut self, stream_id: StreamId) {
99        self.stream_id = Some(stream_id);
100    }
101
102    fn do_read(&self, descriptors: Vec<CopyDescriptor>) -> DynFut<Result<Vec<Bytes>, ServerError>> {
103        let stream_id = self.stream_id();
104        self.device
105            .submit_blocking(move |server| server.read(descriptors, stream_id))
106            .unwrap()
107    }
108
109    /// Given bindings, returns owned resources as bytes.
110    pub fn read_async(
111        &self,
112        handles: Vec<Handle>,
113    ) -> impl Future<Output = Result<Vec<Bytes>, ServerError>> + Send {
114        let shapes = handles
115            .iter()
116            .map(|it| [it.size_in_used() as usize].into())
117            .collect::<Vec<Shape>>();
118        let descriptors = handles
119            .into_iter()
120            .zip(shapes)
121            .map(|(handle, shape)| CopyDescriptor::new(handle.binding(), shape, [1].into(), 1))
122            .collect();
123
124        self.do_read(descriptors)
125    }
126
127    /// Given bindings, returns owned resources as bytes.
128    ///
129    /// # Remarks
130    ///
131    /// Panics if the read operation fails.
132    pub fn read(&self, handles: Vec<Handle>) -> Vec<Bytes> {
133        cubecl_common::reader::read_sync(self.read_async(handles)).expect("TODO")
134    }
135
136    /// Given a binding, returns owned resource as bytes.
137    pub fn read_one(&self, handle: Handle) -> Result<Bytes, ServerError> {
138        Ok(cubecl_common::reader::read_sync(self.read_async(vec![handle]))?.remove(0))
139    }
140
141    /// Given a binding, returns owned resource as bytes.
142    ///
143    /// # Remarks
144    ///
145    /// Panics if the read operation fails. Useful for tests.
146    pub fn read_one_unchecked(&self, handle: Handle) -> Bytes {
147        cubecl_common::reader::read_sync(self.read_async(vec![handle]))
148            .unwrap()
149            .remove(0)
150    }
151
152    /// Given bindings, returns owned resources as bytes.
153    pub fn read_tensor_async(
154        &self,
155        descriptors: Vec<CopyDescriptor>,
156    ) -> impl Future<Output = Result<Vec<Bytes>, ServerError>> + Send {
157        self.do_read(descriptors)
158    }
159
160    /// Given bindings, returns owned resources as bytes.
161    ///
162    /// # Remarks
163    ///
164    /// Panics if the read operation fails.
165    ///
166    /// The tensor must be in the same layout as created by the runtime, or more strict.
167    /// Contiguous tensors are always fine, strided tensors are only ok if the stride is similar to
168    /// the one created by the runtime (i.e. padded on only the last dimension). A way to check
169    /// stride compatibility on the runtime will be added in the future.
170    ///
171    /// Also see [`ComputeClient::create_tensor`].
172    pub fn read_tensor(&self, descriptors: Vec<CopyDescriptor>) -> Vec<Bytes> {
173        cubecl_common::reader::read_sync(self.read_tensor_async(descriptors)).expect("TODO")
174    }
175
176    /// Given a binding, returns owned resource as bytes.
177    /// See [`ComputeClient::read_tensor`]
178    pub fn read_one_tensor_async(
179        &self,
180        descriptor: CopyDescriptor,
181    ) -> impl Future<Output = Result<Bytes, ServerError>> + Send {
182        let fut = self.read_tensor_async(vec![descriptor]);
183
184        async { Ok(fut.await?.remove(0)) }
185    }
186
187    /// Given a binding, returns owned resource as bytes.
188    ///
189    /// # Remarks
190    ///
191    /// Panics if the read operation fails.
192    /// See [`ComputeClient::read_tensor`]
193    pub fn read_one_unchecked_tensor(&self, descriptor: CopyDescriptor) -> Bytes {
194        self.read_tensor(vec![descriptor]).remove(0)
195    }
196
197    /// Given a resource handle, returns the storage resource.
198    pub fn get_resource(
199        &self,
200        handle: Handle,
201    ) -> Result<
202        ManagedResource<<<R::Server as ComputeServer>::Storage as ComputeStorage>::Resource>,
203        ServerError,
204    > {
205        let stream_id = self.stream_id();
206        let binding = handle.binding();
207
208        self.device
209            .submit_blocking(move |state| state.get_resource(binding, stream_id))
210            .unwrap()
211    }
212
213    fn do_create_from_slices(
214        &self,
215        descriptors: Vec<MemoryLayoutDescriptor>,
216        slices: Vec<Vec<u8>>,
217    ) -> Result<Vec<MemoryLayout>, IoError> {
218        let stream_id = self.stream_id();
219        let (handle_base, layouts) = self.utilities.layout_policy.apply(stream_id, &descriptors);
220
221        let descriptors = descriptors
222            .into_iter()
223            .zip(layouts.iter())
224            .zip(slices)
225            .map(|((desc, alloc), data)| {
226                (
227                    CopyDescriptor::new(
228                        alloc.memory.clone().binding(),
229                        desc.shape,
230                        alloc.strides.clone(),
231                        desc.elem_size,
232                    ),
233                    // `data` is already an owned `Vec<u8>` the caller gave up;
234                    // `to_vec` here copied every byte a second time, which on a
235                    // 14 MB expert weight measured 0.48 ms of pure waste per
236                    // call, 1 666 calls to a forward.
237                    Bytes::from_bytes_vec(data),
238                )
239            })
240            .collect::<Vec<_>>();
241
242        let (size, memory) = (handle_base.size(), handle_base.memory);
243        self.device.submit(move |server| {
244            server.initialize_memory(memory, size, stream_id);
245            server.write(descriptors, stream_id);
246        });
247
248        Ok(layouts)
249    }
250
251    fn do_create(
252        &self,
253        descriptors: Vec<MemoryLayoutDescriptor>,
254        mut data: Vec<Bytes>,
255    ) -> Result<Vec<MemoryLayout>, IoError> {
256        self.staging(data.iter_mut(), true);
257
258        let stream_id = self.stream_id();
259        let (handle_base, layouts) = self.utilities.layout_policy.apply(stream_id, &descriptors);
260
261        let descriptors = descriptors
262            .into_iter()
263            .zip(layouts.iter())
264            .zip(data)
265            .map(|((desc, layout), data)| {
266                (
267                    CopyDescriptor::new(
268                        layout.memory.clone().binding(),
269                        desc.shape,
270                        layout.strides.clone(),
271                        desc.elem_size,
272                    ),
273                    Bytes::from_bytes_vec(data.to_vec()),
274                )
275            })
276            .collect::<Vec<_>>();
277
278        let (size, memory) = (handle_base.size(), handle_base.memory);
279        self.device.submit(move |server| {
280            server.initialize_memory(memory, size, stream_id);
281            server.write(descriptors, stream_id);
282        });
283
284        Ok(layouts)
285    }
286
287    /// Returns a resource handle containing the given data.
288    ///
289    /// # Notes
290    ///
291    /// Prefer using the more efficient [`Self::create`] function.
292    pub fn create_from_slice(&self, slice: &[u8]) -> Handle {
293        let shape: Shape = [slice.len()].into();
294
295        self.do_create_from_slices(
296            vec![MemoryLayoutDescriptor::new(
297                MemoryLayoutStrategy::Contiguous,
298                shape,
299                1,
300            )],
301            vec![slice.to_vec()],
302        )
303        .unwrap()
304        .remove(0)
305        .memory
306    }
307
308    /// todo: docs
309    pub fn exclusive<'a, Re: Send + 'static, F: FnOnce() -> Re + Send + 'a>(
310        &'a self,
311        task: F,
312    ) -> Result<Re, ServerError> {
313        // We then launch the task.
314        self.device
315            .exclusive(task)
316            .map_err(|err| ServerError::Generic {
317                reason: format!("Communication channel with the server is down: {err:?}"),
318                backtrace: BackTrace::capture(),
319            })
320    }
321
322    /// dodo: Docs
323    pub fn memory_persistent_allocation<
324        'a,
325        Re: Send,
326        Input: Send,
327        F: FnOnce(Input) -> Re + Send + 'a,
328    >(
329        &'a self,
330        input: Input,
331        task: F,
332    ) -> Result<Re, ServerError> {
333        let stream_id = StreamId::current();
334
335        self.device.submit(move |server| {
336            server.allocation_mode(MemoryAllocationMode::Persistent, stream_id);
337        });
338
339        // All tasks created on the same stream will have persistent memory.
340        let output = task(input);
341
342        self.device.submit(move |server| {
343            server.allocation_mode(MemoryAllocationMode::Auto, stream_id);
344        });
345
346        Ok(output)
347    }
348
349    /// Returns a resource handle containing the given [Bytes].
350    pub fn create(&self, data: Bytes) -> Handle {
351        let shape = [data.len()].into();
352
353        self.do_create(
354            vec![MemoryLayoutDescriptor::new(
355                MemoryLayoutStrategy::Contiguous,
356                shape,
357                1,
358            )],
359            vec![data],
360        )
361        .unwrap()
362        .remove(0)
363        .memory
364    }
365
366    /// Given a resource and shape, stores it and returns the tensor handle and strides.
367    /// This may or may not return contiguous strides. The layout is up to the runtime, and care
368    /// should be taken when indexing.
369    ///
370    /// Currently the tensor may either be contiguous (most runtimes), or "pitched", to use the CUDA
371    /// terminology. This means the last (contiguous) dimension is padded to fit a certain alignment,
372    /// and the strides are adjusted accordingly. This can make memory accesses significantly faster
373    /// since all rows are aligned to at least 16 bytes (the maximum load width), meaning the GPU
374    /// can load as much data as possible in a single instruction. It may be aligned even more to
375    /// also take cache lines into account.
376    ///
377    /// However, the stride must be taken into account when indexing and reading the tensor
378    /// (also see [`ComputeClient::read_tensor`]).
379    ///
380    /// # Notes
381    ///
382    /// Prefer using [`Self::create_tensor`] for better performance.
383    pub fn create_tensor_from_slice(
384        &self,
385        slice: &[u8],
386        shape: Shape,
387        elem_size: usize,
388    ) -> MemoryLayout {
389        self.do_create_from_slices(
390            vec![MemoryLayoutDescriptor::new(
391                MemoryLayoutStrategy::Optimized,
392                shape,
393                elem_size,
394            )],
395            vec![slice.to_vec()],
396        )
397        .unwrap()
398        .remove(0)
399    }
400
401    /// Given a resource and shape, stores it and returns the tensor handle and strides.
402    /// This may or may not return contiguous strides. The layout is up to the runtime, and care
403    /// should be taken when indexing.
404    ///
405    /// Currently the tensor may either be contiguous (most runtimes), or "pitched", to use the CUDA
406    /// terminology. This means the last (contiguous) dimension is padded to fit a certain alignment,
407    /// and the strides are adjusted accordingly. This can make memory accesses significantly faster
408    /// since all rows are aligned to at least 16 bytes (the maximum load width), meaning the GPU
409    /// can load as much data as possible in a single instruction. It may be aligned even more to
410    /// also take cache lines into account.
411    ///
412    /// However, the stride must be taken into account when indexing and reading the tensor
413    /// (also see [`ComputeClient::read_tensor`]).
414    pub fn create_tensor(&self, bytes: Bytes, shape: Shape, elem_size: usize) -> MemoryLayout {
415        self.do_create(
416            vec![MemoryLayoutDescriptor::new(
417                MemoryLayoutStrategy::Optimized,
418                shape,
419                elem_size,
420            )],
421            vec![bytes],
422        )
423        .unwrap()
424        .remove(0)
425    }
426
427    /// Reserves all `shapes` in a single storage buffer, copies the corresponding `data` into each
428    /// handle, and returns the handles for them.
429    /// See [`ComputeClient::create_tensor`]
430    ///
431    /// # Notes
432    ///
433    /// Prefer using [`Self::create_tensors`] for better performance.
434    pub fn create_tensors_from_slices(
435        &self,
436        descriptors: Vec<(MemoryLayoutDescriptor, &[u8])>,
437    ) -> Vec<MemoryLayout> {
438        let mut data = Vec::with_capacity(descriptors.len());
439        let mut descriptors_ = Vec::with_capacity(descriptors.len());
440        for (a, b) in descriptors {
441            data.push(b.to_vec());
442            descriptors_.push(a);
443        }
444
445        self.do_create_from_slices(descriptors_, data).unwrap()
446    }
447
448    /// Reserves all `shapes` in a single storage buffer, copies the corresponding `data` into each
449    /// handle, and returns the handles for them.
450    /// See [`ComputeClient::create_tensor`]
451    pub fn create_tensors(
452        &self,
453        descriptors: Vec<(MemoryLayoutDescriptor, Bytes)>,
454    ) -> Vec<MemoryLayout> {
455        let (descriptors, data) = descriptors.into_iter().unzip();
456
457        self.do_create(descriptors, data).unwrap()
458    }
459
460    fn do_empty(
461        &self,
462        descriptors: Vec<MemoryLayoutDescriptor>,
463    ) -> Result<Vec<MemoryLayout>, IoError> {
464        let stream_id = self.stream_id();
465        let (handle_base, layouts) = self.utilities.layout_policy.apply(stream_id, &descriptors);
466
467        let (size, memory) = (handle_base.size(), handle_base.memory);
468        self.device.submit(move |server| {
469            server.initialize_memory(memory, size, stream_id);
470        });
471
472        Ok(layouts)
473    }
474
475    /// Reserves `size` bytes in the storage, and returns a handle over them.
476    pub fn empty(&self, size: usize) -> Handle {
477        let shape: Shape = [size].into();
478        let descriptor = MemoryLayoutDescriptor::new(MemoryLayoutStrategy::Contiguous, shape, 1);
479        self.do_empty(vec![descriptor]).unwrap().remove(0).memory
480    }
481
482    /// See [`ComputeServer::graph_capture_supported`].
483    pub fn graph_capture_supported(&self) -> bool {
484        self.device
485            .submit_blocking(move |server| server.graph_capture_supported())
486            .unwrap()
487    }
488
489    /// See [`ComputeServer::graph_defer_frees`].
490    pub fn graph_defer_frees(&self, defer: bool) {
491        let stream_id = self.stream_id();
492        self.device
493            .submit_blocking(move |server| server.graph_defer_frees(defer, stream_id))
494            .unwrap()
495    }
496
497    /// See [`ComputeServer::graph_arena_begin`].
498    pub fn graph_arena_begin(&self) -> u64 {
499        let stream_id = self.stream_id();
500        self.device
501            .submit_blocking(move |server| server.graph_arena_begin(stream_id))
502            .unwrap()
503    }
504
505    /// See [`ComputeServer::graph_arena_end`].
506    pub fn graph_arena_end(&self) {
507        let stream_id = self.stream_id();
508        self.device
509            .submit_blocking(move |server| server.graph_arena_end(stream_id))
510            .unwrap()
511    }
512
513    /// See [`ComputeServer::graph_arena_stats`].
514    pub fn graph_arena_stats(&self) -> ArenaStats {
515        let stream_id = self.stream_id();
516        self.device
517            .submit_blocking(move |server| server.graph_arena_stats(stream_id))
518            .unwrap()
519    }
520
521    /// See [`ComputeServer::graph_arena_reset_counters`].
522    pub fn graph_arena_reset_counters(&self) {
523        let stream_id = self.stream_id();
524        self.device
525            .submit_blocking(move |server| server.graph_arena_reset_counters(stream_id))
526            .unwrap()
527    }
528
529    /// See [`ComputeServer::reserve_timing`].
530    pub fn reserve_timing(&self) -> (u64, u64) {
531        let stream_id = self.stream_id();
532        self.device
533            .submit_blocking(move |server| server.reserve_timing(stream_id))
534            .unwrap()
535    }
536
537    /// See [`ComputeServer::reserve_timing_reset`].
538    pub fn reserve_timing_reset(&self) {
539        let stream_id = self.stream_id();
540        self.device
541            .submit_blocking(move |server| server.reserve_timing_reset(stream_id))
542            .unwrap()
543    }
544
545    /// See [`ComputeServer::graph_capture_begin`].
546    pub fn graph_capture_begin(&self) {
547        let stream_id = self.stream_id();
548        self.device
549            .submit_blocking(move |server| server.graph_capture_begin(stream_id))
550            .unwrap()
551    }
552
553    /// See [`ComputeServer::graph_capture_end`].
554    pub fn graph_capture_end(&self) -> u64 {
555        let stream_id = self.stream_id();
556        self.device
557            .submit_blocking(move |server| server.graph_capture_end(stream_id))
558            .unwrap()
559    }
560
561    /// See [`ComputeServer::graph_replay`].
562    ///
563    /// Fire-and-forget, exactly like a kernel launch: a replay is enqueued
564    /// work, and making the caller wait for the device thread to pick it up
565    /// would hand back most of what the graph was supposed to save. The whole
566    /// point of the call is that ONE of these replaces the hundreds of
567    /// dispatches the captured region used to cost.
568    pub fn graph_replay(&self, id: u64) {
569        let stream_id = self.stream_id();
570        self.device
571            .submit(move |server| server.graph_replay(id, stream_id));
572    }
573
574    /// See [`ComputeServer::graph_capture_status`].
575    pub fn graph_capture_status(&self) -> u32 {
576        let stream_id = self.stream_id();
577        self.device
578            .submit_blocking(move |server| server.graph_capture_status(stream_id))
579            .unwrap()
580    }
581
582    /// See [`ComputeServer::graph_node_count`].
583    pub fn graph_node_count(&self, id: u64) -> usize {
584        self.device
585            .submit_blocking(move |server| server.graph_node_count(id))
586            .unwrap()
587    }
588
589    /// See [`ComputeServer::graph_capture_launch_count`].
590    pub fn graph_capture_launch_count(&self) -> usize {
591        self.device
592            .submit_blocking(move |server| server.graph_capture_launch_count())
593            .unwrap()
594    }
595
596    /// See [`ComputeServer::graph_launch_count`].
597    pub fn graph_launch_count(&self, id: u64) -> usize {
598        self.device
599            .submit_blocking(move |server| server.graph_launch_count(id))
600            .unwrap()
601    }
602
603    /// See [`ComputeServer::graph_launch_params`].
604    pub fn graph_launch_params(&self, id: u64, idx: usize) -> GraphLaunchParams {
605        self.device
606            .submit_blocking(move |server| server.graph_launch_params(id, idx))
607            .unwrap()
608    }
609
610    /// See [`ComputeServer::graph_patch_launch`].
611    ///
612    /// Blocking, unlike [`Self::graph_replay`]. A patch must be ORDERED before
613    /// the replay that is supposed to see it, and the two go through the same
614    /// device queue, so fire-and-forget would be correct as well -- but a
615    /// patch that names a bad index has to say so at the call site rather than
616    /// on some later unrelated submit.
617    pub fn graph_patch_launch(&self, id: u64, idx: usize, patch: GraphLaunchPatch) {
618        self.device
619            .submit_blocking(move |server| server.graph_patch_launch(id, idx, patch))
620            .unwrap()
621    }
622
623    /// See [`ComputeServer::graph_node_kinds`].
624    pub fn graph_node_kinds(&self, id: u64) -> Vec<(u32, usize)> {
625        self.device
626            .submit_blocking(move |server| server.graph_node_kinds(id))
627            .unwrap()
628    }
629
630    /// See [`ComputeServer::graph_patch_launches`].
631    pub fn graph_patch_launches(&self, id: u64, patches: Vec<(usize, GraphLaunchPatch)>) {
632        self.device
633            .submit_blocking(move |server| server.graph_patch_launches(id, patches))
634            .unwrap()
635    }
636
637    /// See [`ComputeServer::graph_alloc_regions`].
638    pub fn graph_alloc_regions(&self, id: u64) -> Vec<(u64, u64)> {
639        self.device
640            .submit_blocking(move |server| server.graph_alloc_regions(id))
641            .unwrap()
642    }
643
644    /// See [`ComputeServer::graph_memcpy_specs`].
645    pub fn graph_memcpy_specs(&self, id: u64) -> Vec<(u64, u64, u64, u32)> {
646        self.device
647            .submit_blocking(move |server| server.graph_memcpy_specs(id))
648            .unwrap()
649    }
650
651    /// See [`ComputeServer::graph_destroy`].
652    pub fn graph_destroy(&self, id: u64) {
653        self.device
654            .submit_blocking(move |server| server.graph_destroy(id))
655            .unwrap()
656    }
657
658    /// Register an externally-owned GPU buffer that ALIASES host memory (e.g.
659    /// mmap'd pile pages) as a ZERO-COPY tensor [`Handle`]. `ptr`/`page_len`
660    /// describe a page-aligned superset region; `offset`/`size` locate the tensor
661    /// inside it. No allocation and no copy happen — the GPU reads the host pages
662    /// directly (unified memory). See [`ComputeServer::register_external_aliased`].
663    ///
664    /// # Safety
665    /// `ptr` must point to a live, page-aligned host region of at least `page_len`
666    /// bytes that remains valid and immutable for the entire lifetime of the
667    /// returned handle (and any tensor built from it).
668    pub unsafe fn register_external_aliased(
669        &self,
670        ptr: *mut core::ffi::c_void,
671        page_len: u64,
672        offset: u64,
673        size: u64,
674        keepalive: alloc::sync::Arc<dyn core::any::Any + Send + Sync>,
675    ) -> Handle {
676        let stream_id = self.stream_id();
677        // Raw pointers aren't `Send`; the safety contract above covers the hop to
678        // the device thread.
679        struct SendPtr(*mut core::ffi::c_void);
680        // SAFETY: caller guarantees the region stays valid; we only read it.
681        unsafe impl Send for SendPtr {}
682        let p = SendPtr(ptr);
683        self.device
684            .submit_blocking(move |server| {
685                // Rebind the whole `p` so the closure captures `SendPtr` as a unit
686                // (Send) rather than the raw `p.0` field (edition-2021 disjoint
687                // closure capture would otherwise make the closure non-Send).
688                let p = p;
689                server.register_external_aliased(p.0, page_len, offset, size, keepalive, stream_id)
690            })
691            .unwrap()
692    }
693
694    /// Reserves `shape` in the storage, and returns a tensor handle for it.
695    /// See [`ComputeClient::create_tensor`]
696    pub fn empty_tensor(&self, shape: Shape, elem_size: usize) -> MemoryLayout {
697        let descriptor =
698            MemoryLayoutDescriptor::new(MemoryLayoutStrategy::Optimized, shape, elem_size);
699        self.do_empty(vec![descriptor]).unwrap().remove(0)
700    }
701
702    /// Reserves all `shapes` in a single storage buffer, and returns the handles for them.
703    /// See [`ComputeClient::create_tensor`]
704    pub fn empty_tensors(&self, descriptors: Vec<MemoryLayoutDescriptor>) -> Vec<MemoryLayout> {
705        self.do_empty(descriptors).unwrap()
706    }
707
708    /// Marks the given [Bytes] as being a staging buffer, maybe transferring it to pinned memory
709    /// for faster data transfer with compute device.
710    ///
711    /// TODO: This blocks the compute queue, so it will drop the compute utilization.
712    pub fn staging<'a, I>(&self, bytes: I, file_only: bool)
713    where
714        I: Iterator<Item = &'a mut Bytes>,
715    {
716        let has_staging = |b: &Bytes| match b.property() {
717            AllocationProperty::Pinned => false,
718            AllocationProperty::File => true,
719            AllocationProperty::Native | AllocationProperty::Other => !file_only,
720        };
721
722        let mut to_be_updated = Vec::new();
723        let sizes = bytes
724            .filter_map(|b| match has_staging(b) {
725                true => {
726                    let len = b.len();
727                    to_be_updated.push(b);
728                    Some(len)
729                }
730                false => None,
731            })
732            .collect::<Vec<usize>>();
733
734        if sizes.is_empty() {
735            return;
736        }
737
738        let stream_id = self.stream_id();
739        let sizes = sizes.to_vec();
740        let stagings = self
741            .device
742            .submit_blocking(move |server| server.staging(&sizes, stream_id))
743            .unwrap();
744
745        let stagings = match stagings {
746            Ok(val) => val,
747            Err(_) => return,
748        };
749
750        to_be_updated
751            .into_iter()
752            .zip(stagings)
753            .for_each(|(b, mut staging)| {
754                b.copy_into(&mut staging);
755                core::mem::swap(b, &mut staging);
756            });
757    }
758
759    /// Transfer data from one client to another
760    #[cfg_attr(
761        feature = "tracing",
762        tracing::instrument(level = "trace", skip(self, src, dst_server))
763    )]
764    pub fn to_client(&mut self, src: Handle, dst_server: &Self, dtype: ElemType) -> Handle {
765        let shape = [src.size_in_used() as usize];
766        let src_descriptor = src.copy_descriptor(shape.into(), [1].into(), 1);
767
768        if R::Server::SERVER_COMM_ENABLED {
769            self.to_client_tensor(src_descriptor, dst_server, dtype)
770        } else {
771            let alloc_desc = MemoryLayoutDescriptor::new(
772                MemoryLayoutStrategy::Contiguous,
773                src_descriptor.shape.clone(),
774                src_descriptor.elem_size,
775            );
776            self.change_client_sync(src_descriptor, alloc_desc, dst_server)
777                .memory
778        }
779    }
780
781    /// Perform an `all_reduce` operation on the given devices.
782    #[cfg_attr(
783        feature = "tracing",
784        tracing::instrument(level = "trace", skip(self, device_ids))
785    )]
786    pub fn ensure_init_collective(&mut self, device_ids: Vec<DeviceId>) {
787        let comm_id = CommunicationId::from(device_ids.clone());
788        let is_comms_init = self
789            .utilities
790            .initialized_comms
791            .read()
792            .unwrap()
793            .contains(&comm_id);
794        if !is_comms_init {
795            self.device
796                .submit(move |server| server.comm_init(device_ids).unwrap());
797            let mut initialized_comms = self.utilities.initialized_comms.write().unwrap();
798            initialized_comms.insert(comm_id);
799            // Flush immediately so other devices aren't blocked waiting on this initialization.
800            self.device.flush_queue();
801        }
802    }
803
804    /// Wait on the communication stream.
805    #[cfg_attr(feature = "tracing", tracing::instrument(level = "trace", skip(self)))]
806    pub fn sync_collective(&self) {
807        if DeviceHandle::<R::Server>::is_blocking() {
808            panic!("Can't use `sync_collective` with a blocking device handle");
809        }
810        let stream_id = self.stream_id();
811
812        self.device.submit(move |server| {
813            server.sync_collective(stream_id).unwrap();
814        });
815
816        // We don't actually need or want to sync the server here, but we need to make sure any
817        // task enqueued on the communication channel is done.
818        self.device.flush_queue();
819    }
820
821    /// Perform an `all_reduce` operation on the given devices.
822    #[cfg_attr(
823        feature = "tracing",
824        tracing::instrument(level = "trace", skip(self, src, dst, dtype, device_ids, op))
825    )]
826    pub fn all_reduce(
827        &mut self,
828        src: Handle,
829        dst: Handle,
830        dtype: ElemType,
831        device_ids: Vec<DeviceId>,
832        op: ReduceOperation,
833    ) {
834        if DeviceHandle::<R::Server>::is_blocking() {
835            panic!("Can't use `all_reduce` with a blocking device handle");
836        }
837
838        let stream_id = self.stream_id();
839        let src = src.binding();
840        let dst = dst.binding();
841
842        self.ensure_init_collective(device_ids.clone());
843
844        self.device.submit(move |server| {
845            server
846                .all_reduce(src, dst, dtype, stream_id, op, device_ids)
847                .unwrap();
848        });
849    }
850
851    /// Transfer data from one client to another
852    ///
853    /// Make sure the source description can be read in a contiguous manner.
854    #[cfg_attr(
855        feature = "tracing",
856        tracing::instrument(level = "trace", skip(self, src_descriptor, dst_server))
857    )]
858    pub fn to_client_tensor(
859        &mut self,
860        src_descriptor: CopyDescriptor,
861        dst_server: &Self,
862        dtype: ElemType,
863    ) -> Handle {
864        let stream_id_src = self.stream_id();
865        let stream_id_dst = dst_server.stream_id();
866
867        let device_id_src = self.device.device_id();
868        let device_id_dst = dst_server.device.device_id();
869
870        let mut dst_server = dst_server.clone();
871        let handle = Handle::new(stream_id_dst, src_descriptor.handle.size_in_used());
872        let handle_cloned = handle.clone();
873
874        let device_ids = vec![device_id_src, device_id_dst];
875        self.ensure_init_collective(device_ids.clone());
876        dst_server.ensure_init_collective(device_ids);
877
878        self.device.submit(move |server_src| {
879            server_src
880                .send(src_descriptor, dtype, stream_id_src, device_id_dst)
881                .unwrap()
882        });
883
884        dst_server.device.submit(move |server_dst| {
885            server_dst
886                .recv(handle_cloned, dtype, stream_id_dst, device_id_src)
887                .unwrap();
888            server_dst.sync_collective(stream_id_dst).unwrap();
889        });
890
891        // `ServerCommunication::send` and`ServerCommunication::recv` are blocking: they each wait for the corresponding recv/send
892        // call to be made. We flush the operations right away so that the neither server ends up in a deadlock.
893        // The actual data transfer is still executed asynchronously on the communication stream.
894        self.device.flush_queue();
895        dst_server.device.flush_queue();
896
897        handle
898    }
899
900    #[track_caller]
901    #[cfg_attr(feature = "tracing", tracing::instrument(level="trace",
902        skip(self, kernel, bindings),
903        fields(
904            kernel.name = %kernel.name(),
905            kernel.id = %kernel.id(),
906        )
907    ))]
908    unsafe fn launch_inner(
909        &self,
910        kernel: <R::Server as ComputeServer>::Kernel,
911        count: CubeCount,
912        bindings: KernelArguments,
913        mode: ExecutionMode,
914        stream_id: StreamId,
915    ) {
916        let level = self.utilities.logger.profile_level();
917
918        match level {
919            None | Some(ProfileLevel::ExecutionOnly) => {
920                let utilities = self.utilities.clone();
921                self.device.submit(move |state| {
922                    let name = kernel.name();
923                    unsafe { state.launch(kernel, count, bindings, mode, stream_id) };
924
925                    if matches!(level, Some(ProfileLevel::ExecutionOnly)) {
926                        let info = type_name_format(name, TypeNameFormatLevel::Balanced);
927                        utilities.logger.register_execution(info);
928                    }
929                });
930            }
931            Some(level) => {
932                let name = kernel.name();
933                let kernel_id = kernel.id();
934                let context = self.device.clone();
935                let count_moved = count.clone();
936                let (result, profile) = self
937                    .profile(
938                        move || {
939                            context
940                                .submit_blocking(move |state| unsafe {
941                                    state.launch(kernel, count_moved, bindings, mode, stream_id)
942                                })
943                                .unwrap()
944                        },
945                        name,
946                    )
947                    .unwrap();
948                let info = match level {
949                    ProfileLevel::Full => {
950                        format!("{name}: {kernel_id} CubeCount {count:?}")
951                    }
952                    _ => type_name_format(name, TypeNameFormatLevel::Balanced),
953                };
954                self.utilities.logger.register_profiled(info, profile);
955                result
956            }
957        }
958    }
959
960    /// Launches the `kernel` with the given `bindings`.
961    #[track_caller]
962    pub fn launch(
963        &self,
964        kernel: <R::Server as ComputeServer>::Kernel,
965        count: CubeCount,
966        bindings: KernelArguments,
967    ) {
968        // SAFETY: Using checked execution mode.
969        unsafe {
970            self.launch_inner(
971                kernel,
972                count,
973                bindings,
974                ExecutionMode::Checked,
975                self.stream_id(),
976            )
977        }
978    }
979
980    /// Launches the `kernel` with the given `bindings` without performing any bound checks.
981    ///
982    /// # Safety
983    ///
984    /// To ensure this is safe, you must verify your kernel:
985    /// - Has no out-of-bound reads and writes that can happen.
986    /// - Has no infinite loops that might never terminate.
987    #[track_caller]
988    pub unsafe fn launch_unchecked(
989        &self,
990        kernel: <R::Server as ComputeServer>::Kernel,
991        count: CubeCount,
992        bindings: KernelArguments,
993    ) {
994        // SAFETY: Caller has to uphold kernel being safe.
995        unsafe {
996            self.launch_inner(
997                kernel,
998                count,
999                bindings,
1000                match self.utilities.check_mode {
1001                    crate::config::compilation::BoundsCheckMode::Enforce => ExecutionMode::Checked,
1002                    crate::config::compilation::BoundsCheckMode::Validate => {
1003                        ExecutionMode::Validate
1004                    }
1005                    crate::config::compilation::BoundsCheckMode::Auto => ExecutionMode::Unchecked,
1006                },
1007                self.stream_id(),
1008            )
1009        }
1010    }
1011
1012    /// Flush all outstanding commands.
1013    pub fn flush(&self) -> Result<(), ServerError> {
1014        let stream_id = self.stream_id();
1015
1016        self.device
1017            .submit_blocking(move |server| server.flush(stream_id))
1018            .unwrap()
1019    }
1020
1021    /// Wait for the completion of every task in the server.
1022    pub fn sync(&self) -> DynFut<Result<(), ServerError>> {
1023        let stream_id = self.stream_id();
1024
1025        let fut = self
1026            .device
1027            .submit_blocking(move |server| server.sync(stream_id))
1028            .unwrap();
1029
1030        self.utilities.logger.profile_summary();
1031
1032        fut
1033    }
1034
1035    /// Get the features supported by the compute server.
1036    pub fn properties(&self) -> &DeviceProperties {
1037        &self.utilities.properties
1038    }
1039
1040    /// Get the features supported by the compute server.
1041    pub fn features(&self) -> &Features {
1042        &self.utilities.properties.features
1043    }
1044
1045    /// # Warning
1046    ///
1047    /// For private use only.
1048    pub fn properties_mut(&mut self) -> Option<&mut DeviceProperties> {
1049        Arc::get_mut(&mut self.utilities).map(|state| &mut state.properties)
1050    }
1051
1052    /// Get the current memory usage of this client.
1053    pub fn memory_usage(&self) -> Result<MemoryUsage, ServerError> {
1054        let stream_id = self.stream_id();
1055        self.device
1056            .submit_blocking(move |server| server.memory_usage(stream_id))
1057            .unwrap()
1058    }
1059
1060    /// Get all devices of a specific type available to this runtime
1061    pub fn enumerate_devices(&self, type_id: u16) -> Vec<DeviceId> {
1062        R::enumerate_devices(type_id, self.info())
1063    }
1064
1065    /// Get all devices available to this runtime
1066    pub fn enumerate_all_devices(&self) -> Vec<DeviceId> {
1067        R::enumerate_all_devices(self.info())
1068    }
1069
1070    /// Get the number of devices of a specific type available to this runtime
1071    pub fn device_count(&self, type_id: u16) -> usize {
1072        self.enumerate_devices(type_id).len()
1073    }
1074
1075    /// Get the number of devices of a specific type available to this runtime
1076    pub fn device_count_total(&self) -> usize {
1077        self.enumerate_all_devices().len()
1078    }
1079
1080    /// Change the memory allocation mode.
1081    ///
1082    /// # Safety
1083    ///
1084    /// This function isn't thread safe and might create memory leaks.
1085    pub unsafe fn allocation_mode(&self, mode: MemoryAllocationMode) {
1086        let stream_id = self.stream_id();
1087        self.device
1088            .submit(move |server| server.allocation_mode(mode, stream_id));
1089    }
1090
1091    /// Ask the client to release memory that it can release.
1092    ///
1093    /// Nb: Results will vary on what the memory allocator deems beneficial,
1094    /// so it's not guaranteed any memory is freed.
1095    pub fn memory_cleanup(&self) {
1096        let stream_id = self.stream_id();
1097        self.device
1098            .submit(move |server| server.memory_cleanup(stream_id));
1099    }
1100
1101    /// Measure the execution time of some inner operations.
1102    #[track_caller]
1103    pub fn profile<O: Send + 'static>(
1104        &self,
1105        func: impl FnOnce() -> O + Send,
1106        #[allow(unused)] func_name: &str,
1107    ) -> Result<(O, ProfileDuration), ProfileError> {
1108        // Get the outer caller. For execute() this points straight to the
1109        // cube kernel. For general profiling it points to whoever calls profile.
1110        #[cfg(feature = "profile-tracy")]
1111        let location = std::panic::Location::caller();
1112
1113        // Make a CPU span. If the server has system profiling this is all you need.
1114        #[cfg(feature = "profile-tracy")]
1115        let _span = tracy_client::Client::running().unwrap().span_alloc(
1116            None,
1117            func_name,
1118            location.file(),
1119            location.line(),
1120            0,
1121        );
1122
1123        let stream_id = self.stream_id();
1124
1125        #[cfg(feature = "profile-tracy")]
1126        let gpu_span = if self.utilities.properties.timing_method == TimingMethod::Device {
1127            let gpu_span = self
1128                .utilities
1129                .gpu_client
1130                .span_alloc(func_name, "profile", location.file(), location.line())
1131                .unwrap();
1132            Some(gpu_span)
1133        } else {
1134            None
1135        };
1136
1137        let device = self.device.clone();
1138        #[allow(unused_mut, reason = "Used in profile-tracy")]
1139        let mut result = self
1140            .device
1141            .exclusive(move || {
1142                // We first get mut access to the server to create a token.
1143                // Then we free to server, since it's going to be accessed in `func()`.
1144                let token =
1145                    match device.submit_blocking(move |server| server.start_profile(stream_id)) {
1146                        Ok(token) => match token {
1147                            Ok(token) => token,
1148                            Err(err) => return Err(err),
1149                        },
1150                        Err(err) => {
1151                            return Err(ServerError::Generic {
1152                                reason: alloc::format!(
1153                                    "Can't start profiling because of a call error: {err:?}"
1154                                ),
1155                                backtrace: BackTrace::capture(),
1156                            });
1157                        }
1158                    };
1159
1160                // We execute `func()` which will recursibly access the server.
1161                let out = func();
1162
1163                // Finally we get the result from the token.
1164                let result = device
1165                    .submit_blocking(move |server| {
1166                        let mut result = server.end_profile(stream_id, token);
1167
1168                        match result {
1169                            Ok(result) => Ok((out, result)),
1170                            Err(err) => Err(err),
1171                        }
1172                    })
1173                    .unwrap();
1174
1175                Ok(result)
1176            })
1177            .unwrap()
1178            .map_err(|err| ProfileError::Unknown {
1179                reason: alloc::format!("{err:?}"),
1180                backtrace: BackTrace::capture(),
1181            })?;
1182
1183        #[cfg(feature = "profile-tracy")]
1184        if let Some(mut gpu_span) = gpu_span {
1185            gpu_span.end_zone();
1186            let epoch = self.utilities.epoch_time;
1187            // Add in the work to upload the timestamp data.
1188            result = result.map(|(o, result)| {
1189                (
1190                    o,
1191                    ProfileDuration::new(
1192                        alloc::boxed::Box::pin(async move {
1193                            let ticks = result.resolve().await;
1194                            let start_duration =
1195                                ticks.start_duration_since(epoch).as_nanos() as i64;
1196                            let end_duration = ticks.end_duration_since(epoch).as_nanos() as i64;
1197                            gpu_span.upload_timestamp_start(start_duration);
1198                            gpu_span.upload_timestamp_end(end_duration);
1199                            ticks
1200                        }),
1201                        TimingMethod::Device,
1202                    ),
1203                )
1204            });
1205        }
1206
1207        result
1208    }
1209
1210    /// Transfer data from one client to another
1211    #[cfg_attr(
1212        feature = "tracing",
1213        tracing::instrument(
1214            level = "trace",
1215            skip(self, src_descriptor, alloc_descriptor, dst_server)
1216        )
1217    )]
1218    fn change_client_sync(
1219        &self,
1220        src_descriptor: CopyDescriptor,
1221        alloc_descriptor: MemoryLayoutDescriptor,
1222        dst_server: &Self,
1223    ) -> MemoryLayout {
1224        let shape = src_descriptor.shape.clone();
1225        let elem_size = src_descriptor.elem_size;
1226        let stream_id = self.stream_id();
1227
1228        let read = self
1229            .device
1230            .submit_blocking(move |server| server.read(vec![src_descriptor], stream_id))
1231            .unwrap();
1232
1233        let mut data = cubecl_common::future::block_on(read).unwrap();
1234
1235        let (handle_base, mut layouts) = self
1236            .utilities
1237            .layout_policy
1238            .apply(stream_id, &[alloc_descriptor]);
1239        let alloc = layouts.remove(0);
1240
1241        let desc_descriptor = CopyDescriptor {
1242            handle: handle_base.clone().binding(),
1243            shape,
1244            strides: alloc.strides.clone(),
1245            elem_size,
1246        };
1247
1248        let (size, memory) = (handle_base.size(), handle_base.memory);
1249        dst_server.device.submit(move |server| {
1250            server.initialize_memory(memory, size, stream_id);
1251            server.write(vec![(desc_descriptor, data.remove(0))], stream_id)
1252        });
1253
1254        alloc
1255    }
1256
1257    /// Returns all vector sizes that are useful to perform optimal IO operation on the given element.
1258    pub fn io_optimized_vector_sizes(
1259        &self,
1260        size: usize,
1261    ) -> impl Iterator<Item = VectorSize> + Clone {
1262        let load_width = self.properties().hardware.load_width as usize;
1263        let size_bits = size * 8;
1264        let max = load_width / size_bits;
1265        let max = usize::min(self.properties().hardware.max_vector_size, max);
1266
1267        // If the max is 8, we want to test 1, 2, 4, 8 which is log2(8) + 1.
1268        let num_candidates = max.trailing_zeros() + 1;
1269
1270        (0..num_candidates).map(|i| 2usize.pow(i)).rev()
1271    }
1272}