1use crate::{
2 config::{TypeNameFormatLevel, type_name_format},
3 kernel::KernelMetadata,
4 logging::ProfileLevel,
5 memory_management::{ArenaStats, MemoryAllocationMode, MemoryUsage},
6 runtime::Runtime,
7 server::{
8 CommunicationId, ComputeServer, CopyDescriptor, CubeCount, ExecutionMode,
9 GraphLaunchParams, GraphLaunchPatch, Handle, IoError,
10 KernelArguments, MemoryLayout, MemoryLayoutDescriptor, MemoryLayoutPolicy,
11 MemoryLayoutStrategy, ProfileError, ReduceOperation, ServerCommunication, ServerError,
12 ServerUtilities,
13 },
14 storage::{ComputeStorage, ManagedResource},
15};
16use alloc::{format, sync::Arc, vec, vec::Vec};
17use cubecl_common::{
18 backtrace::BackTrace,
19 bytes::{AllocationProperty, Bytes},
20 device::{Device, DeviceId},
21 device_handle::DeviceHandle,
22 future::DynFut,
23 profile::ProfileDuration,
24};
25use cubecl_ir::{DeviceProperties, ElemType, VectorSize, features::Features};
26use cubecl_zspace::Shape;
27
28#[allow(unused)]
29use cubecl_common::profile::TimingMethod;
30use cubecl_common::stream_id::StreamId;
31
32pub struct ComputeClient<R: Runtime> {
35 device: DeviceHandle<R::Server>,
36 utilities: Arc<ServerUtilities<R::Server>>,
37 stream_id: Option<StreamId>,
38}
39
40impl<R: Runtime> Clone for ComputeClient<R> {
41 fn clone(&self) -> Self {
42 Self {
43 device: self.device.clone(),
44 utilities: self.utilities.clone(),
45 stream_id: self.stream_id,
46 }
47 }
48}
49
50impl<R: Runtime> ComputeClient<R> {
51 pub fn info(&self) -> &<R::Server as ComputeServer>::Info {
53 &self.utilities.info
54 }
55
56 pub fn init<D: Device>(device: &D, server: R::Server) -> Self {
58 let utilities = server.utilities();
59 let context = DeviceHandle::<R::Server>::insert(device.to_id(), server)
60 .expect("Can't create a new client on an already registered server");
61
62 Self {
63 device: context,
64 utilities,
65 stream_id: None,
66 }
67 }
68
69 pub fn load<D: Device>(device: &D) -> Self {
71 let context = DeviceHandle::<R::Server>::new(device.to_id());
72
73 let utilities = context
75 .utilities()
76 .downcast::<ServerUtilities<R::Server>>()
77 .expect("Can downcast to `ServerUtilities`");
78
79 Self {
80 device: context,
81 utilities,
82 stream_id: None,
83 }
84 }
85
86 fn stream_id(&self) -> StreamId {
87 match self.stream_id {
88 Some(val) => val,
89 None => StreamId::current(),
90 }
91 }
92
93 pub unsafe fn set_stream(&mut self, stream_id: StreamId) {
99 self.stream_id = Some(stream_id);
100 }
101
102 fn do_read(&self, descriptors: Vec<CopyDescriptor>) -> DynFut<Result<Vec<Bytes>, ServerError>> {
103 let stream_id = self.stream_id();
104 self.device
105 .submit_blocking(move |server| server.read(descriptors, stream_id))
106 .unwrap()
107 }
108
109 pub fn read_async(
111 &self,
112 handles: Vec<Handle>,
113 ) -> impl Future<Output = Result<Vec<Bytes>, ServerError>> + Send {
114 let shapes = handles
115 .iter()
116 .map(|it| [it.size_in_used() as usize].into())
117 .collect::<Vec<Shape>>();
118 let descriptors = handles
119 .into_iter()
120 .zip(shapes)
121 .map(|(handle, shape)| CopyDescriptor::new(handle.binding(), shape, [1].into(), 1))
122 .collect();
123
124 self.do_read(descriptors)
125 }
126
127 pub fn read(&self, handles: Vec<Handle>) -> Vec<Bytes> {
133 cubecl_common::reader::read_sync(self.read_async(handles)).expect("TODO")
134 }
135
136 pub fn read_one(&self, handle: Handle) -> Result<Bytes, ServerError> {
138 Ok(cubecl_common::reader::read_sync(self.read_async(vec![handle]))?.remove(0))
139 }
140
141 pub fn read_one_unchecked(&self, handle: Handle) -> Bytes {
147 cubecl_common::reader::read_sync(self.read_async(vec![handle]))
148 .unwrap()
149 .remove(0)
150 }
151
152 pub fn read_tensor_async(
154 &self,
155 descriptors: Vec<CopyDescriptor>,
156 ) -> impl Future<Output = Result<Vec<Bytes>, ServerError>> + Send {
157 self.do_read(descriptors)
158 }
159
160 pub fn read_tensor(&self, descriptors: Vec<CopyDescriptor>) -> Vec<Bytes> {
173 cubecl_common::reader::read_sync(self.read_tensor_async(descriptors)).expect("TODO")
174 }
175
176 pub fn read_one_tensor_async(
179 &self,
180 descriptor: CopyDescriptor,
181 ) -> impl Future<Output = Result<Bytes, ServerError>> + Send {
182 let fut = self.read_tensor_async(vec![descriptor]);
183
184 async { Ok(fut.await?.remove(0)) }
185 }
186
187 pub fn read_one_unchecked_tensor(&self, descriptor: CopyDescriptor) -> Bytes {
194 self.read_tensor(vec![descriptor]).remove(0)
195 }
196
197 pub fn get_resource(
199 &self,
200 handle: Handle,
201 ) -> Result<
202 ManagedResource<<<R::Server as ComputeServer>::Storage as ComputeStorage>::Resource>,
203 ServerError,
204 > {
205 let stream_id = self.stream_id();
206 let binding = handle.binding();
207
208 self.device
209 .submit_blocking(move |state| state.get_resource(binding, stream_id))
210 .unwrap()
211 }
212
213 fn do_create_from_slices(
214 &self,
215 descriptors: Vec<MemoryLayoutDescriptor>,
216 slices: Vec<Vec<u8>>,
217 ) -> Result<Vec<MemoryLayout>, IoError> {
218 let stream_id = self.stream_id();
219 let (handle_base, layouts) = self.utilities.layout_policy.apply(stream_id, &descriptors);
220
221 let descriptors = descriptors
222 .into_iter()
223 .zip(layouts.iter())
224 .zip(slices)
225 .map(|((desc, alloc), data)| {
226 (
227 CopyDescriptor::new(
228 alloc.memory.clone().binding(),
229 desc.shape,
230 alloc.strides.clone(),
231 desc.elem_size,
232 ),
233 Bytes::from_bytes_vec(data),
238 )
239 })
240 .collect::<Vec<_>>();
241
242 let (size, memory) = (handle_base.size(), handle_base.memory);
243 self.device.submit(move |server| {
244 server.initialize_memory(memory, size, stream_id);
245 server.write(descriptors, stream_id);
246 });
247
248 Ok(layouts)
249 }
250
251 fn do_create(
252 &self,
253 descriptors: Vec<MemoryLayoutDescriptor>,
254 mut data: Vec<Bytes>,
255 ) -> Result<Vec<MemoryLayout>, IoError> {
256 self.staging(data.iter_mut(), true);
257
258 let stream_id = self.stream_id();
259 let (handle_base, layouts) = self.utilities.layout_policy.apply(stream_id, &descriptors);
260
261 let descriptors = descriptors
262 .into_iter()
263 .zip(layouts.iter())
264 .zip(data)
265 .map(|((desc, layout), data)| {
266 (
267 CopyDescriptor::new(
268 layout.memory.clone().binding(),
269 desc.shape,
270 layout.strides.clone(),
271 desc.elem_size,
272 ),
273 Bytes::from_bytes_vec(data.to_vec()),
274 )
275 })
276 .collect::<Vec<_>>();
277
278 let (size, memory) = (handle_base.size(), handle_base.memory);
279 self.device.submit(move |server| {
280 server.initialize_memory(memory, size, stream_id);
281 server.write(descriptors, stream_id);
282 });
283
284 Ok(layouts)
285 }
286
287 pub fn create_from_slice(&self, slice: &[u8]) -> Handle {
293 let shape: Shape = [slice.len()].into();
294
295 self.do_create_from_slices(
296 vec![MemoryLayoutDescriptor::new(
297 MemoryLayoutStrategy::Contiguous,
298 shape,
299 1,
300 )],
301 vec![slice.to_vec()],
302 )
303 .unwrap()
304 .remove(0)
305 .memory
306 }
307
308 pub fn exclusive<'a, Re: Send + 'static, F: FnOnce() -> Re + Send + 'a>(
310 &'a self,
311 task: F,
312 ) -> Result<Re, ServerError> {
313 self.device
315 .exclusive(task)
316 .map_err(|err| ServerError::Generic {
317 reason: format!("Communication channel with the server is down: {err:?}"),
318 backtrace: BackTrace::capture(),
319 })
320 }
321
322 pub fn memory_persistent_allocation<
324 'a,
325 Re: Send,
326 Input: Send,
327 F: FnOnce(Input) -> Re + Send + 'a,
328 >(
329 &'a self,
330 input: Input,
331 task: F,
332 ) -> Result<Re, ServerError> {
333 let stream_id = StreamId::current();
334
335 self.device.submit(move |server| {
336 server.allocation_mode(MemoryAllocationMode::Persistent, stream_id);
337 });
338
339 let output = task(input);
341
342 self.device.submit(move |server| {
343 server.allocation_mode(MemoryAllocationMode::Auto, stream_id);
344 });
345
346 Ok(output)
347 }
348
349 pub fn create(&self, data: Bytes) -> Handle {
351 let shape = [data.len()].into();
352
353 self.do_create(
354 vec![MemoryLayoutDescriptor::new(
355 MemoryLayoutStrategy::Contiguous,
356 shape,
357 1,
358 )],
359 vec![data],
360 )
361 .unwrap()
362 .remove(0)
363 .memory
364 }
365
366 pub fn create_tensor_from_slice(
384 &self,
385 slice: &[u8],
386 shape: Shape,
387 elem_size: usize,
388 ) -> MemoryLayout {
389 self.do_create_from_slices(
390 vec![MemoryLayoutDescriptor::new(
391 MemoryLayoutStrategy::Optimized,
392 shape,
393 elem_size,
394 )],
395 vec![slice.to_vec()],
396 )
397 .unwrap()
398 .remove(0)
399 }
400
401 pub fn create_tensor(&self, bytes: Bytes, shape: Shape, elem_size: usize) -> MemoryLayout {
415 self.do_create(
416 vec![MemoryLayoutDescriptor::new(
417 MemoryLayoutStrategy::Optimized,
418 shape,
419 elem_size,
420 )],
421 vec![bytes],
422 )
423 .unwrap()
424 .remove(0)
425 }
426
427 pub fn create_tensors_from_slices(
435 &self,
436 descriptors: Vec<(MemoryLayoutDescriptor, &[u8])>,
437 ) -> Vec<MemoryLayout> {
438 let mut data = Vec::with_capacity(descriptors.len());
439 let mut descriptors_ = Vec::with_capacity(descriptors.len());
440 for (a, b) in descriptors {
441 data.push(b.to_vec());
442 descriptors_.push(a);
443 }
444
445 self.do_create_from_slices(descriptors_, data).unwrap()
446 }
447
448 pub fn create_tensors(
452 &self,
453 descriptors: Vec<(MemoryLayoutDescriptor, Bytes)>,
454 ) -> Vec<MemoryLayout> {
455 let (descriptors, data) = descriptors.into_iter().unzip();
456
457 self.do_create(descriptors, data).unwrap()
458 }
459
460 fn do_empty(
461 &self,
462 descriptors: Vec<MemoryLayoutDescriptor>,
463 ) -> Result<Vec<MemoryLayout>, IoError> {
464 let stream_id = self.stream_id();
465 let (handle_base, layouts) = self.utilities.layout_policy.apply(stream_id, &descriptors);
466
467 let (size, memory) = (handle_base.size(), handle_base.memory);
468 self.device.submit(move |server| {
469 server.initialize_memory(memory, size, stream_id);
470 });
471
472 Ok(layouts)
473 }
474
475 pub fn empty(&self, size: usize) -> Handle {
477 let shape: Shape = [size].into();
478 let descriptor = MemoryLayoutDescriptor::new(MemoryLayoutStrategy::Contiguous, shape, 1);
479 self.do_empty(vec![descriptor]).unwrap().remove(0).memory
480 }
481
482 pub fn graph_capture_supported(&self) -> bool {
484 self.device
485 .submit_blocking(move |server| server.graph_capture_supported())
486 .unwrap()
487 }
488
489 pub fn graph_defer_frees(&self, defer: bool) {
491 let stream_id = self.stream_id();
492 self.device
493 .submit_blocking(move |server| server.graph_defer_frees(defer, stream_id))
494 .unwrap()
495 }
496
497 pub fn graph_arena_begin(&self) -> u64 {
499 let stream_id = self.stream_id();
500 self.device
501 .submit_blocking(move |server| server.graph_arena_begin(stream_id))
502 .unwrap()
503 }
504
505 pub fn graph_arena_end(&self) {
507 let stream_id = self.stream_id();
508 self.device
509 .submit_blocking(move |server| server.graph_arena_end(stream_id))
510 .unwrap()
511 }
512
513 pub fn graph_arena_stats(&self) -> ArenaStats {
515 let stream_id = self.stream_id();
516 self.device
517 .submit_blocking(move |server| server.graph_arena_stats(stream_id))
518 .unwrap()
519 }
520
521 pub fn graph_arena_reset_counters(&self) {
523 let stream_id = self.stream_id();
524 self.device
525 .submit_blocking(move |server| server.graph_arena_reset_counters(stream_id))
526 .unwrap()
527 }
528
529 pub fn reserve_timing(&self) -> (u64, u64) {
531 let stream_id = self.stream_id();
532 self.device
533 .submit_blocking(move |server| server.reserve_timing(stream_id))
534 .unwrap()
535 }
536
537 pub fn reserve_timing_reset(&self) {
539 let stream_id = self.stream_id();
540 self.device
541 .submit_blocking(move |server| server.reserve_timing_reset(stream_id))
542 .unwrap()
543 }
544
545 pub fn graph_capture_begin(&self) {
547 let stream_id = self.stream_id();
548 self.device
549 .submit_blocking(move |server| server.graph_capture_begin(stream_id))
550 .unwrap()
551 }
552
553 pub fn graph_capture_end(&self) -> u64 {
555 let stream_id = self.stream_id();
556 self.device
557 .submit_blocking(move |server| server.graph_capture_end(stream_id))
558 .unwrap()
559 }
560
561 pub fn graph_replay(&self, id: u64) {
569 let stream_id = self.stream_id();
570 self.device
571 .submit(move |server| server.graph_replay(id, stream_id));
572 }
573
574 pub fn graph_capture_status(&self) -> u32 {
576 let stream_id = self.stream_id();
577 self.device
578 .submit_blocking(move |server| server.graph_capture_status(stream_id))
579 .unwrap()
580 }
581
582 pub fn graph_node_count(&self, id: u64) -> usize {
584 self.device
585 .submit_blocking(move |server| server.graph_node_count(id))
586 .unwrap()
587 }
588
589 pub fn graph_capture_launch_count(&self) -> usize {
591 self.device
592 .submit_blocking(move |server| server.graph_capture_launch_count())
593 .unwrap()
594 }
595
596 pub fn graph_launch_count(&self, id: u64) -> usize {
598 self.device
599 .submit_blocking(move |server| server.graph_launch_count(id))
600 .unwrap()
601 }
602
603 pub fn graph_launch_params(&self, id: u64, idx: usize) -> GraphLaunchParams {
605 self.device
606 .submit_blocking(move |server| server.graph_launch_params(id, idx))
607 .unwrap()
608 }
609
610 pub fn graph_patch_launch(&self, id: u64, idx: usize, patch: GraphLaunchPatch) {
618 self.device
619 .submit_blocking(move |server| server.graph_patch_launch(id, idx, patch))
620 .unwrap()
621 }
622
623 pub fn graph_node_kinds(&self, id: u64) -> Vec<(u32, usize)> {
625 self.device
626 .submit_blocking(move |server| server.graph_node_kinds(id))
627 .unwrap()
628 }
629
630 pub fn graph_patch_launches(&self, id: u64, patches: Vec<(usize, GraphLaunchPatch)>) {
632 self.device
633 .submit_blocking(move |server| server.graph_patch_launches(id, patches))
634 .unwrap()
635 }
636
637 pub fn graph_alloc_regions(&self, id: u64) -> Vec<(u64, u64)> {
639 self.device
640 .submit_blocking(move |server| server.graph_alloc_regions(id))
641 .unwrap()
642 }
643
644 pub fn graph_memcpy_specs(&self, id: u64) -> Vec<(u64, u64, u64, u32)> {
646 self.device
647 .submit_blocking(move |server| server.graph_memcpy_specs(id))
648 .unwrap()
649 }
650
651 pub fn graph_destroy(&self, id: u64) {
653 self.device
654 .submit_blocking(move |server| server.graph_destroy(id))
655 .unwrap()
656 }
657
658 pub unsafe fn register_external_aliased(
669 &self,
670 ptr: *mut core::ffi::c_void,
671 page_len: u64,
672 offset: u64,
673 size: u64,
674 keepalive: alloc::sync::Arc<dyn core::any::Any + Send + Sync>,
675 ) -> Handle {
676 let stream_id = self.stream_id();
677 struct SendPtr(*mut core::ffi::c_void);
680 unsafe impl Send for SendPtr {}
682 let p = SendPtr(ptr);
683 self.device
684 .submit_blocking(move |server| {
685 let p = p;
689 server.register_external_aliased(p.0, page_len, offset, size, keepalive, stream_id)
690 })
691 .unwrap()
692 }
693
694 pub fn empty_tensor(&self, shape: Shape, elem_size: usize) -> MemoryLayout {
697 let descriptor =
698 MemoryLayoutDescriptor::new(MemoryLayoutStrategy::Optimized, shape, elem_size);
699 self.do_empty(vec![descriptor]).unwrap().remove(0)
700 }
701
702 pub fn empty_tensors(&self, descriptors: Vec<MemoryLayoutDescriptor>) -> Vec<MemoryLayout> {
705 self.do_empty(descriptors).unwrap()
706 }
707
708 pub fn staging<'a, I>(&self, bytes: I, file_only: bool)
713 where
714 I: Iterator<Item = &'a mut Bytes>,
715 {
716 let has_staging = |b: &Bytes| match b.property() {
717 AllocationProperty::Pinned => false,
718 AllocationProperty::File => true,
719 AllocationProperty::Native | AllocationProperty::Other => !file_only,
720 };
721
722 let mut to_be_updated = Vec::new();
723 let sizes = bytes
724 .filter_map(|b| match has_staging(b) {
725 true => {
726 let len = b.len();
727 to_be_updated.push(b);
728 Some(len)
729 }
730 false => None,
731 })
732 .collect::<Vec<usize>>();
733
734 if sizes.is_empty() {
735 return;
736 }
737
738 let stream_id = self.stream_id();
739 let sizes = sizes.to_vec();
740 let stagings = self
741 .device
742 .submit_blocking(move |server| server.staging(&sizes, stream_id))
743 .unwrap();
744
745 let stagings = match stagings {
746 Ok(val) => val,
747 Err(_) => return,
748 };
749
750 to_be_updated
751 .into_iter()
752 .zip(stagings)
753 .for_each(|(b, mut staging)| {
754 b.copy_into(&mut staging);
755 core::mem::swap(b, &mut staging);
756 });
757 }
758
759 #[cfg_attr(
761 feature = "tracing",
762 tracing::instrument(level = "trace", skip(self, src, dst_server))
763 )]
764 pub fn to_client(&mut self, src: Handle, dst_server: &Self, dtype: ElemType) -> Handle {
765 let shape = [src.size_in_used() as usize];
766 let src_descriptor = src.copy_descriptor(shape.into(), [1].into(), 1);
767
768 if R::Server::SERVER_COMM_ENABLED {
769 self.to_client_tensor(src_descriptor, dst_server, dtype)
770 } else {
771 let alloc_desc = MemoryLayoutDescriptor::new(
772 MemoryLayoutStrategy::Contiguous,
773 src_descriptor.shape.clone(),
774 src_descriptor.elem_size,
775 );
776 self.change_client_sync(src_descriptor, alloc_desc, dst_server)
777 .memory
778 }
779 }
780
781 #[cfg_attr(
783 feature = "tracing",
784 tracing::instrument(level = "trace", skip(self, device_ids))
785 )]
786 pub fn ensure_init_collective(&mut self, device_ids: Vec<DeviceId>) {
787 let comm_id = CommunicationId::from(device_ids.clone());
788 let is_comms_init = self
789 .utilities
790 .initialized_comms
791 .read()
792 .unwrap()
793 .contains(&comm_id);
794 if !is_comms_init {
795 self.device
796 .submit(move |server| server.comm_init(device_ids).unwrap());
797 let mut initialized_comms = self.utilities.initialized_comms.write().unwrap();
798 initialized_comms.insert(comm_id);
799 self.device.flush_queue();
801 }
802 }
803
804 #[cfg_attr(feature = "tracing", tracing::instrument(level = "trace", skip(self)))]
806 pub fn sync_collective(&self) {
807 if DeviceHandle::<R::Server>::is_blocking() {
808 panic!("Can't use `sync_collective` with a blocking device handle");
809 }
810 let stream_id = self.stream_id();
811
812 self.device.submit(move |server| {
813 server.sync_collective(stream_id).unwrap();
814 });
815
816 self.device.flush_queue();
819 }
820
821 #[cfg_attr(
823 feature = "tracing",
824 tracing::instrument(level = "trace", skip(self, src, dst, dtype, device_ids, op))
825 )]
826 pub fn all_reduce(
827 &mut self,
828 src: Handle,
829 dst: Handle,
830 dtype: ElemType,
831 device_ids: Vec<DeviceId>,
832 op: ReduceOperation,
833 ) {
834 if DeviceHandle::<R::Server>::is_blocking() {
835 panic!("Can't use `all_reduce` with a blocking device handle");
836 }
837
838 let stream_id = self.stream_id();
839 let src = src.binding();
840 let dst = dst.binding();
841
842 self.ensure_init_collective(device_ids.clone());
843
844 self.device.submit(move |server| {
845 server
846 .all_reduce(src, dst, dtype, stream_id, op, device_ids)
847 .unwrap();
848 });
849 }
850
851 #[cfg_attr(
855 feature = "tracing",
856 tracing::instrument(level = "trace", skip(self, src_descriptor, dst_server))
857 )]
858 pub fn to_client_tensor(
859 &mut self,
860 src_descriptor: CopyDescriptor,
861 dst_server: &Self,
862 dtype: ElemType,
863 ) -> Handle {
864 let stream_id_src = self.stream_id();
865 let stream_id_dst = dst_server.stream_id();
866
867 let device_id_src = self.device.device_id();
868 let device_id_dst = dst_server.device.device_id();
869
870 let mut dst_server = dst_server.clone();
871 let handle = Handle::new(stream_id_dst, src_descriptor.handle.size_in_used());
872 let handle_cloned = handle.clone();
873
874 let device_ids = vec![device_id_src, device_id_dst];
875 self.ensure_init_collective(device_ids.clone());
876 dst_server.ensure_init_collective(device_ids);
877
878 self.device.submit(move |server_src| {
879 server_src
880 .send(src_descriptor, dtype, stream_id_src, device_id_dst)
881 .unwrap()
882 });
883
884 dst_server.device.submit(move |server_dst| {
885 server_dst
886 .recv(handle_cloned, dtype, stream_id_dst, device_id_src)
887 .unwrap();
888 server_dst.sync_collective(stream_id_dst).unwrap();
889 });
890
891 self.device.flush_queue();
895 dst_server.device.flush_queue();
896
897 handle
898 }
899
900 #[track_caller]
901 #[cfg_attr(feature = "tracing", tracing::instrument(level="trace",
902 skip(self, kernel, bindings),
903 fields(
904 kernel.name = %kernel.name(),
905 kernel.id = %kernel.id(),
906 )
907 ))]
908 unsafe fn launch_inner(
909 &self,
910 kernel: <R::Server as ComputeServer>::Kernel,
911 count: CubeCount,
912 bindings: KernelArguments,
913 mode: ExecutionMode,
914 stream_id: StreamId,
915 ) {
916 let level = self.utilities.logger.profile_level();
917
918 match level {
919 None | Some(ProfileLevel::ExecutionOnly) => {
920 let utilities = self.utilities.clone();
921 self.device.submit(move |state| {
922 let name = kernel.name();
923 unsafe { state.launch(kernel, count, bindings, mode, stream_id) };
924
925 if matches!(level, Some(ProfileLevel::ExecutionOnly)) {
926 let info = type_name_format(name, TypeNameFormatLevel::Balanced);
927 utilities.logger.register_execution(info);
928 }
929 });
930 }
931 Some(level) => {
932 let name = kernel.name();
933 let kernel_id = kernel.id();
934 let context = self.device.clone();
935 let count_moved = count.clone();
936 let (result, profile) = self
937 .profile(
938 move || {
939 context
940 .submit_blocking(move |state| unsafe {
941 state.launch(kernel, count_moved, bindings, mode, stream_id)
942 })
943 .unwrap()
944 },
945 name,
946 )
947 .unwrap();
948 let info = match level {
949 ProfileLevel::Full => {
950 format!("{name}: {kernel_id} CubeCount {count:?}")
951 }
952 _ => type_name_format(name, TypeNameFormatLevel::Balanced),
953 };
954 self.utilities.logger.register_profiled(info, profile);
955 result
956 }
957 }
958 }
959
960 #[track_caller]
962 pub fn launch(
963 &self,
964 kernel: <R::Server as ComputeServer>::Kernel,
965 count: CubeCount,
966 bindings: KernelArguments,
967 ) {
968 unsafe {
970 self.launch_inner(
971 kernel,
972 count,
973 bindings,
974 ExecutionMode::Checked,
975 self.stream_id(),
976 )
977 }
978 }
979
980 #[track_caller]
988 pub unsafe fn launch_unchecked(
989 &self,
990 kernel: <R::Server as ComputeServer>::Kernel,
991 count: CubeCount,
992 bindings: KernelArguments,
993 ) {
994 unsafe {
996 self.launch_inner(
997 kernel,
998 count,
999 bindings,
1000 match self.utilities.check_mode {
1001 crate::config::compilation::BoundsCheckMode::Enforce => ExecutionMode::Checked,
1002 crate::config::compilation::BoundsCheckMode::Validate => {
1003 ExecutionMode::Validate
1004 }
1005 crate::config::compilation::BoundsCheckMode::Auto => ExecutionMode::Unchecked,
1006 },
1007 self.stream_id(),
1008 )
1009 }
1010 }
1011
1012 pub fn flush(&self) -> Result<(), ServerError> {
1014 let stream_id = self.stream_id();
1015
1016 self.device
1017 .submit_blocking(move |server| server.flush(stream_id))
1018 .unwrap()
1019 }
1020
1021 pub fn sync(&self) -> DynFut<Result<(), ServerError>> {
1023 let stream_id = self.stream_id();
1024
1025 let fut = self
1026 .device
1027 .submit_blocking(move |server| server.sync(stream_id))
1028 .unwrap();
1029
1030 self.utilities.logger.profile_summary();
1031
1032 fut
1033 }
1034
1035 pub fn properties(&self) -> &DeviceProperties {
1037 &self.utilities.properties
1038 }
1039
1040 pub fn features(&self) -> &Features {
1042 &self.utilities.properties.features
1043 }
1044
1045 pub fn properties_mut(&mut self) -> Option<&mut DeviceProperties> {
1049 Arc::get_mut(&mut self.utilities).map(|state| &mut state.properties)
1050 }
1051
1052 pub fn memory_usage(&self) -> Result<MemoryUsage, ServerError> {
1054 let stream_id = self.stream_id();
1055 self.device
1056 .submit_blocking(move |server| server.memory_usage(stream_id))
1057 .unwrap()
1058 }
1059
1060 pub fn enumerate_devices(&self, type_id: u16) -> Vec<DeviceId> {
1062 R::enumerate_devices(type_id, self.info())
1063 }
1064
1065 pub fn enumerate_all_devices(&self) -> Vec<DeviceId> {
1067 R::enumerate_all_devices(self.info())
1068 }
1069
1070 pub fn device_count(&self, type_id: u16) -> usize {
1072 self.enumerate_devices(type_id).len()
1073 }
1074
1075 pub fn device_count_total(&self) -> usize {
1077 self.enumerate_all_devices().len()
1078 }
1079
1080 pub unsafe fn allocation_mode(&self, mode: MemoryAllocationMode) {
1086 let stream_id = self.stream_id();
1087 self.device
1088 .submit(move |server| server.allocation_mode(mode, stream_id));
1089 }
1090
1091 pub fn memory_cleanup(&self) {
1096 let stream_id = self.stream_id();
1097 self.device
1098 .submit(move |server| server.memory_cleanup(stream_id));
1099 }
1100
1101 #[track_caller]
1103 pub fn profile<O: Send + 'static>(
1104 &self,
1105 func: impl FnOnce() -> O + Send,
1106 #[allow(unused)] func_name: &str,
1107 ) -> Result<(O, ProfileDuration), ProfileError> {
1108 #[cfg(feature = "profile-tracy")]
1111 let location = std::panic::Location::caller();
1112
1113 #[cfg(feature = "profile-tracy")]
1115 let _span = tracy_client::Client::running().unwrap().span_alloc(
1116 None,
1117 func_name,
1118 location.file(),
1119 location.line(),
1120 0,
1121 );
1122
1123 let stream_id = self.stream_id();
1124
1125 #[cfg(feature = "profile-tracy")]
1126 let gpu_span = if self.utilities.properties.timing_method == TimingMethod::Device {
1127 let gpu_span = self
1128 .utilities
1129 .gpu_client
1130 .span_alloc(func_name, "profile", location.file(), location.line())
1131 .unwrap();
1132 Some(gpu_span)
1133 } else {
1134 None
1135 };
1136
1137 let device = self.device.clone();
1138 #[allow(unused_mut, reason = "Used in profile-tracy")]
1139 let mut result = self
1140 .device
1141 .exclusive(move || {
1142 let token =
1145 match device.submit_blocking(move |server| server.start_profile(stream_id)) {
1146 Ok(token) => match token {
1147 Ok(token) => token,
1148 Err(err) => return Err(err),
1149 },
1150 Err(err) => {
1151 return Err(ServerError::Generic {
1152 reason: alloc::format!(
1153 "Can't start profiling because of a call error: {err:?}"
1154 ),
1155 backtrace: BackTrace::capture(),
1156 });
1157 }
1158 };
1159
1160 let out = func();
1162
1163 let result = device
1165 .submit_blocking(move |server| {
1166 let mut result = server.end_profile(stream_id, token);
1167
1168 match result {
1169 Ok(result) => Ok((out, result)),
1170 Err(err) => Err(err),
1171 }
1172 })
1173 .unwrap();
1174
1175 Ok(result)
1176 })
1177 .unwrap()
1178 .map_err(|err| ProfileError::Unknown {
1179 reason: alloc::format!("{err:?}"),
1180 backtrace: BackTrace::capture(),
1181 })?;
1182
1183 #[cfg(feature = "profile-tracy")]
1184 if let Some(mut gpu_span) = gpu_span {
1185 gpu_span.end_zone();
1186 let epoch = self.utilities.epoch_time;
1187 result = result.map(|(o, result)| {
1189 (
1190 o,
1191 ProfileDuration::new(
1192 alloc::boxed::Box::pin(async move {
1193 let ticks = result.resolve().await;
1194 let start_duration =
1195 ticks.start_duration_since(epoch).as_nanos() as i64;
1196 let end_duration = ticks.end_duration_since(epoch).as_nanos() as i64;
1197 gpu_span.upload_timestamp_start(start_duration);
1198 gpu_span.upload_timestamp_end(end_duration);
1199 ticks
1200 }),
1201 TimingMethod::Device,
1202 ),
1203 )
1204 });
1205 }
1206
1207 result
1208 }
1209
1210 #[cfg_attr(
1212 feature = "tracing",
1213 tracing::instrument(
1214 level = "trace",
1215 skip(self, src_descriptor, alloc_descriptor, dst_server)
1216 )
1217 )]
1218 fn change_client_sync(
1219 &self,
1220 src_descriptor: CopyDescriptor,
1221 alloc_descriptor: MemoryLayoutDescriptor,
1222 dst_server: &Self,
1223 ) -> MemoryLayout {
1224 let shape = src_descriptor.shape.clone();
1225 let elem_size = src_descriptor.elem_size;
1226 let stream_id = self.stream_id();
1227
1228 let read = self
1229 .device
1230 .submit_blocking(move |server| server.read(vec![src_descriptor], stream_id))
1231 .unwrap();
1232
1233 let mut data = cubecl_common::future::block_on(read).unwrap();
1234
1235 let (handle_base, mut layouts) = self
1236 .utilities
1237 .layout_policy
1238 .apply(stream_id, &[alloc_descriptor]);
1239 let alloc = layouts.remove(0);
1240
1241 let desc_descriptor = CopyDescriptor {
1242 handle: handle_base.clone().binding(),
1243 shape,
1244 strides: alloc.strides.clone(),
1245 elem_size,
1246 };
1247
1248 let (size, memory) = (handle_base.size(), handle_base.memory);
1249 dst_server.device.submit(move |server| {
1250 server.initialize_memory(memory, size, stream_id);
1251 server.write(vec![(desc_descriptor, data.remove(0))], stream_id)
1252 });
1253
1254 alloc
1255 }
1256
1257 pub fn io_optimized_vector_sizes(
1259 &self,
1260 size: usize,
1261 ) -> impl Iterator<Item = VectorSize> + Clone {
1262 let load_width = self.properties().hardware.load_width as usize;
1263 let size_bits = size * 8;
1264 let max = load_width / size_bits;
1265 let max = usize::min(self.properties().hardware.max_vector_size, max);
1266
1267 let num_candidates = max.trailing_zeros() + 1;
1269
1270 (0..num_candidates).map(|i| 2usize.pow(i)).rev()
1271 }
1272}