1use crate::context::{GpuContext, GpuContextConfig};
7use crate::error::{GpuError, GpuResult};
8use std::collections::HashMap;
9use std::sync::atomic::{AtomicUsize, Ordering};
10use std::sync::{Arc, Mutex};
11use tracing::{debug, info, warn};
12use wgpu::{Adapter, AdapterInfo, Backend, Backends, BufferUsages, Instance, PollType};
13
14#[derive(Debug, Clone)]
16pub struct MultiGpuConfig {
17 pub backends: Backends,
19 pub min_devices: usize,
21 pub max_devices: usize,
23 pub auto_load_balance: bool,
25 pub enable_p2p: bool,
27}
28
29impl Default for MultiGpuConfig {
30 fn default() -> Self {
31 Self {
32 backends: Backends::all(),
33 min_devices: 1,
34 max_devices: 8,
35 auto_load_balance: true,
36 enable_p2p: false,
37 }
38 }
39}
40
41#[derive(Debug, Clone)]
43pub struct GpuDeviceInfo {
44 pub index: usize,
46 pub adapter_info: AdapterInfo,
48 pub backend: Backend,
50 pub vram_bytes: Option<u64>,
60 pub active: bool,
62}
63
64impl GpuDeviceInfo {
65 pub fn description(&self) -> String {
67 format!(
68 "GPU {} : {} ({:?})",
69 self.index, self.adapter_info.name, self.backend
70 )
71 }
72}
73
74pub struct MultiGpuManager {
76 devices: Vec<Arc<GpuContext>>,
78 device_info: Vec<GpuDeviceInfo>,
80 config: MultiGpuConfig,
82 load_state: Arc<Mutex<LoadBalanceState>>,
84 round_robin: AtomicUsize,
87}
88
89impl MultiGpuManager {
90 #[cfg(test)]
96 pub(crate) fn new_empty_for_testing() -> Self {
97 Self {
98 devices: Vec::new(),
99 device_info: Vec::new(),
100 config: MultiGpuConfig::default(),
101 load_state: Arc::new(Mutex::new(LoadBalanceState::new(0))),
102 round_robin: AtomicUsize::new(0),
103 }
104 }
105}
106
107#[derive(Debug, Clone)]
108struct LoadBalanceState {
109 task_counts: HashMap<usize, usize>,
111 workload: HashMap<usize, f64>,
113}
114
115impl LoadBalanceState {
116 fn new(num_devices: usize) -> Self {
117 let mut task_counts = HashMap::new();
118 let mut workload = HashMap::new();
119
120 for i in 0..num_devices {
121 task_counts.insert(i, 0);
122 workload.insert(i, 0.0);
123 }
124
125 Self {
126 task_counts,
127 workload,
128 }
129 }
130
131 fn select_device(&self) -> usize {
132 self.workload
134 .iter()
135 .min_by(|a, b| a.1.partial_cmp(b.1).unwrap_or(std::cmp::Ordering::Equal))
136 .map(|(idx, _)| *idx)
137 .unwrap_or(0)
138 }
139
140 fn add_task(&mut self, device: usize, workload: f64) {
141 *self.task_counts.entry(device).or_insert(0) += 1;
142 *self.workload.entry(device).or_insert(0.0) += workload;
143 }
144
145 fn complete_task(&mut self, device: usize, workload: f64) {
146 if let Some(count) = self.task_counts.get_mut(&device) {
147 *count = count.saturating_sub(1);
148 }
149 if let Some(load) = self.workload.get_mut(&device) {
150 *load = load.max(workload) - workload;
151 }
152 }
153}
154
155impl MultiGpuManager {
156 pub async fn new(config: MultiGpuConfig) -> GpuResult<Self> {
162 info!("Initializing multi-GPU manager");
163
164 let instance = Instance::new(wgpu::InstanceDescriptor {
165 backends: config.backends,
166 ..wgpu::InstanceDescriptor::new_without_display_handle()
167 });
168
169 let adapters = Self::enumerate_adapters(&instance).await;
171
172 if adapters.len() < config.min_devices {
173 return Err(GpuError::no_adapter(format!(
174 "Found {} GPUs, but {} required",
175 adapters.len(),
176 config.min_devices
177 )));
178 }
179
180 let num_devices = adapters.len().min(config.max_devices);
181 info!(
182 "Found {} compatible GPUs, using {}",
183 adapters.len(),
184 num_devices
185 );
186
187 let mut devices = Vec::new();
189 let mut device_info = Vec::new();
190
191 for (index, adapter) in adapters.into_iter().take(num_devices).enumerate() {
192 match Self::create_device_context(adapter, index).await {
193 Ok((context, info)) => {
194 devices.push(Arc::new(context));
195 device_info.push(info);
196 info!(
197 "Initialized: {}",
198 device_info
199 .last()
200 .map(|i| i.description())
201 .unwrap_or_default()
202 );
203 }
204 Err(e) => {
205 warn!("Failed to initialize GPU {}: {}", index, e);
206 }
207 }
208 }
209
210 if devices.len() < config.min_devices {
211 return Err(GpuError::device_request(format!(
212 "Successfully initialized {} GPUs, but {} required",
213 devices.len(),
214 config.min_devices
215 )));
216 }
217
218 let load_state = Arc::new(Mutex::new(LoadBalanceState::new(devices.len())));
219
220 Ok(Self {
221 devices,
222 device_info,
223 config,
224 load_state,
225 round_robin: AtomicUsize::new(0),
226 })
227 }
228
229 pub fn num_devices(&self) -> usize {
231 self.devices.len()
232 }
233
234 pub fn device(&self, index: usize) -> Option<&Arc<GpuContext>> {
236 self.devices.get(index)
237 }
238
239 pub fn devices(&self) -> &[Arc<GpuContext>] {
241 &self.devices
242 }
243
244 pub fn device_info(&self, index: usize) -> Option<&GpuDeviceInfo> {
246 self.device_info.get(index)
247 }
248
249 pub fn all_device_info(&self) -> &[GpuDeviceInfo] {
251 &self.device_info
252 }
253
254 pub fn select_device(&self) -> usize {
262 let num_devices = self.devices.len();
263 if num_devices == 0 {
264 return 0;
265 }
266
267 if !self.config.auto_load_balance {
268 return self.next_round_robin(num_devices);
271 }
272
273 self.load_state
274 .lock()
275 .map(|state| state.select_device())
276 .unwrap_or(0)
277 }
278
279 fn next_round_robin(&self, num_devices: usize) -> usize {
283 if num_devices == 0 {
284 return 0;
285 }
286 self.round_robin.fetch_add(1, Ordering::Relaxed) % num_devices
287 }
288
289 pub fn dispatch<F, T>(&self, workload: f64, f: F) -> GpuResult<T>
291 where
292 F: FnOnce(&GpuContext) -> GpuResult<T>,
293 {
294 let device_idx = self.select_device();
295
296 if let Ok(mut state) = self.load_state.lock() {
297 state.add_task(device_idx, workload);
298 }
299
300 let context = self
301 .devices
302 .get(device_idx)
303 .ok_or_else(|| GpuError::internal("Invalid device index"))?;
304
305 let result = f(context);
306
307 if let Ok(mut state) = self.load_state.lock() {
308 state.complete_task(device_idx, workload);
309 }
310
311 result
312 }
313
314 pub async fn distribute<F, T>(&self, items: Vec<(f64, F)>) -> Vec<GpuResult<T>>
316 where
317 F: FnOnce(&GpuContext) -> GpuResult<T> + Send + 'static,
318 T: Send + 'static,
319 {
320 let mut tasks = Vec::new();
321
322 for (workload, work_fn) in items {
323 let device_idx = self.select_device();
324
325 if let Ok(mut state) = self.load_state.lock() {
326 state.add_task(device_idx, workload);
327 }
328
329 let context = match self.devices.get(device_idx) {
330 Some(ctx) => Arc::clone(ctx),
331 None => continue,
332 };
333
334 let load_state = Arc::clone(&self.load_state);
335
336 let task = tokio::spawn(async move {
337 let result = work_fn(&context);
338
339 if let Ok(mut state) = load_state.lock() {
340 state.complete_task(device_idx, workload);
341 }
342
343 result
344 });
345
346 tasks.push(task);
347 }
348
349 let mut results = Vec::new();
351 for task in tasks {
352 match task.await {
353 Ok(result) => results.push(result),
354 Err(e) => results.push(Err(GpuError::internal(e.to_string()))),
355 }
356 }
357
358 results
359 }
360
361 pub fn load_stats(&self) -> HashMap<usize, (usize, f64)> {
363 self.load_state
364 .lock()
365 .map(|state| {
366 let mut stats = HashMap::new();
367 for i in 0..self.devices.len() {
368 let tasks = *state.task_counts.get(&i).unwrap_or(&0);
369 let workload = *state.workload.get(&i).unwrap_or(&0.0);
370 stats.insert(i, (tasks, workload));
371 }
372 stats
373 })
374 .unwrap_or_default()
375 }
376
377 async fn enumerate_adapters(_instance: &Instance) -> Vec<Adapter> {
378 let mut adapters = Vec::new();
379
380 for backend in &[
382 Backends::VULKAN,
383 Backends::METAL,
384 Backends::DX12,
385 Backends::BROWSER_WEBGPU,
386 ] {
387 let backend_instance = Instance::new(wgpu::InstanceDescriptor {
388 backends: *backend,
389 ..wgpu::InstanceDescriptor::new_without_display_handle()
390 });
391
392 if let Ok(adapter) = backend_instance
393 .request_adapter(&wgpu::RequestAdapterOptions {
394 power_preference: wgpu::PowerPreference::HighPerformance,
395 force_fallback_adapter: false,
396 compatible_surface: None,
397 apply_limit_buckets: false,
398 })
399 .await
400 {
401 adapters.push(adapter);
402 }
403 }
404
405 adapters
406 }
407
408 async fn create_device_context(
409 adapter: Adapter,
410 index: usize,
411 ) -> GpuResult<(GpuContext, GpuDeviceInfo)> {
412 let adapter_info = adapter.get_info();
413 let backend = adapter_info.backend;
414
415 let vram_bytes = Self::estimate_vram_coarse(&adapter_info);
418
419 let config = GpuContextConfig::default().with_label(format!("GPU {}", index));
420
421 let context = GpuContext::with_config(config).await?;
422
423 let info = GpuDeviceInfo {
424 index,
425 adapter_info,
426 backend,
427 vram_bytes,
428 active: true,
429 };
430
431 Ok((context, info))
432 }
433
434 fn estimate_vram_coarse(adapter_info: &AdapterInfo) -> Option<u64> {
441 match adapter_info.device_type {
443 wgpu::DeviceType::DiscreteGpu => Some(8 * 1024 * 1024 * 1024), wgpu::DeviceType::IntegratedGpu => Some(2 * 1024 * 1024 * 1024), wgpu::DeviceType::VirtualGpu => Some(4 * 1024 * 1024 * 1024), _ => None,
447 }
448 }
449}
450
451pub struct InterGpuTransfer {
459 manager: Arc<MultiGpuManager>,
460 per_device_buffers: Vec<Arc<Mutex<Option<(Arc<wgpu::Buffer>, u64)>>>>,
463}
464
465impl InterGpuTransfer {
466 pub fn new(manager: Arc<MultiGpuManager>) -> Self {
468 let num_devices = manager.num_devices();
469 let per_device_buffers = (0..num_devices)
470 .map(|_| Arc::new(Mutex::new(None)))
471 .collect();
472 Self {
473 manager,
474 per_device_buffers,
475 }
476 }
477
478 pub fn set_device_buffer(
488 &self,
489 device_idx: usize,
490 buffer: Arc<wgpu::Buffer>,
491 size_bytes: u64,
492 ) -> GpuResult<()> {
493 let slot = self
494 .per_device_buffers
495 .get(device_idx)
496 .ok_or_else(|| GpuError::invalid_buffer("device_idx out of range"))?;
497 *slot
498 .lock()
499 .map_err(|_| GpuError::internal("per_device_buffers lock poisoned"))? =
500 Some((buffer, size_bytes));
501 Ok(())
502 }
503
504 pub async fn copy_buffer(
510 &self,
511 src_device: usize,
512 dst_device: usize,
513 data: &[u8],
514 ) -> GpuResult<()> {
515 let _src_ctx = self
516 .manager
517 .device(src_device)
518 .ok_or_else(|| GpuError::invalid_buffer("Invalid source device"))?;
519
520 let dst_ctx = self
521 .manager
522 .device(dst_device)
523 .ok_or_else(|| GpuError::invalid_buffer("Invalid destination device"))?;
524
525 let dst_buffer = dst_ctx.device().create_buffer(&wgpu::BufferDescriptor {
527 label: Some("Inter-GPU Transfer"),
528 size: data.len() as u64,
529 usage: BufferUsages::COPY_DST | BufferUsages::STORAGE,
530 mapped_at_creation: false,
531 });
532
533 dst_ctx.queue().write_buffer(&dst_buffer, 0, data);
535
536 debug!(
537 "Transferred {} bytes from GPU {} to GPU {}",
538 data.len(),
539 src_device,
540 dst_device
541 );
542
543 Ok(())
544 }
545
546 pub async fn broadcast(&self, data: &[u8]) -> GpuResult<()> {
552 for i in 1..self.manager.num_devices() {
553 self.copy_buffer(0, i, data).await?;
554 }
555
556 Ok(())
557 }
558
559 pub async fn gather(&self, dst_device: usize) -> GpuResult<Vec<Vec<u8>>> {
584 let num_devices = self.per_device_buffers.len();
589
590 if dst_device >= num_devices {
591 return Err(GpuError::invalid_buffer(format!(
592 "dst_device {} is out of range (num_devices = {})",
593 dst_device, num_devices
594 )));
595 }
596
597 let mut results: Vec<Vec<u8>> = Vec::with_capacity(num_devices.saturating_sub(1));
598
599 for src_idx in 0..num_devices {
600 if src_idx == dst_device {
601 continue;
602 }
603
604 let slot = self
608 .per_device_buffers
609 .get(src_idx)
610 .ok_or_else(|| GpuError::internal("per_device_buffers shorter than num_devices"))?;
611
612 let maybe_buffer_info: Option<(Arc<wgpu::Buffer>, u64)> = {
616 let guard = slot
617 .lock()
618 .map_err(|_| GpuError::internal("per_device_buffers lock poisoned"))?;
619 guard.as_ref().map(|(buf, sz)| (Arc::clone(buf), *sz))
620 };
622
623 let (src_buffer, size_bytes) = match maybe_buffer_info {
625 Some(pair) => pair,
626 None => {
627 debug!(
628 "gather: device {} has no registered buffer, returning empty slice",
629 src_idx
630 );
631 results.push(Vec::new());
632 continue;
633 }
634 };
635
636 let ctx = self
641 .manager
642 .device(src_idx)
643 .ok_or_else(|| GpuError::invalid_buffer("source device context missing"))?;
644
645 let wgpu_device = ctx.device();
650 let wgpu_queue = ctx.queue();
651
652 let staging = wgpu_device.create_buffer(&wgpu::BufferDescriptor {
653 label: Some("gather_staging"),
654 size: size_bytes,
655 usage: BufferUsages::MAP_READ | BufferUsages::COPY_DST,
656 mapped_at_creation: false,
657 });
658
659 let mut encoder = wgpu_device.create_command_encoder(&wgpu::CommandEncoderDescriptor {
661 label: Some("gather_copy_encoder"),
662 });
663 encoder.copy_buffer_to_buffer(&src_buffer, 0, &staging, 0, size_bytes);
664 ctx.check_device_lost()?;
665 wgpu_queue.submit(std::iter::once(encoder.finish()));
666
667 wgpu_device
671 .poll(PollType::wait_indefinitely())
672 .map_err(|e| GpuError::execution_failed(format!("device poll error: {e:?}")))?;
673
674 let (tx, rx) =
676 futures::channel::oneshot::channel::<Result<(), wgpu::BufferAsyncError>>();
677 staging.slice(..).map_async(wgpu::MapMode::Read, move |r| {
678 let _ = tx.send(r);
679 });
680
681 wgpu_device
683 .poll(PollType::wait_indefinitely())
684 .map_err(|e| {
685 GpuError::execution_failed(format!("device poll (map) error: {e:?}"))
686 })?;
687
688 rx.await
689 .map_err(|_| GpuError::buffer_mapping("gather: oneshot channel closed"))?
690 .map_err(|e| GpuError::buffer_mapping(format!("gather: map_async failed: {e}")))?;
691
692 let raw_data: Vec<u8> = staging
693 .slice(..)
694 .get_mapped_range()
695 .map_err(|e| {
696 GpuError::buffer_mapping(format!("gather: get_mapped_range failed: {e}"))
697 })?
698 .to_vec();
699 staging.unmap();
700
701 debug!(
702 "gather: read {} bytes from device {} (staging buffer)",
703 raw_data.len(),
704 src_idx
705 );
706
707 results.push(raw_data);
708 }
709
710 Ok(results)
711 }
712
713 pub fn gather_blocking(&self, dst_device: usize) -> GpuResult<Vec<Vec<u8>>> {
723 pollster::block_on(self.gather(dst_device))
724 }
725}
726
727pub struct GpuAffinityManager {
729 affinity_groups: HashMap<usize, Vec<usize>>,
731}
732
733impl GpuAffinityManager {
734 pub fn new() -> Self {
736 Self {
737 affinity_groups: HashMap::new(),
738 }
739 }
740
741 pub fn set_affinity_group(&mut self, group_id: usize, devices: Vec<usize>) {
743 self.affinity_groups.insert(group_id, devices);
744 }
745
746 pub fn get_affinity_group(&self, device: usize) -> Vec<usize> {
748 for (_, devices) in &self.affinity_groups {
749 if devices.contains(&device) {
750 return devices.clone();
751 }
752 }
753 vec![device]
754 }
755
756 pub fn same_affinity(&self, device_a: usize, device_b: usize) -> bool {
758 let group_a = self.get_affinity_group(device_a);
759 group_a.contains(&device_b)
760 }
761
762 pub fn optimal_device(&self, data_device: usize, available: &[usize]) -> Option<usize> {
764 let group = self.get_affinity_group(data_device);
766
767 for device in available {
768 if group.contains(device) {
769 return Some(*device);
770 }
771 }
772
773 available.first().copied()
775 }
776}
777
778impl Default for GpuAffinityManager {
779 fn default() -> Self {
780 Self::new()
781 }
782}
783
784#[derive(Debug, Clone, Copy, PartialEq, Eq)]
786pub enum DistributionStrategy {
787 RoundRobin,
789 LoadBalanced,
791 DataLocal,
793 SingleDevice,
795}
796
797pub struct WorkDistributor {
799 manager: Arc<MultiGpuManager>,
800 strategy: DistributionStrategy,
801 affinity: GpuAffinityManager,
802}
803
804impl WorkDistributor {
805 pub fn new(manager: Arc<MultiGpuManager>, strategy: DistributionStrategy) -> Self {
807 Self {
808 manager,
809 strategy,
810 affinity: GpuAffinityManager::new(),
811 }
812 }
813
814 pub fn set_affinity_group(&mut self, group_id: usize, devices: Vec<usize>) {
816 self.affinity.set_affinity_group(group_id, devices);
817 }
818
819 pub fn distribute_work<T>(&self, items: Vec<T>) -> Vec<(usize, Vec<T>)> {
821 match self.strategy {
822 DistributionStrategy::RoundRobin => self.round_robin(items),
823 DistributionStrategy::LoadBalanced => self.load_balanced(items),
824 DistributionStrategy::DataLocal => self.data_local(items),
825 DistributionStrategy::SingleDevice => self.single_device(items),
826 }
827 }
828
829 fn round_robin<T>(&self, items: Vec<T>) -> Vec<(usize, Vec<T>)> {
830 let num_devices = self.manager.num_devices();
831 if num_devices == 0 {
835 return Vec::new();
836 }
837 let mut device_items: Vec<Vec<T>> = (0..num_devices).map(|_| Vec::new()).collect();
838
839 for (idx, item) in items.into_iter().enumerate() {
840 device_items[idx % num_devices].push(item);
841 }
842
843 device_items
844 .into_iter()
845 .enumerate()
846 .filter(|(_, items)| !items.is_empty())
847 .collect()
848 }
849
850 fn load_balanced<T>(&self, items: Vec<T>) -> Vec<(usize, Vec<T>)> {
851 let stats = self.manager.load_stats();
852 let num_devices = self.manager.num_devices();
853 if num_devices == 0 {
856 return Vec::new();
857 }
858 let items_len = items.len();
859
860 let mut weights: Vec<f64> = (0..num_devices)
862 .map(|i| {
863 let (_, load) = stats.get(&i).unwrap_or(&(0, 0.0));
864 1.0 / (1.0 + load)
865 })
866 .collect();
867
868 let total: f64 = weights.iter().sum();
870 if total > 0.0 {
871 for w in &mut weights {
872 *w /= total;
873 }
874 }
875
876 let mut device_items: Vec<Vec<T>> = (0..num_devices).map(|_| Vec::new()).collect();
878
879 for (idx, item) in items.into_iter().enumerate() {
880 let target = (idx as f64 + 0.5) / items_len as f64;
881 let mut device = 0;
882 let mut cumulative = 0.0;
883
884 for (dev, weight) in weights.iter().enumerate() {
885 cumulative += weight;
886 if cumulative >= target {
887 device = dev;
888 break;
889 }
890 }
891
892 device_items[device].push(item);
893 }
894
895 device_items
896 .into_iter()
897 .enumerate()
898 .filter(|(_, items)| !items.is_empty())
899 .collect()
900 }
901
902 fn data_local<T>(&self, items: Vec<T>) -> Vec<(usize, Vec<T>)> {
903 self.round_robin(items)
906 }
907
908 fn single_device<T>(&self, items: Vec<T>) -> Vec<(usize, Vec<T>)> {
909 vec![(0, items)]
910 }
911}
912
913#[cfg(test)]
914mod tests {
915 use super::*;
916
917 #[test]
918 fn test_multi_gpu_config() {
919 let config = MultiGpuConfig::default();
920 assert_eq!(config.min_devices, 1);
921 assert_eq!(config.max_devices, 8);
922 assert!(config.auto_load_balance);
923 }
924
925 #[test]
926 fn test_load_balance_state() {
927 let mut state = LoadBalanceState::new(3);
928
929 state.add_task(0, 100.0);
930 state.add_task(1, 50.0);
931 state.add_task(2, 75.0);
932
933 assert_eq!(state.select_device(), 1);
935
936 state.complete_task(1, 50.0);
937 assert_eq!(state.select_device(), 1);
938 }
939
940 #[test]
941 fn test_affinity_manager() {
942 let mut manager = GpuAffinityManager::new();
943
944 manager.set_affinity_group(0, vec![0, 1]);
945 manager.set_affinity_group(1, vec![2, 3]);
946
947 assert!(manager.same_affinity(0, 1));
948 assert!(manager.same_affinity(2, 3));
949 assert!(!manager.same_affinity(0, 2));
950
951 let group = manager.get_affinity_group(0);
952 assert_eq!(group, vec![0, 1]);
953 }
954
955 #[test]
956 fn test_work_distributor_zero_devices_no_panic() {
957 let manager = Arc::new(MultiGpuManager::new_empty_for_testing());
961 assert_eq!(manager.num_devices(), 0);
962
963 for strategy in [
964 DistributionStrategy::RoundRobin,
965 DistributionStrategy::LoadBalanced,
966 DistributionStrategy::DataLocal,
967 ] {
968 let distributor = WorkDistributor::new(Arc::clone(&manager), strategy);
969 let work: Vec<u32> = (0..10).collect();
970 let distribution = distributor.distribute_work(work);
971 assert!(
972 distribution.is_empty(),
973 "strategy {strategy:?} must return an empty distribution with zero devices"
974 );
975 }
976 }
977
978 #[test]
979 fn test_round_robin_cycles_through_all_devices() {
980 let manager = MultiGpuManager::new_empty_for_testing();
983 let num_devices = 3;
984
985 let seq: Vec<usize> = (0..7)
986 .map(|_| manager.next_round_robin(num_devices))
987 .collect();
988 assert_eq!(seq, vec![0, 1, 2, 0, 1, 2, 0]);
989 }
990
991 #[test]
992 fn test_round_robin_zero_devices_returns_zero() {
993 let manager = MultiGpuManager::new_empty_for_testing();
994 assert_eq!(manager.next_round_robin(0), 0);
995 assert_eq!(manager.next_round_robin(0), 0);
996 }
997
998 #[test]
999 fn test_distribution_strategy() {
1000 assert_eq!(
1001 DistributionStrategy::RoundRobin,
1002 DistributionStrategy::RoundRobin
1003 );
1004 }
1005
1006 fn zero_device_transfer() -> InterGpuTransfer {
1022 InterGpuTransfer {
1023 manager: Arc::new(MultiGpuManager::new_empty_for_testing()),
1024 per_device_buffers: Vec::new(),
1025 }
1026 }
1027
1028 fn n_slot_transfer(n: usize) -> InterGpuTransfer {
1033 let per_device_buffers = (0..n).map(|_| Arc::new(Mutex::new(None))).collect();
1034 InterGpuTransfer {
1035 manager: Arc::new(MultiGpuManager::new_empty_for_testing()),
1036 per_device_buffers,
1037 }
1038 }
1039
1040 #[test]
1044 fn test_set_device_buffer_out_of_range_returns_error() {
1045 let transfer = zero_device_transfer();
1046 let result = transfer.per_device_buffers.get(0);
1055 assert!(result.is_none(), "zero-device transfer must have no slots");
1056 }
1062
1063 #[test]
1066 fn test_gather_blocking_available() {
1067 let transfer = zero_device_transfer();
1070 let result = transfer.gather_blocking(0);
1071 assert!(
1072 result.is_err(),
1073 "gather_blocking with 0 devices must error on any dst_device"
1074 );
1075 }
1076
1077 #[test]
1080 fn test_gather_no_source_devices_returns_empty() {
1081 let transfer = n_slot_transfer(1);
1085 let result = pollster::block_on(transfer.gather(0));
1086 let gathered = result.expect("gather with single-device stub must succeed");
1087 assert!(
1088 gathered.is_empty(),
1089 "no source devices → gather result must be empty"
1090 );
1091 }
1092
1093 #[test]
1095 fn test_gather_invalid_dst_device_returns_error() {
1096 let transfer = n_slot_transfer(1);
1098 let result = pollster::block_on(transfer.gather(1));
1099 assert!(
1100 result.is_err(),
1101 "dst_device=1 when num_devices=1 must be an error"
1102 );
1103 }
1104
1105 #[ignore]
1110 #[tokio::test]
1111 async fn test_gather_real_readback_returns_nonzero_bytes() {
1112 let config = MultiGpuConfig {
1115 min_devices: 2,
1116 max_devices: 8,
1117 ..MultiGpuConfig::default()
1118 };
1119 let manager = match MultiGpuManager::new(config).await {
1120 Ok(m) => Arc::new(m),
1121 Err(_) => {
1122 return;
1124 }
1125 };
1126
1127 let transfer = InterGpuTransfer::new(Arc::clone(&manager));
1128
1129 let src_ctx = manager
1132 .device(1)
1133 .expect("device 1 must exist when num_devices >= 2");
1134
1135 const PAYLOAD: &[u8] = b"OxiGeo-gather-test-payload-12345678";
1136 let payload_size = PAYLOAD.len() as u64;
1137 let aligned = ((payload_size + 3) / 4) * 4;
1139
1140 let src_buffer = Arc::new(src_ctx.device().create_buffer(&wgpu::BufferDescriptor {
1141 label: Some("test_src_buffer"),
1142 size: aligned,
1143 usage: BufferUsages::COPY_SRC | BufferUsages::COPY_DST | BufferUsages::STORAGE,
1144 mapped_at_creation: false,
1145 }));
1146
1147 src_ctx.queue().write_buffer(&src_buffer, 0, PAYLOAD);
1149
1150 transfer
1152 .set_device_buffer(1, Arc::clone(&src_buffer), aligned)
1153 .expect("set_device_buffer must succeed for device 1");
1154
1155 let gathered = transfer
1157 .gather(0)
1158 .await
1159 .expect("gather must succeed when GPU is available");
1160
1161 assert_eq!(
1162 gathered.len(),
1163 1,
1164 "should have exactly 1 result (device 1 → device 0)"
1165 );
1166 assert!(!gathered[0].is_empty(), "gathered bytes must not be empty");
1167 assert_eq!(
1169 &gathered[0][..PAYLOAD.len()],
1170 PAYLOAD,
1171 "gathered payload must match the uploaded data"
1172 );
1173 }
1174}