sklears_utils/

gpu_computing.rs

//! GPU computing integration utilities
//!
//! This module provides utilities for GPU computing integration including device detection,
//! memory management, kernel execution, and performance optimization for ML workloads.

use std::collections::HashMap;
use std::sync::{Arc, RwLock};
use std::time::Instant;

/// GPU device information
#[derive(Debug, Clone)]
pub struct GpuDevice {
    pub id: u32,
    pub name: String,
    pub memory_total: u64,
    pub memory_available: u64,
    pub compute_capability: (u32, u32),
    pub cores: u32,
    pub clock_rate: u32,
    pub memory_bandwidth: u64,
    pub is_integrated: bool,
}

/// GPU memory allocation tracking
#[derive(Debug, Clone)]
pub struct GpuMemoryAllocation {
    pub ptr: u64,
    pub size: u64,
    pub device_id: u32,
    pub allocated_at: Instant,
    pub name: String,
}

/// GPU kernel execution info
#[derive(Debug, Clone)]
pub struct GpuKernelExecution {
    pub kernel_name: String,
    pub device_id: u32,
    pub grid_size: (u32, u32, u32),
    pub block_size: (u32, u32, u32),
    pub shared_memory: u32,
    pub execution_time: f64,
    pub parameters: HashMap<String, String>,
}

/// GPU computing utilities
#[derive(Debug)]
pub struct GpuUtils {
    devices: Vec<GpuDevice>,
    allocations: Arc<RwLock<HashMap<u64, GpuMemoryAllocation>>>,
    kernel_executions: Arc<RwLock<Vec<GpuKernelExecution>>>,
    performance_counters: Arc<RwLock<HashMap<String, f64>>>,
}

impl GpuUtils {
    /// Create new GPU utilities
    pub fn new() -> Self {
        Self {
            devices: Vec::new(),
            allocations: Arc::new(RwLock::new(HashMap::new())),
            kernel_executions: Arc::new(RwLock::new(Vec::new())),
            performance_counters: Arc::new(RwLock::new(HashMap::new())),
        }
    }

    /// Initialize GPU devices
    pub fn init_devices(&mut self) -> Result<(), GpuError> {
        // Mock device initialization (in real implementation, this would use CUDA/OpenCL)
        let mock_devices = vec![
            GpuDevice {
                id: 0,
                name: "NVIDIA GeForce RTX 3080".to_string(),
                memory_total: 10_737_418_240,    // 10 GB
                memory_available: 9_663_676_416, // 9 GB
                compute_capability: (8, 6),
                cores: 8704,
                clock_rate: 1710,
                memory_bandwidth: 760_000_000_000, // 760 GB/s
                is_integrated: false,
            },
            GpuDevice {
                id: 1,
                name: "Intel UHD Graphics 770".to_string(),
                memory_total: 2_147_483_648,     // 2 GB
                memory_available: 1_610_612_736, // 1.5 GB
                compute_capability: (0, 0),
                cores: 256,
                clock_rate: 1550,
                memory_bandwidth: 68_000_000_000, // 68 GB/s
                is_integrated: true,
            },
        ];

        self.devices = mock_devices;
        Ok(())
    }

    /// Get available GPU devices
    pub fn get_devices(&self) -> &[GpuDevice] {
        &self.devices
    }

    /// Get device by ID
    pub fn get_device(&self, id: u32) -> Option<&GpuDevice> {
        self.devices.iter().find(|d| d.id == id)
    }

    /// Get best device for ML workloads
    pub fn get_best_device(&self) -> Option<&GpuDevice> {
        self.devices
            .iter()
            .filter(|d| !d.is_integrated)
            .max_by_key(|d| d.cores * d.clock_rate)
            .or_else(|| self.devices.first())
    }

    /// Allocate GPU memory
    pub fn allocate_memory(&self, size: u64, device_id: u32, name: &str) -> Result<u64, GpuError> {
        let device = self.get_device(device_id).ok_or(GpuError::DeviceNotFound)?;

        if size > device.memory_available {
            return Err(GpuError::OutOfMemory);
        }

        // Mock allocation (in real implementation, this would use CUDA/OpenCL)
        let ptr = (std::ptr::null::<u8>() as u64) + size; // Mock pointer
        let allocation = GpuMemoryAllocation {
            ptr,
            size,
            device_id,
            allocated_at: Instant::now(),
            name: name.to_string(),
        };

        self.allocations
            .write()
            .expect("operation should succeed")
            .insert(ptr, allocation);
        Ok(ptr)
    }

    /// Free GPU memory
    pub fn free_memory(&self, ptr: u64) -> Result<(), GpuError> {
        let mut allocations = self.allocations.write().expect("operation should succeed");
        allocations.remove(&ptr).ok_or(GpuError::InvalidPointer)?;
        Ok(())
    }

    /// Get memory usage statistics
    pub fn get_memory_stats(&self) -> HashMap<u32, MemoryStats> {
        let allocations = self.allocations.read().expect("operation should succeed");
        let mut stats = HashMap::new();

        for device in &self.devices {
            let device_allocations: Vec<_> = allocations
                .values()
                .filter(|a| a.device_id == device.id)
                .collect();

            let total_allocated = device_allocations.iter().map(|a| a.size).sum();
            let num_allocations = device_allocations.len();

            stats.insert(
                device.id,
                MemoryStats {
                    total_memory: device.memory_total,
                    available_memory: device.memory_available,
                    allocated_memory: total_allocated,
                    free_memory: device.memory_available - total_allocated,
                    num_allocations,
                    largest_allocation: device_allocations
                        .iter()
                        .map(|a| a.size)
                        .max()
                        .unwrap_or(0),
                    fragmentation_ratio: if num_allocations > 0 {
                        (num_allocations as f64) / (total_allocated as f64 / 1024.0)
                    } else {
                        0.0
                    },
                },
            );
        }

        stats
    }

    /// Execute GPU kernel
    pub fn execute_kernel(&self, kernel: &GpuKernelInfo) -> Result<GpuKernelExecution, GpuError> {
        let _device = self
            .get_device(kernel.device_id)
            .ok_or(GpuError::DeviceNotFound)?;

        let start_time = Instant::now();

        // Mock kernel execution (in real implementation, this would use CUDA/OpenCL)
        std::thread::sleep(std::time::Duration::from_millis(1));

        let execution_time = start_time.elapsed().as_secs_f64() * 1000.0; // ms

        let execution = GpuKernelExecution {
            kernel_name: kernel.name.clone(),
            device_id: kernel.device_id,
            grid_size: kernel.grid_size,
            block_size: kernel.block_size,
            shared_memory: kernel.shared_memory,
            execution_time,
            parameters: kernel.parameters.clone(),
        };

        self.kernel_executions
            .write()
            .expect("operation should succeed")
            .push(execution.clone());
        Ok(execution)
    }

    /// Get kernel execution history
    pub fn get_kernel_history(&self) -> Vec<GpuKernelExecution> {
        self.kernel_executions
            .read()
            .expect("operation should succeed")
            .clone()
    }

    /// Get performance counters
    pub fn get_performance_counters(&self) -> HashMap<String, f64> {
        self.performance_counters
            .read()
            .expect("operation should succeed")
            .clone()
    }

    /// Update performance counter
    pub fn update_counter(&self, name: &str, value: f64) {
        self.performance_counters
            .write()
            .expect("operation should succeed")
            .insert(name.to_string(), value);
    }

    /// Get throughput estimate for array operations
    pub fn estimate_throughput(&self, device_id: u32, array_size: usize, operation: &str) -> f64 {
        let device = match self.get_device(device_id) {
            Some(d) => d,
            None => return 0.0,
        };

        let base_throughput = match operation {
            "add" | "subtract" | "multiply" => device.memory_bandwidth as f64 * 0.8,
            "divide" | "sqrt" | "exp" | "log" => device.memory_bandwidth as f64 * 0.6,
            "matrix_multiply" => (device.cores as f64 * device.clock_rate as f64 * 1e6) * 0.5,
            "fft" => (device.cores as f64 * device.clock_rate as f64 * 1e6) * 0.3,
            _ => device.memory_bandwidth as f64 * 0.5,
        };

        let array_factor = (array_size as f64).log2() / 20.0; // Efficiency decreases with size
        base_throughput * (1.0 - array_factor.min(0.5))
    }

    /// Check if operation should use GPU
    pub fn should_use_gpu(&self, array_size: usize, operation: &str) -> bool {
        if self.devices.is_empty() {
            return false;
        }

        let min_size = match operation {
            "add" | "subtract" | "multiply" | "divide" => 1000,
            "matrix_multiply" => 100,
            "fft" | "conv" => 512,
            _ => 1000,
        };

        array_size >= min_size
    }

    /// Get GPU utilization
    pub fn get_utilization(&self) -> HashMap<u32, f64> {
        let mut utilization = HashMap::new();

        for device in &self.devices {
            // Mock utilization calculation
            let recent_executions = self
                .kernel_executions
                .read()
                .expect("operation should succeed")
                .iter()
                .filter(|e| e.device_id == device.id)
                .filter(|e| e.execution_time > 0.0)
                .count();

            let util = (recent_executions as f64 / 10.0).min(1.0);
            utilization.insert(device.id, util);
        }

        utilization
    }

    /// Cleanup all resources
    pub fn cleanup(&self) -> Result<(), GpuError> {
        let allocations = self.allocations.read().expect("operation should succeed");
        if !allocations.is_empty() {
            return Err(GpuError::ResourcesNotFreed);
        }

        // Clear history
        self.kernel_executions
            .write()
            .expect("operation should succeed")
            .clear();
        self.performance_counters
            .write()
            .expect("operation should succeed")
            .clear();

        Ok(())
    }
}

/// GPU kernel execution information
#[derive(Debug, Clone)]
pub struct GpuKernelInfo {
    pub name: String,
    pub device_id: u32,
    pub grid_size: (u32, u32, u32),
    pub block_size: (u32, u32, u32),
    pub shared_memory: u32,
    pub parameters: HashMap<String, String>,
}

/// GPU memory usage statistics
#[derive(Debug, Clone)]
pub struct MemoryStats {
    pub total_memory: u64,
    pub available_memory: u64,
    pub allocated_memory: u64,
    pub free_memory: u64,
    pub num_allocations: usize,
    pub largest_allocation: u64,
    pub fragmentation_ratio: f64,
}

/// GPU array operations
pub struct GpuArrayOps;

impl GpuArrayOps {
    /// Add two arrays on GPU
    pub fn add_arrays(a: &[f32], b: &[f32], _device_id: u32) -> Result<Vec<f32>, GpuError> {
        if a.len() != b.len() {
            return Err(GpuError::ShapeMismatch);
        }

        // Mock GPU computation
        let result: Vec<f32> = a.iter().zip(b.iter()).map(|(x, y)| x + y).collect();
        Ok(result)
    }

    /// Multiply two arrays on GPU
    pub fn multiply_arrays(a: &[f32], b: &[f32], _device_id: u32) -> Result<Vec<f32>, GpuError> {
        if a.len() != b.len() {
            return Err(GpuError::ShapeMismatch);
        }

        // Mock GPU computation
        let result: Vec<f32> = a.iter().zip(b.iter()).map(|(x, y)| x * y).collect();
        Ok(result)
    }

    /// Matrix multiplication on GPU
    pub fn matrix_multiply(
        a: &[f32],
        b: &[f32],
        m: usize,
        n: usize,
        k: usize,
        _device_id: u32,
    ) -> Result<Vec<f32>, GpuError> {
        if a.len() != m * k || b.len() != k * n {
            return Err(GpuError::ShapeMismatch);
        }

        // Mock GPU computation
        let mut result = vec![0.0f32; m * n];

        for i in 0..m {
            for j in 0..n {
                for l in 0..k {
                    result[i * n + j] += a[i * k + l] * b[l * n + j];
                }
            }
        }

        Ok(result)
    }

    /// Apply activation function on GPU
    pub fn apply_activation(
        input: &[f32],
        activation: ActivationFunction,
        _device_id: u32,
    ) -> Result<Vec<f32>, GpuError> {
        // Mock GPU computation
        let result: Vec<f32> = input
            .iter()
            .map(|&x| {
                match activation {
                    ActivationFunction::ReLU => x.max(0.0),
                    ActivationFunction::Sigmoid => 1.0 / (1.0 + (-x).exp()),
                    ActivationFunction::Tanh => x.tanh(),
                    ActivationFunction::Softmax => x.exp(), // Simplified, would need proper normalization
                }
            })
            .collect();

        Ok(result)
    }

    /// Compute reduction on GPU
    pub fn reduce_sum(input: &[f32], _device_id: u32) -> Result<f32, GpuError> {
        // Mock GPU computation
        Ok(input.iter().sum())
    }

    /// Compute reduction max on GPU
    pub fn reduce_max(input: &[f32], _device_id: u32) -> Result<f32, GpuError> {
        // Mock GPU computation
        input
            .iter()
            .fold(f32::NEG_INFINITY, |a, &b| a.max(b))
            .is_finite()
            .then_some(input.iter().fold(f32::NEG_INFINITY, |a, &b| a.max(b)))
            .ok_or(GpuError::ComputationError)
    }
}

/// GPU activation functions
#[derive(Debug, Clone, Copy)]
pub enum ActivationFunction {
    ReLU,
    Sigmoid,
    Tanh,
    Softmax,
}

/// GPU computing errors
#[derive(Debug, thiserror::Error)]
pub enum GpuError {
    #[error("GPU device not found")]
    DeviceNotFound,
    #[error("Out of GPU memory")]
    OutOfMemory,
    #[error("Invalid GPU pointer")]
    InvalidPointer,
    #[error("GPU computation error")]
    ComputationError,
    #[error("Array shape mismatch")]
    ShapeMismatch,
    #[error("GPU resources not freed")]
    ResourcesNotFreed,
    #[error("GPU initialization failed: {0}")]
    InitializationFailed(String),
}

/// GPU performance profiler
#[derive(Debug)]
pub struct GpuProfiler {
    kernel_times: HashMap<String, Vec<f64>>,
    memory_transfers: Vec<(Instant, u64, String)>,
    device_utilization: HashMap<u32, Vec<(Instant, f64)>>,
}

impl GpuProfiler {
    /// Create new GPU profiler
    pub fn new() -> Self {
        Self {
            kernel_times: HashMap::new(),
            memory_transfers: Vec::new(),
            device_utilization: HashMap::new(),
        }
    }

    /// Record kernel execution time
    pub fn record_kernel_time(&mut self, kernel_name: &str, time_ms: f64) {
        self.kernel_times
            .entry(kernel_name.to_string())
            .or_default()
            .push(time_ms);
    }

    /// Record memory transfer
    pub fn record_memory_transfer(&mut self, size: u64, direction: &str) {
        self.memory_transfers
            .push((Instant::now(), size, direction.to_string()));
    }

    /// Record device utilization
    pub fn record_utilization(&mut self, device_id: u32, utilization: f64) {
        self.device_utilization
            .entry(device_id)
            .or_default()
            .push((Instant::now(), utilization));
    }

    /// Get kernel statistics
    pub fn get_kernel_stats(&self) -> HashMap<String, KernelStats> {
        let mut stats = HashMap::new();

        for (kernel_name, times) in &self.kernel_times {
            let count = times.len();
            let total_time: f64 = times.iter().sum();
            let avg_time = total_time / count as f64;
            let min_time = times.iter().fold(f64::INFINITY, |a, &b| a.min(b));
            let max_time = times.iter().fold(f64::NEG_INFINITY, |a, &b| a.max(b));

            stats.insert(
                kernel_name.clone(),
                KernelStats {
                    count,
                    total_time,
                    avg_time,
                    min_time,
                    max_time,
                },
            );
        }

        stats
    }

    /// Get memory transfer statistics
    pub fn get_memory_transfer_stats(&self) -> MemoryTransferStats {
        let total_transfers = self.memory_transfers.len();
        let total_bytes: u64 = self.memory_transfers.iter().map(|(_, size, _)| size).sum();

        let host_to_device = self
            .memory_transfers
            .iter()
            .filter(|(_, _, dir)| dir == "host_to_device")
            .count();

        let device_to_host = self
            .memory_transfers
            .iter()
            .filter(|(_, _, dir)| dir == "device_to_host")
            .count();

        MemoryTransferStats {
            total_transfers,
            total_bytes,
            host_to_device_transfers: host_to_device,
            device_to_host_transfers: device_to_host,
        }
    }

    /// Clear all profiling data
    pub fn clear(&mut self) {
        self.kernel_times.clear();
        self.memory_transfers.clear();
        self.device_utilization.clear();
    }
}

/// Kernel execution statistics
#[derive(Debug, Clone)]
pub struct KernelStats {
    pub count: usize,
    pub total_time: f64,
    pub avg_time: f64,
    pub min_time: f64,
    pub max_time: f64,
}

/// Memory transfer statistics
#[derive(Debug, Clone)]
pub struct MemoryTransferStats {
    pub total_transfers: usize,
    pub total_bytes: u64,
    pub host_to_device_transfers: usize,
    pub device_to_host_transfers: usize,
}

impl Default for GpuUtils {
    fn default() -> Self {
        Self::new()
    }
}

impl Default for GpuProfiler {
    fn default() -> Self {
        Self::new()
    }
}

/// Multi-GPU coordinator for distributed computing
pub struct MultiGpuCoordinator {
    gpus: HashMap<u32, GpuUtils>,
    load_balancer: LoadBalancer,
    #[allow(dead_code)]
    communication_topology: CommunicationTopology,
    #[allow(dead_code)]
    synchronization_barriers: Vec<SynchronizationBarrier>,
}

impl Default for MultiGpuCoordinator {
    fn default() -> Self {
        Self::new()
    }
}

impl MultiGpuCoordinator {
    /// Create new multi-GPU coordinator
    pub fn new() -> Self {
        Self {
            gpus: HashMap::new(),
            load_balancer: LoadBalancer::new(),
            communication_topology: CommunicationTopology::Ring,
            synchronization_barriers: Vec::new(),
        }
    }

    /// Initialize all available GPUs
    pub fn init_all_gpus(&mut self) -> Result<(), GpuError> {
        for gpu_id in 0..8 {
            // Check up to 8 GPUs
            let mut gpu = GpuUtils::new();
            if gpu.init_devices().is_ok() && !gpu.devices.is_empty() {
                self.gpus.insert(gpu_id, gpu);
            }
        }

        if self.gpus.is_empty() {
            return Err(GpuError::InitializationFailed("No GPUs found".to_string()));
        }

        Ok(())
    }

    /// Get optimal GPU assignment for workload
    pub fn get_optimal_assignment(&self, workload: &DistributedWorkload) -> Vec<GpuAssignment> {
        self.load_balancer.assign_workload(workload, &self.gpus)
    }

    /// Execute distributed operation across multiple GPUs
    pub fn execute_distributed(
        &self,
        operation: &DistributedOperation,
    ) -> Result<DistributedResult, GpuError> {
        let assignments = self.get_optimal_assignment(&operation.workload);
        let mut results = Vec::new();

        // Execute on each GPU
        for assignment in assignments {
            let gpu = self
                .gpus
                .get(&assignment.gpu_id)
                .ok_or(GpuError::DeviceNotFound)?;

            let kernel_info = GpuKernelInfo {
                name: operation.kernel_name.clone(),
                device_id: assignment.gpu_id,
                grid_size: assignment.grid_size,
                block_size: assignment.block_size,
                shared_memory: assignment.shared_memory,
                parameters: assignment.parameters.clone(),
            };

            let execution = gpu.execute_kernel(&kernel_info)?;
            results.push(execution);
        }

        // Aggregate results
        let total_time: f64 = results.iter().map(|e| e.execution_time).sum();
        Ok(DistributedResult {
            executions: results,
            total_time,
            communication_overhead: 0.0, // Mock value
        })
    }

    /// Synchronize all GPUs
    pub fn synchronize_all(&self) -> Result<(), GpuError> {
        // Mock synchronization
        std::thread::sleep(std::time::Duration::from_millis(1));
        Ok(())
    }

    /// Get cluster-wide memory statistics
    pub fn get_cluster_memory_stats(&self) -> ClusterMemoryStats {
        let mut total_memory = 0;
        let mut total_allocated = 0;
        let mut device_stats = HashMap::new();

        for (gpu_id, gpu) in &self.gpus {
            let stats = gpu.get_memory_stats();
            if let Some(stat) = stats.get(gpu_id) {
                total_memory += stat.total_memory;
                total_allocated += stat.allocated_memory;
                device_stats.insert(*gpu_id, stat.clone());
            }
        }

        ClusterMemoryStats {
            total_memory,
            total_allocated,
            total_free: total_memory - total_allocated,
            num_devices: self.gpus.len(),
            device_stats,
        }
    }
}

/// GPU memory pool for efficient allocation
pub struct GpuMemoryPool {
    pools: HashMap<u32, Vec<MemoryBlock>>,
    #[allow(dead_code)]
    allocation_strategy: AllocationStrategy,
    #[allow(dead_code)]
    fragmentation_threshold: f64,
}

impl GpuMemoryPool {
    /// Create new memory pool
    pub fn new(strategy: AllocationStrategy) -> Self {
        Self {
            pools: HashMap::new(),
            allocation_strategy: strategy,
            fragmentation_threshold: 0.3,
        }
    }

    /// Allocate memory from pool
    pub fn allocate(&mut self, size: u64, device_id: u32) -> Result<u64, GpuError> {
        // First, try to find a suitable block
        let pool = self.pools.entry(device_id).or_default();

        // Find suitable block
        for (i, block) in pool.iter().enumerate() {
            if !block.is_allocated && block.size >= size {
                // Split block if too large
                if block.size > size * 2 {
                    let new_block = MemoryBlock {
                        ptr: block.ptr + size,
                        size: block.size - size,
                        is_allocated: false,
                        allocation_time: None,
                    };
                    pool.push(new_block);

                    pool[i].size = size;
                }

                pool[i].is_allocated = true;
                pool[i].allocation_time = Some(Instant::now());
                return Ok(pool[i].ptr);
            }
        }

        // No suitable block found, allocate new one
        let ptr = self.allocate_new_block(size, device_id)?;

        // Add to pool
        let pool = self.pools.entry(device_id).or_default();
        pool.push(MemoryBlock {
            ptr,
            size,
            is_allocated: true,
            allocation_time: Some(Instant::now()),
        });

        Ok(ptr)
    }

    /// Free memory back to pool
    pub fn free(&mut self, ptr: u64, device_id: u32) -> Result<(), GpuError> {
        let pool = self
            .pools
            .get_mut(&device_id)
            .ok_or(GpuError::DeviceNotFound)?;

        for block in pool.iter_mut() {
            if block.ptr == ptr {
                block.is_allocated = false;
                block.allocation_time = None;
                self.try_merge_blocks(device_id);
                return Ok(());
            }
        }

        Err(GpuError::InvalidPointer)
    }

    /// Defragment memory pool
    pub fn defragment(&mut self, device_id: u32) -> Result<DefragmentationResult, GpuError> {
        let before_fragmentation = self.calculate_fragmentation(device_id);

        let pool = self
            .pools
            .get_mut(&device_id)
            .ok_or(GpuError::DeviceNotFound)?;
        let before_blocks = pool.len();

        // Sort blocks by address
        pool.sort_by_key(|b| b.ptr);

        // Merge adjacent free blocks
        let mut i = 0;
        while i < pool.len() - 1 {
            if !pool[i].is_allocated
                && !pool[i + 1].is_allocated
                && pool[i].ptr + pool[i].size == pool[i + 1].ptr
            {
                pool[i].size += pool[i + 1].size;
                pool.remove(i + 1);
            } else {
                i += 1;
            }
        }

        let after_blocks = pool.len();
        let after_fragmentation = self.calculate_fragmentation(device_id);

        Ok(DefragmentationResult {
            blocks_before: before_blocks,
            blocks_after: after_blocks,
            fragmentation_before: before_fragmentation,
            fragmentation_after: after_fragmentation,
        })
    }

    fn allocate_new_block(&self, size: u64, _device_id: u32) -> Result<u64, GpuError> {
        // Mock allocation
        let ptr = (std::ptr::null::<u8>() as u64) + size;
        Ok(ptr)
    }

    fn try_merge_blocks(&mut self, device_id: u32) {
        if let Some(pool) = self.pools.get_mut(&device_id) {
            pool.sort_by_key(|b| b.ptr);

            let mut i = 0;
            while i < pool.len() - 1 {
                if !pool[i].is_allocated
                    && !pool[i + 1].is_allocated
                    && pool[i].ptr + pool[i].size == pool[i + 1].ptr
                {
                    pool[i].size += pool[i + 1].size;
                    pool.remove(i + 1);
                } else {
                    i += 1;
                }
            }
        }
    }

    fn calculate_fragmentation(&self, device_id: u32) -> f64 {
        let empty_pool = Vec::new();
        let pool = self.pools.get(&device_id).unwrap_or(&empty_pool);
        let free_blocks = pool.iter().filter(|b| !b.is_allocated).count();
        let total_blocks = pool.len();

        if total_blocks == 0 {
            0.0
        } else {
            free_blocks as f64 / total_blocks as f64
        }
    }
}

/// Asynchronous GPU operations
pub struct AsyncGpuOps {
    streams: HashMap<u32, Vec<GpuStream>>,
    pending_operations: Vec<AsyncOperation>,
}

impl Default for AsyncGpuOps {
    fn default() -> Self {
        Self::new()
    }
}

impl AsyncGpuOps {
    /// Create new async GPU operations manager
    pub fn new() -> Self {
        Self {
            streams: HashMap::new(),
            pending_operations: Vec::new(),
        }
    }

    /// Create new GPU stream
    pub fn create_stream(&mut self, device_id: u32) -> Result<u32, GpuError> {
        let stream_id = self.streams.get(&device_id).map_or(0, |s| s.len() as u32);
        let stream = GpuStream {
            id: stream_id,
            device_id,
            is_busy: false,
            priority: StreamPriority::Normal,
        };

        self.streams.entry(device_id).or_default().push(stream);
        Ok(stream_id)
    }

    /// Launch asynchronous kernel
    pub fn launch_kernel_async(
        &mut self,
        kernel: &GpuKernelInfo,
        stream_id: u32,
    ) -> Result<AsyncOperationHandle, GpuError> {
        let operation = AsyncOperation {
            id: self.pending_operations.len() as u32,
            kernel_info: kernel.clone(),
            stream_id,
            start_time: Instant::now(),
            status: OperationStatus::Pending,
        };

        let handle = AsyncOperationHandle {
            operation_id: operation.id,
            device_id: kernel.device_id,
        };

        self.pending_operations.push(operation);
        Ok(handle)
    }

    /// Wait for operation completion
    pub fn wait_for_completion(
        &mut self,
        handle: &AsyncOperationHandle,
    ) -> Result<GpuKernelExecution, GpuError> {
        // Mock completion
        std::thread::sleep(std::time::Duration::from_millis(1));

        if let Some(op) = self
            .pending_operations
            .iter_mut()
            .find(|op| op.id == handle.operation_id)
        {
            op.status = OperationStatus::Completed;

            Ok(GpuKernelExecution {
                kernel_name: op.kernel_info.name.clone(),
                device_id: op.kernel_info.device_id,
                grid_size: op.kernel_info.grid_size,
                block_size: op.kernel_info.block_size,
                shared_memory: op.kernel_info.shared_memory,
                execution_time: op.start_time.elapsed().as_secs_f64() * 1000.0,
                parameters: op.kernel_info.parameters.clone(),
            })
        } else {
            Err(GpuError::ComputationError)
        }
    }

    /// Check if operation is complete
    pub fn is_complete(&self, handle: &AsyncOperationHandle) -> bool {
        self.pending_operations
            .iter()
            .find(|op| op.id == handle.operation_id)
            .is_some_and(|op| matches!(op.status, OperationStatus::Completed))
    }
}

/// GPU optimization advisor
pub struct GpuOptimizationAdvisor {
    performance_history: HashMap<String, Vec<PerformanceMetric>>,
    optimization_rules: Vec<OptimizationRule>,
}

impl Default for GpuOptimizationAdvisor {
    fn default() -> Self {
        Self::new()
    }
}

impl GpuOptimizationAdvisor {
    /// Create new optimization advisor
    pub fn new() -> Self {
        let mut advisor = Self {
            performance_history: HashMap::new(),
            optimization_rules: Vec::new(),
        };

        advisor.init_default_rules();
        advisor
    }

    /// Analyze performance and provide recommendations
    pub fn analyze_performance(
        &mut self,
        kernel_name: &str,
        execution: &GpuKernelExecution,
        workload_size: usize,
    ) -> Vec<OptimizationRecommendation> {
        let metric = PerformanceMetric {
            execution_time: execution.execution_time,
            throughput: workload_size as f64 / execution.execution_time,
            memory_bandwidth: 0.0, // Would be calculated from actual memory transfers
            occupancy: self.calculate_occupancy(execution),
        };

        self.performance_history
            .entry(kernel_name.to_string())
            .or_default()
            .push(metric.clone());

        let mut recommendations = Vec::new();

        for rule in &self.optimization_rules {
            if let Some(recommendation) = rule.evaluate(&metric, execution) {
                recommendations.push(recommendation);
            }
        }

        recommendations
    }

    fn init_default_rules(&mut self) {
        self.optimization_rules.push(OptimizationRule {
            name: "Low Occupancy".to_string(),
            condition: Box::new(|metric, _| metric.occupancy < 0.5),
            recommendation: "Consider increasing block size or reducing register usage".to_string(),
            priority: RecommendationPriority::High,
        });

        self.optimization_rules.push(OptimizationRule {
            name: "Memory Bandwidth".to_string(),
            condition: Box::new(|metric, _| metric.memory_bandwidth < 0.7),
            recommendation: "Optimize memory access patterns for better coalescing".to_string(),
            priority: RecommendationPriority::Medium,
        });

        self.optimization_rules.push(OptimizationRule {
            name: "Small Grid Size".to_string(),
            condition: Box::new(|_, execution| {
                let total_threads = execution.grid_size.0
                    * execution.grid_size.1
                    * execution.grid_size.2
                    * execution.block_size.0
                    * execution.block_size.1
                    * execution.block_size.2;
                total_threads < 1024
            }),
            recommendation: "Consider increasing grid size to better utilize GPU cores".to_string(),
            priority: RecommendationPriority::Low,
        });
    }

    fn calculate_occupancy(&self, execution: &GpuKernelExecution) -> f64 {
        let threads_per_block =
            execution.block_size.0 * execution.block_size.1 * execution.block_size.2;
        let blocks_per_sm = 2048 / threads_per_block.max(1); // Simplified calculation
        (blocks_per_sm as f64 / 32.0).min(1.0) // Assume 32 max blocks per SM
    }
}

// Additional data structures for the new features

#[derive(Debug, Clone)]
pub struct DistributedWorkload {
    pub total_elements: usize,
    pub operation_type: String,
    pub memory_requirement: u64,
    pub computation_complexity: f64,
}

#[derive(Debug, Clone)]
pub struct DistributedOperation {
    pub kernel_name: String,
    pub workload: DistributedWorkload,
}

#[derive(Debug, Clone)]
pub struct DistributedResult {
    pub executions: Vec<GpuKernelExecution>,
    pub total_time: f64,
    pub communication_overhead: f64,
}

#[derive(Debug, Clone)]
pub struct GpuAssignment {
    pub gpu_id: u32,
    pub grid_size: (u32, u32, u32),
    pub block_size: (u32, u32, u32),
    pub shared_memory: u32,
    pub parameters: HashMap<String, String>,
}

#[derive(Debug, Clone)]
pub struct LoadBalancer {
    #[allow(dead_code)]
    strategy: LoadBalancingStrategy,
}

impl Default for LoadBalancer {
    fn default() -> Self {
        Self::new()
    }
}

impl LoadBalancer {
    pub fn new() -> Self {
        Self {
            strategy: LoadBalancingStrategy::WorkloadProportional,
        }
    }

    pub fn assign_workload(
        &self,
        workload: &DistributedWorkload,
        gpus: &HashMap<u32, GpuUtils>,
    ) -> Vec<GpuAssignment> {
        let mut assignments = Vec::new();
        let num_gpus = gpus.len() as u32;

        if num_gpus == 0 {
            return assignments;
        }

        let elements_per_gpu = workload.total_elements / num_gpus as usize;

        for (gpu_id, _) in gpus.iter() {
            let assignment = GpuAssignment {
                gpu_id: *gpu_id,
                grid_size: (elements_per_gpu as u32 / 256, 1, 1),
                block_size: (256, 1, 1),
                shared_memory: 0,
                parameters: HashMap::new(),
            };
            assignments.push(assignment);
        }

        assignments
    }
}

#[derive(Debug, Clone)]
pub enum LoadBalancingStrategy {
    RoundRobin,
    WorkloadProportional,
    MemoryAware,
    PerformanceBased,
}

#[derive(Debug, Clone)]
pub enum CommunicationTopology {
    Ring,
    Tree,
    AllToAll,
    Custom(Vec<Vec<u32>>),
}

#[derive(Debug, Clone)]
pub struct SynchronizationBarrier {
    pub id: u32,
    pub participating_gpus: Vec<u32>,
    pub barrier_type: BarrierType,
}

#[derive(Debug, Clone)]
pub enum BarrierType {
    Global,
    Local(Vec<u32>),
    Hierarchical,
}

#[derive(Debug, Clone)]
pub struct ClusterMemoryStats {
    pub total_memory: u64,
    pub total_allocated: u64,
    pub total_free: u64,
    pub num_devices: usize,
    pub device_stats: HashMap<u32, MemoryStats>,
}

#[derive(Debug, Clone)]
pub struct MemoryBlock {
    pub ptr: u64,
    pub size: u64,
    pub is_allocated: bool,
    pub allocation_time: Option<Instant>,
}

#[derive(Debug, Clone)]
pub enum AllocationStrategy {
    FirstFit,
    BestFit,
    WorstFit,
    BuddySystem,
}

#[derive(Debug, Clone)]
pub struct DefragmentationResult {
    pub blocks_before: usize,
    pub blocks_after: usize,
    pub fragmentation_before: f64,
    pub fragmentation_after: f64,
}

#[derive(Debug, Clone)]
pub struct GpuStream {
    pub id: u32,
    pub device_id: u32,
    pub is_busy: bool,
    pub priority: StreamPriority,
}

#[derive(Debug, Clone)]
pub enum StreamPriority {
    Low,
    Normal,
    High,
}

#[derive(Debug, Clone)]
pub struct AsyncOperation {
    pub id: u32,
    pub kernel_info: GpuKernelInfo,
    pub stream_id: u32,
    pub start_time: Instant,
    pub status: OperationStatus,
}

#[derive(Debug, Clone)]
pub enum OperationStatus {
    Pending,
    Running,
    Completed,
    Failed,
}

#[derive(Debug, Clone)]
pub struct AsyncOperationHandle {
    pub operation_id: u32,
    pub device_id: u32,
}

#[derive(Debug, Clone)]
pub struct PerformanceMetric {
    pub execution_time: f64,
    pub throughput: f64,
    pub memory_bandwidth: f64,
    pub occupancy: f64,
}

type OptimizationCondition =
    Box<dyn Fn(&PerformanceMetric, &GpuKernelExecution) -> bool + Send + Sync>;

pub struct OptimizationRule {
    pub name: String,
    pub condition: OptimizationCondition,
    pub recommendation: String,
    pub priority: RecommendationPriority,
}

impl std::fmt::Debug for OptimizationRule {
    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
        f.debug_struct("OptimizationRule")
            .field("name", &self.name)
            .field("condition", &"<function>")
            .field("recommendation", &self.recommendation)
            .field("priority", &self.priority)
            .finish()
    }
}

impl Clone for OptimizationRule {
    fn clone(&self) -> Self {
        // Note: We can't clone the function pointer directly,
        // so we create a new rule with a placeholder condition
        // This is a limitation when working with function pointers
        OptimizationRule {
            name: self.name.clone(),
            condition: Box::new(|_metric, _execution| false), // Safe default
            recommendation: self.recommendation.clone(),
            priority: self.priority.clone(),
        }
    }
}

impl OptimizationRule {
    pub fn evaluate(
        &self,
        metric: &PerformanceMetric,
        execution: &GpuKernelExecution,
    ) -> Option<OptimizationRecommendation> {
        if (self.condition)(metric, execution) {
            Some(OptimizationRecommendation {
                rule_name: self.name.clone(),
                recommendation: self.recommendation.clone(),
                priority: self.priority.clone(),
                estimated_improvement: 0.0, // Default value
            })
        } else {
            None
        }
    }
}

#[derive(Debug, Clone)]
pub struct OptimizationRecommendation {
    pub rule_name: String,
    pub recommendation: String,
    pub priority: RecommendationPriority,
    pub estimated_improvement: f64,
}

#[derive(Debug, Clone)]
pub enum RecommendationPriority {
    Low,
    Medium,
    High,
    Critical,
}

#[allow(non_snake_case)]
#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn test_gpu_utils_creation() {
        let utils = GpuUtils::new();
        assert!(utils.devices.is_empty());
        assert!(utils
            .allocations
            .read()
            .expect("operation should succeed")
            .is_empty());
    }

    #[test]
    fn test_device_initialization() {
        let mut utils = GpuUtils::new();
        assert!(utils.init_devices().is_ok());
        assert!(!utils.devices.is_empty());
    }

    #[test]
    fn test_device_selection() {
        let mut utils = GpuUtils::new();
        utils.init_devices().expect("operation should succeed");

        let best_device = utils.get_best_device();
        assert!(best_device.is_some());
        assert!(!best_device.expect("operation should succeed").is_integrated);
    }

    #[test]
    fn test_memory_allocation() {
        let mut utils = GpuUtils::new();
        utils.init_devices().expect("operation should succeed");

        let ptr = utils
            .allocate_memory(1024, 0, "test")
            .expect("operation should succeed");
        assert!(ptr > 0);

        assert!(utils.free_memory(ptr).is_ok());
    }

    #[test]
    fn test_kernel_execution() {
        let mut utils = GpuUtils::new();
        utils.init_devices().expect("operation should succeed");

        let kernel_info = GpuKernelInfo {
            name: "test_kernel".to_string(),
            device_id: 0,
            grid_size: (1, 1, 1),
            block_size: (256, 1, 1),
            shared_memory: 0,
            parameters: HashMap::new(),
        };

        let execution = utils
            .execute_kernel(&kernel_info)
            .expect("operation should succeed");
        assert_eq!(execution.kernel_name, "test_kernel");
        assert!(execution.execution_time > 0.0);
    }

    #[test]
    fn test_array_operations() {
        let a = vec![1.0, 2.0, 3.0, 4.0];
        let b = vec![5.0, 6.0, 7.0, 8.0];

        let result = GpuArrayOps::add_arrays(&a, &b, 0).expect("operation should succeed");
        assert_eq!(result, vec![6.0, 8.0, 10.0, 12.0]);

        let result = GpuArrayOps::multiply_arrays(&a, &b, 0).expect("operation should succeed");
        assert_eq!(result, vec![5.0, 12.0, 21.0, 32.0]);
    }

    #[test]
    fn test_matrix_multiplication() {
        let a = vec![1.0, 2.0, 3.0, 4.0]; // 2x2
        let b = vec![5.0, 6.0, 7.0, 8.0]; // 2x2

        let result =
            GpuArrayOps::matrix_multiply(&a, &b, 2, 2, 2, 0).expect("operation should succeed");
        assert_eq!(result, vec![19.0, 22.0, 43.0, 50.0]);
    }

    #[test]
    fn test_activation_functions() {
        let input = vec![-1.0, 0.0, 1.0, 2.0];

        let result = GpuArrayOps::apply_activation(&input, ActivationFunction::ReLU, 0)
            .expect("operation should succeed");
        assert_eq!(result, vec![0.0, 0.0, 1.0, 2.0]);

        let result = GpuArrayOps::apply_activation(&input, ActivationFunction::Sigmoid, 0)
            .expect("operation should succeed");
        assert!(result.iter().all(|&x| (0.0..=1.0).contains(&x)));
    }

    #[test]
    fn test_reduction_operations() {
        let input = vec![1.0, 2.0, 3.0, 4.0, 5.0];

        let sum = GpuArrayOps::reduce_sum(&input, 0).expect("operation should succeed");
        assert_eq!(sum, 15.0);

        let max = GpuArrayOps::reduce_max(&input, 0).expect("operation should succeed");
        assert_eq!(max, 5.0);
    }

    #[test]
    fn test_gpu_profiler() {
        let mut profiler = GpuProfiler::new();

        profiler.record_kernel_time("test_kernel", 1.5);
        profiler.record_kernel_time("test_kernel", 2.0);
        profiler.record_memory_transfer(1024, "host_to_device");

        let stats = profiler.get_kernel_stats();
        assert!(stats.contains_key("test_kernel"));
        assert_eq!(stats["test_kernel"].count, 2);
        assert_eq!(stats["test_kernel"].avg_time, 1.75);

        let mem_stats = profiler.get_memory_transfer_stats();
        assert_eq!(mem_stats.total_transfers, 1);
        assert_eq!(mem_stats.total_bytes, 1024);
    }

    #[test]
    fn test_throughput_estimation() {
        let mut utils = GpuUtils::new();
        utils.init_devices().expect("operation should succeed");

        let throughput = utils.estimate_throughput(0, 1000, "add");
        assert!(throughput > 0.0);

        let should_use = utils.should_use_gpu(1000, "add");
        assert!(should_use);

        let should_not_use = utils.should_use_gpu(100, "add");
        assert!(!should_not_use);
    }

    #[test]
    fn test_memory_stats() {
        let mut utils = GpuUtils::new();
        utils.init_devices().expect("operation should succeed");

        let _ptr = utils
            .allocate_memory(1024, 0, "test")
            .expect("operation should succeed");
        let stats = utils.get_memory_stats();

        assert!(stats.contains_key(&0));
        assert_eq!(stats[&0].allocated_memory, 1024);
        assert_eq!(stats[&0].num_allocations, 1);
    }

    #[test]
    fn test_error_handling() {
        let utils = GpuUtils::new();

        // Test device not found
        let result = utils.allocate_memory(1024, 999, "test");
        assert!(matches!(result, Err(GpuError::DeviceNotFound)));

        // Test invalid pointer
        let result = utils.free_memory(0);
        assert!(matches!(result, Err(GpuError::InvalidPointer)));

        // Test shape mismatch
        let a = vec![1.0, 2.0];
        let b = vec![3.0, 4.0, 5.0];
        let result = GpuArrayOps::add_arrays(&a, &b, 0);
        assert!(matches!(result, Err(GpuError::ShapeMismatch)));
    }

    // Tests for new GPU computing features

    #[test]
    fn test_multi_gpu_coordinator() {
        let mut coordinator = MultiGpuCoordinator::new();

        // Test GPU initialization
        let result = coordinator.init_all_gpus();
        assert!(result.is_ok() || matches!(result, Err(GpuError::InitializationFailed(_))));

        // Test workload assignment
        let workload = DistributedWorkload {
            total_elements: 10_000,
            operation_type: "matrix_multiply".to_string(),
            memory_requirement: 1024 * 1024,
            computation_complexity: 1.0,
        };

        let assignments = coordinator.get_optimal_assignment(&workload);
        assert!(!assignments.is_empty() || coordinator.gpus.is_empty());
    }

    #[test]
    fn test_distributed_operation() {
        let mut coordinator = MultiGpuCoordinator::new();
        let init_result = coordinator.init_all_gpus();

        let operation = DistributedOperation {
            kernel_name: "test_kernel".to_string(),
            workload: DistributedWorkload {
                total_elements: 1000,
                operation_type: "add".to_string(),
                memory_requirement: 4000,
                computation_complexity: 0.5,
            },
        };

        if init_result.is_ok() && !coordinator.gpus.is_empty() {
            let result = coordinator.execute_distributed(&operation);

            // In a test environment, GPU operations might fail due to mock limitations
            // This is acceptable as we're testing the logic, not actual GPU execution
            if let Ok(dist_result) = result {
                assert!(!dist_result.executions.is_empty());
                assert!(dist_result.total_time >= 0.0);
            } else {
                // GPU execution failed, which is acceptable in test environment
                // Just verify that we have the right number of GPUs
                assert!(!coordinator.gpus.is_empty());
            }
        } else {
            // If no GPUs are available (which is expected in test environment),
            // test should pass as this is a valid scenario
            assert!(coordinator.gpus.is_empty());
        }
    }

    #[test]
    fn test_cluster_memory_stats() {
        let mut coordinator = MultiGpuCoordinator::new();
        let _ = coordinator.init_all_gpus();

        let stats = coordinator.get_cluster_memory_stats();
        assert_eq!(stats.num_devices, coordinator.gpus.len());
        assert_eq!(stats.total_free, stats.total_memory - stats.total_allocated);
    }

    #[test]
    fn test_gpu_memory_pool() {
        let mut pool = GpuMemoryPool::new(AllocationStrategy::FirstFit);

        // Test allocation
        let ptr1 = pool.allocate(1024, 0);
        assert!(ptr1.is_ok());

        let ptr2 = pool.allocate(2048, 0);
        assert!(ptr2.is_ok());

        // Test freeing
        let free_result = pool.free(ptr1.expect("operation should succeed"), 0);
        assert!(free_result.is_ok());

        // Test defragmentation
        let defrag_result = pool.defragment(0);
        assert!(defrag_result.is_ok());

        let defrag = defrag_result.expect("operation should succeed");
        assert!(defrag.fragmentation_after <= defrag.fragmentation_before);
    }

    #[test]
    fn test_memory_pool_strategies() {
        let strategies = vec![
            AllocationStrategy::FirstFit,
            AllocationStrategy::BestFit,
            AllocationStrategy::WorstFit,
            AllocationStrategy::BuddySystem,
        ];

        for strategy in strategies {
            let mut pool = GpuMemoryPool::new(strategy);
            let ptr = pool.allocate(1024, 0);
            assert!(ptr.is_ok());
        }
    }

    #[test]
    fn test_async_gpu_operations() {
        let mut async_ops = AsyncGpuOps::new();

        // Test stream creation
        let stream_id = async_ops.create_stream(0);
        assert!(stream_id.is_ok());

        // Test async kernel launch
        let kernel_info = GpuKernelInfo {
            name: "async_test".to_string(),
            device_id: 0,
            grid_size: (1, 1, 1),
            block_size: (256, 1, 1),
            shared_memory: 0,
            parameters: HashMap::new(),
        };

        let handle = async_ops
            .launch_kernel_async(&kernel_info, stream_id.expect("operation should succeed"));
        assert!(handle.is_ok());

        let operation_handle = handle.expect("operation should succeed");

        // Test completion checking
        let _is_complete_before = async_ops.is_complete(&operation_handle);

        // Test waiting for completion
        let execution = async_ops.wait_for_completion(&operation_handle);
        assert!(execution.is_ok());

        let is_complete_after = async_ops.is_complete(&operation_handle);
        assert!(is_complete_after);
    }

    #[test]
    fn test_gpu_optimization_advisor() {
        let mut advisor = GpuOptimizationAdvisor::new();

        // Test performance analysis
        let execution = GpuKernelExecution {
            kernel_name: "test_kernel".to_string(),
            device_id: 0,
            grid_size: (10, 1, 1),  // Small grid size
            block_size: (32, 1, 1), // Small block size
            shared_memory: 0,
            execution_time: 5.0,
            parameters: HashMap::new(),
        };

        let recommendations = advisor.analyze_performance("test_kernel", &execution, 1000);
        assert!(!recommendations.is_empty());

        // Should recommend increasing grid size due to low thread count
        let has_grid_size_recommendation = recommendations
            .iter()
            .any(|r| r.rule_name.contains("Grid Size"));
        assert!(has_grid_size_recommendation);
    }

    #[test]
    fn test_load_balancer() {
        let balancer = LoadBalancer::new();
        let mut gpus = HashMap::new();

        // Mock GPU setup
        let mut gpu1 = GpuUtils::new();
        let mut gpu2 = GpuUtils::new();
        let _ = gpu1.init_devices();
        let _ = gpu2.init_devices();

        gpus.insert(0, gpu1);
        gpus.insert(1, gpu2);

        let workload = DistributedWorkload {
            total_elements: 10_000,
            operation_type: "matrix_multiply".to_string(),
            memory_requirement: 1024 * 1024,
            computation_complexity: 1.0,
        };

        let assignments = balancer.assign_workload(&workload, &gpus);
        assert_eq!(assignments.len(), gpus.len());

        // Verify assignments distribute workload
        let total_elements: u32 = assignments
            .iter()
            .map(|a| a.grid_size.0 * a.block_size.0)
            .sum();
        assert!(total_elements > 0);
    }

    #[test]
    fn test_stream_priorities() {
        let mut async_ops = AsyncGpuOps::new();
        let _stream_id = async_ops
            .create_stream(0)
            .expect("operation should succeed");

        // Verify stream was created with default priority
        let streams = async_ops.streams.get(&0).expect("operation should succeed");
        assert_eq!(streams.len(), 1);
        assert!(matches!(streams[0].priority, StreamPriority::Normal));
    }

    #[test]
    fn test_memory_block_operations() {
        let block1 = MemoryBlock {
            ptr: 1000,
            size: 1024,
            is_allocated: false,
            allocation_time: None,
        };

        let block2 = MemoryBlock {
            ptr: 2024,
            size: 2048,
            is_allocated: true,
            allocation_time: Some(Instant::now()),
        };

        assert!(!block1.is_allocated);
        assert!(block2.is_allocated);
        assert!(block1.allocation_time.is_none());
        assert!(block2.allocation_time.is_some());
    }

    #[test]
    fn test_distributed_workload() {
        let workload = DistributedWorkload {
            total_elements: 1_000_000,
            operation_type: "fft".to_string(),
            memory_requirement: 8 * 1_000_000, // 8 bytes per element
            computation_complexity: 2.5,       // O(n log n) for FFT
        };

        assert_eq!(workload.total_elements, 1_000_000);
        assert_eq!(workload.operation_type, "fft");
        assert!(workload.computation_complexity > 1.0);
    }

    #[test]
    fn test_communication_topology() {
        let ring_topology = CommunicationTopology::Ring;
        let tree_topology = CommunicationTopology::Tree;
        let all_to_all_topology = CommunicationTopology::AllToAll;
        let custom_topology =
            CommunicationTopology::Custom(vec![vec![1, 2], vec![0, 3], vec![0, 3], vec![1, 2]]);

        // Test that all topology types can be created
        match ring_topology {
            CommunicationTopology::Ring => {}
            _ => panic!(),
        }
        match tree_topology {
            CommunicationTopology::Tree => {}
            _ => panic!(),
        }
        match all_to_all_topology {
            CommunicationTopology::AllToAll => {}
            _ => panic!(),
        }
        match custom_topology {
            CommunicationTopology::Custom(_) => {}
            _ => panic!(),
        }
    }

    #[test]
    fn test_synchronization_barrier() {
        let barrier = SynchronizationBarrier {
            id: 1,
            participating_gpus: vec![0, 1, 2, 3],
            barrier_type: BarrierType::Global,
        };

        assert_eq!(barrier.id, 1);
        assert_eq!(barrier.participating_gpus.len(), 4);
        assert!(matches!(barrier.barrier_type, BarrierType::Global));
    }

    #[test]
    fn test_optimization_recommendation_priorities() {
        let low_priority = RecommendationPriority::Low;
        let medium_priority = RecommendationPriority::Medium;
        let high_priority = RecommendationPriority::High;
        let critical_priority = RecommendationPriority::Critical;

        // Test that all priority levels can be created
        match low_priority {
            RecommendationPriority::Low => {}
            _ => panic!(),
        }
        match medium_priority {
            RecommendationPriority::Medium => {}
            _ => panic!(),
        }
        match high_priority {
            RecommendationPriority::High => {}
            _ => panic!(),
        }
        match critical_priority {
            RecommendationPriority::Critical => {}
            _ => panic!(),
        }
    }

    #[test]
    fn test_performance_metric_calculations() {
        let metric = PerformanceMetric {
            execution_time: 10.0,  // ms
            throughput: 1000.0,    // elements/ms
            memory_bandwidth: 0.8, // 80% utilization
            occupancy: 0.75,       // 75% occupancy
        };

        assert!(metric.execution_time > 0.0);
        assert!(metric.throughput > 0.0);
        assert!(metric.memory_bandwidth <= 1.0);
        assert!(metric.occupancy <= 1.0);
    }

    #[test]
    fn test_operation_status_transitions() {
        let mut operation = AsyncOperation {
            id: 0,
            kernel_info: GpuKernelInfo {
                name: "test".to_string(),
                device_id: 0,
                grid_size: (1, 1, 1),
                block_size: (1, 1, 1),
                shared_memory: 0,
                parameters: HashMap::new(),
            },
            stream_id: 0,
            start_time: Instant::now(),
            status: OperationStatus::Pending,
        };

        assert!(matches!(operation.status, OperationStatus::Pending));

        operation.status = OperationStatus::Running;
        assert!(matches!(operation.status, OperationStatus::Running));

        operation.status = OperationStatus::Completed;
        assert!(matches!(operation.status, OperationStatus::Completed));
    }
}