primal-barracuda 0.4.0

barraCuda — BARrier-free Rust Abstracted Cross-platform Unified Dimensional Algebra. Vendor-agnostic GPU compute via WGSL.
Documentation
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
// SPDX-License-Identifier: AGPL-3.0-or-later
//! Pure WGSL device - hardware-agnostic compute via WebGPU
//!
//! **Pure WGSL Architecture**:
//! - WGSL shaders ONLY (no separate CPU code!)
//! - wgpu handles execution on ANY device (GPU/CPU/NPU/TPU)
//!
//! ## Adapter Selection
//!
//! Set `BARRACUDA_GPU_ADAPTER` environment variable:
//! - `BARRACUDA_GPU_ADAPTER=0` — Select first adapter
//! - `BARRACUDA_GPU_ADAPTER=titan` — Select adapter containing "titan"
//! - `BARRACUDA_GPU_ADAPTER=auto` — Use wgpu `HighPerformance` (default)

mod buffers;
mod capabilities;
mod compilation;
mod creation;
mod dispatch;
mod guard;
mod submission;

pub(crate) use dispatch::{DispatchPermit, DispatchSemaphore, concurrency_budget};
pub use guard::{GuardedDeviceHandle, GuardedEncoder};

use super::autotune::{GLOBAL_TUNER, GpuCalibration, GpuDeviceForCalibration};
use crate::env_keys;
use crate::error::Result;
use std::sync::Arc;
use std::sync::atomic::{AtomicBool, Ordering};
use std::time::Duration;

/// Maximum time to wait for a single GPU poll to complete.
///
/// Configurable via `BARRACUDA_POLL_TIMEOUT_SECS`. Defaults to 120 seconds,
/// which is generous for any real GPU operation. This prevents indefinite
/// hangs under instrumentation (llvm-cov) or when a software rasterizer
/// stalls. The CPU sends work to the GPU and expects results within this
/// window; exceeding it signals a driver-level stall, not a compute issue.
pub(crate) fn poll_timeout() -> Option<Duration> {
    static TIMEOUT: std::sync::LazyLock<Option<Duration>> = std::sync::LazyLock::new(|| {
        let secs: u64 = std::env::var(env_keys::BARRACUDA_POLL_TIMEOUT_SECS)
            .ok()
            .and_then(|v| v.parse().ok())
            .unwrap_or(120);
        if secs == 0 {
            None
        } else {
            Some(Duration::from_secs(secs))
        }
    });
    *TIMEOUT
}

/// WebGPU device - executes WGSL on any hardware
///
/// Concurrency model (atomic barrier + Mutex):
/// - `active_encoders` is an `AtomicU32` counting live `GuardedEncoder`s.
///   Encoding is **never blocked** — just an atomic increment on creation
///   and decrement on finish/drop.
/// - `gpu_lock` is a `std::sync::Mutex<()>` that serializes submit and poll
///   against each other. `std::sync::Mutex` is deliberately chosen over
///   `tokio::sync::Mutex` — the lock is **never held across `.await`** and
///   protects wgpu FFI calls that are inherently synchronous. `std::sync::Mutex`
///   avoids the overhead of an async lock for these microsecond-scale critical
///   sections. Poll additionally spin-waits until `active_encoders == 0`
///   to prevent wgpu-core storage races between poll cleanup and encoding.
/// - `dispatch_semaphore` limits how many operations can be in-flight
///   simultaneously based on device type (prevents driver overload).
#[derive(Debug, Clone)]
pub struct WgpuDevice {
    pub(crate) device: GuardedDeviceHandle,
    pub(crate) queue: wgpu::Queue,
    pub(crate) adapter_info: wgpu::AdapterInfo,
    calibration: Option<GpuCalibration>,
    /// Vulkan pipeline cache — avoids re-compiling identical SPIR-V to machine code.
    pipeline_cache: Option<Arc<wgpu::PipelineCache>>,
    /// Set when the GPU reports a device-lost error.
    pub(crate) lost: Arc<AtomicBool>,
    /// Set when the GPU reports an out-of-memory error. Used by multi-GPU pool
    /// to migrate work to another device.
    pub(crate) oom: Arc<AtomicBool>,
    /// Serializes submit+poll to prevent wgpu-core storage races.
    /// `std::sync::Mutex` is correct: never held across `.await`.
    gpu_lock: Arc<std::sync::Mutex<()>>,
    /// Counts live `GuardedEncoders` — poll waits for this to reach zero.
    active_encoders: Arc<std::sync::atomic::AtomicU32>,
    /// Limits concurrent dispatches based on device capability.
    dispatch_semaphore: Arc<DispatchSemaphore>,
    /// Optional VRAM quota tracker — when set, buffer allocations are checked
    /// against the budget before proceeding. `None` means unlimited.
    quota_tracker: Option<Arc<crate::resource_quota::QuotaTracker>>,
}

impl GpuDeviceForCalibration for WgpuDevice {
    fn device(&self) -> &wgpu::Device {
        &self.device
    }

    fn name(&self) -> &str {
        &self.adapter_info.name
    }

    fn submit_and_poll_calibration(&self, commands: impl IntoIterator<Item = wgpu::CommandBuffer>) {
        self.submit_and_poll_inner(commands);
    }
}

impl WgpuDevice {
    /// Get device name
    #[must_use]
    pub fn name(&self) -> &str {
        &self.adapter_info.name
    }

    /// Get device type
    #[must_use]
    pub fn device_type(&self) -> wgpu::DeviceType {
        self.adapter_info.device_type
    }

    /// Check if running on CPU fallback
    #[must_use]
    pub fn is_cpu(&self) -> bool {
        self.adapter_info.device_type == wgpu::DeviceType::Cpu
    }

    /// Check if the GPU driver has reported this device as lost.
    #[must_use]
    pub fn is_lost(&self) -> bool {
        self.lost.load(Ordering::Acquire)
    }

    /// Check if an out-of-memory condition was signaled by the GPU driver.
    ///
    /// When `true`, the device's VRAM is exhausted — multi-GPU pools should
    /// attempt to migrate workloads to another device. The flag can be cleared
    /// with [`clear_oom`](Self::clear_oom) after recovery (e.g. buffers freed).
    #[must_use]
    pub fn is_oom(&self) -> bool {
        self.oom.load(Ordering::Acquire)
    }

    /// Clear the OOM flag after recovery (e.g. buffers freed, workload migrated).
    pub fn clear_oom(&self) {
        self.oom.store(false, Ordering::Release);
    }

    /// Signal that this device has hit an out-of-memory condition.
    ///
    /// Called by the multi-GPU pool's migration logic when a workload fails
    /// with OOM. The flag persists until [`clear_oom`](Self::clear_oom) is
    /// called after recovery.
    pub fn set_oom(&self) {
        self.oom.store(true, Ordering::Release);
    }

    /// Attach a VRAM quota tracker to this device.
    ///
    /// Once set, all buffer allocations through the canonical helpers
    /// (`create_buffer_f32`, `create_buffer_f64`, etc.) will be checked
    /// against the budget before proceeding.
    pub fn set_quota_tracker(&mut self, tracker: Arc<crate::resource_quota::QuotaTracker>) {
        self.quota_tracker = Some(tracker);
    }

    /// Current quota tracker, if any.
    #[must_use]
    pub fn quota_tracker(&self) -> Option<&Arc<crate::resource_quota::QuotaTracker>> {
        self.quota_tracker.as_ref()
    }

    /// Check the VRAM quota and record the allocation. No-op if no tracker is set.
    pub(crate) fn quota_try_allocate(&self, bytes: u64) -> Result<()> {
        if let Some(tracker) = &self.quota_tracker {
            tracker.try_allocate(bytes)?;
        }
        Ok(())
    }

    /// Check if f64 shaders are truly available on this device.
    /// Returns `true` only when BOTH conditions hold:
    /// 1. `wgpu::Features::SHADER_F64` was granted at device creation.
    /// 2. The runtime probe (if completed) confirms basic f64 compilation works.
    ///    groundSpring V37 discovered that NVK/NAK advertise `SHADER_F64` but fail
    ///    to compile even basic f64 WGSL. This method consults the probe cache to
    ///    avoid silent shader compilation failures.
    #[must_use]
    pub fn has_f64_shaders(&self) -> bool {
        if !self.device.features().contains(wgpu::Features::SHADER_F64) {
            return false;
        }
        super::probe::cached_f64_builtins(self).is_none_or(|caps| caps.can_compile_f64())
    }

    /// Whether the device supports WGSL subgroup intrinsics (`subgroupAdd`, etc).
    ///
    /// True when `wgpu::Features::SUBGROUP` was granted at device creation.
    /// Enables subgroup-accelerated reduction shaders (fewer barriers, fewer
    /// shared memory steps). Do NOT add `enable subgroups;` to WGSL — naga 28
    /// generates broken SPIR-V when the directive is present; the feature flag
    /// is sufficient.
    #[must_use]
    pub fn has_subgroups(&self) -> bool {
        self.device.features().contains(wgpu::Features::SUBGROUP)
    }

    /// Check if the Sovereign Compiler's SPIR-V passthrough path is available.
    /// In wgpu 28 SPIR-V passthrough is gated by the `spirv` cargo feature
    /// on the wgpu crate (always enabled in our workspace). The Vulkan
    /// backend is required at runtime for the driver to accept SPIR-V.
    #[must_use]
    pub fn has_spirv_passthrough(&self) -> bool {
        self.adapter_info.backend == wgpu::Backend::Vulkan
    }

    /// Access underlying wgpu device
    #[must_use]
    pub fn device(&self) -> &wgpu::Device {
        &self.device
    }

    /// Get a clone of the wgpu device (cheap — internally Arc-wrapped in wgpu 28).
    #[must_use]
    pub fn device_clone(&self) -> wgpu::Device {
        (*self.device).clone()
    }

    /// Get the shared GPU lock (for components that call `device.poll` directly).
    pub(crate) fn gpu_lock_arc(&self) -> Arc<std::sync::Mutex<()>> {
        self.gpu_lock.clone()
    }

    /// Get the active encoder counter (for `BufferPool` poll coordination).
    pub(crate) fn active_encoders_arc(&self) -> Arc<std::sync::atomic::AtomicU32> {
        self.active_encoders.clone()
    }

    /// Increment active encoder count (encoding phase barrier).
    /// While any encoder is active, `device.poll()` will spin-wait.
    /// This is lock-free and never blocks the caller.
    pub fn encoding_guard(&self) {
        self.active_encoders.fetch_add(1, Ordering::Acquire);
    }

    /// Decrement active encoder count (encoding phase complete).
    pub fn encoding_complete(&self) {
        self.active_encoders.fetch_sub(1, Ordering::Release);
    }

    /// Bounded wait for active encoders to finish.
    /// Encoding is CPU work (recording commands into a buffer), so it
    /// completes in microseconds. A brief yield loop avoids the
    /// wgpu-core race where poll/cleanup overlaps with encoding, without
    /// risking starvation even at 128+ threads.
    pub(crate) fn brief_encoder_wait(&self) {
        for _ in 0..256 {
            if self.active_encoders.load(Ordering::Acquire) == 0 {
                return;
            }
            std::thread::yield_now();
        }
    }

    /// Create a command encoder protected by the active-encoder barrier.
    /// The returned `GuardedEncoder` increments the active encoder count,
    /// preventing `device.poll()` from running. Encoding is lock-free and
    /// never blocks other threads. Call `.finish()` to release.
    #[must_use]
    pub fn create_encoder_guarded(
        &self,
        desc: &wgpu::CommandEncoderDescriptor<'_>,
    ) -> GuardedEncoder {
        self.active_encoders.fetch_add(1, Ordering::Acquire);
        let encoder = self.device.create_command_encoder(desc);
        GuardedEncoder::new(encoder, self.active_encoders.clone())
    }

    /// Get adapter info (for capability detection)
    #[must_use]
    pub fn adapter_info(&self) -> &wgpu::AdapterInfo {
        &self.adapter_info
    }

    /// Get the device's pipeline cache for `create_compute_pipeline` calls.
    /// Returns `None` only when the Vulkan driver does not support pipeline caching.
    #[must_use]
    pub fn pipeline_cache(&self) -> Option<&wgpu::PipelineCache> {
        self.pipeline_cache.as_deref()
    }

    /// Access command queue
    #[must_use]
    pub fn queue(&self) -> &wgpu::Queue {
        &self.queue
    }

    /// Get a clone of the command queue (cheap — internally Arc-wrapped in wgpu 28).
    #[must_use]
    pub fn queue_clone(&self) -> wgpu::Queue {
        self.queue.clone()
    }

    /// Acquire a dispatch permit from the device's concurrency budget.
    /// Hold this for the entire compile→bind→submit lifecycle of an
    /// operation. The device automatically sizes the budget to what the
    /// hardware can handle (2 for CPU/llvmpipe, 8 for discrete GPU, etc.).
    #[must_use]
    pub fn acquire_dispatch(&self) -> DispatchPermit<'_> {
        self.dispatch_semaphore.acquire()
    }

    /// The device's concurrency budget (max simultaneous dispatches).
    #[must_use]
    pub fn max_concurrent_dispatches(&self) -> usize {
        self.dispatch_semaphore.max_permits()
    }

    /// Execute WGSL compute shader
    /// Acquires a dispatch permit for the full compile→submit lifecycle.
    /// Holds encoding read-guard during compile/encode, releases before submit.
    /// # Errors
    /// Returns [`Err`] if shader compilation fails, pipeline creation fails,
    /// or GPU dispatch fails (e.g., device lost).
    pub fn execute_compute(
        &self,
        shader_source: &str,
        bind_groups: &[&wgpu::BindGroup],
        workgroups: (u32, u32, u32),
    ) -> Result<()> {
        let _permit = self.acquire_dispatch();
        let commands = {
            self.encoding_guard();
            let shader = self.compile_shader(shader_source, Some("barraCuda Operation"));
            let pipeline = self
                .device
                .create_compute_pipeline(&wgpu::ComputePipelineDescriptor {
                    label: Some("barraCuda Pipeline"),
                    layout: None,
                    module: &shader,
                    entry_point: Some("main"),
                    cache: self.pipeline_cache(),
                    compilation_options: Default::default(),
                });

            let mut encoder = self
                .device
                .create_command_encoder(&wgpu::CommandEncoderDescriptor {
                    label: Some("barraCuda Encoder"),
                });

            {
                let mut pass = encoder.begin_compute_pass(&wgpu::ComputePassDescriptor {
                    label: Some("barraCuda Compute"),
                    timestamp_writes: None,
                });
                pass.set_pipeline(&pipeline);
                for (i, bg) in bind_groups.iter().enumerate() {
                    pass.set_bind_group(i as u32, Some(*bg), &[]);
                }
                pass.dispatch_workgroups(workgroups.0, workgroups.1, workgroups.2);
            }

            let cmds = encoder.finish();
            self.encoding_complete();
            cmds
        };
        self.submit_and_poll_inner(Some(commands));
        Ok(())
    }

    /// Get calibration for this device
    pub fn get_calibration(&self) -> GpuCalibration {
        GLOBAL_TUNER.get_or_calibrate(self)
    }

    /// Force recalibration
    pub fn recalibrate(&self) -> GpuCalibration {
        GLOBAL_TUNER.recalibrate(self)
    }

    /// Get optimal workgroup size for this device
    #[must_use]
    pub fn optimal_workgroup_size(&self) -> u32 {
        self.calibration.as_ref().map_or_else(
            || GLOBAL_TUNER.get_or_calibrate(self).optimal_workgroup_size,
            |c| c.optimal_workgroup_size,
        )
    }

    /// Get measured peak bandwidth for this device (GB/s)
    #[must_use]
    pub fn peak_bandwidth_gbps(&self) -> f64 {
        self.get_calibration().peak_bandwidth_gbps
    }

    /// Get measured dispatch overhead for this device (μs)
    #[must_use]
    pub fn dispatch_overhead_us(&self) -> f64 {
        self.get_calibration().dispatch_overhead_us
    }
}

#[cfg(test)]
mod tests {
    use super::*;

    #[tokio::test]
    async fn test_wgpu_device_creation() {
        let Some(device) = crate::device::test_pool::get_test_device_if_gpu_available().await
        else {
            return;
        };
        let _ = (device.name(), device.is_cpu());
    }

    #[tokio::test]
    async fn test_enumerate_adapters() {
        let adapters = WgpuDevice::enumerate_adapters().await;
        assert!(
            !adapters.is_empty(),
            "WGPU should find at least one adapter"
        );
    }

    #[tokio::test]
    async fn test_buffer_operations() {
        let Some(device) = crate::device::test_pool::get_test_device_if_gpu_available().await
        else {
            return;
        };
        let buffer = device.create_buffer_f32(10).unwrap();
        let data = vec![1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0, 10.0];
        device.write_buffer_f32(&buffer, &data).unwrap();
        let read_data = device.read_buffer_f32(&buffer, 10).unwrap();
        assert_eq!(read_data, data);
    }

    #[tokio::test]
    async fn test_adapter_selector_auto() {
        let _ = WgpuDevice::with_adapter_selector("auto").await;
    }

    #[tokio::test]
    async fn test_adapter_selector_index() {
        let adapters = WgpuDevice::enumerate_adapters().await;
        if adapters.is_empty() {
            return;
        }
        if let Ok(device) = WgpuDevice::with_adapter_selector("0").await {
            assert_eq!(device.name(), adapters[0].name);
        }
    }

    #[tokio::test]
    async fn test_adapter_selector_name_match() {
        let adapters = WgpuDevice::enumerate_adapters().await;
        if adapters.is_empty() {
            return;
        }
        let partial = adapters[0]
            .name
            .chars()
            .take(4)
            .collect::<String>()
            .to_lowercase();
        let _ = WgpuDevice::with_adapter_selector(&partial).await;
    }

    #[tokio::test]
    async fn test_adapter_selector_fallthrough() {
        let adapters = WgpuDevice::enumerate_adapters().await;
        let large_index = (adapters.len() + 1000).to_string();
        if let Err(e) = WgpuDevice::with_adapter_selector(&large_index).await {
            assert!(e.to_string().contains("No adapter matches"));
        }
    }

    #[tokio::test]
    async fn test_from_env_default() {
        // Tests the "auto" selection path — same code as from_env() when
        // BARRACUDA_GPU_ADAPTER is unset (the typical case).
        let _ = WgpuDevice::with_adapter_selector("auto").await;
    }

    #[tokio::test]
    async fn test_driver_detection_apis() {
        let Some(device) = crate::device::test_pool::get_test_device_if_gpu_available().await
        else {
            return;
        };
        let _ = device.is_nvk();
        let _ = device.is_radv();
        let _ = device.is_nvidia_proprietary();
    }

    #[test]
    fn test_driver_detection_logic() {
        fn contains_nvk_markers(driver: &str) -> bool {
            let lower = driver.to_lowercase();
            lower.contains("nvk") || lower.contains("nouveau") || lower.contains("mesa")
        }
        fn contains_radv_markers(driver: &str) -> bool {
            driver.to_lowercase().contains("radv")
        }
        assert!(contains_nvk_markers("NVK"));
        assert!(contains_nvk_markers("nouveau"));
        assert!(!contains_nvk_markers("NVIDIA"));
        assert!(contains_radv_markers("RADV"));
        assert!(!contains_radv_markers("NVIDIA"));
    }
}