use std::collections::HashMap;
use std::hash::{Hash, Hasher};
use std::sync::{Arc, RwLock, RwLockReadGuard, RwLockWriteGuard};
use wgpu::{BindGroupLayout, ComputePipeline, Device, ShaderModule};
fn read_or_recover<T>(lock: &RwLock<T>) -> RwLockReadGuard<'_, T> {
lock.read()
.unwrap_or_else(std::sync::PoisonError::into_inner)
}
fn write_or_recover<T>(lock: &RwLock<T>) -> RwLockWriteGuard<'_, T> {
lock.write()
.unwrap_or_else(std::sync::PoisonError::into_inner)
}
#[derive(Clone, Hash, PartialEq, Eq, Debug)]
pub struct DeviceFingerprint {
name_hash: u64,
backend_hash: u64,
}
impl DeviceFingerprint {
#[must_use]
pub fn from_device_info(device: &Device, adapter_info: &wgpu::AdapterInfo) -> Self {
use std::hash::Hasher;
let mut name_hasher = std::collections::hash_map::DefaultHasher::new();
adapter_info.name.hash(&mut name_hasher);
let name_hash = name_hasher.finish();
let mut backend_hasher = std::collections::hash_map::DefaultHasher::new();
std::mem::discriminant(&adapter_info.backend).hash(&mut backend_hasher);
std::mem::discriminant(&adapter_info.device_type).hash(&mut backend_hasher);
device.hash(&mut backend_hasher);
let backend_hash = backend_hasher.finish();
Self {
name_hash,
backend_hash,
}
}
#[must_use]
pub fn from_adapter_info(adapter_info: &wgpu::AdapterInfo) -> Self {
use std::hash::Hasher;
let mut name_hasher = std::collections::hash_map::DefaultHasher::new();
adapter_info.name.hash(&mut name_hasher);
let name_hash = name_hasher.finish();
let mut backend_hasher = std::collections::hash_map::DefaultHasher::new();
std::mem::discriminant(&adapter_info.backend).hash(&mut backend_hasher);
std::mem::discriminant(&adapter_info.device_type).hash(&mut backend_hasher);
let backend_hash = backend_hasher.finish();
Self {
name_hash,
backend_hash,
}
}
}
#[derive(Clone, Hash, PartialEq, Eq)]
pub struct ShaderKey {
source_hash: u64,
device_fingerprint: DeviceFingerprint,
}
impl ShaderKey {
#[must_use]
pub fn new(source: &str, device_fingerprint: DeviceFingerprint) -> Self {
let mut hasher = std::collections::hash_map::DefaultHasher::new();
source.hash(&mut hasher);
Self {
source_hash: hasher.finish(),
device_fingerprint,
}
}
}
#[derive(Clone, Copy, Hash, PartialEq, Eq)]
pub struct BindGroupLayoutSignature {
pub read_only_buffers: u32,
pub read_write_buffers: u32,
pub uniform_buffers: u32,
}
impl BindGroupLayoutSignature {
#[must_use]
pub fn elementwise_binary() -> Self {
Self {
read_only_buffers: 2,
read_write_buffers: 1,
uniform_buffers: 0,
}
}
#[must_use]
pub fn elementwise_unary() -> Self {
Self {
read_only_buffers: 1,
read_write_buffers: 1,
uniform_buffers: 0,
}
}
#[must_use]
pub fn reduction() -> Self {
Self {
read_only_buffers: 1,
read_write_buffers: 1,
uniform_buffers: 1,
}
}
#[must_use]
pub fn matmul() -> Self {
Self {
read_only_buffers: 2,
read_write_buffers: 1,
uniform_buffers: 1,
}
}
#[must_use]
pub fn elementwise_ternary() -> Self {
Self {
read_only_buffers: 3,
read_write_buffers: 1,
uniform_buffers: 0,
}
}
#[must_use]
pub fn two_input_reduction() -> Self {
Self {
read_only_buffers: 2,
read_write_buffers: 1,
uniform_buffers: 1,
}
}
#[must_use]
pub fn three_input_reduction() -> Self {
Self {
read_only_buffers: 3,
read_write_buffers: 1,
uniform_buffers: 1,
}
}
}
#[derive(Clone, Hash, PartialEq, Eq)]
pub struct BindGroupLayoutKey {
signature: BindGroupLayoutSignature,
device_fingerprint: DeviceFingerprint,
}
impl BindGroupLayoutKey {
#[must_use]
pub fn new(signature: BindGroupLayoutSignature, device_fingerprint: DeviceFingerprint) -> Self {
Self {
signature,
device_fingerprint,
}
}
}
#[derive(Clone, Hash, PartialEq, Eq)]
pub struct PipelineKey {
source_hash: u64,
layout_signature: BindGroupLayoutSignature,
entry_point_hash: u64,
device_fingerprint: DeviceFingerprint,
}
impl PipelineKey {
#[must_use]
pub fn new(
shader_source: &str,
layout_signature: BindGroupLayoutSignature,
entry_point: &str,
device_fingerprint: DeviceFingerprint,
) -> Self {
let mut src_hasher = std::collections::hash_map::DefaultHasher::new();
shader_source.hash(&mut src_hasher);
let mut ep_hasher = std::collections::hash_map::DefaultHasher::new();
entry_point.hash(&mut ep_hasher);
Self {
source_hash: src_hasher.finish(),
layout_signature,
entry_point_hash: ep_hasher.finish(),
device_fingerprint,
}
}
}
#[derive(Default)]
pub struct PipelineCache {
shaders: RwLock<HashMap<ShaderKey, Arc<ShaderModule>>>,
shaders_f64: RwLock<HashMap<ShaderKey, Arc<ShaderModule>>>,
layouts: RwLock<HashMap<BindGroupLayoutKey, Arc<BindGroupLayout>>>,
pipelines: RwLock<HashMap<PipelineKey, Arc<ComputePipeline>>>,
pipelines_f64: RwLock<HashMap<PipelineKey, Arc<ComputePipeline>>>,
}
impl PipelineCache {
#[must_use]
pub fn new() -> Self {
Self {
shaders: RwLock::new(HashMap::new()),
shaders_f64: RwLock::new(HashMap::new()),
layouts: RwLock::new(HashMap::new()),
pipelines: RwLock::new(HashMap::new()),
pipelines_f64: RwLock::new(HashMap::new()),
}
}
pub fn get_or_compile_shader(
&self,
device: &Device,
adapter_info: &wgpu::AdapterInfo,
source: &str,
label: Option<&str>,
) -> Arc<ShaderModule> {
let fingerprint = DeviceFingerprint::from_device_info(device, adapter_info);
let key = ShaderKey::new(source, fingerprint);
if let Some(m) = read_or_recover(&self.shaders).get(&key) {
return m.clone();
}
let resolved: std::borrow::Cow<'_, str> =
if crate::shaders::precision::compiler::source_is_f64(source) {
std::borrow::Cow::Owned(
crate::shaders::precision::downcast_f64_to_f32_with_transcendentals(source),
)
} else {
std::borrow::Cow::Borrowed(source)
};
let module = Arc::new(device.create_shader_module(wgpu::ShaderModuleDescriptor {
label,
source: wgpu::ShaderSource::Wgsl((&*resolved).into()),
}));
write_or_recover(&self.shaders)
.entry(key)
.or_insert(module)
.clone()
}
pub fn get_or_create_layout(
&self,
device: &Device,
adapter_info: &wgpu::AdapterInfo,
signature: BindGroupLayoutSignature,
label: Option<&str>,
) -> Arc<BindGroupLayout> {
let fingerprint = DeviceFingerprint::from_device_info(device, adapter_info);
let key = BindGroupLayoutKey::new(signature, fingerprint);
if let Some(l) = read_or_recover(&self.layouts).get(&key) {
return l.clone();
}
let layout = {
let mut entries = Vec::new();
let mut binding = 0u32;
for _ in 0..signature.read_only_buffers {
entries.push(wgpu::BindGroupLayoutEntry {
binding,
visibility: wgpu::ShaderStages::COMPUTE,
ty: wgpu::BindingType::Buffer {
ty: wgpu::BufferBindingType::Storage { read_only: true },
has_dynamic_offset: false,
min_binding_size: None,
},
count: None,
});
binding += 1;
}
for _ in 0..signature.read_write_buffers {
entries.push(wgpu::BindGroupLayoutEntry {
binding,
visibility: wgpu::ShaderStages::COMPUTE,
ty: wgpu::BindingType::Buffer {
ty: wgpu::BufferBindingType::Storage { read_only: false },
has_dynamic_offset: false,
min_binding_size: None,
},
count: None,
});
binding += 1;
}
for _ in 0..signature.uniform_buffers {
entries.push(wgpu::BindGroupLayoutEntry {
binding,
visibility: wgpu::ShaderStages::COMPUTE,
ty: wgpu::BindingType::Buffer {
ty: wgpu::BufferBindingType::Uniform,
has_dynamic_offset: false,
min_binding_size: None,
},
count: None,
});
binding += 1;
}
let layout = device.create_bind_group_layout(&wgpu::BindGroupLayoutDescriptor {
label,
entries: &entries,
});
Arc::new(layout)
};
write_or_recover(&self.layouts)
.entry(key)
.or_insert(layout)
.clone()
}
pub fn get_or_create_pipeline(
&self,
device: &Device,
adapter_info: &wgpu::AdapterInfo,
shader_source: &str,
layout_signature: BindGroupLayoutSignature,
entry_point: &str,
label: Option<&str>,
) -> Arc<ComputePipeline> {
let fingerprint = DeviceFingerprint::from_device_info(device, adapter_info);
let key = PipelineKey::new(shader_source, layout_signature, entry_point, fingerprint);
if let Some(p) = read_or_recover(&self.pipelines).get(&key) {
return p.clone();
}
let shader = self.get_or_compile_shader(device, adapter_info, shader_source, label);
let layout = self.get_or_create_layout(device, adapter_info, layout_signature, label);
let pipeline_layout = device.create_pipeline_layout(&wgpu::PipelineLayoutDescriptor {
label,
bind_group_layouts: &[&layout],
immediate_size: 0,
});
let pipeline = Arc::new(
device.create_compute_pipeline(&wgpu::ComputePipelineDescriptor {
label,
layout: Some(&pipeline_layout),
module: &shader,
entry_point: Some(entry_point),
cache: None,
compilation_options: Default::default(),
}),
);
write_or_recover(&self.pipelines)
.entry(key)
.or_insert(pipeline)
.clone()
}
pub fn get_or_compile_shader_f64_native(
&self,
device: &Device,
adapter_info: &wgpu::AdapterInfo,
source: &str,
label: Option<&str>,
) -> Arc<ShaderModule> {
let fingerprint = DeviceFingerprint::from_device_info(device, adapter_info);
let key = ShaderKey::new(source, fingerprint);
if let Some(m) = read_or_recover(&self.shaders_f64).get(&key) {
return m.clone();
}
let stripped: std::borrow::Cow<'_, str> = if source.contains("enable f64;") {
source
.lines()
.filter(|l| l.trim() != "enable f64;")
.collect::<Vec<_>>()
.join("\n")
.into()
} else {
source.into()
};
let module = Arc::new(device.create_shader_module(wgpu::ShaderModuleDescriptor {
label,
source: wgpu::ShaderSource::Wgsl((&*stripped).into()),
}));
write_or_recover(&self.shaders_f64)
.entry(key)
.or_insert(module)
.clone()
}
pub fn get_or_create_pipeline_f64_native(
&self,
device: &Device,
adapter_info: &wgpu::AdapterInfo,
shader_source: &str,
layout_signature: BindGroupLayoutSignature,
entry_point: &str,
label: Option<&str>,
) -> Arc<ComputePipeline> {
let fingerprint = DeviceFingerprint::from_device_info(device, adapter_info);
let key = PipelineKey::new(shader_source, layout_signature, entry_point, fingerprint);
if let Some(p) = read_or_recover(&self.pipelines_f64).get(&key) {
return p.clone();
}
let shader =
self.get_or_compile_shader_f64_native(device, adapter_info, shader_source, label);
let layout = self.get_or_create_layout(device, adapter_info, layout_signature, label);
let pipeline_layout = device.create_pipeline_layout(&wgpu::PipelineLayoutDescriptor {
label,
bind_group_layouts: &[&layout],
immediate_size: 0,
});
let pipeline = Arc::new(
device.create_compute_pipeline(&wgpu::ComputePipelineDescriptor {
label,
layout: Some(&pipeline_layout),
module: &shader,
entry_point: Some(entry_point),
cache: None,
compilation_options: Default::default(),
}),
);
write_or_recover(&self.pipelines_f64)
.entry(key)
.or_insert(pipeline)
.clone()
}
pub fn stats(&self) -> CacheStats {
CacheStats {
shaders: read_or_recover(&self.shaders).len()
+ read_or_recover(&self.shaders_f64).len(),
layouts: read_or_recover(&self.layouts).len(),
pipelines: read_or_recover(&self.pipelines).len()
+ read_or_recover(&self.pipelines_f64).len(),
}
}
pub fn clear(&self) {
write_or_recover(&self.shaders).clear();
write_or_recover(&self.shaders_f64).clear();
write_or_recover(&self.layouts).clear();
write_or_recover(&self.pipelines).clear();
write_or_recover(&self.pipelines_f64).clear();
}
}
#[derive(Debug, Clone)]
pub struct CacheStats {
pub shaders: usize,
pub layouts: usize,
pub pipelines: usize,
}
pub static GLOBAL_CACHE: std::sync::LazyLock<PipelineCache> =
std::sync::LazyLock::new(PipelineCache::new);
pub fn create_f64_data_pipeline(
device: &crate::device::WgpuDevice,
shader_src: &str,
layout_sig: BindGroupLayoutSignature,
entry: &str,
label: Option<&str>,
) -> std::sync::Arc<wgpu::ComputePipeline> {
let adapter_info = device.adapter_info();
let needs_native = device
.device()
.features()
.contains(wgpu::Features::SHADER_F64)
&& crate::shaders::precision::compiler::source_is_f64(shader_src);
if needs_native {
GLOBAL_CACHE.get_or_create_pipeline_f64_native(
device.device(),
adapter_info,
shader_src,
layout_sig,
entry,
label,
)
} else {
GLOBAL_CACHE.get_or_create_pipeline(
device.device(),
adapter_info,
shader_src,
layout_sig,
entry,
label,
)
}
}
pub fn clear_global_cache() {
GLOBAL_CACHE.clear();
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_fingerprint_consistency() {
let adapter_info = wgpu::AdapterInfo {
name: "Test GPU".to_string(),
vendor: 0,
device: 0,
device_type: wgpu::DeviceType::DiscreteGpu,
driver: "test".to_string(),
driver_info: "1.0".to_string(),
backend: wgpu::Backend::Vulkan,
device_pci_bus_id: String::new(),
subgroup_min_size: 1,
subgroup_max_size: 128,
transient_saves_memory: false,
};
let fp1 = DeviceFingerprint::from_adapter_info(&adapter_info);
let fp2 = DeviceFingerprint::from_adapter_info(&adapter_info);
assert_eq!(fp1, fp2);
}
#[test]
fn test_different_gpus_different_fingerprints() {
let nvidia = wgpu::AdapterInfo {
name: "NVIDIA RTX 3090".to_string(),
vendor: 0,
device: 0,
device_type: wgpu::DeviceType::DiscreteGpu,
driver: "nvidia".to_string(),
driver_info: "1.0".to_string(),
backend: wgpu::Backend::Vulkan,
device_pci_bus_id: String::new(),
subgroup_min_size: 1,
subgroup_max_size: 128,
transient_saves_memory: false,
};
let amd = wgpu::AdapterInfo {
name: "AMD RX 6950 XT".to_string(),
vendor: 0,
device: 0,
device_type: wgpu::DeviceType::DiscreteGpu,
driver: "radv".to_string(),
driver_info: "1.0".to_string(),
backend: wgpu::Backend::Vulkan,
device_pci_bus_id: String::new(),
subgroup_min_size: 1,
subgroup_max_size: 128,
transient_saves_memory: false,
};
let fp_nvidia = DeviceFingerprint::from_adapter_info(&nvidia);
let fp_amd = DeviceFingerprint::from_adapter_info(&amd);
assert_ne!(
fp_nvidia, fp_amd,
"Different GPUs should have different fingerprints"
);
}
#[test]
fn test_layout_signature_presets() {
let binary = BindGroupLayoutSignature::elementwise_binary();
assert_eq!(binary.read_only_buffers, 2);
assert_eq!(binary.read_write_buffers, 1);
}
}