use crate::device::capabilities::WORKGROUP_SIZE_1D;
use std::collections::HashMap;
use std::path::PathBuf;
use std::sync::RwLock;
use std::time::Instant;
const CALIBRATION_WG_SIZES: [u32; 4] = [32, 64, 128, 256];
const CALIBRATION_TEST_SIZE: usize = 4_000_000;
use wgpu::util::DeviceExt;
pub(crate) trait GpuDeviceForCalibration {
fn device(&self) -> &wgpu::Device;
fn name(&self) -> &str;
fn submit_and_poll_calibration(&self, commands: impl IntoIterator<Item = wgpu::CommandBuffer>);
}
#[derive(Debug, Clone)]
#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))]
pub struct GpuCalibration {
pub device_id: String,
pub device_name: String,
pub optimal_workgroup_size: u32,
pub peak_bandwidth_gbps: f64,
pub dispatch_overhead_us: f64,
pub calibrated_at: u64,
}
impl Default for GpuCalibration {
fn default() -> Self {
Self {
device_id: String::new(),
device_name: String::new(),
optimal_workgroup_size: WORKGROUP_SIZE_1D,
peak_bandwidth_gbps: 0.0,
dispatch_overhead_us: 0.0,
calibrated_at: 0,
}
}
}
#[derive(Default)]
pub struct AutoTuner {
calibrations: RwLock<HashMap<String, GpuCalibration>>,
cache_path: Option<PathBuf>,
}
impl AutoTuner {
#[must_use]
pub fn new() -> Self {
Self {
calibrations: RwLock::new(HashMap::new()),
cache_path: None,
}
}
#[must_use]
pub fn with_cache(cache_path: PathBuf) -> Self {
let mut tuner = Self::new();
tuner.cache_path = Some(cache_path.clone());
#[cfg(feature = "serde")]
if let Ok(contents) = std::fs::read_to_string(&cache_path) {
if let Ok(cals) = serde_json::from_str::<Vec<GpuCalibration>>(&contents) {
let mut map = tuner
.calibrations
.write()
.unwrap_or_else(std::sync::PoisonError::into_inner);
for cal in cals {
map.insert(cal.device_name.clone(), cal);
}
}
}
#[cfg(not(feature = "serde"))]
if let Err(e) = std::fs::read_to_string(&cache_path) {
tracing::debug!(
"autotune cache at {}: {e} (serde disabled, cannot parse)",
cache_path.display()
);
}
tuner
}
pub(crate) fn get_or_calibrate(
&self,
wgpu_device: &impl GpuDeviceForCalibration,
) -> GpuCalibration {
let device_name = wgpu_device.name();
{
let cals = self
.calibrations
.read()
.unwrap_or_else(std::sync::PoisonError::into_inner);
if let Some(cal) = cals.get(device_name) {
return cal.clone();
}
}
let cal = self.calibrate_device(wgpu_device);
{
let mut cals = self
.calibrations
.write()
.unwrap_or_else(std::sync::PoisonError::into_inner);
cals.insert(device_name.to_string(), cal.clone());
}
self.save_cache();
cal
}
fn calibrate_device(&self, wgpu_device: &impl GpuDeviceForCalibration) -> GpuCalibration {
let device = wgpu_device.device();
let device_name = wgpu_device.name();
let mut best_wg = WORKGROUP_SIZE_1D;
let mut best_bw = 0.0f64;
for &wg_size in &CALIBRATION_WG_SIZES {
if let Some(bw) = self.measure_bandwidth(wgpu_device, wg_size, CALIBRATION_TEST_SIZE)
&& bw > best_bw
{
best_bw = bw;
best_wg = wg_size;
}
}
let overhead = self.measure_overhead(wgpu_device, best_wg);
let now = std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.map_or(0, |d| d.as_secs());
GpuCalibration {
device_id: format!("{device:?}"),
device_name: device_name.to_string(),
optimal_workgroup_size: best_wg,
peak_bandwidth_gbps: best_bw,
dispatch_overhead_us: overhead,
calibrated_at: now,
}
}
fn measure_bandwidth(
&self,
wgpu_device: &impl GpuDeviceForCalibration,
workgroup_size: u32,
size: usize,
) -> Option<f64> {
let device = wgpu_device.device();
let shader_src = format!(
r"
@group(0) @binding(0) var<storage, read> a: array<f32>;
@group(0) @binding(1) var<storage, read> b: array<f32>;
@group(0) @binding(2) var<storage, read_write> output: array<f32>;
@compute @workgroup_size({workgroup_size})
fn main(@builtin(global_invocation_id) global_id: vec3<u32>) {{
let idx = global_id.x;
if (idx >= arrayLength(&output)) {{
return;
}}
output[idx] = a[idx] + b[idx];
}}
"
);
let data: Vec<f32> = (0..size).map(|i| i as f32 * 0.001).collect();
let buf_a = device.create_buffer_init(&wgpu::util::BufferInitDescriptor {
label: Some("A"),
contents: bytemuck::cast_slice(&data),
usage: wgpu::BufferUsages::STORAGE,
});
let buf_b = device.create_buffer_init(&wgpu::util::BufferInitDescriptor {
label: Some("B"),
contents: bytemuck::cast_slice(&data),
usage: wgpu::BufferUsages::STORAGE,
});
let buf_out = device.create_buffer(&wgpu::BufferDescriptor {
label: Some("Out"),
size: (size * 4) as u64,
usage: wgpu::BufferUsages::STORAGE,
mapped_at_creation: false,
});
let shader = device.create_shader_module(wgpu::ShaderModuleDescriptor {
label: None,
source: wgpu::ShaderSource::Wgsl(shader_src.into()),
});
let bgl = device.create_bind_group_layout(&wgpu::BindGroupLayoutDescriptor {
label: None,
entries: &[
wgpu::BindGroupLayoutEntry {
binding: 0,
visibility: wgpu::ShaderStages::COMPUTE,
ty: wgpu::BindingType::Buffer {
ty: wgpu::BufferBindingType::Storage { read_only: true },
has_dynamic_offset: false,
min_binding_size: None,
},
count: None,
},
wgpu::BindGroupLayoutEntry {
binding: 1,
visibility: wgpu::ShaderStages::COMPUTE,
ty: wgpu::BindingType::Buffer {
ty: wgpu::BufferBindingType::Storage { read_only: true },
has_dynamic_offset: false,
min_binding_size: None,
},
count: None,
},
wgpu::BindGroupLayoutEntry {
binding: 2,
visibility: wgpu::ShaderStages::COMPUTE,
ty: wgpu::BindingType::Buffer {
ty: wgpu::BufferBindingType::Storage { read_only: false },
has_dynamic_offset: false,
min_binding_size: None,
},
count: None,
},
],
});
let bind_group = device.create_bind_group(&wgpu::BindGroupDescriptor {
label: None,
layout: &bgl,
entries: &[
wgpu::BindGroupEntry {
binding: 0,
resource: buf_a.as_entire_binding(),
},
wgpu::BindGroupEntry {
binding: 1,
resource: buf_b.as_entire_binding(),
},
wgpu::BindGroupEntry {
binding: 2,
resource: buf_out.as_entire_binding(),
},
],
});
let pl = device.create_pipeline_layout(&wgpu::PipelineLayoutDescriptor {
label: None,
bind_group_layouts: &[&bgl],
immediate_size: 0,
});
let pipeline = device.create_compute_pipeline(&wgpu::ComputePipelineDescriptor {
label: None,
layout: Some(&pl),
module: &shader,
entry_point: Some("main"),
cache: None,
compilation_options: Default::default(),
});
let workgroups = (size as u32).div_ceil(workgroup_size).min(65_535);
for _ in 0..3 {
let mut encoder =
device.create_command_encoder(&wgpu::CommandEncoderDescriptor::default());
{
let mut pass = encoder.begin_compute_pass(&wgpu::ComputePassDescriptor::default());
pass.set_pipeline(&pipeline);
pass.set_bind_group(0, Some(&bind_group), &[]);
pass.dispatch_workgroups(workgroups, 1, 1);
}
wgpu_device.submit_and_poll_calibration(Some(encoder.finish()));
}
let iterations = 10;
let start = Instant::now();
for _ in 0..iterations {
let mut encoder =
device.create_command_encoder(&wgpu::CommandEncoderDescriptor::default());
{
let mut pass = encoder.begin_compute_pass(&wgpu::ComputePassDescriptor::default());
pass.set_pipeline(&pipeline);
pass.set_bind_group(0, Some(&bind_group), &[]);
pass.dispatch_workgroups(workgroups, 1, 1);
}
wgpu_device.submit_and_poll_calibration(Some(encoder.finish()));
}
let elapsed = start.elapsed();
let time_per_op = elapsed.as_secs_f64() / iterations as f64;
let bytes = size * 3 * 4;
let bandwidth_gbps = bytes as f64 / time_per_op / 1e9;
Some(bandwidth_gbps)
}
fn measure_overhead(
&self,
wgpu_device: &impl GpuDeviceForCalibration,
workgroup_size: u32,
) -> f64 {
let device = wgpu_device.device();
let size = 1024usize;
let shader_src = format!(
r"
@group(0) @binding(0) var<storage, read_write> data: array<f32>;
@compute @workgroup_size({workgroup_size})
fn main(@builtin(global_invocation_id) global_id: vec3<u32>) {{
let idx = global_id.x;
if (idx < arrayLength(&data)) {{
data[idx] = data[idx] + 1.0;
}}
}}
"
);
let data: Vec<f32> = vec![0.0; size];
let buffer = device.create_buffer_init(&wgpu::util::BufferInitDescriptor {
label: Some("Data"),
contents: bytemuck::cast_slice(&data),
usage: wgpu::BufferUsages::STORAGE,
});
let shader = device.create_shader_module(wgpu::ShaderModuleDescriptor {
label: None,
source: wgpu::ShaderSource::Wgsl(shader_src.into()),
});
let bgl = device.create_bind_group_layout(&wgpu::BindGroupLayoutDescriptor {
label: None,
entries: &[wgpu::BindGroupLayoutEntry {
binding: 0,
visibility: wgpu::ShaderStages::COMPUTE,
ty: wgpu::BindingType::Buffer {
ty: wgpu::BufferBindingType::Storage { read_only: false },
has_dynamic_offset: false,
min_binding_size: None,
},
count: None,
}],
});
let bind_group = device.create_bind_group(&wgpu::BindGroupDescriptor {
label: None,
layout: &bgl,
entries: &[wgpu::BindGroupEntry {
binding: 0,
resource: buffer.as_entire_binding(),
}],
});
let pl = device.create_pipeline_layout(&wgpu::PipelineLayoutDescriptor {
label: None,
bind_group_layouts: &[&bgl],
immediate_size: 0,
});
let pipeline = device.create_compute_pipeline(&wgpu::ComputePipelineDescriptor {
label: None,
layout: Some(&pl),
module: &shader,
entry_point: Some("main"),
cache: None,
compilation_options: Default::default(),
});
let workgroups = (size as u32).div_ceil(workgroup_size);
for _ in 0..10 {
let mut encoder =
device.create_command_encoder(&wgpu::CommandEncoderDescriptor::default());
{
let mut pass = encoder.begin_compute_pass(&wgpu::ComputePassDescriptor::default());
pass.set_pipeline(&pipeline);
pass.set_bind_group(0, Some(&bind_group), &[]);
pass.dispatch_workgroups(workgroups, 1, 1);
}
wgpu_device.submit_and_poll_calibration(Some(encoder.finish()));
}
let iterations = 100;
let start = Instant::now();
for _ in 0..iterations {
let mut encoder =
device.create_command_encoder(&wgpu::CommandEncoderDescriptor::default());
{
let mut pass = encoder.begin_compute_pass(&wgpu::ComputePassDescriptor::default());
pass.set_pipeline(&pipeline);
pass.set_bind_group(0, Some(&bind_group), &[]);
pass.dispatch_workgroups(workgroups, 1, 1);
}
wgpu_device.submit_and_poll_calibration(Some(encoder.finish()));
}
let elapsed = start.elapsed();
elapsed.as_secs_f64() * 1e6 / iterations as f64
}
#[cfg(feature = "serde")]
fn save_cache(&self) {
if let Some(ref path) = self.cache_path {
let cals = self
.calibrations
.read()
.unwrap_or_else(std::sync::PoisonError::into_inner);
let cal_vec: Vec<&GpuCalibration> = cals.values().collect();
if let Ok(json) = serde_json::to_string_pretty(&cal_vec) {
if let Err(e) = std::fs::write(path, &json) {
tracing::debug!(path = %path.display(), error = %e, "autotune cache write failed");
}
}
}
}
#[cfg(not(feature = "serde"))]
fn save_cache(&self) {
}
pub(crate) fn recalibrate(&self, wgpu_device: &impl GpuDeviceForCalibration) -> GpuCalibration {
let cal = self.calibrate_device(wgpu_device);
{
let mut cals = self
.calibrations
.write()
.unwrap_or_else(std::sync::PoisonError::into_inner);
cals.insert(wgpu_device.name().to_string(), cal.clone());
}
self.save_cache();
cal
}
}
pub static GLOBAL_TUNER: std::sync::LazyLock<AutoTuner> = std::sync::LazyLock::new(|| {
let cache_dir = std::env::var("XDG_CACHE_HOME").map_or_else(
|_| {
std::env::var("HOME").map_or_else(
|_| std::env::temp_dir(),
|h| PathBuf::from(h).join(".cache"),
)
},
PathBuf::from,
);
let cache_path = cache_dir
.join(env!("CARGO_PKG_NAME"))
.join("gpu_calibrations.json");
if let Some(parent) = cache_path.parent()
&& let Err(e) = std::fs::create_dir_all(parent)
{
tracing::debug!(path = %parent.display(), error = %e, "autotune cache dir creation failed");
}
AutoTuner::with_cache(cache_path)
});
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_default_calibration() {
let cal = GpuCalibration::default();
assert_eq!(cal.optimal_workgroup_size, 256);
}
#[test]
fn test_tuner_creation() {
let tuner = AutoTuner::new();
assert!(
tuner
.calibrations
.read()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.is_empty()
);
}
}