pub mod compiler;
pub mod cpu;
pub mod eps;
mod math_f64;
pub mod polyfill;
mod preambles;
pub use compiler::{
downcast_f64_to_df64, downcast_f64_to_f32, downcast_f64_to_f32_with_transcendentals,
};
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
pub enum Precision {
Binary,
Int2,
Q4,
Q8,
Fp8E5M2,
Fp8E4M3,
Bf16,
F16,
F32,
F64,
Df64,
Qf128,
Df128,
}
impl Precision {
#[must_use]
pub fn scalar(&self) -> &'static str {
match self {
Self::Binary | Self::Int2 | Self::Q4 | Self::Q8 => "u32",
Self::Fp8E5M2 | Self::Fp8E4M3 => "u32",
Self::Bf16 => "u32",
Self::F16 => "f16",
Self::F32 => "f32",
Self::F64 => "f64",
Self::Df64 => "vec2<f32>",
Self::Qf128 => "vec4<f32>",
Self::Df128 => "vec2<f64>",
}
}
#[must_use]
pub fn vec2(&self) -> &'static str {
match self {
Self::F16 => "vec2<f16>",
Self::F32 => "vec2<f32>",
Self::F64 => "f64",
Self::Df64 => "vec2<f32>",
Self::Qf128 => "vec4<f32>",
Self::Df128 => "vec2<f64>",
_ => "u32",
}
}
#[must_use]
pub fn vec4(&self) -> &'static str {
match self {
Self::F16 => "vec4<f16>",
Self::F32 => "vec4<f32>",
Self::F64 => "f64",
Self::Df64 => "vec2<f32>",
Self::Qf128 => "vec4<f32>",
Self::Df128 => "vec2<f64>",
_ => "u32",
}
}
#[must_use]
pub fn has_vec4(&self) -> bool {
matches!(self, Self::F16 | Self::F32)
}
#[must_use]
pub fn bytes_per_element(&self) -> usize {
match self {
Self::Binary | Self::Int2 | Self::Q4 | Self::Q8 => 1,
Self::Fp8E5M2 | Self::Fp8E4M3 => 1,
Self::Bf16 | Self::F16 => 2,
Self::F32 => 4,
Self::F64 | Self::Df64 => 8,
Self::Qf128 | Self::Df128 => 16,
}
}
#[must_use]
pub fn required_feature(&self) -> Option<wgpu::Features> {
match self {
Self::F16 => Some(wgpu::Features::SHADER_F16),
Self::F64 | Self::Df128 => Some(wgpu::Features::SHADER_F64),
_ => None,
}
}
#[must_use]
pub fn is_f64_class(&self) -> bool {
matches!(self, Self::F64 | Self::Df64 | Self::Df128)
}
#[must_use]
pub fn is_reduced(&self) -> bool {
matches!(
self,
Self::Binary
| Self::Int2
| Self::Q4
| Self::Q8
| Self::Fp8E5M2
| Self::Fp8E4M3
| Self::Bf16
| Self::F16
)
}
#[must_use]
pub fn is_extended(&self) -> bool {
matches!(self, Self::Qf128 | Self::Df128)
}
#[must_use]
pub fn is_quantized(&self) -> bool {
matches!(self, Self::Binary | Self::Int2 | Self::Q4 | Self::Q8)
}
#[must_use]
pub fn op_preamble(&self) -> &'static str {
match self {
Self::Binary | Self::Int2 | Self::Q4 | Self::Q8 => OP_PREAMBLE_QUANTIZED,
Self::Fp8E5M2 => OP_PREAMBLE_FP8_E5M2,
Self::Fp8E4M3 => OP_PREAMBLE_FP8_E4M3,
Self::Bf16 => OP_PREAMBLE_BF16,
Self::F16 => OP_PREAMBLE_F16,
Self::F32 => OP_PREAMBLE_F32,
Self::F64 => OP_PREAMBLE_F64,
Self::Df64 => OP_PREAMBLE_DF64,
Self::Qf128 => OP_PREAMBLE_QF128,
Self::Df128 => OP_PREAMBLE_DF128,
}
}
}
pub use preambles::DF64_PACK_UNPACK;
use preambles::{
OP_PREAMBLE_BF16, OP_PREAMBLE_DF128, OP_PREAMBLE_DF64, OP_PREAMBLE_F16, OP_PREAMBLE_F32,
OP_PREAMBLE_F64, OP_PREAMBLE_FP8_E4M3, OP_PREAMBLE_FP8_E5M2, OP_PREAMBLE_QF128,
OP_PREAMBLE_QUANTIZED,
};
pub struct ShaderTemplate;
impl ShaderTemplate {
#[must_use]
pub fn math_f64_preamble() -> String {
polyfill::math_f64_preamble()
}
#[must_use]
pub fn with_math_f64(shader_body: &str) -> String {
format!(
"{}\n\n// User shader:\n{}",
Self::math_f64_preamble(),
shader_body
)
}
#[must_use]
pub fn for_device(shader_body: &str, device: &crate::device::WgpuDevice) -> String {
Self::for_driver_auto(shader_body, device.needs_f64_exp_log_workaround())
}
#[must_use]
pub fn for_device_auto(shader_body: &str, device: &crate::device::WgpuDevice) -> String {
Self::for_driver_auto(shader_body, device.needs_f64_exp_log_workaround())
}
#[must_use]
pub fn patch_warp_size(shader_body: &str, wave_size: u32) -> String {
shader_body
.replace(
"const WARP_SIZE: u32 = 32u;",
&format!("const WARP_SIZE: u32 = {wave_size}u;"),
)
.replace(
"@workgroup_size(32, 1, 1)",
&format!("@workgroup_size({wave_size}, 1, 1)"),
)
}
#[must_use]
pub fn substitute_fossil_f64(shader_body: &str) -> String {
polyfill::substitute_fossil_f64(shader_body)
}
#[must_use]
pub fn for_driver_auto(shader_body: &str, needs_exp_log_workaround: bool) -> String {
let stripped = shader_body
.lines()
.filter(|l| l.trim() != "enable f64;")
.collect::<Vec<_>>()
.join("\n");
let substituted = polyfill::substitute_fossil_f64(&stripped);
let patched = polyfill::apply_transcendental_workaround_with_sin_cos(
&substituted,
needs_exp_log_workaround,
false,
);
let injected = polyfill::inject_f64_polyfills(&patched, None);
crate::shaders::optimizer::WgslOptimizer::default().optimize(&injected)
}
#[must_use]
pub fn for_device_capabilities(
shader_body: &str,
needs_exp_log_workaround: bool,
caps: &crate::device::capabilities::DeviceCapabilities,
) -> String {
let stripped = shader_body
.lines()
.filter(|l| l.trim() != "enable f64;")
.collect::<Vec<_>>()
.join("\n");
let use_sin_cos_taylor = caps.needs_sin_f64_workaround() || caps.needs_cos_f64_workaround();
let substituted = polyfill::substitute_fossil_f64(&stripped);
let patched = polyfill::apply_transcendental_workaround_with_sin_cos(
&substituted,
needs_exp_log_workaround,
use_sin_cos_taylor,
);
let extra_preamble = if use_sin_cos_taylor
&& (patched.contains("sin_f64_safe(") || patched.contains("cos_f64_safe("))
{
Some(polyfill::SIN_COS_F64_SAFE_PREAMBLE)
} else {
None
};
let injected = polyfill::inject_f64_polyfills(&patched, extra_preamble);
crate::shaders::optimizer::WgslOptimizer::new(caps.latency_model()).optimize(&injected)
}
#[must_use]
pub fn with_math_f64_auto(shader_body: &str) -> String {
use math_f64::F64_FUNCTION_ORDER;
let mut used_functions: Vec<&str> = F64_FUNCTION_ORDER
.iter()
.filter(|func_name| {
let call_pattern = format!("{func_name}(");
let call_pattern_space = format!("{func_name} (");
shader_body.contains(&call_pattern) || shader_body.contains(&call_pattern_space)
})
.copied()
.collect();
if shader_body.contains("round_f64") && !used_functions.contains(&"round_f64") {
used_functions.push("round_f64");
}
if used_functions.is_empty() {
return shader_body.to_string();
}
format!(
"{}\n\n// User shader:\n{}",
polyfill::math_f64_subset(&used_functions),
shader_body
)
}
#[must_use]
pub fn math_f64_subset(functions: &[&str]) -> String {
polyfill::math_f64_subset(functions)
}
#[must_use]
pub fn shader_defines_function(shader_body: &str, func_name: &str) -> bool {
polyfill::shader_defines_function(shader_body, func_name)
}
#[must_use]
pub fn shader_defines_module_var(shader_body: &str, var_name: &str) -> bool {
polyfill::shader_defines_module_var(shader_body, var_name)
}
#[must_use]
pub fn with_math_f64_safe(shader_body: &str) -> String {
polyfill::inject_f64_polyfills(shader_body, None)
}
#[must_use]
pub fn with_math_f64_auto_safe(shader_body: &str) -> String {
polyfill::inject_f64_polyfills(shader_body, None)
}
}
#[cfg(test)]
#[path = "precision_tests.rs"]
mod tests;