use naga::{Arena, BinaryOperator, Expression, Handle, MathFunction};
fn mark_f64_expressions(expressions: &Arena<Expression>) -> Vec<bool> {
let mut is_f64 = vec![false; expressions.len()];
for (handle, expr) in expressions.iter() {
is_f64[handle.index()] = match *expr {
Expression::Literal(naga::Literal::F64(_)) => true,
Expression::As {
kind: naga::ScalarKind::Float,
convert: Some(8),
..
} => true,
Expression::Binary { left, right, .. } => is_f64[left.index()] || is_f64[right.index()],
Expression::Math { arg, .. } => is_f64[arg.index()],
Expression::Unary { expr, .. } => is_f64[expr.index()],
Expression::Select { accept, reject, .. } => {
is_f64[accept.index()] || is_f64[reject.index()]
}
_ => false,
};
}
is_f64
}
pub fn fuse_multiply_add(expressions: &mut Arena<Expression>) -> usize {
let len = expressions.len();
if len < 2 {
return 0;
}
let is_f64 = mark_f64_expressions(expressions);
let mut ref_counts = vec![0u32; len];
for (_handle, expr) in expressions.iter() {
visit_operands(expr, |h| {
ref_counts[h.index()] += 1;
});
}
let replacements: Vec<(Handle<Expression>, Expression)> = expressions
.iter()
.filter_map(|(handle, expr)| {
match *expr {
Expression::Binary {
op: BinaryOperator::Add,
left,
right,
} => try_fuse_add(expressions, left, right, &ref_counts, &is_f64)
.or_else(|| try_fuse_add(expressions, right, left, &ref_counts, &is_f64))
.map(|fma| (handle, fma)),
Expression::Binary {
op: BinaryOperator::Subtract,
left: mul_candidate,
right: addend,
} => try_fuse_sub_left(expressions, mul_candidate, addend, &ref_counts, &is_f64)
.map(|fma| (handle, fma)),
_ => None,
}
})
.collect();
let count = replacements.len();
for (handle, replacement) in replacements {
expressions[handle] = replacement;
}
count
}
fn try_fuse_add(
expressions: &Arena<Expression>,
mul_candidate: Handle<Expression>,
addend: Handle<Expression>,
ref_counts: &[u32],
is_f64: &[bool],
) -> Option<Expression> {
if ref_counts[mul_candidate.index()] != 1 {
return None;
}
if is_f64[mul_candidate.index()] {
return None;
}
match expressions[mul_candidate] {
Expression::Binary {
op: BinaryOperator::Multiply,
left: a,
right: b,
} => Some(Expression::Math {
fun: MathFunction::Fma,
arg: a,
arg1: Some(b),
arg2: Some(addend),
arg3: None,
}),
_ => None,
}
}
fn try_fuse_sub_left(
expressions: &Arena<Expression>,
mul_candidate: Handle<Expression>,
subtrahend: Handle<Expression>,
ref_counts: &[u32],
is_f64: &[bool],
) -> Option<Expression> {
if ref_counts[mul_candidate.index()] != 1 {
return None;
}
if is_f64[mul_candidate.index()] {
return None;
}
match expressions[mul_candidate] {
Expression::Binary {
op: BinaryOperator::Multiply,
left: a,
right: b,
} => {
let neg_handle = find_negate(expressions, subtrahend)?;
Some(Expression::Math {
fun: MathFunction::Fma,
arg: a,
arg1: Some(b),
arg2: Some(neg_handle),
arg3: None,
})
}
_ => None,
}
}
fn find_negate(
expressions: &Arena<Expression>,
target: Handle<Expression>,
) -> Option<Handle<Expression>> {
for (handle, expr) in expressions.iter() {
if let Expression::Unary {
op: naga::UnaryOperator::Negate,
expr: inner,
} = *expr
&& inner == target
{
return Some(handle);
}
}
None
}
pub(crate) fn visit_operands<F: FnMut(Handle<Expression>)>(expr: &Expression, mut f: F) {
match *expr {
Expression::Access { base, index } => {
f(base);
f(index);
}
Expression::AccessIndex { base, .. } => f(base),
Expression::Splat { value, .. } => f(value),
Expression::Swizzle { vector, .. } => f(vector),
Expression::Compose { ref components, .. } => {
for &c in components {
f(c);
}
}
Expression::Load { pointer } => f(pointer),
Expression::Unary { expr, .. } => f(expr),
Expression::Binary { left, right, .. } => {
f(left);
f(right);
}
Expression::Select {
condition,
accept,
reject,
} => {
f(condition);
f(accept);
f(reject);
}
Expression::Derivative { expr, .. } => f(expr),
Expression::Relational { argument, .. } => f(argument),
Expression::Math {
arg,
arg1,
arg2,
arg3,
..
} => {
f(arg);
if let Some(a) = arg1 {
f(a);
}
if let Some(a) = arg2 {
f(a);
}
if let Some(a) = arg3 {
f(a);
}
}
Expression::As { expr, .. } => f(expr),
Expression::ArrayLength(expr) => f(expr),
Expression::ImageSample {
image,
sampler,
coordinate,
array_index,
offset,
depth_ref,
..
} => {
f(image);
f(sampler);
f(coordinate);
if let Some(a) = array_index {
f(a);
}
if let Some(o) = offset {
f(o);
}
if let Some(d) = depth_ref {
f(d);
}
}
Expression::ImageLoad {
image,
coordinate,
array_index,
sample,
level,
} => {
f(image);
f(coordinate);
if let Some(a) = array_index {
f(a);
}
if let Some(s) = sample {
f(s);
}
if let Some(l) = level {
f(l);
}
}
Expression::ImageQuery { image, .. } => f(image),
Expression::RayQueryGetIntersection { query, .. } => f(query),
Expression::RayQueryVertexPositions { .. } => {}
Expression::Literal(_)
| Expression::Constant(_)
| Expression::Override(_)
| Expression::ZeroValue(_)
| Expression::FunctionArgument(_)
| Expression::GlobalVariable(_)
| Expression::LocalVariable(_)
| Expression::CallResult(_)
| Expression::AtomicResult { .. }
| Expression::WorkGroupUniformLoadResult { .. }
| Expression::RayQueryProceedResult
| Expression::SubgroupBallotResult
| Expression::SubgroupOperationResult { .. } => {}
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_fma_fusion_on_add_pattern() {
let wgsl = r"
@group(0) @binding(0) var<storage, read> a_buf: array<f32>;
@group(0) @binding(1) var<storage, read> b_buf: array<f32>;
@group(0) @binding(2) var<storage, read> c_buf: array<f32>;
@group(0) @binding(3) var<storage, read_write> out: array<f32>;
@compute @workgroup_size(64)
fn main(@builtin(global_invocation_id) gid: vec3<u32>) {
let i = gid.x;
let a = a_buf[i];
let b = b_buf[i];
let c = c_buf[i];
let product = a * b;
let result = product + c;
out[i] = result;
}
";
let mut module = naga::front::wgsl::parse_str(wgsl).expect("parse");
let mut total_fusions = 0usize;
for (_h, func) in module.functions.iter_mut() {
total_fusions += fuse_multiply_add(&mut func.expressions);
}
for ep in &mut module.entry_points {
total_fusions += fuse_multiply_add(&mut ep.function.expressions);
}
assert!(
total_fusions >= 1,
"expected at least 1 FMA fusion, got {total_fusions}"
);
let mut validator = naga::valid::Validator::new(
naga::valid::ValidationFlags::all(),
naga::valid::Capabilities::all(),
);
validator
.validate(&module)
.expect("fused module should validate");
}
#[test]
fn test_no_fusion_when_mul_has_multiple_consumers() {
let wgsl = r"
@group(0) @binding(0) var<storage, read> input: array<f32>;
@group(0) @binding(1) var<storage, read_write> out: array<f32>;
@compute @workgroup_size(64)
fn main(@builtin(global_invocation_id) gid: vec3<u32>) {
let i = gid.x;
let a = input[i];
let b = input[i + 1u];
let c = input[i + 2u];
let product = a * b;
let r1 = product + c;
let r2 = product + 1.0;
out[i] = r1 + r2;
}
";
let mut module = naga::front::wgsl::parse_str(wgsl).expect("parse");
let mut total_fusions = 0usize;
for (_h, func) in module.functions.iter_mut() {
total_fusions += fuse_multiply_add(&mut func.expressions);
}
for ep in &mut module.entry_points {
total_fusions += fuse_multiply_add(&mut ep.function.expressions);
}
assert_eq!(
total_fusions, 0,
"should not fuse when mul has multiple consumers"
);
}
#[test]
fn test_no_fusion_on_f64_multiply_add() {
let wgsl = r"
@group(0) @binding(0) var<storage, read_write> out: array<f64>;
@compute @workgroup_size(1)
fn main(@builtin(global_invocation_id) gid: vec3<u32>) {
let x: f64 = f64(3.0);
let y: f64 = x * f64(2.0) + f64(1.0);
out[gid.x] = y;
}
";
let mut module = naga::front::wgsl::parse_str(wgsl).expect("parse");
let mut total_fusions = 0usize;
for (_h, func) in module.functions.iter_mut() {
total_fusions += fuse_multiply_add(&mut func.expressions);
}
for ep in &mut module.entry_points {
total_fusions += fuse_multiply_add(&mut ep.function.expressions);
}
assert_eq!(
total_fusions, 0,
"must not fuse f64 multiply-add: WGSL fma() is f32/f16 only"
);
}
}