use crate::registry::{
BackwardKind, FusedOpEntry, FusedOpFamily, FusedOpParams, FusedOps, PatternMatch,
SubgraphPattern, decompose_via_recipe,
};
use crate::{Graph, NodeId};
use fuel_ir::{DType, Shape};
use fuel_kernel_seam_types::{OpAttrs, OpTag, PatternNode};
pub fn entry() -> FusedOpEntry {
FusedOpEntry {
destructive_input: None,
id: FusedOps::CONV_TRANSPOSE2D,
name: "ConvTranspose2D",
family: FusedOpFamily::Forward,
pattern: SubgraphPattern::Callable(canonical_pattern),
decompose,
backward: BackwardKind::NotDifferentiable,
shape_rule,
dtype_rule,
output_views: None,
}
}
fn shape_rule(input_shapes: &[Shape], params: &FusedOpParams) -> Shape {
debug_assert!(
input_shapes.len() == 2 || input_shapes.len() == 3,
"ConvTranspose2D takes 2 or 3 inputs (x, weight, [bias])",
);
let (stride, padding, output_padding, dilation, groups) = match params {
FusedOpParams::ConvTranspose2D {
stride,
padding,
output_padding,
dilation,
groups,
} => (*stride, *padding, *output_padding, *dilation, *groups),
_ => panic!("conv_transpose_2d::shape_rule got non-ConvTranspose2D params: {params:?}"),
};
let x_dims = input_shapes[0].dims();
let w_dims = input_shapes[1].dims();
debug_assert_eq!(x_dims.len(), 4, "ConvTranspose2D x must be rank 4");
debug_assert_eq!(w_dims.len(), 4, "ConvTranspose2D weight must be rank 4");
let (n, _cin, h_in, w_in) = (x_dims[0], x_dims[1], x_dims[2], x_dims[3]);
let (_cin_w, cout_per_g, kh, kw) = (w_dims[0], w_dims[1], w_dims[2], w_dims[3]);
let cout = cout_per_g * groups;
let (sh, sw) = stride;
let (ph, pw) = padding;
let (oph, opw) = output_padding;
let (dh, dw) = dilation;
let h_out = (h_in.saturating_sub(1)) * sh + dh * (kh.saturating_sub(1)) + oph + 1;
let h_out = h_out.saturating_sub(2 * ph);
let w_out = (w_in.saturating_sub(1)) * sw + dw * (kw.saturating_sub(1)) + opw + 1;
let w_out = w_out.saturating_sub(2 * pw);
Shape::from_dims(&[n, cout, h_out, w_out])
}
fn dtype_rule(input_dtypes: &[DType], _params: &FusedOpParams) -> DType {
debug_assert!(
input_dtypes.len() == 2 || input_dtypes.len() == 3,
"ConvTranspose2D takes 2 or 3 inputs (x, weight, [bias])",
);
input_dtypes[0]
}
fn scatter_flat_index(
kh: usize,
kw: usize,
hin: usize,
win: usize,
sh: usize,
sw: usize,
dh: usize,
dw: usize,
wbuf: usize,
) -> PatternNode {
use OpTag as T;
let op = |op, attrs, operands| PatternNode::Op {
op,
attrs,
operands,
};
let shape_attr = |dims: Vec<i64>| OpAttrs {
target_shape: dims,
..OpAttrs::default()
};
let l = kh * kw * hin * win;
let idx4_shape = vec![kh as i64, kw as i64, hin as i64, win as i64];
let term = |axis_len: usize, reshape_dims: Vec<i64>, factor: f64| -> PatternNode {
let iota = op(
T::Iota,
OpAttrs {
target_shape: vec![axis_len as i64],
..OpAttrs::default()
},
vec![],
);
let re = op(T::Reshape, shape_attr(reshape_dims), vec![iota]);
let scaled = op(
T::MulScalar,
OpAttrs {
scalars: vec![factor],
..OpAttrs::default()
},
vec![re],
);
op(T::BroadcastTo, shape_attr(idx4_shape.clone()), vec![scaled])
};
let term_ky = term(kh, vec![kh as i64, 1, 1, 1], (dh * wbuf) as f64);
let term_kx = term(kw, vec![1, kw as i64, 1, 1], dw as f64);
let term_ih = term(hin, vec![1, 1, hin as i64, 1], (sh * wbuf) as f64);
let term_iw = term(win, vec![1, 1, 1, win as i64], sw as f64);
let add = |a, b| op(T::Add, OpAttrs::default(), vec![a, b]);
let idx4 = add(add(add(term_ky, term_kx), term_ih), term_iw);
let idx1 = op(T::Reshape, shape_attr(vec![l as i64]), vec![idx4]);
op(
T::Cast,
OpAttrs {
cast_dtype: Some("u32".to_string()),
..OpAttrs::default()
},
vec![idx1],
)
}
fn col2im_tail(
csrc: PatternNode,
sidx: PatternNode,
n: usize,
cout: usize,
hbuf: usize,
wbuf: usize,
ph: usize,
pw: usize,
hout: usize,
wout: usize,
) -> PatternNode {
use OpTag as T;
let op = |op, attrs, operands| PatternNode::Op {
op,
attrs,
operands,
};
let shape_attr = |dims: Vec<i64>| OpAttrs {
target_shape: dims,
..OpAttrs::default()
};
let sbuf = (hbuf * wbuf) as i64;
let zero_col = op(
T::Slice,
OpAttrs {
axis: Some(2),
slice_start: Some(0),
slice_len: Some(1),
..OpAttrs::default()
},
vec![csrc.clone()],
);
let base_b = op(
T::BroadcastTo,
shape_attr(vec![n as i64, cout as i64, sbuf]),
vec![zero_col],
);
let base0 = op(
T::MulScalar,
OpAttrs {
scalars: vec![0.0],
..OpAttrs::default()
},
vec![base_b],
);
let acc = op(
T::IndexAdd,
OpAttrs {
axis: Some(2),
..OpAttrs::default()
},
vec![base0, sidx, csrc],
);
let acc4 = op(
T::Reshape,
shape_attr(vec![n as i64, cout as i64, hbuf as i64, wbuf as i64]),
vec![acc],
);
let crop_h = op(
T::Slice,
OpAttrs {
axis: Some(2),
slice_start: Some(ph as u64),
slice_len: Some(hout as u64),
..OpAttrs::default()
},
vec![acc4],
);
op(
T::Slice,
OpAttrs {
axis: Some(3),
slice_start: Some(pw as u64),
slice_len: Some(wout as u64),
..OpAttrs::default()
},
vec![crop_h],
)
}
fn recipe(
n: usize,
cin: usize,
hin: usize,
win: usize,
cout: usize,
kh: usize,
kw: usize,
sh: usize,
sw: usize,
dh: usize,
dw: usize,
ph: usize,
pw: usize,
hbuf: usize,
wbuf: usize,
hout: usize,
wout: usize,
) -> PatternNode {
use OpTag as T;
let op = |op, attrs, operands| PatternNode::Op {
op,
attrs,
operands,
};
let bind = |i| PatternNode::Bind { index: i };
let shape_attr = |dims: Vec<i64>| OpAttrs {
target_shape: dims,
..OpAttrs::default()
};
let m = cout * kh * kw; let s = hin * win; let l = kh * kw * hin * win;
let xf = op(
T::Reshape,
shape_attr(vec![n as i64, cin as i64, s as i64]),
vec![bind(0)],
);
let wp = op(
T::Permute,
OpAttrs {
perm: vec![1, 2, 3, 0],
..OpAttrs::default()
},
vec![bind(1)],
);
let wr = op(
T::Reshape,
shape_attr(vec![1, m as i64, cin as i64]),
vec![wp],
);
let wb = op(
T::BroadcastTo,
shape_attr(vec![n as i64, m as i64, cin as i64]),
vec![wr],
);
let cols = op(T::MatMul, OpAttrs::default(), vec![wb, xf]);
let csrc = op(
T::Reshape,
shape_attr(vec![n as i64, cout as i64, l as i64]),
vec![cols],
);
let sidx = scatter_flat_index(kh, kw, hin, win, sh, sw, dh, dw, wbuf);
col2im_tail(csrc, sidx, n, cout, hbuf, wbuf, ph, pw, hout, wout)
}
fn recipe_grouped(
n: usize,
cin: usize,
hin: usize,
win: usize,
cout: usize,
groups: usize,
kh: usize,
kw: usize,
sh: usize,
sw: usize,
dh: usize,
dw: usize,
ph: usize,
pw: usize,
hbuf: usize,
wbuf: usize,
hout: usize,
wout: usize,
) -> PatternNode {
use OpTag as T;
let op = |op, attrs, operands| PatternNode::Op {
op,
attrs,
operands,
};
let bind = |i| PatternNode::Bind { index: i };
let shape_attr = |dims: Vec<i64>| OpAttrs {
target_shape: dims,
..OpAttrs::default()
};
let cin_per_g = cin / groups;
let cout_per_g = cout / groups;
let m = cout_per_g * kh * kw; let s = hin * win; let l = kh * kw * hin * win;
let xf = op(
T::Reshape,
shape_attr(vec![n as i64, groups as i64, cin_per_g as i64, s as i64]),
vec![bind(0)],
);
let wr5 = op(
T::Reshape,
shape_attr(vec![
groups as i64,
cin_per_g as i64,
cout_per_g as i64,
kh as i64,
kw as i64,
]),
vec![bind(1)],
);
let wp = op(
T::Permute,
OpAttrs {
perm: vec![0, 2, 3, 4, 1],
..OpAttrs::default()
},
vec![wr5],
);
let wr = op(
T::Reshape,
shape_attr(vec![1, groups as i64, m as i64, cin_per_g as i64]),
vec![wp],
);
let wb = op(
T::BroadcastTo,
shape_attr(vec![n as i64, groups as i64, m as i64, cin_per_g as i64]),
vec![wr],
);
let cols = op(T::MatMul, OpAttrs::default(), vec![wb, xf]);
let csrc = op(
T::Reshape,
shape_attr(vec![n as i64, cout as i64, l as i64]),
vec![cols],
);
let sidx = scatter_flat_index(kh, kw, hin, win, sh, sw, dh, dw, wbuf);
col2im_tail(csrc, sidx, n, cout, hbuf, wbuf, ph, pw, hout, wout)
}
pub fn decompose(graph: &mut Graph, id: NodeId, params: &FusedOpParams) -> NodeId {
let (stride, padding, output_padding, dilation, groups) = match params {
FusedOpParams::ConvTranspose2D {
stride,
padding,
output_padding,
dilation,
groups,
} => (*stride, *padding, *output_padding, *dilation, *groups),
_ => return id,
};
if groups == 0 {
return id;
}
let (x_shape, w_shape) = {
let node = graph.node(id);
if node.inputs.len() != 2 {
return id;
}
(
graph.node(node.inputs[0]).shape.clone(),
graph.node(node.inputs[1]).shape.clone(),
)
};
let x_dims = x_shape.dims();
let w_dims = w_shape.dims();
if x_dims.len() != 4 || w_dims.len() != 4 {
return id;
}
let (n, cin, hin, win) = (x_dims[0], x_dims[1], x_dims[2], x_dims[3]);
let (cin_w, cout_per_g, kh, kw) = (w_dims[0], w_dims[1], w_dims[2], w_dims[3]);
if cin_w != cin || cin % groups != 0 {
return id;
}
let cout = cout_per_g * groups;
let (sh, sw) = stride;
let (ph, pw) = padding;
let (oph, opw) = output_padding;
let (dh, dw) = dilation;
if sh == 0 || sw == 0 || dh == 0 || dw == 0 {
return id;
}
let hout = ((hin.saturating_sub(1)) * sh + dh * (kh.saturating_sub(1)) + oph + 1)
.saturating_sub(2 * ph);
let wout = ((win.saturating_sub(1)) * sw + dw * (kw.saturating_sub(1)) + opw + 1)
.saturating_sub(2 * pw);
if hout == 0 || wout == 0 {
return id;
}
let hbuf = (hin.saturating_sub(1)) * sh + dh * (kh.saturating_sub(1)) + 1 + oph;
let wbuf = (win.saturating_sub(1)) * sw + dw * (kw.saturating_sub(1)) + 1 + opw;
if ph + hout > hbuf || pw + wout > wbuf {
return id;
}
let recipe_node = if groups == 1 {
recipe(
n, cin, hin, win, cout, kh, kw, sh, sw, dh, dw, ph, pw, hbuf, wbuf, hout, wout,
)
} else {
recipe_grouped(
n, cin, hin, win, cout, groups, kh, kw, sh, sw, dh, dw, ph, pw, hbuf, wbuf, hout, wout,
)
};
decompose_via_recipe(graph, id, &recipe_node, Some(Vec::new()))
}
pub fn canonical_pattern(_graph: &Graph, _root: NodeId) -> Option<PatternMatch> {
None
}
#[cfg(test)]
mod tests {
use super::*;
use crate::{Node, Op};
fn mk_const(g: &mut Graph, dims: &[usize]) -> NodeId {
g.push(Node {
op: Op::Const,
inputs: vec![],
shape: Shape::from_dims(dims),
dtype: DType::F32,
})
}
fn fused_conv_transpose(
g: &mut Graph,
n: usize,
cin: usize,
hin: usize,
win: usize,
cout_per_g: usize,
kh: usize,
kw: usize,
stride: (usize, usize),
padding: (usize, usize),
output_padding: (usize, usize),
dilation: (usize, usize),
groups: usize,
) -> NodeId {
let x = mk_const(g, &[n, cin, hin, win]);
let w = mk_const(g, &[cin, cout_per_g, kh, kw]);
let cout = cout_per_g * groups;
let (sh, sw) = stride;
let (ph, pw) = padding;
let (oph, opw) = output_padding;
let (dh, dw) = dilation;
let hout = (hin - 1) * sh + dh * (kh - 1) + oph + 1 - 2 * ph;
let wout = (win - 1) * sw + dw * (kw - 1) + opw + 1 - 2 * pw;
g.push(Node {
op: Op::Fused(
FusedOps::CONV_TRANSPOSE2D,
FusedOpParams::ConvTranspose2D {
stride,
padding,
output_padding,
dilation,
groups,
},
),
inputs: vec![x, w],
shape: Shape::from_dims(&[n, cout, hout, wout]),
dtype: DType::F32,
})
}
fn reachable_ops(g: &Graph, root: NodeId) -> Vec<Op> {
let mut stack = vec![root];
let mut seen = std::collections::HashSet::new();
let mut ops = Vec::new();
while let Some(id) = stack.pop() {
if !seen.insert(id) {
continue;
}
let node = g.node(id);
ops.push(node.op.clone());
for &inp in &node.inputs {
stack.push(inp);
}
}
ops
}
fn assert_col2im_recipe(g: &Graph, root: NodeId, label: &str) {
let ops = reachable_ops(g, root);
assert!(
!ops.iter()
.any(|o| matches!(o, Op::Fused(fid, _) if *fid == FusedOps::CONV_TRANSPOSE2D)),
"{label}: ConvTranspose2D must lower to primitives, not remain fused",
);
assert!(
ops.iter().any(|o| matches!(o, Op::MatMul)),
"{label}: recipe contains MatMul"
);
assert!(
ops.iter().any(|o| matches!(o, Op::IndexAdd { .. })),
"{label}: col2im overlap-add via IndexAdd",
);
assert!(
ops.iter().any(|o| matches!(o, Op::Iota { .. })),
"{label}: scatter index built from Iota"
);
assert!(
ops.iter()
.any(|o| matches!(o, Op::Cast(dt) if *dt == DType::U32)),
"{label}: scatter index cast to U32",
);
assert!(
ops.iter().any(|o| matches!(o, Op::Slice { .. })),
"{label}: output cropped via Slice",
);
}
#[test]
fn conv_transpose2d_groups1_decompose_lowers_to_col2im() {
#[allow(clippy::type_complexity)]
let cases: [(
usize,
usize,
usize,
usize,
usize,
usize,
usize,
(usize, usize),
(usize, usize),
(usize, usize),
(usize, usize),
); 3] = [
(1, 2, 4, 4, 3, 2, 2, (1, 1), (0, 0), (0, 0), (1, 1)), (1, 3, 5, 4, 2, 3, 3, (2, 2), (1, 1), (1, 1), (1, 1)), (2, 2, 3, 3, 4, 2, 2, (2, 1), (0, 0), (0, 0), (2, 1)), ];
for (n, cin, hin, win, cpg, kh, kw, stride, padding, out_pad, dilation) in cases {
let mut g = Graph::new();
let fused = fused_conv_transpose(
&mut g, n, cin, hin, win, cpg, kh, kw, stride, padding, out_pad, dilation, 1,
);
let out_shape = g.node(fused).shape.clone();
let params = match &g.node(fused).op {
Op::Fused(_, p) => p.clone(),
other => panic!("expected fused node, got {other:?}"),
};
let root = decompose(&mut g, fused, ¶ms);
assert_ne!(
root, fused,
"groups=1 must lower ({stride:?}/{padding:?}/{out_pad:?}/{dilation:?})"
);
assert_eq!(
g.node(root).shape,
out_shape,
"lowered root keeps [N,Cout,Hout,Wout]"
);
assert_eq!(g.node(root).dtype, DType::F32);
assert_col2im_recipe(&g, root, "groups=1");
}
}
#[test]
fn conv_transpose2d_groups_gt_1_decompose_lowers_to_col2im() {
#[allow(clippy::type_complexity)]
let cases: [(
usize,
usize,
usize,
usize,
usize,
usize,
usize,
(usize, usize),
(usize, usize),
(usize, usize),
(usize, usize),
usize,
); 3] = [
(1, 4, 4, 4, 3, 2, 2, (1, 1), (0, 0), (0, 0), (1, 1), 2), (2, 4, 3, 3, 1, 2, 2, (2, 2), (0, 0), (1, 1), (1, 1), 4), (1, 6, 5, 5, 2, 3, 3, (1, 1), (1, 1), (0, 0), (1, 1), 3), ];
for (n, cin, hin, win, cpg, kh, kw, stride, padding, out_pad, dilation, groups) in cases {
let mut g = Graph::new();
let fused = fused_conv_transpose(
&mut g, n, cin, hin, win, cpg, kh, kw, stride, padding, out_pad, dilation, groups,
);
let out_shape = g.node(fused).shape.clone();
let params = match &g.node(fused).op {
Op::Fused(_, p) => p.clone(),
other => panic!("expected fused node, got {other:?}"),
};
let root = decompose(&mut g, fused, ¶ms);
assert_ne!(root, fused, "groups={groups} must lower");
assert_eq!(
g.node(root).shape,
out_shape,
"lowered root keeps [N,Cout,Hout,Wout] (groups={groups})"
);
assert_eq!(g.node(root).dtype, DType::F32);
assert_col2im_recipe(&g, root, "grouped");
}
}
#[test]
fn conv_transpose2d_wrong_params_is_a_fixpoint() {
let mut g = Graph::new();
let fused = fused_conv_transpose(
&mut g,
1,
2,
4,
4,
3,
2,
2,
(1, 1),
(0, 0),
(0, 0),
(1, 1),
1,
);
let before = g.len();
let out = decompose(&mut g, fused, &FusedOpParams::Rope);
assert_eq!(out, fused, "wrong params => typed decline => fixpoint");
assert_eq!(g.len(), before, "declined before any emission");
}
#[test]
fn conv_transpose2d_bad_groups_is_a_fixpoint() {
let mut g = Graph::new();
let x = mk_const(&mut g, &[1, 5, 4, 4]);
let w = mk_const(&mut g, &[5, 3, 2, 2]);
let params = FusedOpParams::ConvTranspose2D {
stride: (1, 1),
padding: (0, 0),
output_padding: (0, 0),
dilation: (1, 1),
groups: 2,
};
let fused = g.push(Node {
op: Op::Fused(FusedOps::CONV_TRANSPOSE2D, params.clone()),
inputs: vec![x, w],
shape: Shape::from_dims(&[1, 6, 5, 5]),
dtype: DType::F32,
});
let before = g.len();
let out = decompose(&mut g, fused, ¶ms);
assert_eq!(out, fused, "Cin not divisible by groups => fixpoint");
assert_eq!(g.len(), before, "declined before any emission");
}
}