use crate::fuse::fuse_dims_bilateral;
use crate::hptt::micro_kernel::{MicroKernel, ScalarKernel};
#[derive(Debug, Clone)]
pub struct ComputeNode {
pub end: usize,
pub lda: isize,
pub ldb: isize,
pub next: Option<Box<ComputeNode>>,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum ExecMode {
Transpose {
dim_a: usize,
dim_b: usize,
},
ConstStride1 {
inner_dim: usize,
},
Scalar,
}
#[derive(Debug)]
pub struct PermutePlan {
pub fused_dims: Vec<usize>,
pub src_strides: Vec<isize>,
pub dst_strides: Vec<isize>,
pub root: Option<ComputeNode>,
pub mode: ExecMode,
pub lda_inner: isize,
pub ldb_inner: isize,
pub block: usize,
}
pub fn build_permute_plan(
dims: &[usize],
src_strides: &[isize],
dst_strides: &[isize],
elem_size: usize,
) -> PermutePlan {
let (fused_dims, fused_src, fused_dst) = fuse_dims_bilateral(dims, src_strides, dst_strides);
let rank = fused_dims.len();
if rank == 0 {
return PermutePlan {
fused_dims,
src_strides: fused_src,
dst_strides: fused_dst,
root: None,
mode: ExecMode::Scalar,
lda_inner: 0,
ldb_inner: 0,
block: 0,
};
}
let dim_a = find_stride1_dim(&fused_dims, &fused_src);
let dim_b = find_stride1_dim(&fused_dims, &fused_dst);
let block = block_for_elem_size(elem_size);
if dim_a == dim_b {
let inner_dim = dim_a;
let mode = ExecMode::ConstStride1 { inner_dim };
let loop_order = compute_loop_order_const(&fused_dims, &fused_src, &fused_dst, inner_dim);
let root = build_compute_nodes(&fused_dims, &fused_src, &fused_dst, &loop_order);
PermutePlan {
fused_dims,
src_strides: fused_src.clone(),
dst_strides: fused_dst.clone(),
root,
mode,
lda_inner: fused_src[inner_dim],
ldb_inner: fused_dst[inner_dim],
block: 0,
}
} else {
let mode = ExecMode::Transpose { dim_a, dim_b };
let lda_inner = fused_src[dim_b];
let ldb_inner = fused_dst[dim_a];
let loop_order =
compute_loop_order_transpose(&fused_dims, &fused_src, &fused_dst, dim_a, dim_b);
let root = build_compute_nodes(&fused_dims, &fused_src, &fused_dst, &loop_order);
PermutePlan {
fused_dims,
src_strides: fused_src,
dst_strides: fused_dst,
root,
mode,
lda_inner,
ldb_inner,
block,
}
}
}
fn find_stride1_dim(dims: &[usize], strides: &[isize]) -> usize {
dims.iter()
.zip(strides.iter())
.enumerate()
.filter(|(_, (&d, _))| d > 1)
.min_by_key(|(_, (_, &s))| s.unsigned_abs())
.map(|(i, _)| i)
.unwrap_or(0)
}
fn block_for_elem_size(elem_size: usize) -> usize {
match elem_size {
8 => <ScalarKernel as MicroKernel<f64>>::BLOCK, 4 => <ScalarKernel as MicroKernel<f32>>::BLOCK, _ => 16, }
}
fn compute_loop_order_transpose(
dims: &[usize],
src_strides: &[isize],
dst_strides: &[isize],
dim_a: usize,
dim_b: usize,
) -> Vec<usize> {
let mut loop_dims: Vec<usize> = (0..dims.len())
.filter(|&d| d != dim_a && d != dim_b && dims[d] > 1)
.collect();
loop_dims.sort_by(|&a, &b| {
let cost_a = src_strides[a].unsigned_abs() + dst_strides[a].unsigned_abs();
let cost_b = src_strides[b].unsigned_abs() + dst_strides[b].unsigned_abs();
cost_b.cmp(&cost_a)
});
loop_dims
}
fn compute_loop_order_const(
dims: &[usize],
_src_strides: &[isize],
dst_strides: &[isize],
inner_dim: usize,
) -> Vec<usize> {
let mut loop_dims: Vec<usize> = (0..dims.len())
.filter(|&d| d != inner_dim && dims[d] > 1)
.collect();
loop_dims.sort_by(|&a, &b| {
dst_strides[b]
.unsigned_abs()
.cmp(&dst_strides[a].unsigned_abs())
});
loop_dims
}
fn build_compute_nodes(
dims: &[usize],
src_strides: &[isize],
dst_strides: &[isize],
loop_order: &[usize],
) -> Option<ComputeNode> {
let mut current: Option<ComputeNode> = None;
for &d in loop_order.iter().rev() {
let node = ComputeNode {
end: dims[d],
lda: src_strides[d],
ldb: dst_strides[d],
next: current.map(Box::new),
};
current = Some(node);
}
current
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_find_stride1_dim_basic() {
assert_eq!(find_stride1_dim(&[4, 5], &[1, 4]), 0);
assert_eq!(find_stride1_dim(&[4, 5], &[5, 1]), 1);
}
#[test]
fn test_find_stride1_dim_skips_size1() {
assert_eq!(find_stride1_dim(&[1, 5], &[1, 2]), 1);
}
#[test]
fn test_build_plan_identity() {
let plan = build_permute_plan(&[2, 3, 4], &[1, 2, 6], &[1, 2, 6], 8);
assert_eq!(plan.fused_dims, vec![24]);
assert!(matches!(plan.mode, ExecMode::ConstStride1 { .. }));
}
#[test]
fn test_build_plan_transpose_2d() {
let plan = build_permute_plan(&[4, 5], &[1, 4], &[5, 1], 8);
assert_eq!(plan.fused_dims, vec![4, 5]);
match plan.mode {
ExecMode::Transpose { dim_a, dim_b } => {
assert_eq!(dim_a, 0); assert_eq!(dim_b, 1); }
_ => panic!("expected Transpose mode"),
}
assert_eq!(plan.block, 16); assert_eq!(plan.lda_inner, 4); assert_eq!(plan.ldb_inner, 5); assert!(plan.root.is_none());
}
#[test]
fn test_build_plan_3d_permute() {
let plan = build_permute_plan(&[4, 2, 3], &[6, 1, 2], &[1, 4, 8], 8);
assert_eq!(plan.fused_dims, vec![4, 6]);
match plan.mode {
ExecMode::Transpose { dim_a, dim_b } => {
assert_eq!(dim_a, 1);
assert_eq!(dim_b, 0);
}
_ => panic!("expected Transpose mode"),
}
assert!(plan.root.is_none());
}
#[test]
fn test_build_plan_scattered_strides() {
let dims = vec![2, 2, 2, 2];
let src_strides = vec![1, 8, 2, 4]; let dst_strides = vec![1, 2, 4, 8];
let plan = build_permute_plan(&dims, &src_strides, &dst_strides, 8);
assert_eq!(plan.fused_dims.len(), 3);
match plan.mode {
ExecMode::Transpose { .. } | ExecMode::ConstStride1 { .. } => {
}
_ => panic!("unexpected mode"),
}
}
#[test]
fn test_build_plan_rank0() {
let plan = build_permute_plan(&[], &[], &[], 8);
assert!(matches!(plan.mode, ExecMode::Scalar));
assert!(plan.root.is_none());
}
#[test]
fn test_compute_loop_order_transpose() {
let dims = [4, 5, 3, 7];
let src_s = [1isize, 4, 100, 300];
let dst_s = [35isize, 1, 7, 21];
let order = compute_loop_order_transpose(&dims, &src_s, &dst_s, 0, 1);
assert_eq!(order, vec![3, 2]);
}
#[test]
fn test_build_compute_nodes_chain() {
let dims = [10, 5, 3];
let src_s = [1isize, 10, 50];
let dst_s = [15isize, 1, 5];
let loop_order = vec![2];
let root = build_compute_nodes(&dims, &src_s, &dst_s, &loop_order);
assert!(root.is_some());
let root = root.unwrap();
assert_eq!(root.end, 3);
assert_eq!(root.lda, 50);
assert_eq!(root.ldb, 5);
assert!(root.next.is_none());
}
}