use crate::fuse::{plan_bilateral_fusion, FusionPlanError};
use crate::hptt::micro_kernel::{MicroKernel, ScalarKernel};
use strided_view::INLINE_RANK;
use tinyvec::TinyVec;
pub(crate) type PlanDimVec = TinyVec<[usize; INLINE_RANK]>;
pub(crate) type PlanStrideVec = TinyVec<[isize; INLINE_RANK]>;
#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
pub(crate) struct LoopLevel {
pub(crate) end: usize,
pub(crate) lda: isize,
pub(crate) ldb: isize,
}
pub(crate) type LoopLevels = TinyVec<[LoopLevel; INLINE_RANK]>;
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub(crate) enum ExecMode {
Transpose {
dim_a: usize,
dim_b: usize,
},
ConstStride1 {
inner_dim: usize,
},
GroupedTranspose,
Scalar,
}
#[derive(Debug, Clone)]
pub(crate) struct OffsetGroup {
pub(crate) src_offsets: TinyVec<[isize; GROUP_OFFSETS_INLINE]>,
pub(crate) dst_offsets: TinyVec<[isize; GROUP_OFFSETS_INLINE]>,
pub(crate) len: usize,
}
pub(crate) const GROUP_OFFSETS_INLINE: usize = 64;
#[derive(Debug, Clone)]
pub(crate) struct GroupedTilePlan {
pub(crate) group_a: OffsetGroup,
pub(crate) group_b: OffsetGroup,
}
#[derive(Debug)]
pub(crate) struct PermutePlan {
pub(crate) fused_dims: PlanDimVec,
pub(crate) src_strides: PlanStrideVec,
pub(crate) dst_strides: PlanStrideVec,
pub(crate) levels: LoopLevels,
pub(crate) mode: ExecMode,
pub(crate) lda_inner: isize,
pub(crate) ldb_inner: isize,
pub(crate) block: usize,
pub(crate) grouped_tile: Option<GroupedTilePlan>,
}
pub(crate) fn build_permute_plan(
dims: &[usize],
src_strides: &[isize],
dst_strides: &[isize],
elem_size: usize,
) -> Result<PermutePlan, FusionPlanError> {
let fusion = plan_bilateral_fusion(dims, src_strides, dst_strides)?;
let fused_dims = fusion.dims;
let fused_src = fusion.src_strides;
let fused_dst = fusion.dst_strides;
let rank = fused_dims.len();
if rank == 0 {
return Ok(PermutePlan {
fused_dims,
src_strides: fused_src,
dst_strides: fused_dst,
levels: LoopLevels::new(),
mode: ExecMode::Scalar,
lda_inner: 0,
ldb_inner: 0,
block: 0,
grouped_tile: None,
});
}
let dim_a = find_stride1_dim(&fused_dims, &fused_src);
let dim_b = find_stride1_dim(&fused_dims, &fused_dst);
let block = block_for_elem_size(elem_size);
if dim_a == dim_b || fused_src[dim_a] != 1 || fused_dst[dim_b] != 1 {
let inner_dim = dim_a;
let mode = ExecMode::ConstStride1 { inner_dim };
let loop_order = compute_loop_order_const(&fused_dims, &fused_src, &fused_dst, inner_dim);
let levels = build_loop_levels(&fused_dims, &fused_src, &fused_dst, &loop_order);
Ok(PermutePlan {
fused_dims,
src_strides: fused_src.clone(),
dst_strides: fused_dst.clone(),
levels,
mode,
lda_inner: fused_src[inner_dim],
ldb_inner: fused_dst[inner_dim],
block: 0,
grouped_tile: None,
})
} else {
let legacy_area = fused_dims[dim_a].saturating_mul(fused_dims[dim_b]);
if legacy_area < block.saturating_mul(block) {
let grouped_target = block.saturating_mul(2);
if let Some((levels, grouped_tile)) = try_build_grouped_tile_plan(
&fused_dims,
&fused_src,
&fused_dst,
grouped_target,
legacy_area,
) {
return Ok(PermutePlan {
fused_dims,
src_strides: fused_src,
dst_strides: fused_dst,
levels,
mode: ExecMode::GroupedTranspose,
lda_inner: 0,
ldb_inner: 0,
block,
grouped_tile: Some(grouped_tile),
});
}
}
let mode = ExecMode::Transpose { dim_a, dim_b };
let lda_inner = fused_src[dim_b];
let ldb_inner = fused_dst[dim_a];
let loop_order =
compute_loop_order_transpose(&fused_dims, &fused_src, &fused_dst, dim_a, dim_b);
let levels = build_loop_levels(&fused_dims, &fused_src, &fused_dst, &loop_order);
Ok(PermutePlan {
fused_dims,
src_strides: fused_src,
dst_strides: fused_dst,
levels,
mode,
lda_inner,
ldb_inner,
block,
grouped_tile: None,
})
}
}
fn find_stride1_dim(dims: &[usize], strides: &[isize]) -> usize {
dims.iter()
.zip(strides.iter())
.enumerate()
.filter(|(_, (&d, _))| d > 1)
.min_by_key(|(_, (_, &s))| s.unsigned_abs())
.map(|(i, _)| i)
.unwrap_or(0)
}
const MAX_GROUP_LEN_MULTIPLE: usize = 64;
fn block_for_elem_size(elem_size: usize) -> usize {
match elem_size {
8 => <ScalarKernel as MicroKernel<f64>>::BLOCK, 4 => <ScalarKernel as MicroKernel<f32>>::BLOCK, _ => 16, }
}
fn try_build_grouped_tile_plan(
dims: &[usize],
src_strides: &[isize],
dst_strides: &[isize],
target: usize,
legacy_area: usize,
) -> Option<(LoopLevels, GroupedTilePlan)> {
let group_a = select_contiguous_group(dims, src_strides, &[], target)?;
let group_a_len = group_len(dims, &group_a);
if group_a_len < 4 {
return None;
}
let group_b = select_contiguous_group(dims, dst_strides, &group_a, target)?;
let group_b_len = group_len(dims, &group_b);
let area = group_a_len.saturating_mul(group_b_len);
if group_b_len < 4 || area < 128 || area <= legacy_area.saturating_mul(4) {
return None;
}
let mut consumed: TinyVec<[bool; INLINE_RANK]> = TinyVec::with_capacity(dims.len());
for _ in 0..dims.len() {
consumed.push(false);
}
for &d in group_a.iter().chain(group_b.iter()) {
consumed[d] = true;
}
let mut outer: PlanDimVec = TinyVec::with_capacity(dims.len());
for d in 0..dims.len() {
if !consumed[d] && dims[d] > 1 {
outer.push(d);
}
}
outer.sort_by(|&a, &b| {
let cost_a = src_strides[a].unsigned_abs() + dst_strides[a].unsigned_abs();
let cost_b = src_strides[b].unsigned_abs() + dst_strides[b].unsigned_abs();
cost_b.cmp(&cost_a)
});
let levels = build_loop_levels(dims, src_strides, dst_strides, &outer);
let grouped_tile = GroupedTilePlan {
group_a: build_offset_group(dims, src_strides, dst_strides, &group_a),
group_b: build_offset_group(dims, src_strides, dst_strides, &group_b),
};
Some((levels, grouped_tile))
}
fn select_contiguous_group(
dims: &[usize],
strides: &[isize],
excluded: &[usize],
target: usize,
) -> Option<PlanDimVec> {
let mut is_excluded: TinyVec<[bool; INLINE_RANK]> = TinyVec::with_capacity(dims.len());
for _ in 0..dims.len() {
is_excluded.push(false);
}
for &d in excluded {
is_excluded[d] = true;
}
let mut order: PlanDimVec = TinyVec::with_capacity(dims.len());
for d in 0..dims.len() {
if dims[d] > 1 && !is_excluded[d] && strides[d] > 0 {
order.push(d);
}
}
order.sort_by_key(|&d| strides[d].unsigned_abs());
let mut best = PlanDimVec::with_capacity(dims.len());
let mut best_len = 0usize;
for start in 0..order.len() {
let first = order[start];
let base = strides[first];
let mut group = PlanDimVec::with_capacity(order.len());
group.push(first);
let mut len = dims[first];
let mut expected = base.saturating_mul(len as isize);
for &d in &order[start + 1..] {
match strides[d].cmp(&expected) {
std::cmp::Ordering::Less => continue,
std::cmp::Ordering::Equal => {
if len >= target {
break;
}
let next_len = len.saturating_mul(dims[d]);
if next_len > target.saturating_mul(MAX_GROUP_LEN_MULTIPLE) {
break;
}
group.push(d);
len = next_len;
expected = base.saturating_mul(len as isize);
}
std::cmp::Ordering::Greater => break,
}
}
if len > best_len {
best_len = len;
best = group;
}
if best_len >= target {
break;
}
}
if best.is_empty() {
None
} else {
Some(best)
}
}
fn group_len(dims: &[usize], group: &[usize]) -> usize {
group.iter().map(|&d| dims[d]).product()
}
fn build_offset_group(
dims: &[usize],
src_strides: &[isize],
dst_strides: &[isize],
group: &[usize],
) -> OffsetGroup {
let len = group_len(dims, group);
let mut src_offsets: TinyVec<[isize; GROUP_OFFSETS_INLINE]> = TinyVec::with_capacity(len);
let mut dst_offsets: TinyVec<[isize; GROUP_OFFSETS_INLINE]> = TinyVec::with_capacity(len);
for flat in 0..len {
let mut rem = flat;
let mut src_offset = 0isize;
let mut dst_offset = 0isize;
for &d in group {
let coord = rem % dims[d];
rem /= dims[d];
src_offset += coord as isize * src_strides[d];
dst_offset += coord as isize * dst_strides[d];
}
src_offsets.push(src_offset);
dst_offsets.push(dst_offset);
}
OffsetGroup {
src_offsets,
dst_offsets,
len,
}
}
fn compute_loop_order_transpose(
dims: &[usize],
src_strides: &[isize],
dst_strides: &[isize],
dim_a: usize,
dim_b: usize,
) -> PlanDimVec {
let mut loop_dims: PlanDimVec = TinyVec::with_capacity(dims.len());
for d in 0..dims.len() {
if d != dim_a && d != dim_b && dims[d] > 1 {
loop_dims.push(d);
}
}
loop_dims.sort_by(|&a, &b| {
let cost_a = src_strides[a].unsigned_abs() + dst_strides[a].unsigned_abs();
let cost_b = src_strides[b].unsigned_abs() + dst_strides[b].unsigned_abs();
cost_b.cmp(&cost_a)
});
loop_dims
}
fn compute_loop_order_const(
dims: &[usize],
_src_strides: &[isize],
dst_strides: &[isize],
inner_dim: usize,
) -> PlanDimVec {
let mut loop_dims: PlanDimVec = TinyVec::with_capacity(dims.len());
for d in 0..dims.len() {
if d != inner_dim && dims[d] > 1 {
loop_dims.push(d);
}
}
loop_dims.sort_by(|&a, &b| {
dst_strides[b]
.unsigned_abs()
.cmp(&dst_strides[a].unsigned_abs())
});
loop_dims
}
fn build_loop_levels(
dims: &[usize],
src_strides: &[isize],
dst_strides: &[isize],
loop_order: &[usize],
) -> LoopLevels {
let mut levels: LoopLevels = TinyVec::with_capacity(loop_order.len());
for &d in loop_order {
levels.push(LoopLevel {
end: dims[d],
lda: src_strides[d],
ldb: dst_strides[d],
});
}
levels
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_find_stride1_dim_basic() {
assert_eq!(find_stride1_dim(&[4, 5], &[1, 4]), 0);
assert_eq!(find_stride1_dim(&[4, 5], &[5, 1]), 1);
}
#[test]
fn test_find_stride1_dim_skips_size1() {
assert_eq!(find_stride1_dim(&[1, 5], &[1, 2]), 1);
}
#[test]
fn test_build_plan_identity() {
let plan = build_permute_plan(&[2, 3, 4], &[1, 2, 6], &[1, 2, 6], 8).unwrap();
assert_eq!(&plan.fused_dims[..], &[24]);
assert!(matches!(plan.mode, ExecMode::ConstStride1 { .. }));
}
#[test]
fn test_build_plan_transpose_2d() {
let plan = build_permute_plan(&[4, 5], &[1, 4], &[5, 1], 8).unwrap();
assert_eq!(&plan.fused_dims[..], &[4, 5]);
match plan.mode {
ExecMode::Transpose { dim_a, dim_b } => {
assert_eq!(dim_a, 0); assert_eq!(dim_b, 1); }
_ => panic!("expected Transpose mode"),
}
assert_eq!(plan.block, 16); assert_eq!(plan.lda_inner, 4); assert_eq!(plan.ldb_inner, 5); assert!(plan.levels.is_empty());
}
#[test]
fn test_build_plan_3d_permute() {
let plan = build_permute_plan(&[4, 2, 3], &[6, 1, 2], &[1, 4, 8], 8).unwrap();
assert_eq!(&plan.fused_dims[..], &[4, 6]);
match plan.mode {
ExecMode::Transpose { dim_a, dim_b } => {
assert_eq!(dim_a, 1);
assert_eq!(dim_b, 0);
}
_ => panic!("expected Transpose mode"),
}
assert!(plan.levels.is_empty());
}
#[test]
fn test_build_plan_scattered_strides() {
let dims = vec![2, 2, 2, 2];
let src_strides = vec![1, 8, 2, 4]; let dst_strides = vec![1, 2, 4, 8];
let plan = build_permute_plan(&dims, &src_strides, &dst_strides, 8).unwrap();
assert_eq!(plan.fused_dims.len(), 3);
match plan.mode {
ExecMode::Transpose { .. } | ExecMode::ConstStride1 { .. } => {
}
_ => panic!("unexpected mode"),
}
}
#[test]
fn test_build_plan_step408_uses_grouped_tile() {
let dims = vec![2usize; 24];
let src_strides = vec![
16, 1024, 8388608, 4096, 1048576, 1, 8, 131072, 2, 4, 64, 128, 256, 512, 2048, 8192,
16384, 32768, 262144, 2097152, 4194304, 32, 65536, 524288,
];
let dst_strides = (0..24).map(|i| 1isize << i).collect::<Vec<_>>();
let plan = build_permute_plan(&dims, &src_strides, &dst_strides, 8).unwrap();
assert!(matches!(plan.mode, ExecMode::GroupedTranspose));
let grouped = plan.grouped_tile.as_ref().expect("grouped tile plan");
assert!(grouped.group_a.len >= 16);
assert!(grouped.group_b.len >= 16);
assert!(grouped.group_a.len * grouped.group_b.len >= 256);
}
#[test]
fn test_build_plan_rank8_scattered_uses_inline_grouped_tile() {
let dims = vec![2usize; 8];
let src_strides = [16isize, 1024, 8388608, 4096, 1048576, 1, 8, 131072];
let dst_strides = [1isize, 2, 4, 8, 16, 32, 64, 128];
let plan = build_permute_plan(&dims, &src_strides, &dst_strides, 8).unwrap();
assert!(matches!(plan.mode, ExecMode::GroupedTranspose));
let grouped = plan.grouped_tile.as_ref().expect("grouped tile plan");
assert!(grouped.group_a.len <= GROUP_OFFSETS_INLINE);
assert!(grouped.group_b.len <= GROUP_OFFSETS_INLINE);
assert!(grouped.group_a.len >= 4 && grouped.group_b.len >= 4);
assert!(!plan.levels.is_empty());
}
#[test]
fn test_build_plan_large_3d_transpose_uses_legacy_tile() {
let dims = vec![256usize, 256, 256];
let src_strides = vec![256isize, 1, 65536];
let dst_strides = vec![1isize, 256, 65536];
let plan = build_permute_plan(&dims, &src_strides, &dst_strides, 8).unwrap();
assert!(matches!(plan.mode, ExecMode::Transpose { .. }));
assert!(plan.grouped_tile.is_none());
}
#[test]
fn test_build_plan_rank0() {
let plan = build_permute_plan(&[], &[], &[], 8).unwrap();
assert!(matches!(plan.mode, ExecMode::Scalar));
assert!(plan.levels.is_empty());
}
#[test]
fn test_compute_loop_order_transpose() {
let dims = [4, 5, 3, 7];
let src_s = [1isize, 4, 100, 300];
let dst_s = [35isize, 1, 7, 21];
let order = compute_loop_order_transpose(&dims, &src_s, &dst_s, 0, 1);
assert_eq!(&order[..], &[3, 2]);
}
#[test]
fn test_build_loop_levels_chain() {
let dims = [10, 5, 3];
let src_s = [1isize, 10, 50];
let dst_s = [15isize, 1, 5];
let loop_order = vec![2];
let levels = build_loop_levels(&dims, &src_s, &dst_s, &loop_order);
assert_eq!(levels.len(), 1);
assert_eq!(levels[0].end, 3);
assert_eq!(levels[0].lda, 50);
assert_eq!(levels[0].ldb, 5);
assert!(build_loop_levels(&dims, &src_s, &dst_s, &[]).is_empty());
}
}