pub fn is_injective_layout(dims: &[usize], strides: &[isize]) -> bool {
is_injective_layout_without_alloc(dims, strides) || is_injective_layout_exact(dims, strides)
}
#[doc(hidden)]
pub fn is_injective_layout_without_alloc(dims: &[usize], strides: &[isize]) -> bool {
let Some(total) = validate_injective_layout_inputs(dims, strides) else {
return false;
};
if total <= 1 {
return true;
}
match interleaved_block(dims, strides) {
None => false,
Some(None) => true,
Some(Some(block)) => {
if !block.may_be_injective() {
return false;
}
if let Some(injective) = two_axis_block_is_injective(dims, strides, &block) {
return injective;
}
if let Some(injective) = block_differences_avoid_zero(dims, strides, &block) {
return injective;
}
block.total <= PAIRWISE_BLOCK_BUDGET as u128
&& block_offsets_unique_pairwise(dims, strides, &block)
}
}
}
fn is_injective_layout_exact(dims: &[usize], strides: &[isize]) -> bool {
let Some(total) = validate_injective_layout_inputs(dims, strides) else {
return false;
};
if total <= 1 {
return true;
}
match interleaved_block(dims, strides) {
None => false,
Some(None) => true,
Some(Some(block)) => {
if !block.may_be_injective() {
return false;
}
if let Some(injective) = two_axis_block_is_injective(dims, strides, &block) {
return injective;
}
block.total <= EXACT_BLOCK_BUDGET as u128 && block_offsets_unique(dims, strides, &block)
}
}
}
const EXACT_BLOCK_BUDGET: usize = 1 << 24;
const PAIRWISE_BLOCK_BUDGET: usize = 4096;
const DIFFERENCE_SEARCH_BUDGET: u128 = 1 << 22;
struct InterleavedBlock {
last_key: (u128, usize),
total: u128,
span: u128,
}
impl InterleavedBlock {
fn contains(&self, axis: usize, dim: usize, stride: isize) -> bool {
dim > 1 && (stride.unsigned_abs() as u128, axis) <= self.last_key
}
fn may_be_injective(&self) -> bool {
self.total <= self.span + 1
}
}
fn two_axis_block_is_injective(
dims: &[usize],
strides: &[isize],
block: &InterleavedBlock,
) -> Option<bool> {
let mut axes = [(0usize, 0usize); 2]; let mut count = 0;
for (axis, (&dim, &stride)) in dims.iter().zip(strides.iter()).enumerate() {
if !block.contains(axis, dim, stride) {
continue;
}
if count == 2 {
return None;
}
axes[count] = (dim, stride.unsigned_abs());
count += 1;
}
if count != 2 {
return None;
}
let (extent_lo, stride_lo, extent_hi, stride_hi) = if axes[0].1 <= axes[1].1 {
(
axes[0].0 as u128,
axes[0].1 as u128,
axes[1].0 as u128,
axes[1].1 as u128,
)
} else {
(
axes[1].0 as u128,
axes[1].1 as u128,
axes[0].0 as u128,
axes[0].1 as u128,
)
};
let g = gcd(stride_lo, stride_hi);
debug_assert!(g > 0, "stride zero on a non-singleton axis was rejected");
Some(stride_hi / g >= extent_lo || stride_lo / g >= extent_hi)
}
fn block_differences_avoid_zero(
dims: &[usize],
strides: &[isize],
block: &InterleavedBlock,
) -> Option<bool> {
let in_block = |axis: usize| block.contains(axis, dims[axis], strides[axis]);
let (mut first, mut second) = (None::<usize>, None::<usize>);
let mut count = 0usize;
for axis in (0..dims.len()).filter(|&axis| in_block(axis)) {
count += 1;
if first.is_none_or(|f| dims[axis] > dims[f]) {
second = first;
first = Some(axis);
} else if second.is_none_or(|s| dims[axis] > dims[s]) {
second = Some(axis);
}
}
if count < 3 {
return None;
}
let (first, second) = (first?, second?);
let enumerated = |axis: usize| axis != first && axis != second && in_block(axis);
let mut boxed = 1u128;
for axis in (0..dims.len()).filter(|&axis| enumerated(axis)) {
boxed = boxed.checked_mul(2 * dims[axis] as u128 - 1)?;
if boxed / 2 > DIFFERENCE_SEARCH_BUDGET {
return None;
}
}
let a = strides[first].unsigned_abs() as i128;
let b = strides[second].unsigned_abs() as i128;
let (x_bound, y_bound) = ((dims[first] - 1) as i128, (dims[second] - 1) as i128);
let (g, inverse) = gcd_and_inverse(a, b);
let (a, b) = (a / g, b / g);
let solvable = |c: i128| -> bool {
let x0 = (c.rem_euclid(b) * inverse).rem_euclid(b);
let lo = (c - y_bound * b).div_euclid(a) + i128::from((c - y_bound * b).rem_euclid(a) != 0);
let hi = (c + y_bound * b).div_euclid(a);
let (lo, hi) = (lo.max(-x_bound), hi.min(x_bound));
lo <= hi && lo + (x0 - lo).rem_euclid(b) <= hi
};
if b <= x_bound && a <= y_bound {
return Some(false);
}
let mut axes: tinyvec::TinyVec<[(i128, i128, i128); crate::INLINE_RANK]> = (0..dims.len())
.filter(|&axis| enumerated(axis))
.map(|axis| {
(
0,
(dims[axis] - 1) as i128,
strides[axis].unsigned_abs() as i128,
)
})
.collect();
let mut r = 0i128;
for _ in 0..boxed / 2 {
for (delta, bound, stride) in axes.iter_mut() {
if *delta < *bound {
*delta += 1;
r += *stride;
break;
}
r -= *stride * (*delta + *bound);
*delta = -*bound;
}
if r % g == 0 && solvable(-r / g) {
return Some(false);
}
}
Some(true)
}
fn gcd_and_inverse(a: i128, b: i128) -> (i128, i128) {
let (mut old_r, mut r) = (a, b);
let (mut old_s, mut s) = (1i128, 0i128);
while r != 0 {
let q = old_r / r;
(old_r, r) = (r, old_r - q * r);
(old_s, s) = (s, old_s - q * s);
}
let modulus = b / old_r;
(old_r, old_s.rem_euclid(modulus))
}
fn gcd(mut a: u128, mut b: u128) -> u128 {
while b != 0 {
let remainder = a % b;
a = b;
b = remainder;
}
a
}
fn interleaved_block(dims: &[usize], strides: &[isize]) -> Option<Option<InterleavedBlock>> {
let mut covered_span = 0u128;
let mut covered_total = 1u128;
let mut previous_key = None;
let mut block = None;
let active_axes = dims.iter().filter(|&&dim| dim > 1).count();
for _ in 0..active_axes {
let mut next = None;
for (axis, (&dim, &stride)) in dims.iter().zip(strides.iter()).enumerate() {
if dim <= 1 {
continue;
}
let key = (stride.checked_abs()? as u128, axis);
if previous_key.is_some_and(|previous| key <= previous) {
continue;
}
if next.is_none_or(|(best, _)| key < best) {
next = Some((key, dim as u128));
}
}
let ((stride, axis), dim) = next?;
let separated = stride > covered_span;
covered_span = stride
.checked_mul(dim - 1)
.and_then(|span| covered_span.checked_add(span))?;
covered_total = covered_total.checked_mul(dim)?;
if !separated {
block = Some(InterleavedBlock {
last_key: (stride, axis),
total: covered_total,
span: covered_span,
});
}
previous_key = Some((stride, axis));
}
Some(block)
}
fn block_offsets_unique(dims: &[usize], strides: &[isize], block: &InterleavedBlock) -> bool {
let axes: Vec<(usize, usize)> = dims
.iter()
.zip(strides.iter())
.enumerate()
.filter(|&(axis, (&dim, &stride))| block.contains(axis, dim, stride))
.map(|(_, (&dim, &stride))| (dim, stride.unsigned_abs()))
.collect();
let (Ok(total), Ok(span)) = (usize::try_from(block.total), usize::try_from(block.span)) else {
return false;
};
let mut indices = vec![0usize; axes.len()];
let mut offset = 0usize;
let mut advance = |offset: &mut usize| {
for (index, &(dim, stride)) in indices.iter_mut().zip(axes.iter()) {
if *index + 1 < dim {
*index += 1;
*offset += stride;
return;
}
*offset -= stride * (dim - 1);
*index = 0;
}
};
if block.span / 64 <= block.total {
let mut seen = vec![0u64; span / 64 + 1];
for _ in 0..total {
let (word, bit) = (offset / 64, 1u64 << (offset % 64));
if seen[word] & bit != 0 {
return false;
}
seen[word] |= bit;
advance(&mut offset);
}
true
} else {
let mut offsets = Vec::with_capacity(total);
for _ in 0..total {
offsets.push(offset);
advance(&mut offset);
}
offsets.sort_unstable();
offsets.windows(2).all(|pair| pair[0] != pair[1])
}
}
fn block_offset_for_linear_index(
dims: &[usize],
strides: &[isize],
block: &InterleavedBlock,
mut linear: usize,
) -> u128 {
let mut offset = 0u128;
for (axis, (&dim, &stride)) in dims.iter().zip(strides.iter()).enumerate() {
if !block.contains(axis, dim, stride) {
continue;
}
offset += stride.unsigned_abs() as u128 * (linear % dim) as u128;
linear /= dim;
}
offset
}
fn block_offsets_unique_pairwise(
dims: &[usize],
strides: &[isize],
block: &InterleavedBlock,
) -> bool {
let Ok(total) = usize::try_from(block.total) else {
return false;
};
for lhs in 0..total {
let lhs_offset = block_offset_for_linear_index(dims, strides, block, lhs);
for rhs in (lhs + 1)..total {
if block_offset_for_linear_index(dims, strides, block, rhs) == lhs_offset {
return false;
}
}
}
true
}
fn validate_injective_layout_inputs(dims: &[usize], strides: &[isize]) -> Option<usize> {
if dims.len() != strides.len() {
return None;
}
let total = total_len(dims)?;
if total <= 1 {
return Some(total);
}
if dims
.iter()
.zip(strides.iter())
.any(|(&dim, &stride)| dim > 1 && stride == 0)
{
return None;
}
let mut min_offset = 0isize;
let mut max_offset = 0isize;
for (&dim, &stride) in dims.iter().zip(strides.iter()) {
if dim <= 1 {
continue;
}
let extent = isize::try_from(dim - 1).ok()?;
let span = stride.checked_mul(extent)?;
if span >= 0 {
max_offset = max_offset.checked_add(span)?;
} else {
min_offset = min_offset.checked_add(span)?;
}
}
Some(total)
}
fn total_len(dims: &[usize]) -> Option<usize> {
if dims.contains(&0) {
return Some(0);
}
dims.iter()
.try_fold(1usize, |acc, &dim| acc.checked_mul(dim))
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn disjoint_axes_are_injective() {
assert!(is_injective_layout(&[2, 3], &[1, 2]));
assert!(is_injective_layout(&[2, 3], &[3, 1]));
assert!(is_injective_layout(&[1, 5], &[0, 1]));
assert!(is_injective_layout(&[], &[]));
}
#[test]
fn overlapping_axes_are_rejected() {
assert!(!is_injective_layout(&[2, 3], &[0, 1]));
assert!(!is_injective_layout(&[2, 2], &[1, 1]));
assert!(!is_injective_layout(&[256, 256], &[256, 256]));
assert!(!is_injective_layout(&[3, 3], &[1, 0]));
}
#[test]
fn three_axis_interleaved_injective_layout_is_decided_without_alloc() {
assert!(is_injective_layout_without_alloc(&[2, 2, 2000], &[4, 5, 6]));
assert!(is_injective_layout(&[2, 2, 2000], &[4, 5, 6]));
assert!(is_injective_layout_without_alloc(&[3, 1500], &[2000, 3]));
assert!(is_injective_layout(&[1500, 3], &[3, 2000]));
assert!(!is_injective_layout(&[3, 1500], &[2000, 2]));
}
#[test]
fn large_two_axis_injective_layout_is_decided_without_enumeration() {
assert!(is_injective_layout(&[3, 6_000_000], &[6_000_001, 3]));
assert!(is_injective_layout_without_alloc(
&[3, 6_000_000],
&[6_000_001, 3]
));
assert!(!is_injective_layout(&[3, 6_000_000], &[6_000_003, 3]));
assert!(!is_injective_layout_without_alloc(
&[3, 6_000_000],
&[6_000_003, 3]
));
}
#[test]
fn large_three_axis_injective_layout_is_decided_without_enumeration() {
assert!(is_injective_layout(&[2, 2, 5_000_000], &[4, 5, 6]));
assert!(is_injective_layout_without_alloc(
&[2, 2, 5_000_000],
&[4, 5, 6]
));
assert!(!is_injective_layout(&[2, 2, 5_000_000], &[4, 6, 5]));
}
#[test]
fn difference_search_matches_enumeration_on_small_layouts() {
let mut checked = 0;
for rank in 3..=4usize {
let cases = 3usize.pow(rank as u32) * 5usize.pow(rank as u32);
for case in 0..cases {
let mut code = case;
let mut dims = [0usize; 4];
let mut strides = [0isize; 4];
for axis in 0..rank {
dims[axis] = 2 + code % 3;
code /= 3;
let stride = 1 + (code % 5) as isize;
code /= 5;
strides[axis] = if axis == 1 { -stride } else { stride };
}
let (dims, strides) = (&dims[..rank], &strides[..rank]);
let Some(Some(block)) = interleaved_block(dims, strides) else {
continue;
};
let Some(fast) = block_differences_avoid_zero(dims, strides, &block) else {
continue;
};
assert_eq!(
fast,
block_offsets_unique(dims, strides, &block),
"{dims:?} {strides:?}"
);
checked += 1;
}
}
assert!(checked > 1000);
}
#[test]
fn checked_total_len_detects_overflow() {
assert_eq!(total_len(&[usize::MAX, 2]), None);
assert_eq!(total_len(&[3, 4]), Some(12));
assert_eq!(total_len(&[usize::MAX, 2, 0]), Some(0));
}
}