pub mod addressing;
pub mod block_paged;
pub mod col_major;
pub mod composite;
pub mod coo;
pub mod csc;
pub mod csf;
pub mod csr;
pub mod hilbert;
pub mod morton;
pub mod private_extension;
pub mod row_major;
pub mod strided;
pub mod tiled;
pub mod unknown;
pub use addressing::{byte_address_from_element_offset, ElementAddress};
pub use block_paged::{BlockPagedLayout, BlockTableIndexType, KvRole};
pub use composite::{CombineOp, CompositeLayout, CompositionRule};
pub use coo::CooLayout;
pub use csc::CscLayout;
pub use csf::CsfLayout;
pub use csr::CsrLayout;
pub use hilbert::HilbertLayout;
pub use morton::MortonLayout;
pub use private_extension::{
PrivateExtensionLayout, PRIVATE_LAYOUT_TAG_MAX, PRIVATE_LAYOUT_TAG_MIN,
};
pub use strided::StridedLayout;
pub use tiled::{InnerStrides, OuterStrides, TiledLayout, MAX_TILED_DEPTH};
pub use unknown::UnknownLayout;
use std::num::NonZeroU8;
use crate::{Error, Result, Shape};
pub const TAG_ROW_MAJOR: u8 = 0x01;
pub const TAG_COL_MAJOR: u8 = 0x02;
pub const TAG_STRIDED: u8 = 0x03;
pub const TAG_TILED: u8 = 0x04;
pub const TAG_MORTON: u8 = 0x05;
pub const TAG_COO: u8 = 0x06;
pub const TAG_CSR: u8 = 0x07;
pub const TAG_CSC: u8 = 0x08;
pub const TAG_CSF: u8 = 0x09;
pub const TAG_BLOCK_PAGED: u8 = 0x0A;
pub const TAG_COMPOSITE: u8 = 0x0B;
pub const TAG_HILBERT: u8 = 0x40;
#[inline]
pub fn is_invalid_tag(tag: u8) -> bool {
tag == 0x00 || tag == 0xFF
}
#[inline]
pub fn is_reserved_tag(tag: u8) -> bool {
matches!(tag, 0x0C..=0x3F | 0x41..=0x7F | 0x80..=0xEF)
}
#[inline]
pub fn is_private_tag(tag: u8) -> bool {
matches!(tag, PRIVATE_LAYOUT_TAG_MIN..=PRIVATE_LAYOUT_TAG_MAX)
}
#[inline]
pub fn is_named_tag(tag: u8) -> bool {
matches!(
tag,
TAG_ROW_MAJOR
| TAG_COL_MAJOR
| TAG_STRIDED
| TAG_TILED
| TAG_MORTON
| TAG_COO
| TAG_CSR
| TAG_CSC
| TAG_CSF
| TAG_BLOCK_PAGED
| TAG_COMPOSITE
| TAG_HILBERT
)
}
pub fn validate_layout_tag_strict(tag: u8) -> Result<()> {
match tag {
0x00 | 0xFF => Err(Error::InvalidLayoutTag(tag)),
t if is_reserved_tag(t) => Err(Error::ReservedLayoutTag(tag)),
t if is_private_tag(t) => Err(Error::PrivateLayoutTag(tag)),
t if is_named_tag(t) => Ok(()),
_ => Err(Error::UnknownLayoutTag(tag)),
}
}
#[derive(Debug, Clone, PartialEq, Eq, Hash)]
#[non_exhaustive]
pub enum LayoutDescriptor {
RowMajor,
ColMajor,
Strided(StridedLayout),
Tiled(Box<TiledLayout>),
Morton(MortonLayout),
Coo(CooLayout),
Csr(CsrLayout),
Csc(CscLayout),
Csf(CsfLayout),
BlockPaged(BlockPagedLayout),
Composite(CompositeLayout),
Hilbert(HilbertLayout),
PrivateExtension(PrivateExtensionLayout),
Unknown(UnknownLayout),
}
impl LayoutDescriptor {
#[inline]
pub fn tag(&self) -> u8 {
match self {
Self::RowMajor => TAG_ROW_MAJOR,
Self::ColMajor => TAG_COL_MAJOR,
Self::Strided(_) => TAG_STRIDED,
Self::Tiled(_) => TAG_TILED,
Self::Morton(_) => TAG_MORTON,
Self::Coo(_) => TAG_COO,
Self::Csr(_) => TAG_CSR,
Self::Csc(_) => TAG_CSC,
Self::Csf(_) => TAG_CSF,
Self::BlockPaged(_) => TAG_BLOCK_PAGED,
Self::Composite(_) => TAG_COMPOSITE,
Self::Hilbert(_) => TAG_HILBERT,
Self::PrivateExtension(p) => p.tag,
Self::Unknown(u) => u.tag,
}
}
#[inline]
pub fn buffer_count(&self) -> Option<NonZeroU8> {
match self {
Self::RowMajor
| Self::ColMajor
| Self::Strided(_)
| Self::Tiled(_)
| Self::Morton(_)
| Self::Hilbert(_) => NonZeroU8::new(1),
Self::Coo(_) => NonZeroU8::new(2),
Self::Csr(_) | Self::Csc(_) | Self::BlockPaged(_) => NonZeroU8::new(3),
Self::Csf(c) => {
let count = (2 * c.mode_order.len() + 1).min(u8::MAX as usize) as u8;
NonZeroU8::new(count)
}
Self::Composite(_) => None,
Self::PrivateExtension(_) | Self::Unknown(_) => None,
}
}
#[inline]
pub fn is_virtual(&self) -> bool {
matches!(self, Self::Composite(_))
}
pub fn validate_against_shape(&self, shape: &Shape) -> Result<()> {
match self {
Self::RowMajor | Self::ColMajor => {
Ok(())
}
Self::Strided(s) => {
if s.strides.len() != shape.rank() {
return Err(Error::InvalidLayout(format!(
"strided layout: strides.len() ({}) != shape.rank() ({})",
s.strides.len(),
shape.rank()
)));
}
if shape.rank() == 0 {
return Err(Error::InvalidLayout(
"strided layout cannot be used for rank-0 (scalar) tensors".to_string(),
));
}
Ok(())
}
Self::Tiled(t) => {
if t.tile_shape.len() != shape.rank() {
return Err(Error::InvalidLayout(format!(
"tiled layout: tile_shape.len() ({}) != shape.rank() ({})",
t.tile_shape.len(),
shape.rank()
)));
}
if shape.rank() == 0 {
return Err(Error::InvalidLayout(
"tiled layout cannot be used for rank-0 (scalar) tensors".to_string(),
));
}
Ok(())
}
Self::Morton(m) => {
if m.morton_bits.len() != shape.rank() {
return Err(Error::InvalidLayout(format!(
"Morton layout: morton_bits.len() ({}) != shape.rank() ({})",
m.morton_bits.len(),
shape.rank()
)));
}
for (k, (&bits, &dim)) in m.morton_bits.iter().zip(shape.dims().iter()).enumerate()
{
if dim == crate::shape::DYNAMIC {
continue;
}
let max_dim = if bits >= 64 { u64::MAX } else { 1u64 << bits };
if dim > max_dim {
return Err(Error::InvalidLayout(format!(
"Morton layout: shape[{k}] ({dim}) > 2^morton_bits[{k}] ({max_dim})"
)));
}
}
Ok(())
}
Self::Coo(_) => {
if shape.rank() == 0 {
return Err(Error::InvalidLayout(
"COO layout cannot be used for rank-0 (scalar) tensors".to_string(),
));
}
Ok(())
}
Self::Csr(_) => {
if shape.rank() != 2 {
return Err(Error::InvalidLayout(format!(
"CSR layout requires rank 2, got rank {}",
shape.rank()
)));
}
Ok(())
}
Self::Csc(_) => {
if shape.rank() != 2 {
return Err(Error::InvalidLayout(format!(
"CSC layout requires rank 2, got rank {}",
shape.rank()
)));
}
Ok(())
}
Self::Csf(c) => {
let rank = shape.rank();
if rank < 3 {
return Err(Error::InvalidLayout(format!(
"CSF layout requires rank >= 3, got rank {rank}"
)));
}
if c.mode_order.len() != rank {
return Err(Error::InvalidLayout(format!(
"CSF layout: mode_order.len() ({}) != shape.rank() ({rank})",
c.mode_order.len()
)));
}
let mut seen = 0u64;
for (level, &dim) in c.mode_order.iter().enumerate() {
if dim as usize >= rank {
return Err(Error::InvalidLayout(format!(
"CSF layout: mode_order[{level}]={dim} out of range [0, {rank})"
)));
}
let bit = 1u64 << dim;
if seen & bit != 0 {
return Err(Error::InvalidLayout(format!(
"CSF layout: mode_order contains duplicate value {dim}"
)));
}
seen |= bit;
}
Ok(())
}
Self::BlockPaged(bp) => {
if shape.rank() != 3 {
return Err(Error::InvalidLayout(format!(
"block-paged layout requires rank 3, got rank {}",
shape.rank()
)));
}
if bp.paged_axis != 0 {
return Err(Error::InvalidLayout(format!(
"block-paged layout: paged_axis must be 0, got {}",
bp.paged_axis
)));
}
if bp.page_size < 1 {
return Err(Error::InvalidLayout(
"block-paged layout: page_size must be >= 1".to_string(),
));
}
Ok(())
}
Self::Composite(c) => {
let needs_static_shape = !matches!(c.rule, CompositionRule::Group);
if needs_static_shape && shape.has_dynamic() {
return Err(Error::InvalidLayout(
"composite head: partition/overlay composition requires a fully \
static shape (no DYNAMIC dimension)"
.to_string(),
));
}
Ok(())
}
Self::Hilbert(h) => {
if h.hilbert_rank as usize != shape.rank() {
return Err(Error::InvalidLayout(format!(
"Hilbert layout: hilbert_rank ({}) != shape.rank() ({})",
h.hilbert_rank,
shape.rank()
)));
}
let expected_dim = 1u64.checked_shl(h.hilbert_order).ok_or_else(|| {
Error::InvalidLayout(format!(
"Hilbert layout: hilbert_order ({}) would require dimension > u64::MAX",
h.hilbert_order
))
})?;
for (k, &dim) in shape.dims().iter().enumerate() {
if dim == crate::shape::DYNAMIC {
continue;
}
if dim != expected_dim {
return Err(Error::InvalidLayout(format!(
"Hilbert layout: shape[{k}] ({dim}) != 2^hilbert_order ({expected_dim})"
)));
}
}
Ok(())
}
Self::PrivateExtension(_) => Ok(()),
Self::Unknown(_) => Ok(()),
}
}
pub fn element_offset(&self, index: &[u64], shape: &Shape) -> crate::Result<u64> {
use addressing::ElementAddress;
match self {
Self::RowMajor => addressing::row_major::element_offset(index, shape),
Self::ColMajor => addressing::col_major::element_offset(index, shape),
Self::Strided(s) => s.element_offset(index, shape),
Self::Tiled(t) => t.element_offset(index, shape),
Self::Morton(m) => m.element_offset(index, shape),
Self::Hilbert(h) => h.element_offset(index, shape),
Self::Coo(_) => Err(crate::Error::LayoutRequiresMultiBuffer {
layout_tag: TAG_COO,
}),
Self::Csr(_) => Err(crate::Error::LayoutRequiresMultiBuffer {
layout_tag: TAG_CSR,
}),
Self::Csc(_) => Err(crate::Error::LayoutRequiresMultiBuffer {
layout_tag: TAG_CSC,
}),
Self::Csf(_) => Err(crate::Error::LayoutRequiresMultiBuffer {
layout_tag: TAG_CSF,
}),
Self::BlockPaged(_) => Err(crate::Error::LayoutRequiresMultiBuffer {
layout_tag: TAG_BLOCK_PAGED,
}),
Self::Composite(_) => Err(crate::Error::LayoutIsVirtual {
layout_tag: TAG_COMPOSITE,
}),
Self::PrivateExtension(p) => Err(crate::Error::PrivateLayoutTag(p.tag)),
Self::Unknown(u) => Err(crate::Error::UnknownLayoutTag(u.tag)),
}
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn tag_values_match_spec() {
assert_eq!(LayoutDescriptor::RowMajor.tag(), 0x01);
assert_eq!(LayoutDescriptor::ColMajor.tag(), 0x02);
assert_eq!(
LayoutDescriptor::Strided(StridedLayout::new(vec![1])).tag(),
0x03
);
assert_eq!(
LayoutDescriptor::Tiled(Box::new(
TiledLayout::new(vec![4], 0x01, 0x01, None, None, None).unwrap()
))
.tag(),
0x04
);
assert_eq!(
LayoutDescriptor::Morton(MortonLayout::new(vec![2]).unwrap()).tag(),
0x05
);
assert_eq!(
LayoutDescriptor::Coo(CooLayout {
nnz: 0,
is_sorted: false
})
.tag(),
0x06
);
assert_eq!(LayoutDescriptor::Csr(CsrLayout { nnz: 0 }).tag(), 0x07);
assert_eq!(LayoutDescriptor::Csc(CscLayout { nnz: 0 }).tag(), 0x08);
assert_eq!(
LayoutDescriptor::Csf(CsfLayout::new(0, vec![0, 1, 2])).tag(),
0x09
);
assert_eq!(
LayoutDescriptor::Hilbert(HilbertLayout::new(2, 2).unwrap()).tag(),
0x40
);
}
#[test]
fn dense_layouts_have_buffer_count_1() {
let dense = [
LayoutDescriptor::RowMajor,
LayoutDescriptor::ColMajor,
LayoutDescriptor::Strided(StridedLayout::new(vec![4, 1])),
LayoutDescriptor::Tiled(Box::new(
TiledLayout::new(vec![4, 4], 0x01, 0x01, None, None, None).unwrap(),
)),
LayoutDescriptor::Morton(MortonLayout::new(vec![2, 2]).unwrap()),
LayoutDescriptor::Hilbert(HilbertLayout::new(2, 2).unwrap()),
];
for layout in &dense {
assert_eq!(
layout.buffer_count(),
NonZeroU8::new(1),
"expected buffer_count=1 for {:?}",
layout.tag()
);
}
}
#[test]
fn coo_has_buffer_count_2() {
assert_eq!(
LayoutDescriptor::Coo(CooLayout {
nnz: 0,
is_sorted: false
})
.buffer_count(),
NonZeroU8::new(2)
);
}
#[test]
fn csr_has_buffer_count_3() {
assert_eq!(
LayoutDescriptor::Csr(CsrLayout { nnz: 0 }).buffer_count(),
NonZeroU8::new(3)
);
}
#[test]
fn csc_has_buffer_count_3() {
assert_eq!(
LayoutDescriptor::Csc(CscLayout { nnz: 0 }).buffer_count(),
NonZeroU8::new(3)
);
}
#[test]
fn private_extension_buffer_count_is_none() {
let layout = LayoutDescriptor::PrivateExtension(
PrivateExtensionLayout::new(0xF0, 0, vec![]).unwrap(),
);
assert!(layout.buffer_count().is_none());
}
#[test]
fn unknown_buffer_count_is_none() {
let layout = LayoutDescriptor::Unknown(UnknownLayout {
tag: 0x0C,
raw_bytes: vec![],
});
assert!(layout.buffer_count().is_none());
}
#[test]
fn known_tags_pass_strict_validation() {
for tag in [
0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0A, 0x40,
] {
assert!(
validate_layout_tag_strict(tag).is_ok(),
"tag 0x{tag:02X} should be valid"
);
}
}
#[test]
fn invalid_sentinels_rejected() {
assert!(matches!(
validate_layout_tag_strict(0x00),
Err(Error::InvalidLayoutTag(0x00))
));
assert!(matches!(
validate_layout_tag_strict(0xFF),
Err(Error::InvalidLayoutTag(0xFF))
));
}
#[test]
fn reserved_tags_rejected() {
for tag in [0x0C_u8, 0x3F, 0x41, 0x7F, 0x80, 0xEF] {
assert!(
matches!(
validate_layout_tag_strict(tag),
Err(Error::ReservedLayoutTag(_))
),
"tag 0x{tag:02X} should be ReservedLayoutTag"
);
}
}
#[test]
fn private_tags_rejected_in_strict_mode() {
for tag in [0xF0_u8, 0xF5, 0xFE] {
assert!(
matches!(
validate_layout_tag_strict(tag),
Err(Error::PrivateLayoutTag(_))
),
"tag 0x{tag:02X} should be PrivateLayoutTag"
);
}
}
#[test]
fn row_major_valid_for_any_rank() {
let shape = Shape::new(vec![3, 4]).unwrap();
assert!(LayoutDescriptor::RowMajor
.validate_against_shape(&shape)
.is_ok());
assert!(LayoutDescriptor::ColMajor
.validate_against_shape(&Shape::scalar())
.is_ok());
}
#[test]
fn strided_valid_when_strides_len_matches_rank() {
let shape = Shape::new(vec![3, 4]).unwrap();
let layout = LayoutDescriptor::Strided(StridedLayout::new(vec![4, 1]));
assert!(layout.validate_against_shape(&shape).is_ok());
}
#[test]
fn strided_invalid_when_strides_len_mismatches_rank() {
let shape = Shape::new(vec![3, 4]).unwrap();
let layout = LayoutDescriptor::Strided(StridedLayout::new(vec![1]));
assert!(matches!(
layout.validate_against_shape(&shape),
Err(Error::InvalidLayout(_))
));
}
#[test]
fn strided_invalid_for_scalar() {
let layout = LayoutDescriptor::Strided(StridedLayout::new(vec![]));
assert!(matches!(
layout.validate_against_shape(&Shape::scalar()),
Err(Error::InvalidLayout(_))
));
}
#[test]
fn csr_valid_for_rank_2() {
let shape = Shape::new(vec![4, 5]).unwrap();
let layout = LayoutDescriptor::Csr(CsrLayout { nnz: 0 });
assert!(layout.validate_against_shape(&shape).is_ok());
}
#[test]
fn csr_invalid_for_rank_3() {
let shape = Shape::new(vec![2, 3, 4]).unwrap();
let layout = LayoutDescriptor::Csr(CsrLayout { nnz: 0 });
assert!(matches!(
layout.validate_against_shape(&shape),
Err(Error::InvalidLayout(_))
));
}
#[test]
fn coo_invalid_for_scalar() {
let layout = LayoutDescriptor::Coo(CooLayout {
nnz: 0,
is_sorted: false,
});
assert!(matches!(
layout.validate_against_shape(&Shape::scalar()),
Err(Error::InvalidLayout(_))
));
}
#[test]
fn coo_valid_for_rank_1() {
let shape = Shape::new(vec![10]).unwrap();
let layout = LayoutDescriptor::Coo(CooLayout {
nnz: 3,
is_sorted: true,
});
assert!(layout.validate_against_shape(&shape).is_ok());
}
#[test]
fn hilbert_valid_for_4x4_order2() {
let shape = Shape::new(vec![4, 4]).unwrap();
let layout = LayoutDescriptor::Hilbert(HilbertLayout::new(2, 2).unwrap());
assert!(layout.validate_against_shape(&shape).is_ok());
}
#[test]
fn hilbert_invalid_when_dim_not_power_of_two_order() {
let shape = Shape::new(vec![3, 4]).unwrap(); let layout = LayoutDescriptor::Hilbert(HilbertLayout::new(2, 2).unwrap());
assert!(matches!(
layout.validate_against_shape(&shape),
Err(Error::InvalidLayout(_))
));
}
#[test]
fn morton_invalid_when_shape_exceeds_bits() {
let shape = Shape::new(vec![3, 2]).unwrap();
let layout = LayoutDescriptor::Morton(MortonLayout::new(vec![1, 1]).unwrap());
assert!(matches!(
layout.validate_against_shape(&shape),
Err(Error::InvalidLayout(_))
));
}
#[test]
fn unknown_always_passes_validation() {
let layout = LayoutDescriptor::Unknown(UnknownLayout {
tag: 0x0C,
raw_bytes: vec![],
});
assert!(layout.validate_against_shape(&Shape::scalar()).is_ok());
assert!(layout
.validate_against_shape(&Shape::new(vec![100, 200]).unwrap())
.is_ok());
}
}