use alloc::vec::Vec;
use core::marker::PhantomData;
use core::{ptr, slice};
use p3_binary_field::{
BinaryField8, BinaryField16, BinaryField32, BinaryField64, BinaryField128, Gf2, PackedGf2,
TowerLevel, Underlier,
};
use p3_field::PrimeCharacteristicRing;
use p3_matrix::dense::RowMajorMatrix;
use p3_maybe_rayon::prelude::*;
use p3_multilinear_util::poly::Poly;
use p3_sumcheck::layout::Table;
use p3_util::log2_strict_usize;
use thiserror::Error;
pub unsafe trait Coordinates: Copy + Send + Sync + 'static {
const COORDINATES: usize;
}
const fn check_coordinates<A: Coordinates>() {
assert!(
A::COORDINATES == 8 * size_of::<A>(),
"a coordinate run must be exactly the bits of its bytes"
);
}
unsafe impl Coordinates for BinaryField8 {
const COORDINATES: usize = 8;
}
unsafe impl Coordinates for BinaryField16 {
const COORDINATES: usize = 16;
}
unsafe impl Coordinates for BinaryField32 {
const COORDINATES: usize = 32;
}
unsafe impl Coordinates for BinaryField64 {
const COORDINATES: usize = 64;
}
unsafe impl Coordinates for BinaryField128 {
const COORDINATES: usize = 128;
}
unsafe impl<U: Underlier> Coordinates for PackedGf2<U> {
const COORDINATES: usize = U::BITS;
}
const PACK_CHUNK_BYTES: usize = 1 << 20;
#[must_use]
pub fn pack<A, EF>(cells: &[A]) -> Vec<EF>
where
A: Coordinates,
EF: Coordinates,
{
const {
check_coordinates::<A>();
check_coordinates::<EF>();
assert!(
cfg!(target_endian = "little"),
"packing a coordinate run needs a little-endian target"
);
}
let coordinates = cells.len() * A::COORDINATES;
assert_eq!(
coordinates % EF::COORDINATES,
0,
"a packed run must fill whole elements"
);
let len = coordinates / EF::COORDINATES;
let mut packed = Vec::<EF>::with_capacity(len);
let elements_per_chunk = (PACK_CHUNK_BYTES / size_of::<EF>()).max(1);
packed.spare_capacity_mut()[..len]
.par_chunks_mut(elements_per_chunk)
.zip(coordinate_bytes(cells).par_chunks(elements_per_chunk * size_of::<EF>()))
.for_each(|(destination, source)| {
debug_assert_eq!(source.len(), size_of_val(destination));
unsafe {
ptr::copy_nonoverlapping(
source.as_ptr(),
destination.as_mut_ptr().cast::<u8>(),
source.len(),
);
}
});
unsafe {
packed.set_len(len);
}
packed
}
#[must_use]
pub fn unpack<A, EF>(elements: &[EF]) -> Vec<A>
where
A: Coordinates,
EF: Coordinates,
{
const {
check_coordinates::<A>();
check_coordinates::<EF>();
assert!(
cfg!(target_endian = "little"),
"unpacking a coordinate run needs a little-endian target"
);
}
let coordinates = elements.len() * EF::COORDINATES;
assert_eq!(
coordinates % A::COORDINATES,
0,
"an unpacked run must fill whole cells"
);
let len = coordinates / A::COORDINATES;
let mut cells = Vec::<A>::with_capacity(len);
unsafe {
ptr::copy_nonoverlapping(
elements.as_ptr().cast::<u8>(),
cells.as_mut_ptr().cast::<u8>(),
len * size_of::<A>(),
);
cells.set_len(len);
}
cells
}
#[must_use]
pub const fn coordinate_bytes<A: Coordinates>(cells: &[A]) -> &[u8] {
const {
check_coordinates::<A>();
assert!(
cfg!(target_endian = "little"),
"viewing coordinate bytes needs a little-endian target"
);
}
unsafe { slice::from_raw_parts(cells.as_ptr().cast::<u8>(), size_of_val(cells)) }
}
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct PackedStack<A, EF> {
elements: Vec<EF>,
column_len: usize,
_alphabet: PhantomData<A>,
}
impl<A, EF> PackedStack<A, EF>
where
A: Coordinates,
EF: Coordinates + TowerLevel,
{
pub const CELLS_PER_ELEMENT: usize = EF::COORDINATES / A::COORDINATES;
pub fn from_columns(columns: &[&[A]]) -> Result<Self, PackError> {
if A::COORDINATES > EF::COORDINATES {
return Err(PackError::CellWiderThanElement {
cell: A::COORDINATES,
element: EF::COORDINATES,
});
}
let Some((&first, rest)) = columns.split_first() else {
return Err(PackError::NoColumns);
};
if let Some(other) = rest.iter().find(|column| column.len() != first.len()) {
return Err(PackError::RaggedColumns {
expected: first.len(),
actual: other.len(),
});
}
if !first.len().is_multiple_of(Self::CELLS_PER_ELEMENT) {
return Err(PackError::PartialElement {
cells: first.len(),
per_element: Self::CELLS_PER_ELEMENT,
});
}
let column_len = first.len() / Self::CELLS_PER_ELEMENT;
if column_len == 0 || !column_len.is_power_of_two() {
return Err(PackError::NotAHypercube {
elements: column_len,
});
}
let elements = if let [column] = columns {
pack::<A, EF>(column)
} else {
let mut elements = Vec::with_capacity(column_len * columns.len());
for column in columns {
elements.extend(pack::<A, EF>(column));
}
elements
};
Ok(Self {
elements,
column_len,
_alphabet: PhantomData,
})
}
#[must_use]
pub fn elements(&self) -> &[EF] {
&self.elements
}
#[must_use]
pub const fn num_columns(&self) -> usize {
self.elements.len() / self.column_len
}
#[must_use]
pub const fn column_num_variables(&self) -> usize {
log2_strict_usize(self.column_len)
}
#[must_use]
pub const fn column_num_cells(&self) -> usize {
self.column_len * Self::CELLS_PER_ELEMENT
}
#[must_use]
pub fn column(&self, index: usize) -> &[EF] {
assert!(index < self.num_columns(), "column index out of range");
&self.elements[index * self.column_len..][..self.column_len]
}
#[must_use]
pub fn locate(&self, column: usize, cell: usize) -> (usize, usize) {
assert!(column < self.num_columns(), "column index out of range");
assert!(cell < self.column_num_cells(), "cell index out of range");
let element = column * self.column_len + cell / Self::CELLS_PER_ELEMENT;
(element, cell % Self::CELLS_PER_ELEMENT)
}
#[must_use]
pub fn cell(&self, column: usize, cell: usize) -> A {
let (element, inside) = self.locate(column, cell);
unpack::<A, EF>(&self.elements[element..=element])[inside]
}
#[must_use]
pub fn unpack_column(&self, index: usize) -> Vec<A> {
unpack::<A, EF>(self.column(index))
}
pub fn into_poly(self) -> Poly<EF> {
Poly::new(self.elements)
}
pub fn into_table(self) -> Table<EF> {
Table::new(RowMajorMatrix::new(self.elements, self.column_len))
}
}
impl<U, EF> PackedStack<PackedGf2<U>, EF>
where
U: Underlier,
EF: Coordinates + TowerLevel,
{
pub fn from_bit_rows(rows: &[PackedGf2<U>], num_columns: usize) -> Result<Self, PackError> {
let width = PackedGf2::<U>::WIDTH;
if !rows.len().is_multiple_of(width) {
return Err(PackError::PartialBitBlock {
rows: rows.len(),
width,
});
}
if num_columns > width {
return Err(PackError::TooManyColumns {
requested: num_columns,
width,
});
}
let blocks = rows.len() / width;
let mut columns = alloc::vec![Vec::with_capacity(blocks); num_columns];
let mut block = alloc::vec![PackedGf2::<U>::from(Gf2::ZERO); width];
for source in rows.chunks_exact(width) {
block.copy_from_slice(source);
PackedGf2::transpose(&mut block);
for (column, &transposed) in columns.iter_mut().zip(&block) {
column.push(transposed);
}
}
let views: Vec<&[PackedGf2<U>]> = columns.iter().map(alloc::vec::Vec::as_slice).collect();
Self::from_columns(&views)
}
}
#[derive(Clone, Copy, Debug, Error, PartialEq, Eq)]
#[non_exhaustive]
pub enum PackError {
#[error("a {cell}-coordinate cell does not fit a {element}-coordinate element")]
CellWiderThanElement {
cell: usize,
element: usize,
},
#[error("a packed stack needs at least one column")]
NoColumns,
#[error("a column holds {actual} cells, expected {expected}")]
RaggedColumns {
expected: usize,
actual: usize,
},
#[error("{cells} cells do not fill whole elements of {per_element} cells each")]
PartialElement {
cells: usize,
per_element: usize,
},
#[error("a packed column holds {elements} elements, which is no hypercube")]
NotAHypercube {
elements: usize,
},
#[error("{rows} bit-sliced rows are no whole number of {width}-row blocks")]
PartialBitBlock {
rows: usize,
width: usize,
},
#[error("{requested} columns exceed the {width} lanes a block holds")]
TooManyColumns {
requested: usize,
width: usize,
},
}
#[cfg(test)]
mod tests {
use p3_binary_field::{PackedGf2x8, PackedGf2x64, PackedGf2x128};
use p3_field::{BasedVectorSpace, PrimeCharacteristicRing};
use proptest::prelude::*;
use rand::rngs::SmallRng;
use rand::{RngExt, SeedableRng};
use super::*;
type EF = BinaryField128;
fn coordinate<A: Coordinates>(cells: &[A], index: usize) -> bool {
let bytes = coordinate_bytes(cells);
(bytes[index / 8] >> (index % 8)) & 1 == 1
}
#[test]
fn packing_preserves_every_coordinate_in_order() {
let mut rng = SmallRng::seed_from_u64(0xC0DE);
let cells: Vec<BinaryField32> = (0..32).map(|_| rng.random()).collect();
let packed = pack::<BinaryField32, EF>(&cells);
assert_eq!(packed.len(), 8);
for index in 0..32 * 32 {
assert_eq!(
coordinate(&packed, index),
coordinate(&cells, index),
"coordinate {index}"
);
}
}
#[test]
fn the_reinterpretation_agrees_with_the_field_basis_accessor() {
let mut rng = SmallRng::seed_from_u64(0xBA515);
let words: Vec<BinaryField32> = (0..64).map(|_| rng.random()).collect();
let by_basis: Vec<EF> = words
.as_chunks::<4>()
.0
.iter()
.map(|chunk| EF::from_basis_coefficients_fn(|index| chunk[index]))
.collect();
assert_eq!(pack::<BinaryField32, EF>(&words), by_basis);
let halves: Vec<BinaryField64> = (0..64).map(|_| rng.random()).collect();
let by_basis: Vec<EF> = halves
.as_chunks::<2>()
.0
.iter()
.map(|chunk| EF::from_basis_coefficients_fn(|index| chunk[index]))
.collect();
assert_eq!(pack::<BinaryField64, EF>(&halves), by_basis);
}
#[test]
fn a_round_trip_through_the_packing_is_the_identity() {
let mut rng = SmallRng::seed_from_u64(0xB1A5);
let words: Vec<BinaryField64> = (0..64).map(|_| rng.random()).collect();
assert_eq!(unpack::<BinaryField64, EF>(&pack::<_, EF>(&words)), words);
let bytes: Vec<BinaryField8> = (0..256).map(|_| rng.random()).collect();
assert_eq!(unpack::<BinaryField8, EF>(&pack::<_, EF>(&bytes)), bytes);
let blocks: Vec<PackedGf2x64> = (0..16)
.map(|_| PackedGf2x64::new(rng.random::<u64>()))
.collect();
assert_eq!(unpack::<PackedGf2x64, EF>(&pack::<_, EF>(&blocks)), blocks);
}
#[test]
fn a_packed_bit_block_carries_the_lanes_it_was_given() {
let mut rng = SmallRng::seed_from_u64(0xB175);
let blocks: Vec<PackedGf2x64> = (0..8)
.map(|_| PackedGf2x64::new(rng.random::<u64>()))
.collect();
let packed = pack::<PackedGf2x64, EF>(&blocks);
assert_eq!(packed.len(), 4);
for (b, block) in blocks.iter().enumerate() {
for lane in 0..PackedGf2x64::WIDTH {
let set = block.get(lane) == Gf2::ONE;
assert_eq!(coordinate(&packed, 64 * b + lane), set, "b={b} lane={lane}");
}
}
}
#[test]
fn the_stack_lays_columns_out_back_to_back() {
let mut rng = SmallRng::seed_from_u64(0x57AC);
let columns: Vec<Vec<BinaryField32>> = (0..3)
.map(|_| (0..8).map(|_| rng.random()).collect())
.collect();
let views: Vec<&[BinaryField32]> = columns.iter().map(Vec::as_slice).collect();
let stack = PackedStack::<BinaryField32, EF>::from_columns(&views).unwrap();
assert_eq!(stack.num_columns(), 3);
assert_eq!(stack.column_num_variables(), 1);
assert_eq!(stack.column_num_cells(), 8);
assert_eq!(stack.elements().len(), 6);
for (index, column) in columns.iter().enumerate() {
assert_eq!(&stack.unpack_column(index), column);
for (cell, &value) in column.iter().enumerate() {
assert_eq!(stack.cell(index, cell), value, "column={index} cell={cell}");
}
}
let (element, inside) = stack.locate(2, 5);
assert_eq!(element, 2 * 2 + 1);
assert_eq!(inside, 1);
}
#[test]
fn the_stacked_table_keeps_one_row_per_packed_column() {
let mut rng = SmallRng::seed_from_u64(0x7AB1);
let columns: Vec<Vec<BinaryField64>> = (0..2)
.map(|_| (0..8).map(|_| rng.random()).collect())
.collect();
let views: Vec<&[BinaryField64]> = columns.iter().map(Vec::as_slice).collect();
let stack = PackedStack::<BinaryField64, EF>::from_columns(&views).unwrap();
let table = stack.clone().into_table();
assert_eq!(table.num_polys(), 2);
assert_eq!(table.num_variables(), 2);
for index in 0..2 {
assert_eq!(table.poly(index).as_slice(), stack.column(index));
}
}
fn transpose_reference<U: Underlier>(
rows: &[PackedGf2<U>],
num_columns: usize,
) -> Vec<Vec<PackedGf2<U>>> {
let width = PackedGf2::<U>::WIDTH;
(0..num_columns)
.map(|column| {
rows.chunks_exact(width)
.map(|block| PackedGf2::<U>::from_fn(|lane| block[lane].get(column)))
.collect()
})
.collect()
}
#[test]
fn a_row_major_bit_block_packs_as_its_own_transpose() {
let mut rng = SmallRng::seed_from_u64(0x7A05);
let rows: Vec<PackedGf2x64> = (0..128)
.map(|_| PackedGf2x64::new(rng.random::<u64>()))
.collect();
let stack = PackedStack::<PackedGf2x64, EF>::from_bit_rows(&rows, 40).unwrap();
let expected = transpose_reference(&rows, 40);
assert_eq!(stack.num_columns(), 40);
for (column, blocks) in expected.iter().enumerate() {
assert_eq!(&stack.unpack_column(column), blocks, "column={column}");
}
}
#[test]
fn a_row_major_block_is_refused_unless_it_is_square_and_wide_enough() {
let rows = alloc::vec![PackedGf2x8::from(Gf2::ZERO); 12];
assert_eq!(
PackedStack::<PackedGf2x8, EF>::from_bit_rows(&rows, 8).unwrap_err(),
PackError::PartialBitBlock { rows: 12, width: 8 }
);
let rows = alloc::vec![PackedGf2x8::from(Gf2::ZERO); 128];
assert_eq!(
PackedStack::<PackedGf2x8, EF>::from_bit_rows(&rows, 9).unwrap_err(),
PackError::TooManyColumns {
requested: 9,
width: 8,
}
);
}
#[test]
fn a_stack_that_describes_no_commitment_is_refused() {
assert_eq!(
PackedStack::<BinaryField32, EF>::from_columns(&[]).unwrap_err(),
PackError::NoColumns
);
let long = alloc::vec![BinaryField32::ZERO; 8];
let short = alloc::vec![BinaryField32::ZERO; 4];
assert_eq!(
PackedStack::<BinaryField32, EF>::from_columns(&[&long, &short]).unwrap_err(),
PackError::RaggedColumns {
expected: 8,
actual: 4,
}
);
let ragged = alloc::vec![BinaryField32::ZERO; 6];
assert_eq!(
PackedStack::<BinaryField32, EF>::from_columns(&[&ragged]).unwrap_err(),
PackError::PartialElement {
cells: 6,
per_element: 4,
}
);
let odd = alloc::vec![BinaryField32::ZERO; 12];
assert_eq!(
PackedStack::<BinaryField32, EF>::from_columns(&[&odd]).unwrap_err(),
PackError::NotAHypercube { elements: 3 }
);
let wide = alloc::vec![PackedGf2x128::from(Gf2::ZERO); 4];
assert_eq!(
PackedStack::<PackedGf2x128, BinaryField64>::from_columns(&[&wide]).unwrap_err(),
PackError::CellWiderThanElement {
cell: 128,
element: 64,
}
);
}
proptest! {
#[test]
fn packing_is_a_bijection_on_coordinates(
log_cells in 4usize..=9,
seed: u64,
) {
let mut rng = SmallRng::seed_from_u64(seed);
let cells = 1usize << log_cells;
let bits: Vec<PackedGf2x8> = (0..cells)
.map(|_| PackedGf2x8::new(rng.random::<u8>()))
.collect();
prop_assert_eq!(unpack::<PackedGf2x8, EF>(&pack::<_, EF>(&bits)), bits);
let bytes: Vec<BinaryField8> = (0..cells).map(|_| rng.random()).collect();
prop_assert_eq!(unpack::<BinaryField8, EF>(&pack::<_, EF>(&bytes)), bytes);
let words: Vec<BinaryField32> = (0..cells).map(|_| rng.random()).collect();
prop_assert_eq!(&unpack::<BinaryField32, EF>(&pack::<_, EF>(&words)), &words);
let packed = pack::<BinaryField32, EF>(&words);
for index in 0..cells * 32 {
prop_assert_eq!(coordinate(&packed, index), coordinate(&words, index));
}
}
}
}