use crate::error::CodingError;
use crate::nibblepacking;
use crate::nibblepack_simd;
use crate::sink::*;
use std::cmp::Ordering;
use core::marker::PhantomData;
use std::ops::{Add, BitXor};
use std::convert::TryFrom;
use enum_dispatch::enum_dispatch;
use num::{PrimInt, Unsigned, Num, Bounded, Float};
use num_enum::{TryFromPrimitive, TryFromPrimitiveError};
use packed_simd::{u32x8, u64x8, f32x8};
use scroll::{ctx, Endian, Pread, Pwrite, LE};
#[repr(u8)]
#[derive(Copy, Clone, Debug, PartialEq, Eq, Hash, TryFromPrimitive)]
pub enum SectionType {
Null = 0,
NibblePackedMedium = 1,
DeltaNPMedium = 3,
Constant = 5,
XorNPMedium = 6,
}
impl From<TryFromPrimitiveError<SectionType>> for CodingError {
fn from(err: TryFromPrimitiveError<SectionType>) -> CodingError {
CodingError::InvalidSectionType(err.number)
}
}
impl SectionType {
pub fn as_num(self) -> u8 { self as u8 }
}
impl<'a> ctx::TryFromCtx<'a, Endian> for SectionType {
type Error = scroll::Error;
fn try_from_ctx (src: &'a [u8], ctx: Endian) -> Result<(SectionType, usize), Self::Error> {
u8::try_from_ctx(src, ctx).and_then(|(n, bytes)| {
SectionType::try_from(n).map(|s| (s, bytes))
.map_err(|e| scroll::Error::Custom(format!("InvalidSectionType {:?}", e.number)))
})
}
}
impl ctx::TryIntoCtx<Endian> for &SectionType {
type Error = scroll::Error;
fn try_into_ctx(self, buf: &mut [u8], ctx: Endian) -> Result<usize, Self::Error> {
u8::try_into_ctx(self.as_num(), buf, ctx)
}
}
#[derive(Copy, Clone, Debug, PartialEq, Pread, Pwrite)]
pub struct SectionHeader {
num_bytes: u16,
num_elements: u16,
typ: SectionType,
}
type CodingResult = Result<(u16, u16), CodingError>;
#[derive(Debug)]
pub struct SectionWriter<'a> {
write_buf: &'a mut [u8],
cur_pos: usize,
cur_header_pos: usize,
max_elements_per_sect: u16,
cur_header: SectionHeader
}
impl<'a> SectionWriter<'a> {
pub fn new(buf: &'a mut [u8], max_elements_per_sect: u16) -> Self {
Self { write_buf: buf,
cur_pos: 0,
cur_header_pos: 0,
max_elements_per_sect,
cur_header: SectionHeader { num_bytes: 0, num_elements: 0, typ: SectionType::Null }
}
}
pub fn cur_pos(&self) -> usize { self.cur_pos }
fn init_new_section(&mut self, sect_type: SectionType) -> CodingResult {
self.cur_header.num_bytes = 0;
self.cur_header.num_elements = 0;
self.cur_header.typ = sect_type;
self.cur_header_pos = self.cur_pos;
let (bytes_written, _) = self.update_sect_header()?;
self.cur_pos += bytes_written as usize;
Ok((bytes_written, 0))
}
fn update_sect_header(&mut self) -> CodingResult {
let bytes_written = self.write_buf.pwrite_with(self.cur_header, self.cur_header_pos, LE)?;
Ok((bytes_written as u16, 0))
}
pub fn add_64kb<F>(&mut self, sect_type: SectionType, filler: F) -> CodingResult
where F: Fn(&mut [u8], usize) -> CodingResult
{
if self.cur_pos == 0 { self.init_new_section(sect_type)?; }
let elements_left = self.max_elements_per_sect - self.cur_header.num_elements;
let bytes_left = std::cmp::min(65535 - self.cur_header.num_bytes as usize,
self.write_buf.len() - self.cur_pos);
let writable_bytes = &mut self.write_buf[self.cur_pos..self.cur_pos + bytes_left];
let filled_res = filler(writable_bytes, elements_left as usize);
match filled_res {
Ok((bytes_written, elements_written)) => {
assert!(elements_written <= elements_left);
self.cur_header.num_bytes += bytes_written;
self.cur_header.num_elements += elements_written;
self.cur_pos += bytes_written as usize;
self.update_sect_header()?;
Ok((bytes_written, elements_written))
},
Err(CodingError::NotEnoughSpace) => {
self.init_new_section(sect_type)?;
self.add_64kb(sect_type, filler)
}
e @ Err(_) => return e,
}
}
}
pub const FIXED_LEN: usize = 256;
#[enum_dispatch]
pub trait FixedSection {
fn num_bytes(&self) -> usize;
fn num_elements(&self) -> usize { FIXED_LEN }
fn sect_bytes(&self) -> Option<&[u8]>;
fn sect_type(&self) -> SectionType;
}
#[enum_dispatch(FixedSection)]
#[derive(Debug, PartialEq)]
pub enum FixedSectEnum<'buf, T: VectBase> {
NullFixedSect,
NibblePackMedFixedSect(NibblePackMedFixedSect<'buf, T>),
DeltaNPMedFixedSect(DeltaNPMedFixedSect<'buf, T>),
ConstFixedSect(ConstFixedSect<'buf, T>),
XorNPMedFixedSect(XorNPMedFixedSect<'buf>),
}
impl<'buf, T: VectBase> FixedSectEnum<'buf, T> {
#[inline]
pub fn decode<S>(self, sink: &mut S) -> Result<(), CodingError>
where S: Sink<T::SI> {
T::Utils::decode_to_sink(self, sink)
}
#[inline]
pub fn is_null(&self) -> bool {
match self {
FixedSectEnum::NullFixedSect(_) => true,
_ => false,
}
}
}
impl<'buf, T: VectBase> TryFrom<&'buf [u8]> for FixedSectEnum<'buf, T> {
type Error = CodingError;
fn try_from(s: &'buf [u8]) -> Result<FixedSectEnum<'buf, T>, CodingError> {
if s.len() <= 0 { return Err(CodingError::InputTooShort) }
let sect_type = SectionType::try_from(s[0])?;
match sect_type {
SectionType::Null => Ok((NullFixedSect {}).into()),
SectionType::NibblePackedMedium =>
NibblePackMedFixedSect::try_from(s).map(|sect| sect.into()),
SectionType::DeltaNPMedium =>
DeltaNPMedFixedSect::try_from(s).map(|sect| sect.into()),
SectionType::Constant =>
ConstFixedSect::try_from(s).map(|sect| sect.into()),
SectionType::XorNPMedium =>
XorNPMedFixedSect::try_from(s).map(|sect| sect.into()),
}
}
}
pub trait FixedSectReader<T: VectBase>: FixedSection {
fn decode_to_sink<Output>(&self, output: &mut Output) -> Result<(), CodingError>
where Output: Sink<T::SI>;
}
pub trait FSUtils<T: VectBase> {
const BYTE_WIDTH: usize;
fn decode_to_sink<Output>(e: FixedSectEnum<T>, output: &mut Output) -> Result<(), CodingError>
where Output: Sink<T::SI>;
fn read_le_offset<'a>(buf: &'a [u8], offset: usize) -> Result<T, scroll::Error>;
fn write_le_offset<'a>(buf: &'a mut [u8], offset: usize, value: T) -> Result<usize, scroll::Error>;
fn nibblepack_decode<'a, S: Sink<T::SI>>(buf: &'a [u8], sink: &mut S) -> Result<&'a [u8], CodingError>;
}
pub struct FSUtilsMarker {}
impl<'buf> FSUtils<u32> for FSUtilsMarker {
const BYTE_WIDTH: usize = 4;
#[inline]
fn decode_to_sink<Output>(e: FixedSectEnum<u32>, output: &mut Output) -> Result<(), CodingError>
where Output: Sink<u32x8> {
match e {
FixedSectEnum::NullFixedSect(nfs) => FixedSectReader::<u32>::decode_to_sink(&nfs, output),
FixedSectEnum::NibblePackMedFixedSect(fs) => fs.decode_to_sink(output),
FixedSectEnum::DeltaNPMedFixedSect(fs) => fs.decode_to_sink(output),
FixedSectEnum::ConstFixedSect(cs) => cs.decode_to_sink(output),
_ => Err(CodingError::InvalidFormat(format!("Section {:?} invalid for u32", e))),
}
}
#[inline]
fn read_le_offset<'a>(buf: &'a [u8], offset: usize) -> Result<u32, scroll::Error> {
buf.pread_with(offset, LE)
}
#[inline]
fn write_le_offset<'a>(buf: &'a mut [u8], offset: usize, value: u32) -> Result<usize, scroll::Error> {
buf.pwrite_with(value, offset, LE)
}
#[inline]
fn nibblepack_decode<'a, S: Sink<u32x8>>(buf: &'a [u8], sink: &mut S) -> Result<&'a [u8], CodingError> {
nibblepack_simd::unpack8_u32_simd(buf, sink)
}
}
impl<'buf> FSUtils<u64> for FSUtilsMarker {
const BYTE_WIDTH: usize = 8;
#[inline]
fn decode_to_sink<Output>(e: FixedSectEnum<u64>, output: &mut Output) -> Result<(), CodingError>
where Output: Sink<u64x8> {
match e {
FixedSectEnum::NullFixedSect(nfs) => FixedSectReader::<u64>::decode_to_sink(&nfs, output),
FixedSectEnum::NibblePackMedFixedSect(fs) => fs.decode_to_sink(output),
FixedSectEnum::DeltaNPMedFixedSect(fs) => fs.decode_to_sink(output),
FixedSectEnum::ConstFixedSect(cs) => cs.decode_to_sink(output),
_ => Err(CodingError::InvalidFormat(format!("Section {:?} invalid for u64", e))),
}
}
#[inline]
fn read_le_offset<'a>(buf: &'a [u8], offset: usize) -> Result<u64, scroll::Error> {
buf.pread_with(offset, LE)
}
#[inline]
fn write_le_offset<'a>(buf: &'a mut [u8], offset: usize, value: u64) -> Result<usize, scroll::Error> {
buf.pwrite_with(value, offset, LE)
}
#[inline]
fn nibblepack_decode<'a, S: Sink<u64x8>>(buf: &'a [u8], sink: &mut S) -> Result<&'a [u8], CodingError> {
nibblepacking::nibble_unpack8(buf, sink)
}
}
impl<'buf> FSUtils<f32> for FSUtilsMarker {
const BYTE_WIDTH: usize = 4;
#[inline]
fn decode_to_sink<Output>(e: FixedSectEnum<f32>, output: &mut Output) -> Result<(), CodingError>
where Output: Sink<f32x8> {
match e {
FixedSectEnum::NullFixedSect(nfs) => FixedSectReader::<f32>::decode_to_sink(&nfs, output),
FixedSectEnum::ConstFixedSect(cs) => cs.decode_to_sink(output),
FixedSectEnum::XorNPMedFixedSect(fs) => fs.decode_to_sink(output),
_ => Err(CodingError::InvalidFormat(format!("Section {:?} invalid for f32", e))),
}
}
#[inline]
fn read_le_offset<'a>(buf: &'a [u8], offset: usize) -> Result<f32, scroll::Error> {
buf.pread_with(offset, LE)
}
#[inline]
fn write_le_offset<'a>(buf: &'a mut [u8], offset: usize, value: f32) -> Result<usize, scroll::Error> {
buf.pwrite_with(value, offset, LE)
}
#[inline]
fn nibblepack_decode<'a, S: Sink<f32x8>>(_buf: &'a [u8], _sink: &mut S) -> Result<&'a [u8], CodingError> {
unimplemented!()
}
}
pub trait VectBase: Num + Bounded + PartialOrd + Copy + std::fmt::Debug {
type SI: SinkInput<Item = Self> + Add<Self::SI, Output = Self::SI>;
type Utils: FSUtils<Self>;
}
impl VectBase for u32 {
type SI = u32x8;
type Utils = FSUtilsMarker;
}
impl VectBase for u64 {
type SI = u64x8;
type Utils = FSUtilsMarker;
}
impl VectBase for f32 {
type SI = f32x8;
type Utils = FSUtilsMarker;
}
#[derive(Debug, PartialEq)]
pub struct NullFixedSect {}
impl NullFixedSect {
pub fn write(out_buf: &mut [u8], offset: usize) -> Result<usize, CodingError> {
out_buf.pwrite_with(SectionType::Null.as_num(), offset, LE)?;
Ok(offset + 1)
}
}
impl FixedSection for NullFixedSect {
fn num_bytes(&self) -> usize { 1 }
fn sect_bytes(&self) -> Option<&[u8]> { None }
fn sect_type(&self) -> SectionType { SectionType::Null }
}
impl<T: VectBase> FixedSectReader<T> for NullFixedSect {
#[inline]
fn decode_to_sink<Output>(&self, output: &mut Output) -> Result<(), CodingError>
where Output: Sink<T::SI> {
for _ in 0..FIXED_LEN/8 {
output.process_zeroes();
}
Ok(())
}
}
#[derive(Debug, Clone, Copy, PartialEq)]
pub struct SectionWriterStats<T: VectBase> {
min: T,
max: T,
}
impl<T: VectBase> SectionWriterStats<T> {
pub fn from_vect(vect: &[T]) -> Self {
Self { min: *vect.iter()
.min_by(|&a, &b| a.partial_cmp(b).unwrap_or(Ordering::Equal))
.unwrap_or(&T::zero()),
max: *vect.iter()
.max_by(|&a, &b| a.partial_cmp(b).unwrap_or(Ordering::Equal))
.unwrap_or(&T::zero()) }
}
#[inline]
pub fn range(&self) -> T {
self.max - self.min
}
}
impl<T: VectBase + PrimInt> SectionWriterStats<T> {
#[inline]
pub fn num_bits_range(&self) -> u8 {
(T::Utils::BYTE_WIDTH * 8) as u8 - self.range().leading_zeros() as u8
}
#[inline]
pub fn num_bits_max(&self) -> u8 {
(T::Utils::BYTE_WIDTH * 8) as u8 - self.max.leading_zeros() as u8
}
}
pub trait FixedSectionWriter<T: VectBase> {
fn write(out_buf: &mut [u8],
offset: usize,
values: &[T],
stats: SectionWriterStats<T>) -> Result<usize, CodingError>;
#[inline]
fn gen_stats_and_write(out_buf: &mut [u8], offset: usize, values: &[T]) -> Result<usize, CodingError> {
let stats = SectionWriterStats::from_vect(values);
Self::write(out_buf, offset, values, stats)
}
}
#[derive(Debug, PartialEq, Copy, Clone)]
pub struct NibblePackMedFixedSect<'buf, T: VectBase> {
sect_bytes: &'buf [u8],
encoded_bytes: u16,
_type: PhantomData<T>,
}
impl<'buf, T: VectBase> NibblePackMedFixedSect<'buf, T> {
pub fn try_from(sect_bytes: &'buf [u8]) -> Result<NibblePackMedFixedSect<T>, CodingError> {
let encoded_bytes = sect_bytes.pread_with(1, LE)
.and_then(|n| {
if (n + 3) <= sect_bytes.len() as u16 { Ok(n) }
else { Err(scroll::Error::Custom("Slice not large enough".to_string())) }
})?;
Ok(Self { sect_bytes, encoded_bytes, _type: PhantomData })
}
}
impl<'buf, T: VectBase> FixedSectReader<T> for NibblePackMedFixedSect<'buf, T> {
#[inline]
fn decode_to_sink<Output>(&self, output: &mut Output) -> Result<(), CodingError>
where Output: Sink<T::SI> {
let mut values_left = FIXED_LEN;
let mut inbuf = &self.sect_bytes[3..];
while values_left > 0 {
inbuf = T::Utils::nibblepack_decode(inbuf, output)?;
values_left -= 8;
}
Ok(())
}
}
impl<'buf, T: VectBase> FixedSection for NibblePackMedFixedSect<'buf, T> {
fn num_bytes(&self) -> usize { self.encoded_bytes as usize + 3 }
fn sect_bytes(&self) -> Option<&[u8]> { Some(self.sect_bytes) }
fn sect_type(&self) -> SectionType { SectionType::NibblePackedMedium }
}
impl<'buf, T> FixedSectionWriter<T> for NibblePackMedFixedSect<'buf, T>
where T: PrimInt + Unsigned + VectBase + num::cast::AsPrimitive<u64> {
fn write(out_buf: &mut [u8],
offset: usize,
values: &[T],
_s: SectionWriterStats<T>) -> Result<usize, CodingError> {
assert_eq!(values.len(), FIXED_LEN);
out_buf.pwrite_with(SectionType::NibblePackedMedium.as_num(), offset, LE)?;
let off = nibblepacking::pack_u64(values.iter().map(|&x| x.as_()),
out_buf,
offset + 3)?;
let num_bytes = off - offset - 3;
if num_bytes <= 65535 {
out_buf.pwrite_with(num_bytes as u16, offset + 1, LE)?;
Ok(off)
} else {
Err(CodingError::NotEnoughSpace)
}
}
}
#[derive(Debug, PartialEq, Copy, Clone)]
pub struct DeltaNPMedFixedSect<'buf, T>
where T: VectBase {
sect_bytes: &'buf [u8],
encoded_bytes: u16,
base: T,
delta_numbits: u8,
}
const DELTA_NP_SECT_HEADER_SIZE: usize = 12;
impl<'buf, T> DeltaNPMedFixedSect<'buf, T>
where T: VectBase {
pub fn try_from(sect_bytes: &'buf [u8]) -> Result<Self, CodingError> {
let encoded_bytes = sect_bytes.pread_with(1, LE)
.and_then(|n| {
if (n + DELTA_NP_SECT_HEADER_SIZE as u16) <= sect_bytes.len() as u16 { Ok(n) }
else { Err(scroll::Error::Custom("Slice not large enough".to_string())) }
})?;
let base: T = T::Utils::read_le_offset(sect_bytes, 4)?;
let delta_numbits: u8 = sect_bytes[3];
Ok(Self { sect_bytes, encoded_bytes, base, delta_numbits })
}
pub fn delta_range(&self) -> u64 {
2u64.pow(self.delta_numbits as u32)
}
}
impl<'buf, T> FixedSectReader<T> for DeltaNPMedFixedSect<'buf, T>
where T: PrimInt + Unsigned + VectBase {
#[inline]
fn decode_to_sink<Output>(&self, output: &mut Output) -> Result<(), CodingError>
where Output: Sink<T::SI> {
let mut values_left = FIXED_LEN;
let mut inbuf = &self.sect_bytes[DELTA_NP_SECT_HEADER_SIZE..];
let mut delta_sink = AddConstSink::new(self.base, output);
while values_left > 0 {
inbuf = T::Utils::nibblepack_decode(inbuf, &mut delta_sink)?;
values_left -= 8;
}
Ok(())
}
}
impl<'buf, T> FixedSectionWriter<T> for DeltaNPMedFixedSect<'buf, T>
where T: PrimInt + Unsigned + VectBase + num::cast::AsPrimitive<u64> {
fn write(out_buf: &mut [u8],
offset: usize,
values: &[T],
stats: SectionWriterStats<T>) -> Result<usize, CodingError> {
assert_eq!(values.len(), FIXED_LEN);
out_buf.pwrite_with(SectionType::DeltaNPMedium.as_num(), offset, LE)?;
let off = nibblepacking::pack_u64(values.iter().map(|&x| (x - stats.min).as_()),
out_buf,
offset + DELTA_NP_SECT_HEADER_SIZE)?;
let num_bytes = off - offset - DELTA_NP_SECT_HEADER_SIZE;
if num_bytes <= 65535 {
out_buf.pwrite_with(num_bytes as u16, offset + 1, LE)?;
T::Utils::write_le_offset(out_buf, offset + 4, stats.min)?;
out_buf[offset + 3] = stats.num_bits_range();
Ok(off)
} else {
Err(CodingError::NotEnoughSpace)
}
}
}
impl<'buf, T> FixedSection for DeltaNPMedFixedSect<'buf, T>
where T: VectBase {
fn num_bytes(&self) -> usize { self.encoded_bytes as usize + DELTA_NP_SECT_HEADER_SIZE }
fn sect_bytes(&self) -> Option<&[u8]> { Some(self.sect_bytes) }
fn sect_type(&self) -> SectionType { SectionType::DeltaNPMedium }
}
#[derive(Debug, Clone, Copy, PartialEq)]
pub struct XorNPMedFixedSect<'buf> {
sect_bytes: &'buf [u8],
total_bytes: u16,
}
impl<'buf> XorNPMedFixedSect<'buf> {
pub fn try_from(sect_bytes: &'buf [u8]) -> Result<Self, CodingError> {
let total_bytes = sect_bytes.pread_with(1, LE)
.and_then(|n| {
if n <= sect_bytes.len() as u16 { Ok(n) }
else { Err(scroll::Error::Custom("Slice not large enough".to_string())) }
})?;
Ok(Self { sect_bytes, total_bytes })
}
}
impl<'buf> FixedSectReader<f32> for XorNPMedFixedSect<'buf> {
#[inline]
fn decode_to_sink<Output>(&self, output: &mut Output) -> Result<(), CodingError>
where Output: Sink<f32x8> {
let mut values_left = FIXED_LEN;
let mut inbuf = &self.sect_bytes[3..];
let mut xor_sink = XorSink::<'_, f32, u32, _>::new(output);
while values_left > 0 {
inbuf = nibblepack_simd::unpack8_u32_simd(inbuf, &mut xor_sink)?;
values_left -= 8;
}
Ok(())
}
}
impl<'buf, T: VectBase + Float> FixedSectionWriter<T> for XorNPMedFixedSect<'buf> {
fn write(out_buf: &mut [u8],
offset: usize,
values: &[T],
stats: SectionWriterStats<T>) -> Result<usize, CodingError> {
assert_eq!(values.len(), FIXED_LEN);
if stats.min == stats.max {
if stats.min == T::zero() {
NullFixedSect::write(out_buf, offset)
} else {
ConstFixedSect::write(out_buf, offset, values, stats)
}
} else {
out_buf.pwrite_with(SectionType::XorNPMedium.as_num(), offset, LE)?;
let mut last_bits = u64x8::splat(0);
let mut off = offset + 3;
for octet in values.chunks(8) {
let octet_bits = T::SI::to_u64x8_bits(octet);
off = nibblepack_simd::pack8_u64_simd(octet_bits.bitxor(last_bits), out_buf, off)?;
last_bits = octet_bits;
}
let total_bytes = off - offset;
if total_bytes <= 65535 {
out_buf.pwrite_with(total_bytes as u16, offset + 1, LE)?;
Ok(off)
} else {
Err(CodingError::NotEnoughSpace)
}
}
}
}
impl<'buf> FixedSection for XorNPMedFixedSect<'buf> {
fn num_bytes(&self) -> usize { self.total_bytes as usize }
fn sect_bytes(&self) -> Option<&[u8]> { Some(self.sect_bytes) }
fn sect_type(&self) -> SectionType { SectionType::XorNPMedium }
}
#[derive(Debug, PartialEq, Copy, Clone)]
pub struct ConstFixedSect<'buf, T: VectBase> {
sect_bytes: &'buf [u8],
value: T
}
impl<'buf, T: VectBase> ConstFixedSect<'buf, T> {
pub fn try_from(sect_bytes: &'buf [u8]) -> Result<Self, CodingError> {
if sect_bytes.len() >= (1 + T::Utils::BYTE_WIDTH) {
let value = T::Utils::read_le_offset(sect_bytes, 1)?;
Ok(Self { sect_bytes, value })
} else {
Err(CodingError::InputTooShort)
}
}
pub fn get_value(&self) -> T { self.value }
}
impl<'buf, T: VectBase> FixedSectReader<T> for ConstFixedSect<'buf, T> {
#[inline]
fn decode_to_sink<Output>(&self, output: &mut Output) -> Result<(), CodingError>
where Output: Sink<T::SI> {
let octet = T::SI::splat(self.value);
for _ in 0..FIXED_LEN/8 {
output.process(octet);
}
Ok(())
}
}
impl<'buf, T: VectBase> FixedSectionWriter<T> for ConstFixedSect<'buf, T> {
fn write(out_buf: &mut [u8],
offset: usize,
values: &[T],
_stats: SectionWriterStats<T>) -> Result<usize, CodingError> {
assert_eq!(values.len(), FIXED_LEN);
out_buf.pwrite_with(SectionType::Constant.as_num(), offset, LE)?;
T::Utils::write_le_offset(out_buf, offset + 1, values[0])?;
Ok(offset + 1 + T::Utils::BYTE_WIDTH)
}
}
impl<'buf, T: VectBase> FixedSection for ConstFixedSect<'buf, T> {
fn num_bytes(&self) -> usize { 1 + T::Utils::BYTE_WIDTH }
fn sect_bytes(&self) -> Option<&[u8]> { Some(self.sect_bytes) }
fn sect_type(&self) -> SectionType { SectionType::Constant }
}
pub struct AutoEncoder {}
impl<'buf, T> FixedSectionWriter<T> for AutoEncoder
where T: VectBase + PrimInt + Unsigned + num::cast::AsPrimitive<u64> {
fn write(out_buf: &mut [u8],
offset: usize,
values: &[T],
stats: SectionWriterStats<T>) -> Result<usize, CodingError> {
if stats.min == stats.max {
if stats.min == T::zero() {
NullFixedSect::write(out_buf, offset)
} else {
ConstFixedSect::write(out_buf, offset, values, stats)
}
} else {
let regular_nibbles = (stats.num_bits_max() + 3) / 4;
let range_nibbles = (stats.num_bits_range() + 3) / 4;
if range_nibbles < regular_nibbles {
DeltaNPMedFixedSect::write(out_buf, offset, values, stats)
} else {
NibblePackMedFixedSect::write(out_buf, offset, values, stats)
}
}
}
}
pub struct FixedSectIterator<'buf, T: VectBase> {
encoded_bytes: &'buf [u8],
_typ: PhantomData<T>,
}
impl<'buf, T: VectBase> FixedSectIterator<'buf, T> {
pub fn new(encoded_bytes: &'buf [u8]) -> Self {
FixedSectIterator { encoded_bytes, _typ: PhantomData }
}
}
impl<'buf, T: VectBase> Iterator for FixedSectIterator<'buf, T> {
type Item = Result<FixedSectEnum<'buf, T>, CodingError>;
fn next(&mut self) -> Option<Self::Item> {
if self.encoded_bytes.is_empty() {
None
} else {
let res = FixedSectEnum::try_from(self.encoded_bytes);
if let Ok(fsreader) = &res {
self.encoded_bytes = &self.encoded_bytes[fsreader.num_bytes()..];
}
Some(res)
}
}
}
pub fn unpack_u32_section(buf: &[u8]) -> [u32; 256] {
let mut sink = U32_256Sink::new();
NibblePackMedFixedSect::<u32>::try_from(buf).unwrap().decode_to_sink(&mut sink).unwrap();
sink.values
}
#[cfg(test)]
mod tests {
use super::*;
use std::time::{SystemTime, UNIX_EPOCH};
#[test]
fn test_sectwriter_cannot_add_sect_header() {
let mut buf = [0u8; 4];
let mut writer = SectionWriter::new(&mut buf, 256);
let res = writer.add_64kb(SectionType::Null, |writebuf: &mut [u8], _| {
if writebuf.len() < 8 { Err(CodingError::NotEnoughSpace) }
else {
for n in 0..8 { writebuf[n] = 0xff; }
Ok((8, 8))
}
});
assert!(res.is_err());
}
#[test]
fn test_sectwriter_fill_section_normal() {
let mut buf = [0u8; 20];
let mut writer = SectionWriter::new(&mut buf, 256);
let res = writer.add_64kb(SectionType::Null, |writebuf: &mut [u8], _| {
if writebuf.len() < 8 { Err(CodingError::NotEnoughSpace) }
else {
for n in 0..8 { writebuf[n] = 0xff; }
Ok((8, 8))
}
});
assert_eq!(res, Ok((8, 8)));
assert_eq!(writer.cur_pos(), 13);
}
#[test]
fn test_npu64med_write_error_no_room() {
let mut buf = [0u8; 2];
let data: Vec<u64> = (0..256).collect();
let res = NibblePackMedFixedSect::gen_stats_and_write(&mut buf, 0, &data[..]);
assert_eq!(res, Err(CodingError::NotEnoughSpace));
let mut buf = [0u8; 100];
let res = NibblePackMedFixedSect::gen_stats_and_write(&mut buf, 0, &data[..]);
assert_eq!(res, Err(CodingError::NotEnoughSpace));
}
#[test]
fn test_fixedsectiterator_write_and_read() {
let mut buf = [0u8; 1024];
let data: Vec<u64> = (0..256).collect();
let mut off = 0;
off = NullFixedSect::write(&mut buf, off).unwrap();
assert_eq!(off, 1);
off = NibblePackMedFixedSect::gen_stats_and_write(&mut buf, off, &data[..]).unwrap();
let sect_iter = FixedSectIterator::<u64>::new(&buf[0..off]);
let sections = sect_iter.map(|x| x.unwrap()).collect::<Vec<FixedSectEnum<u64>>>();
assert_eq!(sections.len(), 2);
let sect = §ions[0];
assert_eq!(sect.num_bytes(), 1);
match sect {
FixedSectEnum::NullFixedSect(..) => {},
_ => panic!("Got the wrong sect: {:?}", sect),
}
let sect = §ions[1];
assert!(sect.num_bytes() <= sect.sect_bytes().unwrap().len());
if let FixedSectEnum::NibblePackMedFixedSect(inner_sect) = sect {
let mut sink = U64_256Sink::new();
inner_sect.decode_to_sink(&mut sink).unwrap();
assert_eq!(sink.values[..data.len()], data[..]);
} else {
panic!("Wrong type obtained at sections[1]")
}
}
#[test]
fn test_fixedsect_u32_write_and_decode() {
let mut buf = [0u8; 1024];
let data: Vec<u32> = (0..256).collect();
let mut off = 0;
off = NibblePackMedFixedSect::gen_stats_and_write(&mut buf, off, &data[..]).unwrap();
let values = unpack_u32_section(&buf[..off]);
assert_eq!(values.iter().count(), 256);
assert_eq!(values.iter().map(|&x| x).collect::<Vec<u32>>(), data);
}
#[test]
fn test_delta_write_and_decode() {
let mut buf = [0u8; 1024];
let now_inst = SystemTime::now();
let base_millis = now_inst.duration_since(UNIX_EPOCH).unwrap().as_millis() as u64;
let data: Vec<u64> = (0..256).map(|x| x + base_millis).collect();
let mut _off = 0;
_off = DeltaNPMedFixedSect::gen_stats_and_write(&mut buf, _off, &data[..]).unwrap();
let mut sink = U64_256Sink::new();
let section = DeltaNPMedFixedSect::<u64>::try_from(&buf).unwrap();
assert!(section.num_bytes() < 350);
section.decode_to_sink(&mut sink).unwrap();
assert_eq!(sink.values[..], data[..]);
assert_eq!(section.delta_range(), 256);
let data: Vec<u32> = (0..256).map(|x| x + 100_000).collect();
_off = 0;
_off = DeltaNPMedFixedSect::<u32>::gen_stats_and_write(&mut buf, _off, &data[..]).unwrap();
let mut sink = U32_256Sink::new();
let section = DeltaNPMedFixedSect::<u32>::try_from(&buf).unwrap();
assert!(section.num_bytes() < 350);
section.decode_to_sink(&mut sink).unwrap();
assert_eq!(sink.values[..], data[..]);
assert_eq!(section.delta_range(), 256);
}
#[test]
fn test_const_write_and_decode() {
let mut buf = [0u8; 256];
let data = [400u64; 256];
let _off = ConstFixedSect::gen_stats_and_write(&mut buf, 0, &data[..]).unwrap();
let mut sink = U64_256Sink::new();
let section = ConstFixedSect::<u64>::try_from(&buf).unwrap();
assert_eq!(section.num_bytes(), 9);
section.decode_to_sink(&mut sink).unwrap();
assert_eq!(sink.values[..], data[..]);
}
#[test]
fn test_autoencoder() {
let mut buf = [0u8; 1024];
let data = [23_000u64; 256];
let _off = AutoEncoder::gen_stats_and_write(&mut buf, 0, &data[..]).unwrap();
let sect = FixedSectEnum::<u64>::try_from(&buf[..]).unwrap();
match sect {
FixedSectEnum::ConstFixedSect(..) => {},
_ => panic!("Got the wrong sect: {:?}", sect),
}
let data = [0u64; 256];
let _off = AutoEncoder::gen_stats_and_write(&mut buf, 0, &data[..]).unwrap();
let sect = FixedSectEnum::<u64>::try_from(&buf[..]).unwrap();
match sect {
FixedSectEnum::NullFixedSect(..) => {},
_ => panic!("Got the wrong sect: {:?}", sect),
}
let data: Vec<u32> = (0..256).collect();
let _off = AutoEncoder::gen_stats_and_write(&mut buf, 0, &data[..]).unwrap();
let sect = FixedSectEnum::<u32>::try_from(&buf[..]).unwrap();
match sect {
FixedSectEnum::NibblePackMedFixedSect(..) => {},
_ => panic!("Got the wrong sect: {:?}", sect),
}
let data: Vec<u32> = (10_000..10_256).collect();
let _off = AutoEncoder::gen_stats_and_write(&mut buf, 0, &data[..]).unwrap();
let sect = FixedSectEnum::<u32>::try_from(&buf[..]).unwrap();
match sect {
FixedSectEnum::DeltaNPMedFixedSect(..) => {},
_ => panic!("Got the wrong sect: {:?}", sect),
}
}
#[test]
fn test_xor_write_and_decode() {
let mut buf = [0u8; 1024];
let data: Vec<f32> = (0..256).map(|x| x as f32 / 1.3).collect();
let _off = XorNPMedFixedSect::gen_stats_and_write(&mut buf, 0, &data[..]).unwrap();
let mut sink = Section256Sink::<f32>::new();
let section = XorNPMedFixedSect::try_from(&buf).unwrap();
dbg!(section.num_bytes());
section.decode_to_sink(&mut sink).unwrap();
assert_eq!(sink.values[..], data[..]);
}
#[test]
fn test_f32_xor_autoencode() {
let mut buf = [0u8; 1024];
let mut sink = Section256Sink::<f32>::new();
let data = [3.5f32; 256];
let _off = XorNPMedFixedSect::gen_stats_and_write(&mut buf, 0, &data[..]).unwrap();
let sect = FixedSectEnum::<f32>::try_from(&buf[..]).unwrap();
match sect {
FixedSectEnum::ConstFixedSect(..) => {},
_ => panic!("Got the wrong sect: {:?}", sect),
}
sect.decode(&mut sink).unwrap();
assert_eq!(sink.values[..], data[..]);
let data = [0f32; 256];
let _off = XorNPMedFixedSect::gen_stats_and_write(&mut buf, 0, &data[..]).unwrap();
let sect = FixedSectEnum::<f32>::try_from(&buf[..]).unwrap();
match sect {
FixedSectEnum::NullFixedSect(..) => {},
_ => panic!("Got the wrong sect: {:?}", sect),
}
sink.reset();
sect.decode(&mut sink).unwrap();
assert_eq!(sink.values[..], data[..]);
let data: Vec<f32> = (0..256).map(|x| x as f32 / 1.6).collect();
let _off = XorNPMedFixedSect::gen_stats_and_write(&mut buf, 0, &data[..]).unwrap();
let sect = FixedSectEnum::<f32>::try_from(&buf[..]).unwrap();
match sect {
FixedSectEnum::XorNPMedFixedSect(..) => {},
_ => panic!("Got the wrong sect: {:?}", sect),
}
sink.reset();
sect.decode(&mut sink).unwrap();
assert_eq!(sink.values[..], data[..]);
}
}