pub const UNPACK_SLACK: usize = 64;
#[cfg(all(lzvolt_asm, any(target_arch = "aarch64", target_arch = "x86_64")))]
extern "C" {
fn keva_unpack(
src: *const u8,
src_len: usize,
dst: *mut u8,
dst_cap: usize,
declared: usize,
start: usize,
) -> u32;
fn keva_unpack_wide(
src: *const u8,
src_len: usize,
dst: *mut u8,
dst_cap: usize,
declared: usize,
start: usize,
) -> u32;
}
#[cfg(all(lzvolt_asm, target_arch = "x86_64"))]
extern "C" {
fn keva_unpack_ssse3(
src: *const u8,
src_len: usize,
dst: *mut u8,
dst_cap: usize,
declared: usize,
start: usize,
) -> u32;
fn keva_unpack_wide_ssse3(
src: *const u8,
src_len: usize,
dst: *mut u8,
dst_cap: usize,
declared: usize,
start: usize,
) -> u32;
fn keva_unpack_xeon(
src: *const u8,
src_len: usize,
dst: *mut u8,
dst_cap: usize,
declared: usize,
start: usize,
) -> u32;
fn keva_unpack_wide_xeon(
src: *const u8,
src_len: usize,
dst: *mut u8,
dst_cap: usize,
declared: usize,
start: usize,
) -> u32;
fn keva_unpack_epyc(
src: *const u8,
src_len: usize,
dst: *mut u8,
dst_cap: usize,
declared: usize,
start: usize,
) -> u32;
fn keva_unpack_wide_epyc(
src: *const u8,
src_len: usize,
dst: *mut u8,
dst_cap: usize,
declared: usize,
start: usize,
) -> u32;
fn keva_unpack_lz4_ssse3(
src: *const u8,
src_len: usize,
dst: *mut u8,
dst_cap: usize,
declared: usize,
start: usize,
) -> u32;
fn keva_unpack_lz4_xeon(
src: *const u8,
src_len: usize,
dst: *mut u8,
dst_cap: usize,
declared: usize,
start: usize,
) -> u32;
fn keva_unpack_lz4_epyc(
src: *const u8,
src_len: usize,
dst: *mut u8,
dst_cap: usize,
declared: usize,
start: usize,
) -> u32;
}
#[cfg(all(lzvolt_asm, any(target_arch = "aarch64", target_arch = "x86_64")))]
extern "C" {
fn keva_unpack_lz4(
src: *const u8,
src_len: usize,
dst: *mut u8,
dst_cap: usize,
declared: usize,
start: usize,
) -> u32;
}
#[cfg(all(lzvolt_asm, target_arch = "aarch64"))]
extern "C" {
fn keva_unpack_lz4_neoverse(
src: *const u8,
src_len: usize,
dst: *mut u8,
dst_cap: usize,
declared: usize,
start: usize,
) -> u32;
}
#[cfg(all(lzvolt_asm, any(target_arch = "aarch64", target_arch = "x86_64")))]
#[inline]
unsafe fn run(
split: Split,
src: *const u8,
src_len: usize,
dst: *mut u8,
dst_cap: usize,
declared: usize,
start: usize,
) -> u32 {
#[cfg(target_arch = "x86_64")]
{
let f = crate::cpu::features();
if f.ssse3 {
return match (f.xeon, f.epyc, split) {
(true, _, Split::Even) => {
keva_unpack_xeon(src, src_len, dst, dst_cap, declared, start)
}
(true, _, Split::WideMatch) => {
keva_unpack_wide_xeon(src, src_len, dst, dst_cap, declared, start)
}
(_, true, Split::Even) => {
keva_unpack_epyc(src, src_len, dst, dst_cap, declared, start)
}
(_, true, Split::WideMatch) => {
keva_unpack_wide_epyc(src, src_len, dst, dst_cap, declared, start)
}
(_, _, Split::Even) => {
keva_unpack_ssse3(src, src_len, dst, dst_cap, declared, start)
}
(_, _, Split::WideMatch) => {
keva_unpack_wide_ssse3(src, src_len, dst, dst_cap, declared, start)
}
};
}
}
match split {
Split::Even => keva_unpack(src, src_len, dst, dst_cap, declared, start),
Split::WideMatch => keva_unpack_wide(src, src_len, dst, dst_cap, declared, start),
}
}
#[derive(Clone, Copy, PartialEq, Eq, Debug)]
pub enum Lz4Body {
Baseline,
Ssse3,
Xeon,
Epyc,
NeoverseV2,
}
impl Lz4Body {
pub fn all() -> &'static [Lz4Body] {
#[cfg(all(lzvolt_asm, target_arch = "x86_64"))]
{
if crate::cpu::features().avx2 {
&[
Lz4Body::Baseline,
Lz4Body::Ssse3,
Lz4Body::Xeon,
Lz4Body::Epyc,
]
} else {
&[Lz4Body::Baseline, Lz4Body::Ssse3]
}
}
#[cfg(all(lzvolt_asm, target_arch = "aarch64"))]
{
&[Lz4Body::Baseline, Lz4Body::NeoverseV2]
}
#[cfg(not(all(lzvolt_asm, any(target_arch = "x86_64", target_arch = "aarch64"))))]
{
&[Lz4Body::Baseline]
}
}
}
#[cfg(all(lzvolt_asm, any(target_arch = "aarch64", target_arch = "x86_64")))]
#[inline]
fn detect_lz4_body() -> Lz4Body {
#[cfg(target_arch = "x86_64")]
{
let f = crate::cpu::features();
if !f.ssse3 {
return Lz4Body::Baseline;
}
if f.avx2 {
if f.xeon {
return Lz4Body::Xeon;
}
if f.epyc {
return Lz4Body::Epyc;
}
}
Lz4Body::Ssse3
}
#[cfg(target_arch = "aarch64")]
{
if crate::cpu::features().neoverse_v2 {
return Lz4Body::NeoverseV2;
}
Lz4Body::Baseline
}
#[cfg(not(any(target_arch = "x86_64", target_arch = "aarch64")))]
{
Lz4Body::Baseline
}
}
pub fn lz4_body() -> Lz4Body {
#[cfg(all(lzvolt_asm, any(target_arch = "aarch64", target_arch = "x86_64")))]
{
detect_lz4_body()
}
#[cfg(not(all(lzvolt_asm, any(target_arch = "aarch64", target_arch = "x86_64"))))]
{
Lz4Body::Baseline
}
}
#[cfg(all(lzvolt_asm, any(target_arch = "aarch64", target_arch = "x86_64")))]
#[inline]
unsafe fn run_lz4(
body: Lz4Body,
src: *const u8,
src_len: usize,
dst: *mut u8,
dst_cap: usize,
declared: usize,
start: usize,
) -> u32 {
#[cfg(target_arch = "x86_64")]
match body {
Lz4Body::Xeon => return keva_unpack_lz4_xeon(src, src_len, dst, dst_cap, declared, start),
Lz4Body::Epyc => return keva_unpack_lz4_epyc(src, src_len, dst, dst_cap, declared, start),
Lz4Body::Ssse3 => {
return keva_unpack_lz4_ssse3(src, src_len, dst, dst_cap, declared, start)
}
Lz4Body::Baseline | Lz4Body::NeoverseV2 => {}
}
#[cfg(target_arch = "aarch64")]
if body == Lz4Body::NeoverseV2 {
return keva_unpack_lz4_neoverse(src, src_len, dst, dst_cap, declared, start);
}
#[cfg(not(any(target_arch = "x86_64", target_arch = "aarch64")))]
let _ = body;
keva_unpack_lz4(src, src_len, dst, dst_cap, declared, start)
}
pub fn unpack_lz4_into_slice_on(
body: Lz4Body,
src: &[u8],
dst: &mut [u8],
declared: usize,
) -> bool {
#[cfg(not(all(lzvolt_asm, any(target_arch = "aarch64", target_arch = "x86_64"))))]
{
let _ = (body, src, dst, declared);
false
}
#[cfg(all(lzvolt_asm, any(target_arch = "aarch64", target_arch = "x86_64")))]
{
if declared == 0 {
return false;
}
let produced = unsafe {
run_lz4(
body,
src.as_ptr(),
src.len(),
dst.as_mut_ptr(),
dst.len(),
declared,
0,
)
} as usize;
produced == declared
}
}
pub fn unpack_lz4_into_slice(src: &[u8], dst: &mut [u8], declared: usize) -> bool {
#[cfg(not(all(lzvolt_asm, any(target_arch = "aarch64", target_arch = "x86_64"))))]
{
let _ = (src, dst, declared);
false
}
#[cfg(all(lzvolt_asm, any(target_arch = "aarch64", target_arch = "x86_64")))]
{
if declared == 0 {
return false;
}
unpack_lz4_into_slice_on(detect_lz4_body(), src, dst, declared)
}
}
#[derive(Clone, Copy, PartialEq, Eq, Debug)]
pub enum Split {
Even,
WideMatch,
}
pub fn unpack_into_slice(src: &[u8], dst: &mut [u8], declared: usize, split: Split) -> bool {
unpack_section(src, dst, declared, 0, split)
}
pub fn unpack_section(
src: &[u8],
dst: &mut [u8],
declared: usize,
start: usize,
split: Split,
) -> bool {
#[cfg(not(all(lzvolt_asm, any(target_arch = "aarch64", target_arch = "x86_64"))))]
{
let _ = (src, dst, declared, start, split);
false
}
#[cfg(all(lzvolt_asm, any(target_arch = "aarch64", target_arch = "x86_64")))]
{
if declared == 0 || start >= declared {
return false;
}
let produced = unsafe {
run(
split,
src.as_ptr(),
src.len(),
dst.as_mut_ptr(),
dst.len(),
declared,
start,
)
} as usize;
produced == declared
}
}
pub fn unpack_asm_hybrid(
body: &[u8],
out: &mut Vec<u8>,
declared: usize,
switch: (usize, usize),
first: Split,
second: Split,
) -> bool {
#[cfg(not(all(lzvolt_asm, any(target_arch = "aarch64", target_arch = "x86_64"))))]
{
let _ = (body, out, declared, switch, first, second);
false
}
#[cfg(all(lzvolt_asm, any(target_arch = "aarch64", target_arch = "x86_64")))]
{
out.clear();
let want = declared + UNPACK_SLACK;
if out.capacity() < want {
out.reserve(want);
}
let cap = out.capacity();
let dst = unsafe { std::slice::from_raw_parts_mut(out.as_mut_ptr(), cap) };
if !unpack_hybrid_into_slice(body, dst, declared, switch, first, second) {
out.clear();
return false;
}
unsafe { out.set_len(declared) };
true
}
}
pub fn unpack_hybrid_into_slice(
body: &[u8],
dst: &mut [u8],
declared: usize,
switch: (usize, usize),
first: Split,
second: Split,
) -> bool {
#[cfg(not(all(lzvolt_asm, any(target_arch = "aarch64", target_arch = "x86_64"))))]
{
let _ = (body, dst, declared, switch, first, second);
false
}
#[cfg(all(lzvolt_asm, any(target_arch = "aarch64", target_arch = "x86_64")))]
{
let (in_at, out_at) = switch;
if declared == 0 || out_at == 0 || out_at >= declared || in_at > body.len() {
return false;
}
if dst.len() < declared {
return false;
}
unsafe {
let cap = dst.len();
let base = dst.as_mut_ptr();
let first_len = run(first, body.as_ptr(), in_at, base, cap, out_at, 0) as usize;
if first_len != out_at {
return false;
}
let tail = body.as_ptr().add(in_at);
let tail_len = body.len() - in_at;
let total = run(second, tail, tail_len, base, cap, declared, out_at) as usize;
total == declared
}
}
}
pub const fn asm_available() -> bool {
cfg!(all(
lzvolt_asm,
any(target_arch = "aarch64", target_arch = "x86_64")
))
}
pub fn unpack_asm(body: &[u8], out: &mut Vec<u8>, declared: usize, split: Split) -> bool {
#[cfg(not(all(lzvolt_asm, any(target_arch = "aarch64", target_arch = "x86_64"))))]
{
let _ = (body, out, declared, split);
false
}
#[cfg(all(lzvolt_asm, any(target_arch = "aarch64", target_arch = "x86_64")))]
{
if declared == 0 {
return false;
}
out.clear();
let want = declared + UNPACK_SLACK;
if out.capacity() < want {
out.reserve(want);
}
let cap = out.capacity();
let produced = unsafe {
run(
split,
body.as_ptr(),
body.len(),
out.as_mut_ptr(),
cap,
declared,
0,
)
} as usize;
if produced != declared {
out.clear();
return false;
}
unsafe { out.set_len(declared) };
true
}
}
pub fn unpack_lz4_asm(block: &[u8], out: &mut Vec<u8>, declared: usize) -> bool {
#[cfg(not(all(lzvolt_asm, any(target_arch = "aarch64", target_arch = "x86_64"))))]
{
let _ = (block, out, declared);
false
}
#[cfg(all(lzvolt_asm, any(target_arch = "aarch64", target_arch = "x86_64")))]
{
if declared == 0 {
return false;
}
out.clear();
let want = declared + UNPACK_SLACK;
if out.capacity() < want {
out.reserve(want);
}
let cap = out.capacity();
let produced = unsafe {
run_lz4(
detect_lz4_body(),
block.as_ptr(),
block.len(),
out.as_mut_ptr(),
cap,
declared,
0,
)
} as usize;
if produced != declared {
out.clear();
return false;
}
unsafe { out.set_len(declared) };
true
}
}