#![allow(unsafe_code)]
#![allow(improper_ctypes_definitions)]
use crate::entropy::bitstream::{BitDStream, BitDStreamStatus};
use crate::entropy::fse::{DState, SequenceDecodeEntry};
#[inline(never)]
#[unsafe(no_mangle)]
pub extern "C" fn asm_read_bits_fast_zero_safe(stream: &mut BitDStream, nb_bits: u32) -> u64 {
stream.read_bits_fast_zero_safe(nb_bits)
}
#[inline(never)]
#[unsafe(no_mangle)]
pub extern "C" fn asm_read_bits_fast(stream: &mut BitDStream, nb_bits: u32) -> u64 {
stream.read_bits_fast(nb_bits)
}
#[inline(never)]
#[unsafe(no_mangle)]
pub extern "C" fn asm_look_bits_fast(stream: &BitDStream, nb_bits: u32) -> u64 {
stream.look_bits_fast(nb_bits)
}
#[inline(never)]
#[unsafe(no_mangle)]
pub extern "C" fn asm_fse_update_state(
state: &mut DState,
stream: &mut BitDStream,
entry: SequenceDecodeEntry,
) {
crate::entropy::fse::update_state_with_seq_entry_fast(state, stream, entry);
}
#[inline(never)]
#[unsafe(no_mangle)]
pub extern "C" fn asm_reload(stream: &mut BitDStream) -> BitDStreamStatus {
stream.reload()
}
#[inline(never)]
#[unsafe(no_mangle)]
pub extern "C" fn asm_reload_fast(stream: &mut BitDStream) -> BitDStreamStatus {
stream.reload_fast()
}
#[inline(never)]
#[unsafe(no_mangle)]
pub extern "C" fn asm_resolve_offset(
offset_value: u32,
literal_length: usize,
rep1: u32,
rep2: u32,
rep3: u32,
) -> (usize, u32, u32, u32) {
let mut r1 = rep1;
let mut r2 = rep2;
let mut r3 = rep3;
let offset: usize = if offset_value > 3 {
let o = offset_value - 3;
r3 = r2;
r2 = r1;
r1 = o;
o as usize
} else if offset_value == 1 && literal_length != 0 {
r1 as usize
} else if offset_value == 0 {
0
} else {
let rep_index = offset_value + (literal_length == 0) as u32;
if rep_index == 2 {
std::mem::swap(&mut r2, &mut r1);
r1 as usize
} else if rep_index == 3 {
let o = r3;
r3 = r2;
r2 = r1;
r1 = o;
o as usize
} else {
let o = r1.wrapping_sub(1);
r3 = r2;
r2 = r1;
r1 = o;
o as usize
}
};
(offset, r1, r2, r3)
}
#[inline(always)]
unsafe fn copy_16_inline(dst: *mut u8, src: *const u8) {
#[cfg(target_arch = "aarch64")]
unsafe {
let v = core::arch::aarch64::vld1q_u8(src);
core::arch::aarch64::vst1q_u8(dst, v);
}
#[cfg(target_arch = "x86_64")]
{
#[cfg(target_feature = "sse2")]
unsafe {
let v = core::arch::x86_64::_mm_loadu_si128(src as *const core::arch::x86_64::__m128i);
core::arch::x86_64::_mm_storeu_si128(dst as *mut core::arch::x86_64::__m128i, v);
}
#[cfg(not(target_feature = "sse2"))]
unsafe {
core::ptr::copy_nonoverlapping(src, dst, 16)
};
}
#[cfg(not(any(target_arch = "aarch64", target_arch = "x86_64")))]
unsafe {
core::ptr::copy_nonoverlapping(src, dst, 16)
};
}
#[inline(never)]
#[unsafe(no_mangle)]
pub unsafe extern "C" fn asm_copy_16(dst: *mut u8, src: *const u8) {
unsafe { copy_16_inline(dst, src) };
}
#[inline(never)]
#[unsafe(no_mangle)]
pub unsafe extern "C" fn asm_copy_literals(dst: *mut u8, src: *const u8, literal_length: usize) {
unsafe {
if literal_length > 0 {
copy_16_inline(dst, src);
if literal_length > 16 {
let mut pos = 16usize;
while pos < literal_length {
copy_16_inline(dst.add(pos), src.add(pos));
pos += 16;
}
}
}
}
}
#[inline(never)]
#[unsafe(no_mangle)]
pub unsafe extern "C" fn asm_copy_literals_no_guard(
dst: *mut u8,
src: *const u8,
literal_length: usize,
) {
unsafe {
copy_16_inline(dst, src);
if literal_length > 16 {
let mut pos = 16usize;
while pos < literal_length {
copy_16_inline(dst.add(pos), src.add(pos));
pos += 16;
}
}
}
}
#[inline(never)]
#[unsafe(no_mangle)]
pub unsafe extern "C" fn asm_copy_match_large_offset(
dst: *mut u8,
src: *const u8,
match_length: usize,
) {
unsafe {
let mut pos = 0usize;
while pos < match_length {
copy_16_inline(dst.add(pos), src.add(pos));
pos += 16;
}
}
}
#[inline(never)]
#[unsafe(no_mangle)]
pub unsafe extern "C" fn asm_copy_match_small_offset(
base: *mut u8,
out_pos: usize,
match_src: usize,
match_length: usize,
) {
let offset = out_pos - match_src;
unsafe {
let dst = base.add(out_pos);
let src = base.add(match_src);
let mut ip = src;
let mut op = dst;
if offset < 8 {
const DEC32TABLE: [u32; 8] = [0, 1, 2, 1, 4, 4, 4, 4];
const DEC64TABLE: [i32; 8] = [8, 8, 8, 7, 8, 9, 10, 11];
*op = *ip;
*op.add(1) = *ip.add(1);
*op.add(2) = *ip.add(2);
*op.add(3) = *ip.add(3);
ip = ip.add(DEC32TABLE[offset] as usize);
core::ptr::copy_nonoverlapping(ip, op.add(4), 4);
ip = ip.offset(8isize - DEC64TABLE[offset] as isize);
} else {
core::ptr::copy_nonoverlapping(ip, op, 8);
ip = ip.add(8);
}
op = op.add(8);
if match_length > 8 {
let end = dst.add(match_length);
while op < end {
core::ptr::copy_nonoverlapping(ip, op, 8);
ip = ip.add(8);
op = op.add(8);
}
}
}
}
#[inline(never)]
#[unsafe(no_mangle)]
pub extern "C" fn asm_bounds_check(
literal_cursor: usize,
literal_length: usize,
lit_len: usize,
out_pos: usize,
match_length: usize,
out_end: usize,
) -> bool {
let literal_end = literal_cursor + literal_length;
let seq_end = out_pos + literal_length + match_length;
literal_end <= lit_len && seq_end <= out_end
}
#[inline(never)]
#[unsafe(no_mangle)]
pub extern "C" fn asm_window_check(
offset: usize,
out_pos: usize,
frame_start: usize,
window_size: usize,
) -> bool {
offset <= out_pos - frame_start && offset <= window_size
}
#[inline(never)]
#[unsafe(no_mangle)]
pub extern "C" fn asm_full_guard(
literal_cursor: usize,
literal_length: usize,
lit_len: usize,
out_pos: usize,
match_length: usize,
out_end: usize,
offset: usize,
frame_start: usize,
window_size: usize,
) -> bool {
let total = literal_length + match_length;
let literal_end = literal_cursor + literal_length;
if total > out_end || literal_end > lit_len {
return false;
}
offset <= out_pos - frame_start && offset <= window_size
}
#[inline(never)]
#[unsafe(no_mangle)]
pub unsafe extern "C" fn asm_prefetch_pair(base: *const u8, pos: usize) {
unsafe { crate::sequence::prefetch_match(base.add(pos)) };
}
#[inline(never)]
#[unsafe(no_mangle)]
pub extern "C" fn asm_full_decode_step(
reader: &mut BitDStream,
literal_state: &mut DState,
match_state: &mut DState,
offset_state: &mut DState,
offset_entry: SequenceDecodeEntry,
match_entry: SequenceDecodeEntry,
literal_entry: SequenceDecodeEntry,
) -> (usize, usize, u32) {
let offset_code = offset_entry.nb_additional_bits as u32;
let match_extra_bits = match_entry.nb_additional_bits as u32;
let literal_extra_bits = literal_entry.nb_additional_bits as u32;
let offset_extra = reader.read_bits_fast_zero_safe(offset_code) as u32;
let match_extra = reader.read_bits_fast_zero_safe(match_extra_bits) as u32;
if offset_code + match_extra_bits + literal_extra_bits >= 31 {
let _ = reader.reload();
}
let literal_extra = reader.read_bits_fast_zero_safe(literal_extra_bits) as u32;
let literal_length = (literal_entry.baseline + literal_extra) as usize;
let match_length = (match_entry.baseline + match_extra) as usize;
let offset_value = (1u32 << offset_code) + offset_extra;
crate::entropy::fse::update_state_with_seq_entry_fast(literal_state, reader, literal_entry);
crate::entropy::fse::update_state_with_seq_entry_fast(match_state, reader, match_entry);
crate::entropy::fse::update_state_with_seq_entry_fast(offset_state, reader, offset_entry);
(literal_length, match_length, offset_value)
}