use super::deflate::{
bulk_hash4, hash4, Compressor, Writer, DEFAULT_COMPRESSION, MAX_MATCH_LENGTH, MIN_MATCH_LENGTH,
};
use super::deflatefast::{DeflateFast, TableEntry, BUFFER_RESET, TABLE_SIZE};
use super::huffman_code::reverse_bits;
use super::inflate;
use super::token::Token;
use crate::bytes;
use crate::io as ggio;
use crate::math::rand;
use std::io::Write;
struct DeflateTest<'a> {
input: &'a [u8],
level: isize,
out: &'a [u8],
}
struct DeflateInflateTest<'a> {
input: &'a [u8],
}
struct ReverseBitsTest {
input: u16,
bit_count: u8,
out: u16,
}
const DEFLATE_TESTS: &[DeflateTest] = &[
DeflateTest {
input: &[],
level: 0,
out: &[1, 0, 0, 255, 255],
},
DeflateTest {
input: &[0x11],
level: -1,
out: &[18, 4, 4, 0, 0, 255, 255],
},
DeflateTest {
input: &[0x11],
level: DEFAULT_COMPRESSION,
out: &[18, 4, 4, 0, 0, 255, 255],
},
DeflateTest {
input: &[0x11],
level: 4,
out: &[18, 4, 4, 0, 0, 255, 255],
},
DeflateTest {
input: &[0x11],
level: 0,
out: &[0, 1, 0, 254, 255, 17, 1, 0, 0, 255, 255],
},
DeflateTest {
input: &[0x11, 0x12],
level: 0,
out: &[0, 2, 0, 253, 255, 17, 18, 1, 0, 0, 255, 255],
},
DeflateTest {
input: &[0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11],
level: 0,
out: &[
0, 8, 0, 247, 255, 17, 17, 17, 17, 17, 17, 17, 17, 1, 0, 0, 255, 255,
],
},
DeflateTest {
input: &[],
level: 2,
out: &[1, 0, 0, 255, 255],
},
DeflateTest {
input: &[0x11],
level: 2,
out: &[18, 4, 4, 0, 0, 255, 255],
},
DeflateTest {
input: &[0x11, 0x12],
level: 2,
out: &[18, 20, 2, 4, 0, 0, 255, 255],
},
DeflateTest {
input: &[0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11],
level: 2,
out: &[18, 132, 2, 64, 0, 0, 0, 255, 255],
},
DeflateTest {
input: &[],
level: 9,
out: &[1, 0, 0, 255, 255],
},
DeflateTest {
input: &[0x11],
level: 9,
out: &[18, 4, 4, 0, 0, 255, 255],
},
DeflateTest {
input: &[0x11, 0x12],
level: 9,
out: &[18, 20, 2, 4, 0, 0, 255, 255],
},
DeflateTest {
input: &[0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11],
level: 9,
out: &[18, 132, 2, 64, 0, 0, 0, 255, 255],
},
];
const DEFLATE_INFLATE_TESTS: &[DeflateInflateTest] = &[
DeflateInflateTest { input: &[] },
DeflateInflateTest { input: &[0x11] },
DeflateInflateTest {
input: &[0x11, 0x12],
},
DeflateInflateTest {
input: &[0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11],
},
DeflateInflateTest {
input: &[
0x11, 0x10, 0x13, 0x41, 0x21, 0x21, 0x41, 0x13, 0x87, 0x78, 0x13,
],
},
];
const REVERSE_BITS_TESTS: &[ReverseBitsTest] = &[
ReverseBitsTest {
input: 1,
bit_count: 1,
out: 1,
},
ReverseBitsTest {
input: 1,
bit_count: 2,
out: 2,
},
ReverseBitsTest {
input: 1,
bit_count: 3,
out: 4,
},
ReverseBitsTest {
input: 1,
bit_count: 4,
out: 8,
},
ReverseBitsTest {
input: 1,
bit_count: 5,
out: 16,
},
ReverseBitsTest {
input: 17,
bit_count: 5,
out: 17,
},
ReverseBitsTest {
input: 257,
bit_count: 9,
out: 257,
},
ReverseBitsTest {
input: 29,
bit_count: 5,
out: 23,
},
];
fn large_data_chunk() -> Vec<u8> {
let mut result = Vec::with_capacity(100000);
#[allow(clippy::needless_range_loop)]
for i in 0..result.len() {
result[i] = ((i * i) & 0xFF) as u8;
}
result
}
#[test]
fn test_bulk_hash4() {
for x in DEFLATE_TESTS {
let mut y = Vec::new();
y.extend_from_slice(x.out);
if y.len() < MIN_MATCH_LENGTH {
continue;
}
y.extend_from_slice(x.out);
let mut j = 4;
while j < y.len() {
let y = &y[..j];
let mut dst = vec![0; y.len() - MIN_MATCH_LENGTH + 1];
#[allow(clippy::needless_range_loop)]
for i in 0..dst.len() {
dst[i] = (i + 100) as u32;
}
bulk_hash4(y, &mut dst);
for (i, got) in dst.iter().enumerate() {
let want = hash4(&y[i..]);
if *got != want && *got == (i as u32) + 100 {
panic!(
"Len:{} Index:{}, want 0x{:08x} but not modified",
y.len(),
i,
want
);
} else if *got != want {
panic!(
"Len:{} Index:{}, got 0x{:08x} want:0x{:08x}",
y.len(),
i,
*got,
want
);
}
}
j += 1;
}
}
}
#[test]
fn test_deflate() {
for h in DEFLATE_TESTS {
let mut buf = bytes::Buffer::new();
let mut w = Writer::new(&mut buf, h.level).unwrap();
w.write_all(h.input).unwrap();
w.close().unwrap();
assert_eq!(h.out, buf.bytes());
}
}
#[test]
fn test_writer_close() {
let mut b = bytes::Buffer::new();
let mut zw = Writer::new(&mut b, 6).unwrap();
let c = zw.write("Test".as_bytes()).unwrap();
assert_eq!(4, c);
zw.close().unwrap();
let res = zw.write_all("Test".as_bytes());
assert!(res.is_err(), "Write to closed writer");
let res = zw.flush();
assert!(res.is_err(), "Flush to closed writer");
let res = zw.close();
assert!(res.is_ok(), "Close on a closed writer is not an error");
}
struct SparseReader {
l: u64,
cur: u64,
}
impl std::io::Read for SparseReader {
fn read(&mut self, p: &mut [u8]) -> std::io::Result<usize> {
if self.cur >= self.l {
return Ok(0);
}
let mut n = p.len();
let mut cur = self.cur + n as u64;
if cur > self.l {
n -= (cur - self.l) as usize;
cur = self.l;
}
for i in 0..p[0..n].len() {
p[i] = if self.cur + i as u64 >= self.l - (1 << 16) {
1
} else {
0
};
}
self.cur = cur;
Ok(n)
}
}
#[test]
#[cfg(not(debug_assertions))]
fn test_very_long_sparse_chunk() {
let mut x = ggio::Discard::new();
let mut w = Writer::new(&mut x, 1).unwrap();
let size = 2_300_000_000;
let mut sr = SparseReader { l: size, cur: 0 };
let n = std::io::copy(&mut sr, &mut w).unwrap();
assert_eq!(size, n);
}
fn test_to_from_with_level_and_limit(level: isize, input: &[u8], name: &str, limit: usize) {
let mut buffer = bytes::Buffer::new();
let mut w = Writer::new(&mut buffer, level).unwrap();
w.write_all(input).unwrap();
w.close().unwrap();
if limit > 0 && buffer.len() > limit {
panic!(
"level: {}, len(compress(data)) = {} > limit = {}",
level,
buffer.len(),
limit
);
}
let mut r = inflate::Reader::new(&mut buffer);
let (data, err) = ggio::read_all(&mut r);
assert!(err.is_none(), "read: {}", err.unwrap());
r.close().unwrap();
assert_eq!(
input,
data.as_slice(),
"decompress(compress(data)) != data: level={} name={}",
level,
name
);
}
fn test_to_from_with_limit(input: &[u8], name: &str, limit: &[usize; 11]) {
for i in 0..10 {
test_to_from_with_level_and_limit(i, input, name, limit[i as usize]);
}
test_to_from_with_level_and_limit(-2, input, name, limit[10]);
}
#[test]
fn test_deflate_inflate() {
let zero_limits = [0; 11];
for (i, h) in DEFLATE_INFLATE_TESTS.iter().enumerate() {
let name = format!("#{}", i);
test_to_from_with_limit(h.input, &name, &zero_limits);
}
let large_chunk = large_data_chunk();
test_to_from_with_limit(&large_chunk, "large chunk", &zero_limits);
}
#[test]
fn test_reverse_bits() {
for h in REVERSE_BITS_TESTS {
let v = reverse_bits(h.input, h.bit_count);
assert_eq!(
h.out, v,
"reverseBits({},{}) = {}, want {}",
h.input, h.bit_count, v, h.out
);
}
}
struct DeflateInflateStringTest {
filename: &'static str,
label: &'static str,
limit: [usize; 11],
}
const DEFLATE_INFLATE_STRING_TESTS: &[DeflateInflateStringTest] = &[
DeflateInflateStringTest {
filename: "src/compress/testdata/e.txt",
label: "2.718281828...",
limit: [
100018, 50650, 50960, 51150, 50930, 50790, 50790, 50790, 50790, 50790, 43683,
],
},
DeflateInflateStringTest {
filename: "src/testdata/Isaac.Newton-Opticks.txt",
label: "Isaac.Newton-Opticks",
limit: [
567248, 218338, 198211, 193152, 181100, 175427, 175427, 173597, 173422, 173422, 325240,
],
},
];
#[test]
fn test_deflate_inflate_string() {
for test in DEFLATE_INFLATE_STRING_TESTS {
let gold = std::fs::read(test.filename).unwrap();
test_to_from_with_limit(&gold, test.label, &test.limit);
}
}
#[test]
fn test_writer_dict() {
let dict = "hello world".as_bytes();
let text = "hello again world".as_bytes();
let mut b = bytes::Buffer::new();
{
let mut w = Writer::new(&mut b, 5).unwrap();
w.write_all(dict).unwrap();
w.flush().unwrap();
w.write_all(text).unwrap();
w.close().unwrap();
}
let mut b1 = bytes::Buffer::new();
{
let mut w = Writer::new_dict(&mut b1, 5, dict).unwrap();
w.write_all(text).unwrap();
w.close().unwrap();
}
assert!(bytes::has_suffix(b.bytes(), b1.bytes()));
}
#[test]
fn test_regression2508() {
let mut discard = ggio::Discard::new();
let mut w = Writer::new(&mut discard, 1).unwrap();
let buf = vec![0; 1024];
for i in 0..131072 {
let res = w.write_all(&buf);
assert!(res.is_ok(), "writer failed: {}, {:?}", i, res.err());
}
w.close().unwrap();
}
#[test]
fn test_writer_reset() {
for level in 0..=9 {
let mut x = ggio::Discard::new();
let mut w = Writer::new(&mut x, level).unwrap();
let buf = b"hello world";
let n = 1024;
for _ in 0..n {
w.write_all(buf).unwrap();
}
let mut x = ggio::Discard::new();
w.reset(&mut x);
}
}
#[test]
fn test_best_speed_match() {
struct Test {
previous: Vec<u8>,
current: Vec<u8>,
t: i32,
s: usize,
want: usize,
}
let cases = [
Test {
previous: vec![0, 0, 0, 1, 2],
current: vec![3, 4, 5, 0, 1, 2, 3, 4, 5],
t: -3,
s: 3,
want: 6,
},
Test {
previous: vec![0, 0, 0, 1, 2],
current: vec![2, 4, 5, 0, 1, 2, 3, 4, 5],
t: -3,
s: 3,
want: 3,
},
Test {
previous: vec![0, 0, 0, 1, 1],
current: vec![3, 4, 5, 0, 1, 2, 3, 4, 5],
t: -3,
s: 3,
want: 2,
},
Test {
previous: vec![0, 0, 0, 1, 2],
current: vec![2, 2, 2, 2, 1, 2, 3, 4, 5],
t: -1,
s: 0,
want: 4,
},
Test {
previous: vec![0, 0, 0, 1, 2, 3, 4, 5, 2, 2],
current: vec![2, 2, 2, 2, 1, 2, 3, 4, 5],
t: -7,
s: 4,
want: 5,
},
Test {
previous: vec![9, 9, 9, 9, 9],
current: vec![2, 2, 2, 2, 1, 2, 3, 4, 5],
t: -1,
s: 0,
want: 0,
},
Test {
previous: vec![9, 9, 9, 9, 9],
current: vec![9, 2, 2, 2, 1, 2, 3, 4, 5],
t: 0,
s: 1,
want: 0,
},
Test {
previous: vec![],
current: vec![9, 2, 2, 2, 1, 2, 3, 4, 5],
t: -5,
s: 1,
want: 0,
},
Test {
previous: vec![],
current: vec![9, 2, 2, 2, 1, 2, 3, 4, 5],
t: -1,
s: 1,
want: 0,
},
Test {
previous: vec![],
current: vec![2, 2, 2, 2, 1, 2, 3, 4, 5],
t: 0,
s: 1,
want: 3,
},
Test {
previous: vec![3, 4, 5],
current: vec![3, 4, 5],
t: -3,
s: 0,
want: 3,
},
Test {
previous: vec![0; 1000],
current: vec![0; 1000],
t: -1000,
s: 0,
want: MAX_MATCH_LENGTH - 4,
},
Test {
previous: vec![0; 200],
current: vec![0; 500],
t: -200,
s: 0,
want: MAX_MATCH_LENGTH - 4,
},
Test {
previous: vec![0; 200],
current: vec![0; 500],
t: 0,
s: 1,
want: MAX_MATCH_LENGTH - 4,
},
Test {
previous: vec![0; MAX_MATCH_LENGTH - 4],
current: vec![0; 500],
t: -(MAX_MATCH_LENGTH as i32 - 4),
s: 0,
want: MAX_MATCH_LENGTH - 4,
},
Test {
previous: vec![0; 200],
current: vec![0; 500],
t: -200,
s: 400,
want: 100,
},
Test {
previous: vec![0; 10],
current: vec![0; 500],
t: 200,
s: 400,
want: 100,
},
];
for c in cases {
let mut e = DeflateFast {
prev: c.previous,
table: vec![TableEntry::default(); TABLE_SIZE as usize],
cur: 0,
};
let got = e.match_len(c.s, c.t, &c.current);
assert_eq!(c.want, got);
}
}
#[test]
fn test_best_speed_shift_offsets() {
let mut enc = DeflateFast::new();
let mut test_data: Vec<u8> = vec![0; 32];
let mut rng = rand::Rand::new(rand::new_source(1));
#[allow(clippy::needless_range_loop)]
for i in 0..test_data.len() {
test_data[i] = rng.uint32() as u8;
}
let mut dest1: Vec<Token> = Vec::new();
enc.encode(&mut dest1, &test_data);
let want_first_tokens = dest1.len();
let mut dest2: Vec<Token> = Vec::new();
enc.encode(&mut dest2, &test_data);
let want_second_tokens = dest2.len();
println!("{}, {}", want_first_tokens, want_second_tokens);
assert!(
want_first_tokens > want_second_tokens,
"test needs matches between inputs to be generated"
);
enc.cur = BUFFER_RESET - test_data.len() as i32;
{
let mut dest: Vec<Token> = Vec::new();
enc.encode(&mut dest, &test_data);
let got = dest.len();
assert_eq!(
want_first_tokens, got,
"got {}, want {} tokens",
got, want_first_tokens
);
assert_eq!(
BUFFER_RESET, enc.cur,
"got {}, want e.cur to be at bufferReset ({})",
enc.cur, BUFFER_RESET
);
}
{
let mut dest1: Vec<Token> = Vec::new();
enc.encode(&mut dest1, &test_data);
let got = dest1.len();
assert_eq!(
want_second_tokens, got,
"got {}, want {} token",
got, want_second_tokens
);
assert!(
enc.cur < BUFFER_RESET,
"want e.cur to be < bufferReset ({}), got {}",
BUFFER_RESET,
enc.cur
);
enc.cur = BUFFER_RESET;
enc.shift_offsets();
let mut dest2: Vec<Token> = Vec::new();
enc.encode(&mut dest2, &test_data);
let got = dest2.len();
assert_eq!(
want_first_tokens, got,
"got {}, want {} tokens",
got, want_first_tokens
);
}
}
#[test]
fn test_structure_sizes() {
let size = std::mem::size_of::<Compressor<std::io::BufWriter<std::fs::File>>>();
assert!(
500 >= size,
"consider reducing size of Compressor ({})",
size
);
}