turnloop_zstd_decoder/encoding/
frame_compressor.rs1use alloc::vec::Vec;
4use core::convert::TryInto;
5#[cfg(feature = "hash")]
6use twox_hash::XxHash64;
7
8#[cfg(feature = "hash")]
9use core::hash::Hasher;
10
11use super::{
12 block_header::BlockHeader, frame_header::FrameHeader, levels::*,
13 match_generator::MatchGeneratorDriver, CompressionLevel, Matcher,
14};
15use crate::fse::fse_encoder::{default_ll_table, default_ml_table, default_of_table, FSETable};
16
17use crate::io::{Read, Write};
18
19pub struct FrameCompressor<R: Read, W: Write, M: Matcher> {
39 uncompressed_data: Option<R>,
40 compressed_data: Option<W>,
41 compression_level: CompressionLevel,
42 state: CompressState<M>,
43 #[cfg(feature = "hash")]
44 hasher: XxHash64,
45}
46
47pub(crate) struct FseTables {
48 pub(crate) ll_default: FSETable,
49 pub(crate) ll_previous: Option<FSETable>,
50 pub(crate) ml_default: FSETable,
51 pub(crate) ml_previous: Option<FSETable>,
52 pub(crate) of_default: FSETable,
53 pub(crate) of_previous: Option<FSETable>,
54}
55
56impl FseTables {
57 pub fn new() -> Self {
58 Self {
59 ll_default: default_ll_table(),
60 ll_previous: None,
61 ml_default: default_ml_table(),
62 ml_previous: None,
63 of_default: default_of_table(),
64 of_previous: None,
65 }
66 }
67}
68
69pub(crate) struct CompressState<M: Matcher> {
70 pub(crate) matcher: M,
71 pub(crate) last_huff_table: Option<crate::huff0::huff0_encoder::HuffmanTable>,
72 pub(crate) fse_tables: FseTables,
73}
74
75impl<R: Read, W: Write> FrameCompressor<R, W, MatchGeneratorDriver> {
76 pub fn new(compression_level: CompressionLevel) -> Self {
78 Self {
79 uncompressed_data: None,
80 compressed_data: None,
81 compression_level,
82 state: CompressState {
83 matcher: MatchGeneratorDriver::new(1024 * 128, 1),
84 last_huff_table: None,
85 fse_tables: FseTables::new(),
86 },
87 #[cfg(feature = "hash")]
88 hasher: XxHash64::with_seed(0),
89 }
90 }
91}
92
93impl<R: Read, W: Write, M: Matcher> FrameCompressor<R, W, M> {
94 pub fn new_with_matcher(matcher: M, compression_level: CompressionLevel) -> Self {
96 Self {
97 uncompressed_data: None,
98 compressed_data: None,
99 state: CompressState {
100 matcher,
101 last_huff_table: None,
102 fse_tables: FseTables::new(),
103 },
104 compression_level,
105 #[cfg(feature = "hash")]
106 hasher: XxHash64::with_seed(0),
107 }
108 }
109
110 pub fn set_source(&mut self, uncompressed_data: R) -> Option<R> {
114 self.uncompressed_data.replace(uncompressed_data)
115 }
116
117 pub fn set_drain(&mut self, compressed_data: W) -> Option<W> {
121 self.compressed_data.replace(compressed_data)
122 }
123
124 pub fn compress(&mut self) {
132 self.state.matcher.reset(self.compression_level);
134 self.state.last_huff_table = None;
135 #[cfg(feature = "hash")]
136 {
137 self.hasher = XxHash64::with_seed(0);
138 }
139 let source = self.uncompressed_data.as_mut().unwrap();
140 let drain = self.compressed_data.as_mut().unwrap();
141 let output: &mut Vec<u8> = &mut Vec::with_capacity(1024 * 130);
143 let header = FrameHeader {
145 frame_content_size: None,
146 single_segment: false,
147 content_checksum: cfg!(feature = "hash"),
148 dictionary_id: None,
149 window_size: Some(self.state.matcher.window_size()),
150 };
151 header.serialize(output);
152 loop {
154 let mut uncompressed_data = self.state.matcher.get_next_space();
156 let mut read_bytes = 0;
157 let last_block;
158 'read_loop: loop {
159 let new_bytes = source.read(&mut uncompressed_data[read_bytes..]).unwrap();
160 if new_bytes == 0 {
161 last_block = true;
162 break 'read_loop;
163 }
164 read_bytes += new_bytes;
165 if read_bytes == uncompressed_data.len() {
166 last_block = false;
167 break 'read_loop;
168 }
169 }
170 uncompressed_data.resize(read_bytes, 0);
171 #[cfg(feature = "hash")]
173 self.hasher.write(&uncompressed_data);
174 if uncompressed_data.is_empty() {
176 let header = BlockHeader {
177 last_block: true,
178 block_type: crate::blocks::block::BlockType::Raw,
179 block_size: 0,
180 };
181 header.serialize(output);
183 drain.write_all(output).unwrap();
184 output.clear();
185 break;
186 }
187
188 match self.compression_level {
189 CompressionLevel::Uncompressed => {
190 let header = BlockHeader {
191 last_block,
192 block_type: crate::blocks::block::BlockType::Raw,
193 block_size: read_bytes.try_into().unwrap(),
194 };
195 header.serialize(output);
197 output.extend_from_slice(&uncompressed_data);
198 }
199 CompressionLevel::Fastest => {
200 compress_fastest(&mut self.state, last_block, uncompressed_data, output)
201 }
202 _ => {
203 unimplemented!();
204 }
205 }
206 drain.write_all(output).unwrap();
207 output.clear();
208 if last_block {
209 break;
210 }
211 }
212
213 #[cfg(feature = "hash")]
216 {
217 let content_checksum = self.hasher.finish();
220 drain
221 .write_all(&(content_checksum as u32).to_le_bytes())
222 .unwrap();
223 }
224 }
225
226 pub fn source_mut(&mut self) -> Option<&mut R> {
228 self.uncompressed_data.as_mut()
229 }
230
231 pub fn drain_mut(&mut self) -> Option<&mut W> {
233 self.compressed_data.as_mut()
234 }
235
236 pub fn source(&self) -> Option<&R> {
238 self.uncompressed_data.as_ref()
239 }
240
241 pub fn drain(&self) -> Option<&W> {
243 self.compressed_data.as_ref()
244 }
245
246 pub fn take_source(&mut self) -> Option<R> {
248 self.uncompressed_data.take()
249 }
250
251 pub fn take_drain(&mut self) -> Option<W> {
253 self.compressed_data.take()
254 }
255
256 pub fn replace_matcher(&mut self, mut match_generator: M) -> M {
258 core::mem::swap(&mut match_generator, &mut self.state.matcher);
259 match_generator
260 }
261
262 pub fn set_compression_level(
264 &mut self,
265 compression_level: CompressionLevel,
266 ) -> CompressionLevel {
267 let old = self.compression_level;
268 self.compression_level = compression_level;
269 old
270 }
271
272 pub fn compression_level(&self) -> CompressionLevel {
274 self.compression_level
275 }
276}
277
278#[cfg(test)]
279mod tests {
280 use alloc::vec;
281
282 use super::FrameCompressor;
283 use crate::common::MAGIC_NUM;
284 use crate::decoding::FrameDecoder;
285 use alloc::vec::Vec;
286
287 #[test]
288 fn frame_starts_with_magic_num() {
289 let mock_data = [1_u8, 2, 3].as_slice();
290 let mut output: Vec<u8> = Vec::new();
291 let mut compressor = FrameCompressor::new(super::CompressionLevel::Uncompressed);
292 compressor.set_source(mock_data);
293 compressor.set_drain(&mut output);
294
295 compressor.compress();
296 assert!(output.starts_with(&MAGIC_NUM.to_le_bytes()));
297 }
298
299 #[test]
300 fn very_simple_raw_compress() {
301 let mock_data = [1_u8, 2, 3].as_slice();
302 let mut output: Vec<u8> = Vec::new();
303 let mut compressor = FrameCompressor::new(super::CompressionLevel::Uncompressed);
304 compressor.set_source(mock_data);
305 compressor.set_drain(&mut output);
306
307 compressor.compress();
308 }
309
310 #[cfg(not(target_arch = "wasm32"))]
311 #[test]
312 fn very_simple_compress() {
313 let mut mock_data = vec![0; 1 << 17];
314 mock_data.extend(vec![1; (1 << 17) - 1]);
315 mock_data.extend(vec![2; (1 << 18) - 1]);
316 mock_data.extend(vec![2; 1 << 17]);
317 mock_data.extend(vec![3; (1 << 17) - 1]);
318 let mut output: Vec<u8> = Vec::new();
319 let mut compressor = FrameCompressor::new(super::CompressionLevel::Uncompressed);
320 compressor.set_source(mock_data.as_slice());
321 compressor.set_drain(&mut output);
322
323 compressor.compress();
324
325 let mut decoder = FrameDecoder::new();
326 let mut decoded = Vec::with_capacity(mock_data.len());
327 decoder.decode_all_to_vec(&output, &mut decoded).unwrap();
328 assert_eq!(mock_data, decoded);
329
330 let mut decoded = Vec::new();
331 zstd::stream::copy_decode(output.as_slice(), &mut decoded).unwrap();
332 assert_eq!(mock_data, decoded);
333 }
334
335 #[test]
336 fn rle_compress() {
337 let mock_data = vec![0; 1 << 19];
338 let mut output: Vec<u8> = Vec::new();
339 let mut compressor = FrameCompressor::new(super::CompressionLevel::Uncompressed);
340 compressor.set_source(mock_data.as_slice());
341 compressor.set_drain(&mut output);
342
343 compressor.compress();
344
345 let mut decoder = FrameDecoder::new();
346 let mut decoded = Vec::with_capacity(mock_data.len());
347 decoder.decode_all_to_vec(&output, &mut decoded).unwrap();
348 assert_eq!(mock_data, decoded);
349 }
350
351 #[cfg(not(target_arch = "wasm32"))]
352 #[test]
353 fn aaa_compress() {
354 let mock_data = vec![0, 1, 3, 4, 5];
355 let mut output: Vec<u8> = Vec::new();
356 let mut compressor = FrameCompressor::new(super::CompressionLevel::Uncompressed);
357 compressor.set_source(mock_data.as_slice());
358 compressor.set_drain(&mut output);
359
360 compressor.compress();
361
362 let mut decoder = FrameDecoder::new();
363 let mut decoded = Vec::with_capacity(mock_data.len());
364 decoder.decode_all_to_vec(&output, &mut decoded).unwrap();
365 assert_eq!(mock_data, decoded);
366
367 let mut decoded = Vec::new();
368 zstd::stream::copy_decode(output.as_slice(), &mut decoded).unwrap();
369 assert_eq!(mock_data, decoded);
370 }
371
372 #[cfg(feature = "hash")]
373 #[test]
374 fn checksum_two_frames_reused_compressor() {
375 let data: Vec<u8> = (0u8..=255).cycle().take(1024).collect();
381
382 let mut compressor = FrameCompressor::new(super::CompressionLevel::Uncompressed);
383
384 let mut compressed1 = Vec::new();
386 compressor.set_source(data.as_slice());
387 compressor.set_drain(&mut compressed1);
388 compressor.compress();
389
390 let mut compressed2 = Vec::new();
392 compressor.set_source(data.as_slice());
393 compressor.set_drain(&mut compressed2);
394 compressor.compress();
395
396 fn decode_and_collect(compressed: &[u8]) -> (Vec<u8>, Option<u32>, Option<u32>) {
397 let mut decoder = FrameDecoder::new();
398 let mut source = compressed;
399 decoder.reset(&mut source).unwrap();
400 while !decoder.is_finished() {
401 decoder
402 .decode_blocks(&mut source, crate::decoding::BlockDecodingStrategy::All)
403 .unwrap();
404 }
405 let mut decoded = Vec::new();
406 decoder.collect_to_writer(&mut decoded).unwrap();
407 (
408 decoded,
409 decoder.get_checksum_from_data(),
410 decoder.get_calculated_checksum(),
411 )
412 }
413
414 let (decoded1, chksum_from_data1, chksum_calculated1) = decode_and_collect(&compressed1);
415 assert_eq!(decoded1, data, "frame 1: decoded data mismatch");
416 assert_eq!(
417 chksum_from_data1, chksum_calculated1,
418 "frame 1: checksum mismatch"
419 );
420
421 let (decoded2, chksum_from_data2, chksum_calculated2) = decode_and_collect(&compressed2);
422 assert_eq!(decoded2, data, "frame 2: decoded data mismatch");
423 assert_eq!(
424 chksum_from_data2, chksum_calculated2,
425 "frame 2: checksum mismatch"
426 );
427
428 assert_eq!(
431 chksum_from_data1, chksum_from_data2,
432 "frame 1 and frame 2 should have the same checksum (same data, hash must reset per frame)"
433 );
434 }
435
436 #[cfg(feature = "std")]
437 #[cfg(not(target_arch = "wasm32"))]
438 #[test]
439 fn fuzz_targets() {
440 use std::io::Read;
441 fn decode_ruzstd(data: &mut dyn std::io::Read) -> Vec<u8> {
442 let mut decoder = crate::decoding::StreamingDecoder::new(data).unwrap();
443 let mut result: Vec<u8> = Vec::new();
444 decoder.read_to_end(&mut result).expect("Decoding failed");
445 result
446 }
447
448 fn decode_ruzstd_writer(mut data: impl Read) -> Vec<u8> {
449 let mut decoder = crate::decoding::FrameDecoder::new();
450 decoder.reset(&mut data).unwrap();
451 let mut result = vec![];
452 while !decoder.is_finished() || decoder.can_collect() > 0 {
453 decoder
454 .decode_blocks(
455 &mut data,
456 crate::decoding::BlockDecodingStrategy::UptoBytes(1024 * 1024),
457 )
458 .unwrap();
459 decoder.collect_to_writer(&mut result).unwrap();
460 }
461 result
462 }
463
464 fn encode_zstd(data: &[u8]) -> Result<Vec<u8>, std::io::Error> {
465 zstd::stream::encode_all(std::io::Cursor::new(data), 3)
466 }
467
468 fn encode_ruzstd_uncompressed(data: &mut dyn std::io::Read) -> Vec<u8> {
469 let mut input = Vec::new();
470 data.read_to_end(&mut input).unwrap();
471
472 crate::encoding::compress_to_vec(
473 input.as_slice(),
474 crate::encoding::CompressionLevel::Uncompressed,
475 )
476 }
477
478 fn encode_ruzstd_compressed(data: &mut dyn std::io::Read) -> Vec<u8> {
479 let mut input = Vec::new();
480 data.read_to_end(&mut input).unwrap();
481
482 crate::encoding::compress_to_vec(
483 input.as_slice(),
484 crate::encoding::CompressionLevel::Fastest,
485 )
486 }
487
488 fn decode_zstd(data: &[u8]) -> Result<Vec<u8>, std::io::Error> {
489 let mut output = Vec::new();
490 zstd::stream::copy_decode(data, &mut output)?;
491 Ok(output)
492 }
493 if std::fs::exists("fuzz/artifacts/interop").unwrap_or(false) {
494 for file in std::fs::read_dir("fuzz/artifacts/interop").unwrap() {
495 if file.as_ref().unwrap().file_type().unwrap().is_file() {
496 let data = std::fs::read(file.unwrap().path()).unwrap();
497 let data = data.as_slice();
498 let compressed = encode_zstd(data).unwrap();
500 let decoded = decode_ruzstd(&mut compressed.as_slice());
501 let decoded2 = decode_ruzstd_writer(&mut compressed.as_slice());
502 assert!(
503 decoded == data,
504 "Decoded data did not match the original input during decompression"
505 );
506 assert_eq!(
507 decoded2, data,
508 "Decoded data did not match the original input during decompression"
509 );
510
511 let mut input = data;
514 let compressed = encode_ruzstd_uncompressed(&mut input);
515 let decoded = decode_zstd(&compressed).unwrap();
516 assert_eq!(
517 decoded, data,
518 "Decoded data did not match the original input during compression"
519 );
520 let mut input = data;
522 let compressed = encode_ruzstd_compressed(&mut input);
523 let decoded = decode_zstd(&compressed).unwrap();
524 assert_eq!(
525 decoded, data,
526 "Decoded data did not match the original input during compression"
527 );
528 }
529 }
530 }
531 }
532}