1#![cfg(not(feature="no-stdlib"))]
2pub use alloc::{AllocatedStackMemory, Allocator, SliceWrapper, SliceWrapperMut, StackAllocator};
3pub use alloc::HeapAlloc;
4use core;
5use std::error;
6use std::io;
7use std::io::{Read};
8use super::interface::{DivansResult, DivansOutputResult, ErrMsg};
9use ::interface::{Compressor, DivansCompressorFactory, Decompressor};
10use ::DivansDecompressorFactory;
11use ::brotli;
12use ::interface;
13use ::StaticCommand;
14use ::brotli::interface::Nop;
15impl core::fmt::Display for ErrMsg {
16 fn fmt(&self, f:&mut core::fmt::Formatter) -> core::result::Result<(), core::fmt::Error> {
17 <Self as core::fmt::Debug>::fmt(self, f)
18 }
19}
20impl error::Error for ErrMsg {
21 fn description(&self) -> &str {
22 "Divans error"
23 }
24 fn cause(&self) -> Option<&error::Error> {None}
25}
26trait Processor {
27 fn process(&mut self, input:&[u8], input_offset:&mut usize, output:&mut [u8], output_offset:&mut usize) -> DivansResult;
28 fn close(&mut self, output:&mut [u8], output_offset:&mut usize) -> DivansOutputResult;
29}
30
31struct GenReader<R: Read,
32 P:Processor,
33 BufferType:SliceWrapperMut<u8>> {
34 compressor: P,
35 input_buffer: BufferType,
36 input_offset: usize,
37 input_len: usize,
38 input_eof: bool,
39 has_flushed: bool,
40 input: R,
41 read_error: Option<io::Error>,
42}
43
44
45impl<R:Read, P:Processor, BufferType:SliceWrapperMut<u8>> Read for GenReader<R,P,BufferType> {
46 fn read(&mut self, buf: &mut [u8]) -> Result<usize, io::Error> {
47 let mut output_offset : usize = 0;
48 let mut avail_out = buf.len() - output_offset;
49 let mut avail_in = self.input_len - self.input_offset;
50 let mut needs_input = false;
51 while avail_out == buf.len() && (!needs_input || !self.input_eof) {
52 if self.input_len < self.input_buffer.slice_mut().len() && !self.input_eof {
53 match self.input.read(&mut self.input_buffer.slice_mut()[self.input_len..]) {
54 Err(e) => {
55 if let io::ErrorKind::Interrupted = e.kind() {
56 continue;
57 }
58 self.read_error = Some(e);
59 self.input_eof = true;
60 },
61 Ok(size) => if size == 0 {
62 self.input_eof = true;
63 } else {
64 needs_input = false;
65 self.input_len += size;
66 avail_in = self.input_len - self.input_offset;
67 },
68 }
69 }
70 let old_output_offset = output_offset;
71 let old_input_offset = self.input_offset;
72 let ret = if avail_in == 0 {
73 self.has_flushed = true;
74 DivansResult::from(self.compressor.close(
75 buf.split_at_mut(output_offset + avail_out).0,
76 &mut output_offset))
77 } else {
78 self.compressor.process(
79 self.input_buffer.slice_mut().split_at_mut(self.input_offset + avail_in).0,
80 &mut self.input_offset,
81 buf.split_at_mut(output_offset + avail_out).0,
82 &mut output_offset)
83 };
84 avail_in -= self.input_offset - old_input_offset;
85 avail_out -= output_offset - old_output_offset;
86 if avail_in == 0 {
87 match self.read_error.take() {
88 Some(err) => return Err(err),
89 None => {
90 needs_input = true;
91 self.copy_to_front();
92 },
93 }
94 }
95 match ret {
96 DivansResult::Failure(m) => {
97 return Err(io::Error::new(io::ErrorKind::InvalidData, m));
98 },
99 DivansResult::Success => {
100 if self.input_eof && avail_in == 0 && self.has_flushed {
101 break;
102 }
103 },
104 DivansResult::NeedsMoreInput | DivansResult::NeedsMoreOutput => {},
105 }
106 }
107 Ok(output_offset)
108 }
109}
110impl<R:Read, C:Processor, BufferType:SliceWrapperMut<u8>> GenReader<R,C,BufferType>{
111 pub fn new(reader:R, compressor:C, buffer:BufferType, needs_flush: bool) ->Self {
112 GenReader {
113 input:reader,
114 compressor:compressor,
115 input_buffer: buffer,
116 input_offset : 0,
117 input_len : 0,
118 input_eof : false,
119 has_flushed: !needs_flush,
120 read_error: None,
121 }
122 }
123 pub fn copy_to_front(&mut self) {
124 let avail_in = self.input_len - self.input_offset;
125 if self.input_offset == self.input_buffer.slice_mut().len() {
126 self.input_offset = 0;
127 self.input_len = 0;
128 } else if self.input_offset + 256 > self.input_buffer.slice_mut().len() && avail_in < self.input_offset {
129 let (first, second) = self.input_buffer.slice_mut().split_at_mut(self.input_offset);
130 first[0..avail_in].clone_from_slice(&second[0..avail_in]);
131 self.input_len -= self.input_offset;
132 self.input_offset = 0;
133 }
134 }
135}
136type DivansBrotliFactory = ::BrotliDivansHybridCompressorFactory<HeapAlloc<u8>,
137 HeapAlloc<u16>,
138 HeapAlloc<u32>,
139 HeapAlloc<i32>,
140 HeapAlloc<u64>,
141 HeapAlloc<brotli::enc::command::Command>,
142 HeapAlloc<::DefaultCDF16>,
143 HeapAlloc<brotli::enc::util::floatX>,
144 HeapAlloc<brotli::enc::vectorization::Mem256f>,
145 HeapAlloc<brotli::enc::PDF>,
146 HeapAlloc<brotli::enc::StaticCommand>,
147 HeapAlloc<brotli::enc::histogram::HistogramLiteral>,
148 HeapAlloc<brotli::enc::histogram::HistogramCommand>,
149 HeapAlloc<brotli::enc::histogram::HistogramDistance>,
150 HeapAlloc<brotli::enc::cluster::HistogramPair>,
151 HeapAlloc<brotli::enc::histogram::ContextType>,
152 HeapAlloc<brotli::enc::entropy_encode::HuffmanTree>,
153 HeapAlloc<brotli::enc::ZopfliNode>>;
154type DivansBrotliConstructedCompressor = <DivansBrotliFactory as ::DivansCompressorFactory<HeapAlloc<u8>,
155 HeapAlloc<u32>,
156 HeapAlloc<::DefaultCDF16>>>::ConstructedCompressor;
157impl<T:Compressor> Processor for T {
158 fn process(&mut self, input:&[u8], input_offset:&mut usize, output:&mut [u8], output_offset:&mut usize) -> DivansResult {
159 self.encode(input, input_offset, output, output_offset)
160 }
161 fn close(&mut self, output:&mut [u8], output_offset:&mut usize) -> DivansOutputResult{
162 self.flush(output, output_offset)
163 }
164
165}
166pub struct DivansBrotliHybridCompressorReader<R:Read>(GenReader<R,
167 DivansBrotliConstructedCompressor,
168 <HeapAlloc<u8> as Allocator<u8>>::AllocatedMemory,
169 >);
170impl<R:Read> Read for DivansBrotliHybridCompressorReader<R> {
171 fn read(&mut self, buf: &mut [u8]) -> Result<usize, io::Error> {
172 self.0.read(buf)
173 }
174}
175impl<R:Read> DivansBrotliHybridCompressorReader<R> {
176 pub fn new(reader: R, opts: interface::DivansCompressorOptions, mut buffer_size: usize) -> Self {
177 if buffer_size == 0 {
178 buffer_size = 4096;
179 }
180 let mut m8 = HeapAlloc::<u8>::new(0);
181 let buffer = m8.alloc_cell(buffer_size);
182 DivansBrotliHybridCompressorReader::<R>(
183 GenReader::<R,
184 DivansBrotliConstructedCompressor,
185 <HeapAlloc<u8> as Allocator<u8>>::AllocatedMemory>::new(
186 reader,
187 DivansBrotliFactory::new(
188 m8,
189 HeapAlloc::<u32>::new(0),
190 HeapAlloc::<::DefaultCDF16>::new(::DefaultCDF16::default()),
191 opts,
192 (
193 HeapAlloc::<u8>::new(0),
194 HeapAlloc::<u16>::new(0),
195 HeapAlloc::<i32>::new(0),
196 HeapAlloc::<brotli::enc::command::Command>::new(brotli::enc::command::Command::default()),
197 HeapAlloc::<u64>::new(0),
198 HeapAlloc::<brotli::enc::util::floatX>::new(0.0 as brotli::enc::util::floatX),
199 HeapAlloc::<brotli::enc::vectorization::Mem256f>::new(brotli::enc::vectorization::Mem256f::default()),
200 HeapAlloc::<brotli::enc::histogram::HistogramLiteral>::new(brotli::enc::histogram::HistogramLiteral::default()),
201 HeapAlloc::<brotli::enc::histogram::HistogramCommand>::new(brotli::enc::histogram::HistogramCommand::default()),
202 HeapAlloc::<brotli::enc::histogram::HistogramDistance>::new(brotli::enc::histogram::HistogramDistance::default()),
203 HeapAlloc::<brotli::enc::cluster::HistogramPair>::new(brotli::enc::cluster::HistogramPair::default()),
204 HeapAlloc::<brotli::enc::histogram::ContextType>::new(brotli::enc::histogram::ContextType::default()),
205 HeapAlloc::<brotli::enc::entropy_encode::HuffmanTree>::new(brotli::enc::entropy_encode::HuffmanTree::default()),
206 HeapAlloc::<brotli::enc::ZopfliNode>::new(brotli::enc::ZopfliNode::default()),
207 HeapAlloc::<brotli::enc::PDF>::new(brotli::enc::PDF::default()),
208 HeapAlloc::<brotli::enc::StaticCommand>::new(brotli::enc::StaticCommand::default()),
209 )),
210 buffer,
211 true,
212 ))
213 }
214}
215
216
217type DivansCustomFactory = ::DivansCompressorFactoryStruct<HeapAlloc<u8>,
218 HeapAlloc<::DefaultCDF16>>;
219type DivansCustomConstructedCompressor = <DivansCustomFactory as ::DivansCompressorFactory<HeapAlloc<u8>,
220 HeapAlloc<u32>,
221 HeapAlloc<::DefaultCDF16>>>::ConstructedCompressor;
222pub struct DivansExperimentalCompressorReader<R:Read>(GenReader<R,
223 DivansCustomConstructedCompressor,
224 <HeapAlloc<u8> as Allocator<u8>>::AllocatedMemory,
225 >);
226impl<R:Read> Read for DivansExperimentalCompressorReader<R> {
227 fn read(&mut self, buf: &mut [u8]) -> Result<usize, io::Error> {
228 self.0.read(buf)
229 }
230}
231impl<R:Read> DivansExperimentalCompressorReader<R> {
232 pub fn new(reader: R, opts: interface::DivansCompressorOptions, mut buffer_size: usize) -> Self {
233 if buffer_size == 0 {
234 buffer_size = 4096;
235 }
236 let mut m8 = HeapAlloc::<u8>::new(0);
237 let buffer = m8.alloc_cell(buffer_size);
238 DivansExperimentalCompressorReader::<R>(
239 GenReader::<R,
240 DivansCustomConstructedCompressor,
241 <HeapAlloc<u8> as Allocator<u8>>::AllocatedMemory>::new(
242 reader,
243 DivansCustomFactory::new(
244 m8,
245 HeapAlloc::<u32>::new(0),
246 HeapAlloc::<::DefaultCDF16>::new(::DefaultCDF16::default()),
247 opts,
248 ()),
249 buffer,
250 true,
251 ))
252 }
253}
254
255
256type StandardDivansDecompressorFactory = ::DivansDecompressorFactoryStruct<HeapAlloc<u8>,
257 HeapAlloc<::DefaultCDF16>,
258 HeapAlloc<StaticCommand>>;
259type DivansConstructedDecompressor = ::DivansDecompressor<<StandardDivansDecompressorFactory as ::DivansDecompressorFactory<HeapAlloc<u8>,
260 HeapAlloc<::DefaultCDF16>,
261 HeapAlloc<StaticCommand>>
262 >::DefaultDecoder,
263 HeapAlloc<u8>,
264 HeapAlloc<::DefaultCDF16>,
265 HeapAlloc<StaticCommand>>;
266impl Processor for DivansConstructedDecompressor {
267 fn process(&mut self, input:&[u8], input_offset:&mut usize, output:&mut [u8], output_offset:&mut usize) -> DivansResult {
268 self.decode(input, input_offset, output, output_offset)
269 }
270 fn close(&mut self, output:&mut [u8], output_offset:&mut usize) -> DivansOutputResult{
271 let mut input_offset = 0usize;
272 match self.decode(&[], &mut input_offset, output, output_offset) {
273 DivansResult::NeedsMoreInput => {
274 DivansOutputResult::Failure(ErrMsg::UnexpectedEof)
275 },
276 DivansResult::Failure(m) => DivansOutputResult::Failure(m),
277 DivansResult::NeedsMoreOutput => DivansOutputResult::NeedsMoreOutput,
278 DivansResult::Success => DivansOutputResult::Success,
279 }
280 }
281
282}
283pub struct DivansDecompressorReader<R:Read>(GenReader<R,
284 DivansConstructedDecompressor,
285 <HeapAlloc<u8> as Allocator<u8>>::AllocatedMemory,
286 >);
287impl<R:Read> Read for DivansDecompressorReader<R> {
288 fn read(&mut self, buf: &mut [u8]) -> Result<usize, io::Error> {
289 self.0.read(buf)
290 }
291}
292impl<R:Read> DivansDecompressorReader<R> {
293 pub fn new(reader: R, mut buffer_size: usize, skip_crc:bool, multithread:bool) -> Self {
294 if buffer_size == 0 {
295 buffer_size = 4096;
296 }
297 let mut m8 = HeapAlloc::<u8>::new(0);
298 let buffer = m8.alloc_cell(buffer_size);
299 DivansDecompressorReader::<R>(
300 GenReader::<R,
301 DivansConstructedDecompressor,
302 <HeapAlloc<u8> as Allocator<u8>>::AllocatedMemory>::new(
303 reader,
304 StandardDivansDecompressorFactory::new(
305 m8,
306 HeapAlloc::<::DefaultCDF16>::new(::DefaultCDF16::default()),
307 HeapAlloc::<StaticCommand>::new(::StaticCommand::nop()),
308 skip_crc,
309 multithread,
310 ),
311 buffer,
312 false,
313 ))
314 }
315}
316#[cfg(test)]
317mod test {
318 use std::vec::Vec;
319 use std::io;
320 use std::io::{Read, Write};
321 use core::cmp;
322 use ::interface;
323 pub struct UnlimitedBuffer {
324 pub data: Vec<u8>,
325 pub read_offset: usize,
326 }
327
328 impl UnlimitedBuffer {
329 pub fn new(buf: &[u8]) -> Self {
330 let mut ret = UnlimitedBuffer {
331 data: Vec::<u8>::new(),
332 read_offset: 0,
333 };
334 ret.data.extend(buf);
335 return ret;
336 }
337 #[allow(unused)]
338 pub fn written(&self) -> &[u8] {
339 &self.data[..]
340 }
341 }
342
343 impl io::Read for UnlimitedBuffer {
344 fn read(self: &mut Self, buf: &mut [u8]) -> io::Result<usize> {
345 let bytes_to_read = cmp::min(buf.len(), self.data.len() - self.read_offset);
346 if bytes_to_read > 0 {
347 buf[0..bytes_to_read].clone_from_slice(&self.data[self.read_offset..
348 self.read_offset + bytes_to_read]);
349 }
350 self.read_offset += bytes_to_read;
351 return Ok(bytes_to_read);
352 }
353 }
354
355 impl io::Write for UnlimitedBuffer {
356 fn write(self: &mut Self, buf: &[u8]) -> io::Result<usize> {
357 self.data.extend(buf);
358 return Ok(buf.len());
359 }
360 fn flush(self: &mut Self) -> io::Result<()> {
361 return Ok(());
362 }
363 }
364
365 struct Tee<'a, R:io::Read> {
366 reader: R,
367 output: &'a mut UnlimitedBuffer,
368 }
369 impl<'a, R:Read> io::Read for Tee<'a, R> {
370 fn read(&mut self, data: &mut[u8]) -> io::Result<usize> {
371 let ret = self.reader.read(data);
372 match ret {
373 Err(_) => {},
374 Ok(size) => {
375 let xret = self.output.write(&data[..size]);
376 if let Ok(xsize) = xret {
377 assert_eq!(xsize, size); } else {
379 unreachable!();
380 }
381 }
382 }
383 ret
384 }
385 }
386 fn hy_reader_tst(data:&[u8], opts: interface::DivansCompressorOptions, buffer_size: usize){
387 let source = UnlimitedBuffer::new(data);
388 let compress = ::DivansBrotliHybridCompressorReader::<UnlimitedBuffer>::new(source, opts, buffer_size);
389 let mut ub = UnlimitedBuffer::new(&mut []);
390 {
391 let tee = Tee::<::DivansBrotliHybridCompressorReader<UnlimitedBuffer>> {
392 reader:compress,
393 output: &mut ub,
394 };
395 let mut decompress = super::DivansDecompressorReader::new(tee, buffer_size, false, true);
396 let mut local_buffer = vec![0u8; buffer_size];
397 let mut offset: usize = 0;
398 loop {
399 match decompress.read(&mut local_buffer[..]) {
400 Err(e) => {
401 panic!(e)
402 },
403 Ok(size) => {
404 if size == 0 {
405 break;
406 }
407 assert_eq!(&data[offset..offset+size], &local_buffer[..size]);
408 offset += size;
409 }
410 }
411 }
412 assert_eq!(offset, data.len());
413 }
414 assert!(ub.data.len() < data.len());
415 print!("Compressed {} to {}...\n", ub.data.len(), data.len());
416 }
417 fn experimental_reader_tst(data:&[u8], opts: interface::DivansCompressorOptions, buffer_size: usize){
418 let source = UnlimitedBuffer::new(data);
419 let compress = ::DivansExperimentalCompressorReader::<UnlimitedBuffer>::new(source, opts, buffer_size);
420 let mut ub = UnlimitedBuffer::new(&mut []);
421 {
422 let tee = Tee::<::DivansExperimentalCompressorReader<UnlimitedBuffer>> {
423 reader:compress,
424 output: &mut ub,
425 };
426 let mut decompress = super::DivansDecompressorReader::new(tee, buffer_size, false, false);
427 let mut local_buffer = vec![0u8; buffer_size];
428 let mut offset: usize = 0;
429 loop {
430 match decompress.read(&mut local_buffer[..]) {
431 Err(e) => panic!(e),
432 Ok(size) => {
433 if size == 0 {
434 break;
435 }
436 assert_eq!(&data[offset..offset+size], &local_buffer[..size]);
437 offset += size;
438 }
439 }
440 }
441 assert_eq!(offset, data.len());
442 }
443 assert!(ub.data.len() < data.len());
444 print!("Compressed {} to {}...\n", ub.data.len(), data.len());
445 }
446 #[test]
447 fn test_hybrid_reader_compressor_on_alice_small_buffer() {
448 hy_reader_tst(include_bytes!("../testdata/alice29"),
449 interface::DivansCompressorOptions{
450 literal_adaptation:None,
451 force_literal_context_mode:None,
452 brotli_literal_byte_score:None,
453 window_size:Some(16),
454 lgblock:Some(16),
455 quality:Some(11),
456 q9_5:true,
457 prior_depth:Some(1),
458 dynamic_context_mixing:None,
459 use_brotli:interface::BrotliCompressionSetting::default(),
460 use_context_map:true,
461 force_stride_value: interface::StrideSelection::default(),
462 speed_detection_quality: None,
463 prior_bitmask_detection: 1,
464 stride_detection_quality: Some(2),
465 divans_ir_optimizer:0,
466 },
467 1);
468 }
469 #[test]
470 fn test_hybrid_reader_compressor_on_alice_full() {
471 hy_reader_tst(include_bytes!("../testdata/alice29"),
472 interface::DivansCompressorOptions{
473 literal_adaptation:None,
474 force_literal_context_mode:None,
475 window_size:Some(22),
476 brotli_literal_byte_score:None,
477 lgblock:None,
478 quality:None,
479 q9_5:false,
480 prior_depth:Some(2),
481 dynamic_context_mixing:Some(2),
482 use_brotli:interface::BrotliCompressionSetting::default(),
483 use_context_map:true,
484 force_stride_value: interface::StrideSelection::Stride1,
485 prior_bitmask_detection: 0,
486 speed_detection_quality: None,
487 stride_detection_quality: None,
488 divans_ir_optimizer:1,
489 },
490 4095);
491 }
492 #[test]
493 fn test_hybrid_reader_compressor_on_unicode_full() {
494 hy_reader_tst(include_bytes!("../testdata/random_then_unicode"),
495 interface::DivansCompressorOptions{
496 literal_adaptation:None,
497 force_literal_context_mode:None,
498 window_size:Some(22),
499 brotli_literal_byte_score:None,
500 lgblock:None,
501 quality:Some(8),
502 q9_5:false,
503 prior_depth:None,
504 dynamic_context_mixing:Some(2),
505 use_brotli:interface::BrotliCompressionSetting::default(),
506 use_context_map:true,
507 force_stride_value: interface::StrideSelection::Stride1,
508 speed_detection_quality: None,
509 prior_bitmask_detection: 1,
510 stride_detection_quality: None,
511 divans_ir_optimizer:0,
512 },
513 4095);
514 }
515 #[test]
516 fn test_experimental_reader_compressor_on_alice_full() {
517 experimental_reader_tst(include_bytes!("../testdata/alice29"),
518 interface::DivansCompressorOptions{
519 literal_adaptation:None,
520 force_literal_context_mode:None,
521 window_size:Some(22),
522 brotli_literal_byte_score:None,
523 lgblock:None,
524 q9_5:true,
525 prior_depth:Some(0),
526 quality:None,
527 dynamic_context_mixing:Some(2),
528 use_brotli:interface::BrotliCompressionSetting::default(),
529 use_context_map:true,
530 speed_detection_quality: None,
531 force_stride_value: interface::StrideSelection::UseBrotliRec,
532 stride_detection_quality: Some(1),
533 prior_bitmask_detection: 1,
534 divans_ir_optimizer:1,
535 },
536 310000);
537 }
538}