niffler_temp/basic/
mod.rs

1pub mod compression;
2
3/* standard use */
4use std::io;
5use std::io::Read;
6use std::path::Path;
7
8/* project use */
9use crate::error::Error;
10use crate::level::Level;
11
12/// Finds out what is the compression format for a stream based on magic numbers
13/// (the first few bytes of the stream).
14///
15/// Return the stream and the compression format detected.
16///
17/// # Example
18/// ```
19/// # fn main() -> Result<(), niffler_temp::Error> {
20///
21/// let data = vec![
22///         0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x03, 0xf3, 0x54, 0xcf, 0x55,
23///         0x48, 0xce, 0xcf, 0x2d, 0x28, 0x4a, 0x2d, 0x2e, 0x56, 0xc8, 0xcc, 0x53, 0x48, 0xaf,
24///         0xca, 0x2c, 0xe0, 0x02, 0x00, 0x45, 0x7c, 0xf4, 0x10, 0x15, 0x00, 0x00, 0x00
25///         ];
26///
27/// let (mut reader, compression) = niffler_temp::sniff(Box::new(&data[..]))?;
28///
29/// let mut contents = Vec::new();
30/// reader.read_to_end(&mut contents).expect("Error durring file reading");
31///
32/// assert_eq!(compression, niffler_temp::compression::Format::Gzip);
33/// assert_eq!(contents, data);
34/// # Ok(())
35/// # }
36/// ```
37pub fn sniff<'a>(
38    in_stream: Box<dyn io::Read + 'a>,
39) -> Result<(Box<dyn io::Read + 'a>, compression::Format), Error> {
40    let (first_bytes, in_stream) = crate::utils::get_first_five(in_stream)?;
41
42    let cursor = io::Cursor::new(first_bytes);
43    match compression::bytes2type(first_bytes) {
44        e @ compression::Format::Gzip
45        | e @ compression::Format::Bzip
46        | e @ compression::Format::Lzma
47        | e @ compression::Format::Zstd => Ok((Box::new(cursor.chain(in_stream)), e)),
48        _ => Ok((Box::new(cursor.chain(in_stream)), compression::Format::No)),
49    }
50}
51
52/// Create a readable stream that can be read transparently even if the original stream is compress.
53/// Also returns the compression type of the original stream.
54///
55/// # Example
56/// ```
57/// use niffler_temp::{Error, get_reader};
58/// # fn main() -> Result<(), Error> {
59///
60/// let probably_compress_stream = std::io::Cursor::new(vec![
61///         0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x03, 0xf3, 0x54, 0xcf, 0x55,
62///         0x48, 0xce, 0xcf, 0x2d, 0x28, 0x4a, 0x2d, 0x2e, 0x56, 0xc8, 0xcc, 0x53, 0x48, 0xaf,
63///         0xca, 0x2c, 0xe0, 0x02, 0x00, 0x45, 0x7c, 0xf4, 0x10, 0x15, 0x00, 0x00, 0x00
64///         ]);
65///
66/// # #[cfg(feature = "gz")] {
67/// let (mut reader, compression) = niffler_temp::get_reader(Box::new(probably_compress_stream))?;
68///
69/// let mut contents = String::new();
70/// reader.read_to_string(&mut contents).expect("Error durring file reading");
71///
72/// assert_eq!(compression, niffler_temp::compression::Format::Gzip);
73/// assert_eq!(contents, "I'm compress in gzip\n");
74/// # }
75/// # Ok(())
76/// # }
77/// ```
78pub fn get_reader<'a>(
79    in_stream: Box<dyn io::Read + 'a>,
80) -> Result<(Box<dyn io::Read + 'a>, compression::Format), Error> {
81    // check compression
82    let (in_stream, compression) = sniff(in_stream)?;
83
84    // return readable and compression status
85    match compression {
86        compression::Format::Gzip => compression::new_gz_decoder(in_stream),
87        compression::Format::Bzip => compression::new_bz2_decoder(in_stream),
88        compression::Format::Lzma => compression::new_lzma_decoder(in_stream),
89        compression::Format::Zstd => compression::new_zstd_decoder(in_stream),
90        compression::Format::No => Ok((in_stream, compression::Format::No)),
91    }
92}
93
94/// Create a new writable stream with the given compression format and level.
95///
96/// # Example
97/// ```
98/// use std::io::Read;
99/// use niffler_temp::{Error, get_writer, compression};
100/// # fn main() -> Result<(), Error> {
101///
102/// # #[cfg(feature = "gz")] {
103/// let mut buffer = vec![];
104/// {
105///   let mut writer = niffler_temp::get_writer(Box::new(&mut buffer), compression::Format::Gzip, niffler_temp::Level::One)?;
106///   writer.write_all("I'm compress in gzip\n".as_bytes())?
107/// }
108///
109/// let mut contents = Vec::new();
110/// buffer.as_slice().read_to_end(&mut contents)?;
111///
112/// assert_eq!(contents, vec![
113///         0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x04, 0xff, 0xf3, 0x54, 0xcf, 0x55,
114///         0x48, 0xce, 0xcf, 0x2d, 0x28, 0x4a, 0x2d, 0x2e, 0x56, 0xc8, 0xcc, 0x53, 0x48, 0xaf,
115///         0xca, 0x2c, 0xe0, 0x02, 0x00, 0x45, 0x7c, 0xf4, 0x10, 0x15, 0x00, 0x00, 0x00
116///         ]);
117/// # }
118/// # Ok(())
119/// # }
120/// ```
121pub fn get_writer<'a>(
122    out_stream: Box<dyn io::Write + 'a>,
123    format: compression::Format,
124    level: Level,
125) -> Result<Box<dyn io::Write + 'a>, Error> {
126    match format {
127        compression::Format::Gzip => compression::new_gz_encoder(out_stream, level),
128        compression::Format::Bzip => compression::new_bz2_encoder(out_stream, level),
129        compression::Format::Lzma => compression::new_lzma_encoder(out_stream, level),
130        compression::Format::Zstd => compression::new_zstd_encoder(out_stream, level),
131        compression::Format::No => Ok(Box::new(out_stream)),
132    }
133}
134
135/// Open a possibly compressed file and decompress it transparently.
136/// ```
137/// use niffler_temp::{Error, compression};
138/// # fn main() -> Result<(), Error> {
139///
140/// # #[cfg(feature = "gz")] {
141/// # let file = tempfile::NamedTempFile::new()?;
142///
143/// # {
144/// #   let mut writer = niffler_temp::to_path(file.path(), compression::Format::Gzip, niffler_temp::Level::Nine)?;
145/// #   writer.write_all(b"hello")?;
146/// # }
147///
148/// let (mut reader, format) = niffler_temp::from_path(file.path())?;
149///
150/// let mut contents = vec![];
151/// reader.read_to_end(&mut contents);
152/// # assert_eq!(&contents, b"hello");
153///
154/// # }
155/// # Ok(())
156/// # }
157/// ```
158pub fn from_path<'a, P: AsRef<Path>>(
159    path: P,
160) -> Result<(Box<dyn io::Read + 'a>, compression::Format), Error> {
161    let readable = io::BufReader::new(std::fs::File::open(path)?);
162    get_reader(Box::new(readable))
163}
164
165/// Create a file with specific compression format.
166/// ```
167/// use niffler_temp::{Error, compression};
168/// # fn main() -> Result<(), Error> {
169///
170/// # #[cfg(feature = "gz")] {
171/// # let file = tempfile::NamedTempFile::new()?;
172///
173/// # {
174/// let mut writer = niffler_temp::to_path(file.path(), compression::Format::Gzip, niffler_temp::Level::Nine)?;
175/// writer.write_all(b"hello")?;
176/// # }
177///
178/// # let (mut reader, format) = niffler_temp::from_path(&file.path())?;
179/// # let mut contents = vec![];
180/// # reader.read_to_end(&mut contents)?;
181/// # assert_eq!(&contents, b"hello");
182/// # }
183/// # Ok(())
184/// # }
185/// ```
186pub fn to_path<'a, P: AsRef<Path>>(
187    path: P,
188    format: compression::Format,
189    level: Level,
190) -> Result<Box<dyn io::Write + 'a>, Error> {
191    let writable = io::BufWriter::new(std::fs::File::create(path)?);
192    get_writer(Box::new(writable), format, level)
193}
194
195#[cfg(test)]
196mod test {
197
198    use super::*;
199    use tempfile::NamedTempFile;
200
201    pub(crate) const SHORT_FILE: &'static [u8] = &[0o037, 0o213, 0o0, 0o0];
202    pub(crate) const GZIP_FILE: &'static [u8] = &[0o037, 0o213, 0o0, 0o0, 0o0];
203    pub(crate) const BZIP_FILE: &'static [u8] = &[0o102, 0o132, 0o0, 0o0, 0o0];
204    pub(crate) const LZMA_FILE: &'static [u8] = &[0o375, 0o067, 0o172, 0o130, 0o132];
205    pub(crate) const ZSTD_FILE: &'static [u8] = &[0x28, 0xb5, 0x2f, 0xfd, 0];
206    pub(crate) const LOREM_IPSUM: &'static [u8] = b"Lorem ipsum dolor sit amet, consectetur adipiscing elit. Ut ultricies scelerisque diam, a scelerisque enim sagittis at.";
207
208    mod compress_uncompress {
209        use super::*;
210
211        #[test]
212        fn no_compression() {
213            let ofile = NamedTempFile::new().expect("Can't create tmpfile");
214
215            {
216                let wfile = ofile.reopen().expect("Can't create tmpfile");
217                let mut writer =
218                    get_writer(Box::new(wfile), compression::Format::No, Level::One).unwrap();
219                writer
220                    .write_all(LOREM_IPSUM)
221                    .expect("Error during write of data");
222            }
223
224            let rfile = ofile.reopen().expect("Can't create tmpfile");
225            let (mut reader, compression) =
226                get_reader(Box::new(rfile)).expect("Error reading from tmpfile");
227
228            assert_eq!(compression, compression::Format::No);
229
230            let mut buffer = Vec::new();
231            reader
232                .read_to_end(&mut buffer)
233                .expect("Error during reading");
234            assert_eq!(LOREM_IPSUM, buffer.as_slice());
235        }
236
237        #[cfg(feature = "gz")]
238        #[test]
239        fn gzip() {
240            let ofile = NamedTempFile::new().expect("Can't create tmpfile");
241
242            {
243                let wfile = ofile.reopen().expect("Can't create tmpfile");
244                let mut writer =
245                    get_writer(Box::new(wfile), compression::Format::Gzip, Level::Six).unwrap();
246                writer
247                    .write_all(LOREM_IPSUM)
248                    .expect("Error during write of data");
249            }
250
251            let rfile = ofile.reopen().expect("Can't create tmpfile");
252            let (mut reader, compression) =
253                get_reader(Box::new(rfile)).expect("Error reading from tmpfile");
254
255            assert_eq!(compression, compression::Format::Gzip);
256
257            let mut buffer = Vec::new();
258            reader
259                .read_to_end(&mut buffer)
260                .expect("Error during reading");
261            assert_eq!(LOREM_IPSUM, buffer.as_slice());
262        }
263
264        #[test]
265        #[cfg(not(feature = "bz2"))]
266        fn no_bzip2_feature() {
267            assert!(
268                get_writer(Box::new(vec![]), compression::Format::Bzip, Level::Six).is_err(),
269                "bz2 disabled, this assertion should fail"
270            );
271
272            assert!(
273                get_reader(Box::new(&BZIP_FILE[..])).is_err(),
274                "bz2 disabled, this assertion should fail"
275            );
276        }
277
278        #[cfg(feature = "bz2")]
279        #[test]
280        fn bzip() {
281            let ofile = NamedTempFile::new().expect("Can't create tmpfile");
282
283            {
284                let wfile = ofile.reopen().expect("Can't create tmpfile");
285                let mut writer =
286                    get_writer(Box::new(wfile), compression::Format::Bzip, Level::Six).unwrap();
287                writer
288                    .write_all(LOREM_IPSUM)
289                    .expect("Error during write of data");
290            }
291
292            let rfile = ofile.reopen().expect("Can't create tmpfile");
293            let (mut reader, compression) =
294                get_reader(Box::new(rfile)).expect("Error reading from tmpfile");
295
296            assert_eq!(compression, compression::Format::Bzip);
297
298            let mut buffer = Vec::new();
299            reader
300                .read_to_end(&mut buffer)
301                .expect("Error during reading");
302            assert_eq!(LOREM_IPSUM, buffer.as_slice());
303        }
304
305        #[cfg(feature = "bz2")]
306        #[test]
307        fn bzip_multidecoder() {
308            let mut buf: Vec<u8> = vec![];
309
310            {
311                let mut writer =
312                    get_writer(Box::new(&mut buf), compression::Format::Bzip, Level::Six).unwrap();
313                writer
314                    .write_all(LOREM_IPSUM)
315                    .expect("Error during write of data");
316            }
317
318            let ofile = NamedTempFile::new().expect("Can't create tmpfile");
319            {
320                use std::io::Write;
321                let mut wfile = ofile.reopen().expect("Can't create tmpfile");
322                wfile
323                    .write_all(buf.as_slice())
324                    .expect("Error during write of data");
325                wfile
326                    .write_all(buf.as_slice())
327                    .expect("Error during write of data");
328            }
329
330            let rfile = ofile.reopen().expect("Can't create tmpfile");
331            let (mut reader, compression) =
332                get_reader(Box::new(rfile)).expect("Error reading from tmpfile");
333
334            assert_eq!(compression, compression::Format::Bzip);
335
336            let mut buffer = Vec::new();
337            reader
338                .read_to_end(&mut buffer)
339                .expect("Error during reading");
340            let mut result: Vec<u8> = LOREM_IPSUM.into();
341            result.extend(LOREM_IPSUM);
342            assert_eq!(result, buffer.as_slice());
343        }
344
345        #[test]
346        #[cfg(not(feature = "lzma"))]
347        fn no_lzma_feature() {
348            assert!(
349                get_writer(Box::new(vec![]), compression::Format::Lzma, Level::Six).is_err(),
350                "lzma disabled, this assertion should fail"
351            );
352
353            assert!(
354                get_reader(Box::new(&LZMA_FILE[..])).is_err(),
355                "lzma disabled, this assertion should fail"
356            );
357        }
358
359        #[cfg(feature = "lzma")]
360        #[test]
361        fn lzma() {
362            let ofile = NamedTempFile::new().expect("Can't create tmpfile");
363
364            {
365                let wfile = ofile.reopen().expect("Can't create tmpfile");
366                let mut writer =
367                    get_writer(Box::new(wfile), compression::Format::Lzma, Level::Six).unwrap();
368                writer
369                    .write_all(LOREM_IPSUM)
370                    .expect("Error during write of data");
371            }
372
373            let rfile = ofile.reopen().expect("Can't create tmpfile");
374            let (mut reader, compression) =
375                get_reader(Box::new(rfile)).expect("Error reading from tmpfile");
376
377            assert_eq!(compression, compression::Format::Lzma);
378
379            let mut buffer = Vec::new();
380            reader
381                .read_to_end(&mut buffer)
382                .expect("Error during reading");
383            assert_eq!(LOREM_IPSUM, buffer.as_slice());
384        }
385
386        #[test]
387        #[cfg(all(not(feature = "xz"), not(feature = "lzma")))]
388        fn no_xz_feature() {
389            assert!(
390                get_writer(Box::new(vec![]), compression::Format::Xz, Level::Six).is_err(),
391                "xz disabled, this assertion should fail"
392            );
393
394            assert!(
395                get_reader(Box::new(&LZMA_FILE[..])).is_err(),
396                "xz disabled, this assertion should fail"
397            );
398        }
399
400        #[cfg(feature = "xz")]
401        #[test]
402        fn xz() {
403            let ofile = NamedTempFile::new().expect("Can't create tmpfile");
404
405            {
406                let wfile = ofile.reopen().expect("Can't create tmpfile");
407                let mut writer =
408                    get_writer(Box::new(wfile), compression::Format::Xz, Level::Six).unwrap();
409                writer
410                    .write_all(LOREM_IPSUM)
411                    .expect("Error during write of data");
412            }
413
414            let rfile = ofile.reopen().expect("Can't create tmpfile");
415            let (mut reader, compression) =
416                get_reader(Box::new(rfile)).expect("Error reading from tmpfile");
417
418            assert_eq!(compression, compression::Format::Xz);
419
420            let mut buffer = Vec::new();
421            reader
422                .read_to_end(&mut buffer)
423                .expect("Error during reading");
424            assert_eq!(LOREM_IPSUM, buffer.as_slice());
425        }
426
427        #[test]
428        #[cfg(not(feature = "zstd"))]
429        fn no_zstd_feature() {
430            assert!(
431                get_writer(Box::new(vec![]), compression::Format::Zstd, Level::Six).is_err(),
432                "zstd disabled, this assertion should fail"
433            );
434
435            assert!(
436                get_reader(Box::new(&ZSTD_FILE[..])).is_err(),
437                "zstd disabled, this assertion should fail"
438            );
439        }
440
441        #[cfg(feature = "zstd")]
442        #[test]
443        fn zstd() {
444            let ofile = NamedTempFile::new().expect("Can't create tmpfile");
445
446            {
447                let wfile = ofile.reopen().expect("Can't create tmpfile");
448                let mut writer =
449                    get_writer(Box::new(wfile), compression::Format::Zstd, Level::Six).unwrap();
450                writer
451                    .write_all(LOREM_IPSUM)
452                    .expect("Error during write of data");
453            }
454
455            let rfile = ofile.reopen().expect("Can't create tmpfile");
456            let (mut reader, compression) =
457                get_reader(Box::new(rfile)).expect("Error reading from tmpfile");
458
459            assert_eq!(compression, compression::Format::Zstd);
460
461            let mut buffer = Vec::new();
462            reader
463                .read_to_end(&mut buffer)
464                .expect("Error during reading");
465            assert_eq!(LOREM_IPSUM, buffer.as_slice());
466        }
467    }
468
469    mod compression_format_detection {
470        use super::*;
471
472        #[test]
473        fn gzip() {
474            let (_, compression) = sniff(Box::new(GZIP_FILE)).expect("Error in read file");
475            assert_eq!(compression, compression::Format::Gzip);
476        }
477
478        #[test]
479        fn bzip() {
480            let (_, compression) = sniff(Box::new(BZIP_FILE)).expect("Error in read file");
481            assert_eq!(compression, compression::Format::Bzip);
482        }
483
484        #[test]
485        fn lzma() {
486            let (_, compression) = sniff(Box::new(LZMA_FILE)).expect("Error in read file");
487            assert_eq!(compression, compression::Format::Lzma);
488        }
489
490        #[test]
491        fn xz() {
492            let (_, compression) = sniff(Box::new(LZMA_FILE)).expect("Error in read file");
493            assert_eq!(compression, compression::Format::Xz);
494        }
495
496        #[test]
497        fn zstd() {
498            let (_, compression) = sniff(Box::new(ZSTD_FILE)).expect("Error in read file");
499            assert_eq!(compression, compression::Format::Zstd);
500        }
501
502        #[test]
503        fn too_short() {
504            let result = sniff(Box::new(SHORT_FILE));
505            assert!(result.is_err());
506        }
507
508        #[test]
509        fn no_compression() {
510            let (_, compression) = sniff(Box::new(LOREM_IPSUM)).expect("Error in read file");
511            assert_eq!(compression, compression::Format::No);
512        }
513    }
514}
niffler_temp/basic/mod.rs

niffler_temp/basic/
mod.rs
