compress_manager/
lib.rs

1use std::{
2    env, fmt,
3    fs::{self, File},
4    io::{self, BufReader, Cursor, Error, ErrorKind, Read, Write},
5    os::unix::fs::PermissionsExt,
6    path::{Path, PathBuf},
7};
8
9use flate2::{
10    bufread::{GzDecoder, GzEncoder},
11    Compression,
12};
13use fs_extra;
14use path_clean::PathClean;
15use tar::{Archive, Builder};
16use walkdir::{DirEntry, WalkDir};
17use zip::{write::FileOptions, ZipArchive, ZipWriter};
18use zstd;
19
20/// Represents the compression encoding algorithm.
21#[derive(Eq, PartialEq, Clone)]
22pub enum Encoder {
23    /// Encodes with "Gzip" compression.
24    Gzip,
25    /// Encodes with "Zstandard" compression.
26    Zstd(i32),
27    /// Encodes with "Zstandard" compression and apply base58.
28    ZstdBase58(i32),
29}
30
31/// ref. https://doc.rust-lang.org/std/string/trait.ToString.html
32/// ref. https://doc.rust-lang.org/std/fmt/trait.Display.html
33/// Use "Self.to_string()" to directly invoke this
34impl fmt::Display for Encoder {
35    /// The last integer is the zstd compression level.
36    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
37        match self {
38            Encoder::Gzip => write!(f, "gzip"),
39            Encoder::Zstd(level) => write!(f, "zstd{}", level),
40            Encoder::ZstdBase58(level) => {
41                write!(f, "zstd-base58{}", level)
42            }
43        }
44    }
45}
46
47impl Encoder {
48    pub fn id(&self) -> &str {
49        match self {
50            Encoder::Gzip => "gzip",
51            Encoder::Zstd(1) => "zstd1",
52            Encoder::Zstd(2) => "zstd2",
53            Encoder::Zstd(3) => "zstd3",
54            Encoder::ZstdBase58(1) => "zstd1-base58",
55            Encoder::ZstdBase58(2) => "zstd2-base58",
56            Encoder::ZstdBase58(3) => "zstd3-base58",
57            _ => "unknown",
58        }
59    }
60
61    pub fn new(id: &str) -> io::Result<Self> {
62        match id {
63            "gzip" => Ok(Encoder::Gzip),
64            "zstd1" => Ok(Encoder::Zstd(1)),
65            "zstd2" => Ok(Encoder::Zstd(2)),
66            "zstd3" => Ok(Encoder::Zstd(3)),
67            "zstd1-base58" => Ok(Encoder::ZstdBase58(1)),
68            "zstd2-base58" => Ok(Encoder::ZstdBase58(2)),
69            "zstd3-base58" => Ok(Encoder::ZstdBase58(3)),
70            _ => Err(Error::new(
71                ErrorKind::InvalidInput,
72                format!("unknown id {}", id),
73            )),
74        }
75    }
76
77    pub fn suffix(&self) -> &str {
78        match self {
79            Encoder::Gzip => "gz",
80            Encoder::Zstd(_) => "zstd",
81            Encoder::ZstdBase58(_) => "zstd.base58",
82        }
83    }
84
85    pub fn ext(&self) -> &str {
86        match self {
87            Encoder::Gzip => ".gz",
88            Encoder::Zstd(_) => ".zstd",
89            Encoder::ZstdBase58(_) => ".zstd.base58",
90        }
91    }
92}
93
94/// Represents the compression decoding algorithm.
95#[derive(Clone)]
96pub enum Decoder {
97    Gzip,
98    Zstd,
99    ZstdBase58,
100}
101
102/// ref. https://doc.rust-lang.org/std/string/trait.ToString.html
103/// ref. https://doc.rust-lang.org/std/fmt/trait.Display.html
104/// Use "Self.to_string()" to directly invoke this
105impl fmt::Display for Decoder {
106    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
107        match self {
108            Decoder::Gzip => write!(f, "gzip"),
109            Decoder::Zstd => write!(f, "zstd"),
110            Decoder::ZstdBase58 => write!(f, "zstd-base58"),
111        }
112    }
113}
114
115impl Decoder {
116    pub fn id(&self) -> &str {
117        match self {
118            Decoder::Gzip => "gzip",
119            Decoder::Zstd => "zstd",
120            Decoder::ZstdBase58 => "zstd-base58",
121        }
122    }
123
124    pub fn new(id: &str) -> io::Result<Self> {
125        match id {
126            "gzip" => Ok(Decoder::Gzip),
127            "zstd" => Ok(Decoder::Zstd),
128            "zstd-base58" => Ok(Decoder::ZstdBase58),
129            _ => Err(Error::new(
130                ErrorKind::InvalidInput,
131                format!("unknown id {}", id),
132            )),
133        }
134    }
135}
136
137pub fn pack(d: &[u8], enc: Encoder) -> io::Result<Vec<u8>> {
138    let size_before = d.len() as f64;
139    log::info!(
140        "packing (algorithm {}, current size {})",
141        enc.to_string(),
142        human_readable::bytes(size_before),
143    );
144
145    let packed = match enc {
146        Encoder::Gzip => {
147            let mut gz = GzEncoder::new(Cursor::new(d), Compression::default());
148            let mut encoded = Vec::new();
149            gz.read_to_end(&mut encoded)?;
150            encoded
151        }
152        Encoder::Zstd(lvl) => zstd::stream::encode_all(Cursor::new(d), lvl)?,
153        Encoder::ZstdBase58(lvl) => {
154            let encoded = zstd::stream::encode_all(Cursor::new(d), lvl)?;
155            bs58::encode(encoded).into_vec()
156        }
157    };
158
159    let size_after = packed.len() as f64;
160    log::info!(
161        "packed to {} (before {}, new size {})",
162        enc.to_string(),
163        human_readable::bytes(size_before),
164        human_readable::bytes(size_after),
165    );
166    Ok(packed)
167}
168
169pub fn unpack(d: &[u8], dec: Decoder) -> io::Result<Vec<u8>> {
170    let size_before = d.len() as f64;
171    log::info!(
172        "unpacking (algorithm {}, current size {})",
173        dec.to_string(),
174        human_readable::bytes(size_before),
175    );
176
177    let unpacked = match dec {
178        Decoder::Gzip => {
179            let mut gz = GzDecoder::new(Cursor::new(d));
180            let mut decoded = Vec::new();
181            gz.read_to_end(&mut decoded)?;
182            decoded
183        }
184        Decoder::Zstd => zstd::stream::decode_all(Cursor::new(d))?,
185        Decoder::ZstdBase58 => {
186            let d_decoded = match bs58::decode(d).into_vec() {
187                Ok(v) => v,
188                Err(e) => {
189                    return Err(Error::new(
190                        ErrorKind::Other,
191                        format!("failed bs58::decode {}", e),
192                    ));
193                }
194            };
195            zstd::stream::decode_all(Cursor::new(d_decoded))?
196        }
197    };
198
199    let size_after = unpacked.len() as f64;
200    log::info!(
201        "unpacked to {} (before {}, new size {})",
202        dec.to_string(),
203        human_readable::bytes(size_before),
204        human_readable::bytes(size_after),
205    );
206    Ok(unpacked)
207}
208
209/// Compresses the contents in "src_path" using compression algorithms
210/// and saves it to "dst_path". Note that even if "dst_path" already exists,
211/// it truncates (overwrites).
212///
213/// Make sure to use stream -- reading the entire file at once may cause OOM!
214///
215///  let d = fs::read(src_path)?;
216///  let compressed = zstd::stream::encode_all(Cursor::new(&d[..]), lvl)?;
217///
218///  let d = fs::read(src_path)?;
219///  let decoded = zstd::stream::decode_all(Cursor::new(&d[..]))?;
220///
221///  let mut dec = zstd::Decoder::new(BufReader::new(f1))?;
222///  let mut decoded = Vec::new();
223///  dec.read_to_end(&mut decoded)?;
224///  f2.write_all(&decoded[..])?;
225///
226pub fn pack_file(src_path: &str, dst_path: &str, enc: Encoder) -> io::Result<()> {
227    let meta = fs::metadata(src_path)?;
228    let size_before = meta.len() as f64;
229    log::info!(
230        "packing file '{}' to '{}' (algorithm {}, current size {})",
231        src_path,
232        dst_path,
233        enc.to_string(),
234        human_readable::bytes(size_before),
235    );
236
237    match enc {
238        Encoder::Gzip => {
239            let f1 = File::open(src_path)?;
240            let mut f2 = File::create(dst_path)?;
241
242            let mut enc = GzEncoder::new(BufReader::new(f1), Compression::default());
243
244            // reads from reader (enc) and writes to writer "f2"
245            io::copy(&mut enc, &mut f2)?;
246        }
247        Encoder::Zstd(lvl) => {
248            let mut f1 = File::open(src_path)?;
249            let f2 = File::create(dst_path)?;
250
251            let mut enc = zstd::Encoder::new(f2, lvl)?;
252
253            // reads from reader (f1) and writes to writer "enc"
254            io::copy(&mut f1, &mut enc)?;
255            enc.finish()?;
256        }
257        Encoder::ZstdBase58(lvl) => {
258            // reading the entire file at once may cause OOM...
259            let d = fs::read(src_path)?;
260            let encoded = pack(&d, Encoder::ZstdBase58(lvl))?;
261            let mut f = File::create(dst_path)?;
262            f.write_all(&encoded[..])?;
263        }
264    };
265
266    let meta = fs::metadata(dst_path)?;
267    let size_after = meta.len() as f64;
268    log::info!(
269        "packed file '{}' to '{}' (algorithm {}, before {}, new size {})",
270        src_path,
271        dst_path,
272        enc.to_string(),
273        human_readable::bytes(size_before),
274        human_readable::bytes(size_after),
275    );
276    Ok(())
277}
278
279/// Decompresses the contents in "src_path" using compression algorithms
280/// and saves it to "dst_path". Note that even if "dst_path" already exists,
281/// it truncates (overwrites).
282///
283/// Make sure to use stream -- reading the entire file at once may cause OOM!
284///
285///  let d = fs::read(src_path)?;
286///  let compressed = zstd::stream::encode_all(Cursor::new(&d[..]), lvl)?;
287///
288///  let d = fs::read(src_path)?;
289///  let decoded = zstd::stream::decode_all(Cursor::new(&d[..]))?;
290///
291///  let mut dec = zstd::Decoder::new(BufReader::new(f1))?;
292///  let mut decoded = Vec::new();
293///  dec.read_to_end(&mut decoded)?;
294///  f2.write_all(&decoded[..])?;
295///
296pub fn unpack_file(src_path: &str, dst_path: &str, dec: Decoder) -> io::Result<()> {
297    let meta = fs::metadata(src_path)?;
298    let size_before = meta.len() as f64;
299    log::info!(
300        "unpacking file '{}' to '{}' (algorithm {}, current size {})",
301        src_path,
302        dst_path,
303        dec.to_string(),
304        human_readable::bytes(size_before),
305    );
306
307    match dec {
308        Decoder::Gzip => {
309            let f1 = File::open(src_path)?;
310            let mut f2 = File::create(dst_path)?;
311
312            let mut dec = GzDecoder::new(BufReader::new(f1));
313
314            // reads from reader (dec) and writes to writer "f2"
315            io::copy(&mut dec, &mut f2)?;
316        }
317        Decoder::Zstd => {
318            let f1 = File::open(src_path)?;
319            let mut f2 = File::create(dst_path)?;
320
321            let mut dec = zstd::Decoder::new(BufReader::new(f1))?;
322
323            // reads from reader (dec) and writes to writer "f2"
324            io::copy(&mut dec, &mut f2)?;
325        }
326        Decoder::ZstdBase58 => {
327            // reading the entire file at once may cause OOM...
328            let d = fs::read(src_path)?;
329            let decoded = unpack(&d, Decoder::ZstdBase58)?;
330            let mut f = File::create(dst_path)?;
331            f.write_all(&decoded[..])?;
332        }
333    };
334
335    let meta = fs::metadata(dst_path)?;
336    let size_after = meta.len() as f64;
337    log::info!(
338        "unpacked file '{}' to '{}' (algorithm {}, before {}, new size {})",
339        src_path,
340        dst_path,
341        dec.to_string(),
342        human_readable::bytes(size_before),
343        human_readable::bytes(size_after),
344    );
345    Ok(())
346}
347
348/// Represents the compression encoding algorithm for directory.
349#[derive(Clone)]
350pub enum DirEncoder {
351    /// Archives the directory with "zip" with no compression.
352    Zip,
353    /// Archives the directory with "tar" and
354    /// and encodes with "Gzip" compression.
355    TarGzip,
356    /// Archives the directory with "zip" and
357    /// and encodes with "Gzip" compression.
358    ZipGzip,
359    /// Archives the directory with "tar" and
360    /// encodes with "Zstandard" compression.
361    TarZstd(i32),
362    /// Archives the directory with "zip" and
363    /// encodes with "Zstandard" compression.
364    ZipZstd(i32),
365}
366
367/// ref. https://doc.rust-lang.org/std/string/trait.ToString.html
368/// ref. https://doc.rust-lang.org/std/fmt/trait.Display.html
369/// Use "Self.to_string()" to directly invoke this
370impl fmt::Display for DirEncoder {
371    /// The last integer is the zstd compression level.
372    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
373        match self {
374            DirEncoder::Zip => write!(f, "zip"),
375            DirEncoder::TarGzip => write!(f, "tar-gzip"),
376            DirEncoder::ZipGzip => write!(f, "zip-gzip"),
377            DirEncoder::TarZstd(level) => write!(f, "tar-zstd{}", level),
378            DirEncoder::ZipZstd(level) => write!(f, "zip-zstd{}", level),
379        }
380    }
381}
382
383impl DirEncoder {
384    pub fn id(&self) -> &str {
385        match self {
386            DirEncoder::Zip => "zip",
387            DirEncoder::TarGzip => "tar-gzip",
388            DirEncoder::ZipGzip => "zip-gzip",
389            DirEncoder::TarZstd(1) => "tar-zstd1",
390            DirEncoder::TarZstd(2) => "tar-zstd2",
391            DirEncoder::TarZstd(3) => "tar-zstd3",
392            DirEncoder::ZipZstd(1) => "zip-zstd1",
393            DirEncoder::ZipZstd(2) => "zip-zstd2",
394            DirEncoder::ZipZstd(3) => "zip-zstd3",
395            _ => "unknown",
396        }
397    }
398
399    pub fn new(id: &str) -> io::Result<Self> {
400        match id {
401            "zip" => Ok(DirEncoder::Zip),
402            "tar-gzip" => Ok(DirEncoder::TarGzip),
403            "zip-gzip" => Ok(DirEncoder::ZipGzip),
404            "tar-zstd1" => Ok(DirEncoder::TarZstd(1)),
405            "tar-zstd2" => Ok(DirEncoder::TarZstd(2)),
406            "tar-zstd3" => Ok(DirEncoder::TarZstd(3)),
407            "zip-zstd1" => Ok(DirEncoder::ZipZstd(1)),
408            "zip-zstd2" => Ok(DirEncoder::ZipZstd(2)),
409            "zip-zstd3" => Ok(DirEncoder::ZipZstd(3)),
410            _ => Err(Error::new(
411                ErrorKind::InvalidInput,
412                format!("unknown id {}", id),
413            )),
414        }
415    }
416
417    pub fn suffix(&self) -> &str {
418        match self {
419            DirEncoder::Zip => ".zip",
420            DirEncoder::TarGzip => "tar.gz",
421            DirEncoder::ZipGzip => "zip.gz",
422            DirEncoder::TarZstd(_) => "tar.zstd",
423            DirEncoder::ZipZstd(_) => "zip.zstd",
424        }
425    }
426
427    pub fn ext(&self) -> &str {
428        match self {
429            DirEncoder::Zip => ".zip",
430            DirEncoder::TarGzip => ".tar.gz",
431            DirEncoder::ZipGzip => ".zip.gz",
432            DirEncoder::TarZstd(_) => ".tar.zstd",
433            DirEncoder::ZipZstd(_) => ".zip.zstd",
434        }
435    }
436}
437
438/// Represents the compression decoding algorithm for directory.
439#[derive(Clone)]
440pub enum DirDecoder {
441    Zip,
442    TarGzip,
443    ZipGzip,
444    TarZstd,
445    ZipZstd,
446}
447
448/// ref. https://doc.rust-lang.org/std/string/trait.ToString.html
449/// ref. https://doc.rust-lang.org/std/fmt/trait.Display.html
450/// Use "Self.to_string()" to directly invoke this
451impl fmt::Display for DirDecoder {
452    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
453        match self {
454            DirDecoder::Zip => write!(f, "zip"),
455            DirDecoder::TarGzip => write!(f, "tar-gzip"),
456            DirDecoder::ZipGzip => write!(f, "zip-gzip"),
457            DirDecoder::TarZstd => write!(f, "tar-zstd"),
458            DirDecoder::ZipZstd => write!(f, "zip-zstd"),
459        }
460    }
461}
462
463impl DirDecoder {
464    pub fn id(&self) -> &str {
465        match self {
466            DirDecoder::Zip => "zip",
467            DirDecoder::TarGzip => "tar-gzip",
468            DirDecoder::ZipGzip => "zip-gzip",
469            DirDecoder::TarZstd => "tar-zstd",
470            DirDecoder::ZipZstd => "zip-zstd",
471        }
472    }
473
474    pub fn new(id: &str) -> io::Result<Self> {
475        match id {
476            "zip" => Ok(DirDecoder::Zip),
477            "tar-gzip" => Ok(DirDecoder::TarGzip),
478            "zip-gzip" => Ok(DirDecoder::ZipGzip),
479            "tar-zstd" => Ok(DirDecoder::TarZstd),
480            "zip-zstd" => Ok(DirDecoder::ZipZstd),
481            _ => Err(Error::new(
482                ErrorKind::InvalidInput,
483                format!("unknown id {}", id),
484            )),
485        }
486    }
487
488    pub fn new_from_file_name(name: &str) -> io::Result<Self> {
489        if name.ends_with(DirDecoder::Zip.suffix()) {
490            Ok(DirDecoder::Zip)
491        } else if name.ends_with(DirDecoder::TarGzip.suffix()) {
492            Ok(DirDecoder::TarGzip)
493        } else if name.ends_with(DirDecoder::ZipGzip.suffix()) {
494            Ok(DirDecoder::ZipGzip)
495        } else if name.ends_with(DirDecoder::TarZstd.suffix()) {
496            Ok(DirDecoder::TarZstd)
497        } else if name.ends_with(DirDecoder::ZipZstd.suffix()) {
498            Ok(DirDecoder::ZipZstd)
499        } else {
500            Err(Error::new(
501                ErrorKind::InvalidInput,
502                format!("unknown suffix {}", name),
503            ))
504        }
505    }
506
507    pub fn suffix(&self) -> &str {
508        match self {
509            DirDecoder::Zip => ".zip",
510            DirDecoder::TarGzip => "tar.gz",
511            DirDecoder::ZipGzip => "zip.gz",
512            DirDecoder::TarZstd => "tar.zstd",
513            DirDecoder::ZipZstd => "zip.zstd",
514        }
515    }
516
517    pub fn ext(&self) -> &str {
518        match self {
519            DirDecoder::Zip => ".zip",
520            DirDecoder::TarGzip => ".tar.gz",
521            DirDecoder::ZipGzip => ".zip.gz",
522            DirDecoder::TarZstd => ".tar.zstd",
523            DirDecoder::ZipZstd => ".zip.zstd",
524        }
525    }
526
527    pub fn compression_ext(&self) -> &str {
528        match self {
529            DirDecoder::Zip => ".zip",
530            DirDecoder::TarGzip => ".gz",
531            DirDecoder::ZipGzip => ".gz",
532            DirDecoder::TarZstd => ".zstd",
533            DirDecoder::ZipZstd => ".zstd",
534        }
535    }
536}
537
538/// Archives the source directory "src_dir_path" with archival method and compression
539/// and saves to "dst_path". If "dst_path" exists, it overwrites.
540pub fn pack_directory(src_dir_path: &str, dst_path: &str, enc: DirEncoder) -> io::Result<()> {
541    if Path::new(src_dir_path).parent().is_none() {
542        return Err(Error::new(
543            ErrorKind::Other,
544            format!("cannot archive root directory {}", src_dir_path),
545        ));
546    };
547    let size = fs_extra::dir::get_size(src_dir_path).map_err(|e| {
548        Error::new(
549            ErrorKind::Other,
550            format!("failed get_size {} for directory {}", e, src_dir_path),
551        )
552    })?;
553    let size_before = size as f64;
554    log::info!(
555        "packing directory from '{}' to '{}' (algorithm {}, current size {})",
556        src_dir_path,
557        dst_path,
558        enc.to_string(),
559        human_readable::bytes(size_before),
560    );
561
562    let parent_dir = Path::new(src_dir_path)
563        .parent()
564        .expect("unexpected no parent dir");
565    let archive_path =
566        parent_dir.join(random_manager::tmp_path(10, None).expect("expected some tmp_path"));
567    let archive_path = archive_path
568        .as_path()
569        .to_str()
570        .expect("unexpected None path");
571    let archive_file = File::create(&archive_path)?;
572    match enc {
573        DirEncoder::Zip => {
574            let mut zip = ZipWriter::new(archive_file);
575
576            let mut buffer = Vec::new();
577            let src_dir = Path::new(src_dir_path);
578            let src_dir_full_path = absolute_path(src_dir)?;
579
580            let options = FileOptions::default()
581                .compression_method(zip::CompressionMethod::Stored)
582                .unix_permissions(0o755);
583            for entry in WalkDir::new(src_dir_path).into_iter() {
584                let entry = match entry {
585                    Ok(v) => v,
586                    Err(e) => {
587                        return Err(Error::new(
588                            ErrorKind::Other,
589                            format!("failed walk dir {} ({})", src_dir_path, e),
590                        ));
591                    }
592                };
593
594                let full_path = absolute_path(entry.path())?;
595                // relative path from source directory
596                // e.g., "text/a/b/c.txt" for absolute path "/tmp/text/a/b/c.txt"
597                let rel_path = match full_path.strip_prefix(&src_dir_full_path) {
598                    Ok(v) => v,
599                    Err(e) => {
600                        return Err(Error::new(
601                            ErrorKind::Other,
602                            format!("failed strip_prefix on {:?} ({})", full_path, e),
603                        ));
604                    }
605                };
606
607                if is_dir(&entry) {
608                    // only if not root
609                    // ref. https://github.com/zip-rs/zip/blob/master/examples/write_dir.rs
610                    if !rel_path.as_os_str().is_empty() {
611                        let dir_name = rel_path
612                            .as_os_str()
613                            .to_str()
614                            .expect("unexpected None os_str");
615                        log::info!("adding directory {}", dir_name);
616                        zip.add_directory(dir_name, options)?;
617                    }
618                    continue;
619                }
620
621                let file_name = rel_path
622                    .as_os_str()
623                    .to_str()
624                    .expect("unexpected None os_str");
625                log::info!("adding file {}", file_name);
626                zip.start_file(file_name, options)?;
627                let mut f = File::open(full_path)?;
628                f.read_to_end(&mut buffer)?;
629                zip.write_all(&*buffer)?;
630                buffer.clear();
631            }
632            zip.finish()?;
633
634            log::info!("renaming archived file {} to {}", archive_path, dst_path);
635            fs::rename(archive_path, dst_path)?;
636        }
637
638        DirEncoder::TarGzip => {
639            // e.g.,
640            // tar -czvf db.tar.gz mainnet/
641            // -c to create a new archive
642            // -z to enable "--gzip" mode
643            // -v to enable verbose mode
644            // -f to specify the file
645            let mut tar = Builder::new(archive_file);
646            let src_dir = Path::new(src_dir_path);
647            let src_dir_full_path = absolute_path(src_dir)?;
648            for entry in WalkDir::new(src_dir_path).into_iter() {
649                let entry = match entry {
650                    Ok(v) => v,
651                    Err(e) => {
652                        return Err(Error::new(
653                            ErrorKind::Other,
654                            format!("failed walk dir {} ({})", src_dir_path, e),
655                        ));
656                    }
657                };
658
659                let full_path = absolute_path(entry.path())?;
660                // relative path from source directory
661                // e.g., "text/a/b/c.txt" for absolute path "/tmp/text/a/b/c.txt"
662                let rel_path = match full_path.strip_prefix(&src_dir_full_path) {
663                    Ok(v) => v,
664                    Err(e) => {
665                        return Err(Error::new(
666                            ErrorKind::Other,
667                            format!("failed strip_prefix on {:?} ({})", full_path, e),
668                        ));
669                    }
670                };
671
672                if is_dir(&entry) {
673                    continue;
674                }
675
676                let file_name = rel_path
677                    .as_os_str()
678                    .to_str()
679                    .expect("unexpected None os_str");
680                log::info!("adding file {}", file_name);
681                let mut f = File::open(&full_path)?;
682                tar.append_file(&file_name, &mut f)?;
683            }
684            pack_file(archive_path, dst_path, Encoder::Gzip)?;
685        }
686
687        DirEncoder::ZipGzip => {
688            let mut zip = ZipWriter::new(archive_file);
689
690            let mut buffer = Vec::new();
691            let src_dir = Path::new(src_dir_path);
692            let src_dir_full_path = absolute_path(src_dir)?;
693
694            let options = FileOptions::default()
695                .compression_method(zip::CompressionMethod::Stored)
696                .unix_permissions(0o755);
697            for entry in WalkDir::new(src_dir_path).into_iter() {
698                let entry = match entry {
699                    Ok(v) => v,
700                    Err(e) => {
701                        return Err(Error::new(
702                            ErrorKind::Other,
703                            format!("failed walk dir {} ({})", src_dir_path, e),
704                        ));
705                    }
706                };
707
708                let full_path = absolute_path(entry.path())?;
709                // relative path from source directory
710                // e.g., "text/a/b/c.txt" for absolute path "/tmp/text/a/b/c.txt"
711                let rel_path = match full_path.strip_prefix(&src_dir_full_path) {
712                    Ok(v) => v,
713                    Err(e) => {
714                        return Err(Error::new(
715                            ErrorKind::Other,
716                            format!("failed strip_prefix on {:?} ({})", full_path, e),
717                        ));
718                    }
719                };
720
721                if is_dir(&entry) {
722                    // only if not root
723                    // ref. https://github.com/zip-rs/zip/blob/master/examples/write_dir.rs
724                    if !rel_path.as_os_str().is_empty() {
725                        let dir_name = rel_path
726                            .as_os_str()
727                            .to_str()
728                            .expect("unexpected None os_str");
729                        log::info!("adding directory {}", dir_name);
730                        zip.add_directory(dir_name, options)?;
731                    }
732                    continue;
733                }
734
735                let file_name = rel_path
736                    .as_os_str()
737                    .to_str()
738                    .expect("unexpected None os_str");
739                log::info!("adding file {}", file_name);
740                zip.start_file(file_name, options)?;
741                let mut f = File::open(full_path)?;
742                f.read_to_end(&mut buffer)?;
743                zip.write_all(&*buffer)?;
744                buffer.clear();
745            }
746            zip.finish()?;
747            pack_file(archive_path, dst_path, Encoder::Gzip)?;
748        }
749
750        DirEncoder::TarZstd(lvl) => {
751            let mut tar = Builder::new(archive_file);
752            let src_dir = Path::new(src_dir_path);
753            let src_dir_full_path = absolute_path(src_dir)?;
754            for entry in WalkDir::new(src_dir_path).into_iter() {
755                let entry = match entry {
756                    Ok(v) => v,
757                    Err(e) => {
758                        return Err(Error::new(
759                            ErrorKind::Other,
760                            format!("failed walk dir {} ({})", src_dir_path, e),
761                        ));
762                    }
763                };
764
765                let full_path = absolute_path(entry.path())?;
766                // relative path from source directory
767                // e.g., "text/a/b/c.txt" for absolute path "/tmp/text/a/b/c.txt"
768                let rel_path = match full_path.strip_prefix(&src_dir_full_path) {
769                    Ok(v) => v,
770                    Err(e) => {
771                        return Err(Error::new(
772                            ErrorKind::Other,
773                            format!("failed strip_prefix on {:?} ({})", full_path, e),
774                        ));
775                    }
776                };
777
778                if is_dir(&entry) {
779                    continue;
780                }
781
782                let file_name = rel_path
783                    .as_os_str()
784                    .to_str()
785                    .expect("unexpected None os_str");
786                log::info!("adding file {}", file_name);
787                let mut f = File::open(&full_path)?;
788                tar.append_file(&file_name, &mut f)?;
789            }
790            pack_file(archive_path, dst_path, Encoder::Zstd(lvl))?;
791        }
792
793        DirEncoder::ZipZstd(lvl) => {
794            let mut zip = ZipWriter::new(archive_file);
795
796            let mut buffer = Vec::new();
797            let src_dir = Path::new(src_dir_path);
798            let src_dir_full_path = absolute_path(src_dir)?;
799
800            let options = FileOptions::default()
801                .compression_method(zip::CompressionMethod::Stored)
802                .unix_permissions(0o755);
803            for entry in WalkDir::new(src_dir_path).into_iter() {
804                let entry = match entry {
805                    Ok(v) => v,
806                    Err(e) => {
807                        return Err(Error::new(
808                            ErrorKind::Other,
809                            format!("failed walk dir {} ({})", src_dir_path, e),
810                        ));
811                    }
812                };
813
814                let full_path = absolute_path(entry.path())?;
815                // relative path from source directory
816                // e.g., "text/a/b/c.txt" for absolute path "/tmp/text/a/b/c.txt"
817                let rel_path = match full_path.strip_prefix(&src_dir_full_path) {
818                    Ok(v) => v,
819                    Err(e) => {
820                        return Err(Error::new(
821                            ErrorKind::Other,
822                            format!("failed strip_prefix on {:?} ({})", full_path, e),
823                        ));
824                    }
825                };
826
827                if is_dir(&entry) {
828                    // only if not root
829                    // ref. https://github.com/zip-rs/zip/blob/master/examples/write_dir.rs
830                    if !rel_path.as_os_str().is_empty() {
831                        let dir_name = rel_path
832                            .as_os_str()
833                            .to_str()
834                            .expect("unexpected None os_str");
835                        log::info!("adding directory {}", dir_name);
836                        zip.add_directory(dir_name, options)?;
837                    }
838                    continue;
839                }
840
841                let file_name = rel_path
842                    .as_os_str()
843                    .to_str()
844                    .expect("unexpected None os_str");
845                log::info!("adding file {}", file_name);
846                zip.start_file(file_name, options)?;
847                let mut f = File::open(full_path)?;
848                f.read_to_end(&mut buffer)?;
849                zip.write_all(&*buffer)?;
850                buffer.clear();
851            }
852            zip.finish()?;
853            pack_file(archive_path, dst_path, Encoder::Zstd(lvl))?;
854        }
855    }
856
857    let meta = fs::metadata(dst_path)?;
858    let size_after = meta.len() as f64;
859    log::info!(
860        "packed directory from '{}' to '{}' (algorithm {}, before {}, new size {})",
861        src_dir_path,
862        dst_path,
863        enc.to_string(),
864        human_readable::bytes(size_before),
865        human_readable::bytes(size_after),
866    );
867    Ok(())
868}
869
870/// Un-archives the file with compression and archival method
871/// to the destination directory "dst_dir_path".
872pub fn unpack_directory(
873    src_archive_path: &str,
874    dst_dir_path: &str,
875    dec: DirDecoder,
876) -> io::Result<()> {
877    let meta = fs::metadata(src_archive_path)?;
878    let size_before = meta.len() as f64;
879    log::info!(
880        "unpacking directory from '{}' to '{}' (algorithm {}, current size {})",
881        src_archive_path,
882        dst_dir_path,
883        dec.to_string(),
884        human_readable::bytes(size_before),
885    );
886    fs::create_dir_all(dst_dir_path)?;
887    let target_dir = Path::new(dst_dir_path);
888
889    // ref. https://stackoverflow.com/questions/48225618/ls-permission-denied-even-though-i-have-read-access-to-the-directory/48225881#48225881
890    fs::set_permissions(target_dir, PermissionsExt::from_mode(0o775))?;
891
892    let decompressed_path = {
893        if src_archive_path.ends_with(dec.compression_ext()) {
894            let p = src_archive_path.replace(dec.compression_ext(), "");
895            if Path::new(&p).exists() {
896                log::info!("decompressed path already exists, removing {}", p);
897                fs::remove_file(&p)?;
898            }
899            p
900        } else {
901            format!("{}.decompressed", src_archive_path)
902        }
903    };
904
905    match dec {
906        DirDecoder::Zip => {
907            log::info!("unarchiving the zip file {}", src_archive_path);
908            let zip_file = File::open(&src_archive_path)?;
909            let mut zip = match ZipArchive::new(zip_file) {
910                Ok(v) => v,
911                Err(e) => {
912                    return Err(Error::new(
913                        ErrorKind::Other,
914                        format!("failed ZipArchive::new on {} ({})", src_archive_path, e),
915                    ));
916                }
917            };
918
919            for i in 0..zip.len() {
920                let mut f = match zip.by_index(i) {
921                    Ok(v) => v,
922                    Err(e) => {
923                        return Err(Error::new(
924                            ErrorKind::Other,
925                            format!("failed zip.by_index ({})", e),
926                        ));
927                    }
928                };
929                let output_path = match f.enclosed_name() {
930                    Some(p) => p.to_owned(),
931                    None => continue,
932                };
933                let output_path = target_dir.join(output_path);
934
935                let is_dir = (*f.name()).ends_with('/');
936                if is_dir {
937                    log::info!("extracting directory {}", output_path.display());
938                    fs::create_dir_all(&output_path)?;
939                } else {
940                    log::info!("extracting file {}", output_path.display());
941                    if let Some(p) = output_path.parent() {
942                        if !p.exists() {
943                            fs::create_dir_all(&p)?;
944                        }
945                    }
946                    let mut f2 = File::create(&output_path)?;
947                    io::copy(&mut f, &mut f2)?;
948                }
949
950                // ref. https://stackoverflow.com/questions/48225618/ls-permission-denied-even-though-i-have-read-access-to-the-directory/48225881#48225881
951                fs::set_permissions(&output_path, PermissionsExt::from_mode(0o775))?;
952            }
953        }
954
955        DirDecoder::TarGzip => {
956            unpack_file(src_archive_path, &decompressed_path, Decoder::Gzip)?;
957
958            log::info!("unarchiving decompressed file {}", decompressed_path);
959            let tar_file = File::open(&decompressed_path)?;
960            let mut tar = Archive::new(tar_file);
961            let entries = tar.entries()?;
962            for file in entries {
963                let mut f = file?;
964                let output_path = f.path()?;
965                let output_path = target_dir.join(output_path);
966                if let Some(p) = output_path.parent() {
967                    if !p.exists() {
968                        fs::create_dir_all(&p)?;
969                    }
970                }
971                if output_path.is_dir()
972                    || output_path
973                        .to_str()
974                        .expect("unexpected None str")
975                        .ends_with('/')
976                {
977                    log::info!("extracting directory {}", output_path.display());
978                    fs::create_dir_all(output_path)?;
979                } else {
980                    log::info!("extracting file {}", output_path.display());
981                    let mut f2 = File::create(&output_path)?;
982                    io::copy(&mut f, &mut f2)?;
983                    fs::set_permissions(&output_path, PermissionsExt::from_mode(0o775))?;
984                }
985            }
986        }
987
988        DirDecoder::ZipGzip => {
989            unpack_file(src_archive_path, &decompressed_path, Decoder::Gzip)?;
990
991            log::info!("unarchiving decompressed file {}", decompressed_path);
992            let zip_file = File::open(&decompressed_path)?;
993            let mut zip = match ZipArchive::new(zip_file) {
994                Ok(v) => v,
995                Err(e) => {
996                    return Err(Error::new(
997                        ErrorKind::Other,
998                        format!("failed ZipArchive::new on {} ({})", decompressed_path, e),
999                    ));
1000                }
1001            };
1002            for i in 0..zip.len() {
1003                let mut f = match zip.by_index(i) {
1004                    Ok(v) => v,
1005                    Err(e) => {
1006                        return Err(Error::new(
1007                            ErrorKind::Other,
1008                            format!("failed zip.by_index ({})", e),
1009                        ));
1010                    }
1011                };
1012                let output_path = match f.enclosed_name() {
1013                    Some(p) => p.to_owned(),
1014                    None => continue,
1015                };
1016                let output_path = target_dir.join(output_path);
1017
1018                let is_dir = (*f.name()).ends_with('/');
1019                if is_dir {
1020                    log::info!("extracting directory {}", output_path.display());
1021                    fs::create_dir_all(&output_path)?;
1022                } else {
1023                    log::info!("extracting file {}", output_path.display());
1024                    if let Some(p) = output_path.parent() {
1025                        if !p.exists() {
1026                            fs::create_dir_all(&p)?;
1027                        }
1028                    }
1029                    let mut f2 = File::create(&output_path)?;
1030                    io::copy(&mut f, &mut f2)?;
1031                }
1032
1033                fs::set_permissions(&output_path, PermissionsExt::from_mode(0o775))?;
1034            }
1035        }
1036
1037        DirDecoder::TarZstd => {
1038            unpack_file(src_archive_path, &decompressed_path, Decoder::Zstd)?;
1039
1040            log::info!("unarchiving decompressed file {}", decompressed_path);
1041            let tar_file = File::open(&decompressed_path)?;
1042            let mut tar = Archive::new(tar_file);
1043            let entries = tar.entries()?;
1044            for file in entries {
1045                let mut f = file?;
1046                let output_path = f.path()?;
1047                let output_path = target_dir.join(output_path);
1048                if let Some(p) = output_path.parent() {
1049                    if !p.exists() {
1050                        fs::create_dir_all(&p)?;
1051                    }
1052                }
1053                if output_path.is_dir()
1054                    || output_path
1055                        .to_str()
1056                        .expect("unexpected None str")
1057                        .ends_with('/')
1058                {
1059                    log::info!("extracting directory {}", output_path.display());
1060                    fs::create_dir_all(output_path)?;
1061                } else {
1062                    log::info!("extracting file {}", output_path.display());
1063                    let mut f2 = File::create(&output_path)?;
1064                    io::copy(&mut f, &mut f2)?;
1065                    fs::set_permissions(&output_path, PermissionsExt::from_mode(0o775))?;
1066                }
1067            }
1068        }
1069
1070        DirDecoder::ZipZstd => {
1071            unpack_file(src_archive_path, &decompressed_path, Decoder::Zstd)?;
1072
1073            log::info!("unarchiving decompressed file {}", decompressed_path);
1074            let zip_file = File::open(&decompressed_path)?;
1075            let mut zip = match ZipArchive::new(zip_file) {
1076                Ok(v) => v,
1077                Err(e) => {
1078                    return Err(Error::new(
1079                        ErrorKind::Other,
1080                        format!("failed ZipArchive::new on {} ({})", decompressed_path, e),
1081                    ));
1082                }
1083            };
1084            for i in 0..zip.len() {
1085                let mut f = match zip.by_index(i) {
1086                    Ok(v) => v,
1087                    Err(e) => {
1088                        return Err(Error::new(
1089                            ErrorKind::Other,
1090                            format!("failed zip.by_index ({})", e),
1091                        ));
1092                    }
1093                };
1094                let output_path = match f.enclosed_name() {
1095                    Some(p) => p.to_owned(),
1096                    None => continue,
1097                };
1098                let output_path = target_dir.join(output_path);
1099
1100                let is_dir = (*f.name()).ends_with('/');
1101                if is_dir {
1102                    log::info!("extracting directory {}", output_path.display());
1103                    fs::create_dir_all(&output_path)?;
1104                } else {
1105                    log::info!("extracting file {}", output_path.display());
1106                    if let Some(p) = output_path.parent() {
1107                        if !p.exists() {
1108                            fs::create_dir_all(&p)?;
1109                        }
1110                    }
1111                    let mut f2 = File::create(&output_path)?;
1112                    io::copy(&mut f, &mut f2)?;
1113                }
1114
1115                fs::set_permissions(&output_path, PermissionsExt::from_mode(0o775))?;
1116            }
1117        }
1118    }
1119
1120    if Path::new(&decompressed_path).exists() {
1121        log::info!(
1122            "removing decompressed file {} after unarchive",
1123            decompressed_path
1124        );
1125        fs::remove_file(decompressed_path)?;
1126    }
1127
1128    let size = fs_extra::dir::get_size(target_dir).map_err(|e| {
1129        Error::new(
1130            ErrorKind::Other,
1131            format!(
1132                "failed get_size {} for directory {}",
1133                e,
1134                target_dir.display()
1135            ),
1136        )
1137    })?;
1138    let size_after = size as f64;
1139    log::info!(
1140        "decompressed directory from '{}' to '{}' (algorithm {}, before {}, new size {})",
1141        src_archive_path,
1142        dst_dir_path,
1143        dec.to_string(),
1144        human_readable::bytes(size_before),
1145        human_readable::bytes(size_after),
1146    );
1147    Ok(())
1148}
1149
1150fn is_dir(entry: &DirEntry) -> bool {
1151    entry.file_type().is_dir()
1152}
1153
1154fn absolute_path(path: impl AsRef<Path>) -> io::Result<PathBuf> {
1155    let p = path.as_ref();
1156
1157    let ap = if p.is_absolute() {
1158        p.to_path_buf()
1159    } else {
1160        env::current_dir()?.join(p)
1161    }
1162    .clean();
1163
1164    Ok(ap)
1165}
1166
1167/// RUST_LOG=debug cargo test --all-features --package avalanche-utils --lib -- compress::test_pack_unpack --exact --show-output
1168#[test]
1169fn test_pack_unpack() {
1170    let _ = env_logger::builder()
1171        .filter_level(log::LevelFilter::Info)
1172        .is_test(true)
1173        .try_init();
1174
1175    let contents = vec![7; 1 * 1024 * 1024];
1176
1177    let encs = vec![
1178        "gzip",
1179        "zstd1",
1180        "zstd2",
1181        "zstd3",
1182        "zstd1-base58",
1183        "zstd2-base58",
1184        "zstd3-base58",
1185    ];
1186    let decs = vec![
1187        "gzip",
1188        "zstd",
1189        "zstd",
1190        "zstd",
1191        "zstd-base58",
1192        "zstd-base58",
1193        "zstd-base58",
1194    ];
1195    for (i, _) in encs.iter().enumerate() {
1196        let encoder = Encoder::new(encs[i]).unwrap();
1197        let decoder = Decoder::new(decs[i]).unwrap();
1198
1199        let packed = pack(&contents, encoder).unwrap();
1200
1201        // compressed should be smaller
1202        if !encs[i].contains("base58") {
1203            assert!(contents.len() > packed.len());
1204        }
1205
1206        let unpacked = unpack(&packed, decoder).unwrap();
1207
1208        // decompressed should be same as original
1209        assert_eq!(contents, unpacked);
1210    }
1211
1212    let mut orig_file = tempfile::NamedTempFile::new().unwrap();
1213    orig_file.write_all(&contents).unwrap();
1214    let orig_path = orig_file.path().to_str().unwrap();
1215    let orig_meta = fs::metadata(orig_path).unwrap();
1216    for (i, _) in encs.iter().enumerate() {
1217        let encoder = Encoder::new(encs[i]).unwrap();
1218        let decoder = Decoder::new(decs[i]).unwrap();
1219
1220        let packed = tempfile::NamedTempFile::new().unwrap();
1221        let packed_path = packed.path().to_str().unwrap();
1222        pack_file(&orig_path, packed_path, encoder).unwrap();
1223
1224        // compressed file should be smaller
1225        if !encs[i].contains("base58") {
1226            let meta_packed = fs::metadata(packed_path).unwrap();
1227            assert!(orig_meta.len() > meta_packed.len());
1228        }
1229
1230        let unpacked = tempfile::NamedTempFile::new().unwrap();
1231        let unpacked_path = unpacked.path().to_str().unwrap();
1232        unpack_file(packed_path, unpacked_path, decoder).unwrap();
1233
1234        // decompressed file should be same as original
1235        let contents_unpacked = fs::read(unpacked_path).unwrap();
1236        assert_eq!(contents, contents_unpacked);
1237    }
1238
1239    let src_dir_path_buf = env::temp_dir().join(random_manager::secure_string(10));
1240    fs::create_dir_all(&src_dir_path_buf).unwrap();
1241    let src_dir_path = src_dir_path_buf.to_str().unwrap();
1242    for _i in 0..20 {
1243        let p = src_dir_path_buf.join(random_manager::secure_string(10));
1244        let mut f = File::create(&p).unwrap();
1245        f.write_all(&contents).unwrap();
1246    }
1247    log::info!("created {}", src_dir_path_buf.display());
1248    let src_dir_size = fs_extra::dir::get_size(src_dir_path_buf.clone()).unwrap();
1249
1250    let encs = vec![
1251        "zip",
1252        "tar-gzip",
1253        "zip-gzip",
1254        "tar-zstd1",
1255        "tar-zstd2",
1256        "tar-zstd3",
1257        "zip-zstd1",
1258        "zip-zstd2",
1259        "zip-zstd3",
1260    ];
1261    let decs = vec![
1262        "zip",      // zip decoder has no level
1263        "tar-gzip", // gzip decoder has no level
1264        "zip-gzip", // gzip decoder has no level
1265        "tar-zstd", // zstd decoder has no level
1266        "tar-zstd", // zstd decoder has no level
1267        "tar-zstd", // zstd decoder has no level
1268        "zip-zstd", // zstd decoder has no level
1269        "zip-zstd", // zstd decoder has no level
1270        "zip-zstd", // zstd decoder has no level
1271    ];
1272    for (i, _) in encs.iter().enumerate() {
1273        let encoder = DirEncoder::new(encs[i]).unwrap();
1274        let decoder = DirDecoder::new(decs[i]).unwrap();
1275
1276        let compressed_path = random_manager::tmp_path(10, Some(encoder.suffix())).unwrap();
1277        pack_directory(src_dir_path, &compressed_path, encoder).unwrap();
1278
1279        if encs[i].ne("zip") {
1280            // archived/compressed file should be smaller
1281            let meta_packed = fs::metadata(&compressed_path).unwrap();
1282            assert!(src_dir_size > meta_packed.len());
1283        }
1284
1285        let dst_dir_path = random_manager::tmp_path(10, None).unwrap();
1286        unpack_directory(&compressed_path, &dst_dir_path, decoder).unwrap();
1287
1288        fs::remove_file(compressed_path).unwrap();
1289        fs::remove_dir_all(dst_dir_path).unwrap();
1290    }
1291    fs::remove_dir_all(src_dir_path).unwrap();
1292}
compress_manager/lib.rs

compress_manager/
lib.rs
