ouch 0.7.1

A command-line utility for easily compressing and decompressing files and directories.
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207

use std::{
    io::{self, BufWriter, Cursor, Seek, Write},
    num::NonZeroU64,
    path::{Path, PathBuf},
};

use fs_err as fs;
use gzp::par::compress::{ParCompress, ParCompressBuilder};

use super::warn_user_about_loading_sevenz_in_memory;
use crate::{
    BUFFER_CAPACITY, QuestionAction, QuestionPolicy, Result, archive,
    commands::warn_user_about_loading_zip_in_memory,
    extension::{CompressionFormat::*, Extension, split_first_compression_format},
    info_accessible,
    utils::{
        BytesFmt, FileVisibilityPolicy, file_size,
        io::lock_and_flush_output_stdio,
        threads::{logical_thread_count, physical_thread_count},
        user_wants_to_continue,
    },
};

/// Compress files into `output_file`.
///
/// # Arguments:
/// - `files`: is the list of paths to be compressed: ["dir/file1.txt", "dir/file2.txt"]
/// - `extensions`: is a list of compression formats for compressing, example: [Tar, Gz] (in compression order)
/// - `output_file` is the resulting compressed file name, example: "archive.tar.gz"
///
/// # Return value
/// - Returns `Ok(true)` if compressed all files normally.
/// - Returns `Ok(false)` if user opted to abort compression mid-way.
pub fn compress_files(
    files: Vec<PathBuf>,
    extensions: Vec<Extension>,
    output_file: fs::File,
    output_path: &Path,
    follow_symlinks: bool,
    question_policy: QuestionPolicy,
    file_visibility_policy: FileVisibilityPolicy,
    level: Option<i16>,
) -> Result<bool> {
    // If the input files contain a directory, then the total size will be underestimated
    let file_writer = BufWriter::with_capacity(BUFFER_CAPACITY, output_file);

    let mut writer: Box<dyn Send + Write> = Box::new(file_writer);

    // Grab previous encoder and wrap it inside of a new one
    let chain_writer_encoder = |format: &_, encoder| -> Result<_> {
        let encoder: Box<dyn Send + Write> = match format {
            Gzip => Box::new({
                // by default, ParCompress uses a default compression level of 3
                // instead of the regular default that flate2 uses
                let parz: ParCompress<gzp::deflate::Gzip, _> = ParCompressBuilder::new()
                    .compression_level(
                        level.map_or_else(Default::default, |l| gzp::Compression::new((l as u32).clamp(0, 9))),
                    )
                    .num_threads(logical_thread_count())
                    .expect("gpz: num_threads must be greater than 0")
                    .from_writer(encoder);
                parz
            }),
            Bzip => Box::new(bzip2::write::BzEncoder::new(
                encoder,
                level.map_or_else(Default::default, |l| bzip2::Compression::new((l as u32).clamp(1, 9))),
            )),
            Bzip3 => {
                #[cfg(not(feature = "bzip3"))]
                return Err(crate::Error::bzip3_no_support());

                #[cfg(feature = "bzip3")]
                Box::new(
                    // Use block size of 16 MiB
                    bzip3::write::Bz3Encoder::new(encoder, 16 * 2_usize.pow(20))?,
                )
            }
            Lz4 => Box::new(lz4_flex::frame::FrameEncoder::new(encoder).auto_finish()),
            Lzma => {
                let options = level.map_or_else(Default::default, |l| {
                    lzma_rust2::LzmaOptions::with_preset((l as u32).clamp(0, 9))
                });
                let writer = lzma_rust2::LzmaWriter::new_use_header(encoder, &options, None)?;
                Box::new(writer.auto_finish())
            }
            Xz => {
                let mut options = level.map_or_else(Default::default, |l| {
                    lzma_rust2::XzOptions::with_preset((l as u32).clamp(0, 9))
                });
                let dict_size = options.lzma_options.dict_size as u64;
                options.set_block_size(NonZeroU64::new(dict_size));
                // Use up to 256 PHYSICAL cores for compression
                let writer = lzma_rust2::XzWriterMt::new(encoder, options, physical_thread_count() as u32)?;
                Box::new(writer.auto_finish())
            }
            Lzip => {
                let options = level.map_or_else(Default::default, |l| {
                    lzma_rust2::LzipOptions::with_preset((l as u32).clamp(0, 9))
                });
                let writer = lzma_rust2::LzipWriter::new(encoder, options);
                Box::new(writer.auto_finish())
            }
            Snappy => Box::new({
                let parz: ParCompress<gzp::snap::Snap, _> = ParCompressBuilder::new()
                    .compression_level(gzp::par::compress::Compression::new(
                        level.map_or_else(Default::default, |l| (l as u32).clamp(0, 9)),
                    ))
                    .num_threads(logical_thread_count())
                    .expect("gpz: num_threads must be greater than 0")
                    .from_writer(encoder);
                parz
            }),
            Zstd => {
                let mut zstd_encoder = zstd::stream::write::Encoder::new(
                    encoder,
                    level.map_or(zstd::DEFAULT_COMPRESSION_LEVEL, |l| {
                        (l as i32).clamp(zstd::zstd_safe::min_c_level(), zstd::zstd_safe::max_c_level())
                    }),
                )?;
                // Use all available PHYSICAL cores for compression
                zstd_encoder.multithread(physical_thread_count() as u32)?;
                Box::new(zstd_encoder.auto_finish())
            }
            Brotli => {
                let default_level = 11; // Same as brotli CLI, default to highest compression
                let level = level.unwrap_or(default_level).clamp(0, 11) as u32;
                let win_size = 22; // default to 2^22 = 4 MiB window size
                Box::new(brotli::CompressorWriter::new(encoder, BUFFER_CAPACITY, level, win_size))
            }
            Tar | Zip | Rar | SevenZip => unreachable!(),
        };
        Ok(encoder)
    };

    let (first_format, formats) = split_first_compression_format(&extensions);

    for format in formats.iter().rev() {
        writer = chain_writer_encoder(format, writer)?;
    }

    match first_format {
        Gzip | Bzip | Bzip3 | Lz4 | Lzma | Xz | Lzip | Snappy | Zstd | Brotli => {
            writer = chain_writer_encoder(&first_format, writer)?;
            let mut reader = fs::File::open(&files[0])?;
            io::copy(&mut reader, &mut writer)?;
            info_accessible!("Input file size: {}", BytesFmt(file_size(&files[0])?));
        }
        Tar => {
            archive::tar::build_archive(
                &files,
                output_path,
                &mut writer,
                file_visibility_policy,
                follow_symlinks,
            )?;
            writer.flush()?;
        }
        Zip => {
            if !formats.is_empty() {
                // Make thread own locks to keep output messages adjacent
                let _locks = lock_and_flush_output_stdio();

                warn_user_about_loading_zip_in_memory();
                if !user_wants_to_continue(output_path, question_policy, QuestionAction::Compression)? {
                    return Ok(false);
                }
            }

            let mut vec_buffer = Cursor::new(vec![]);

            archive::zip::build_archive(
                &files,
                output_path,
                &mut vec_buffer,
                file_visibility_policy,
                follow_symlinks,
            )?;
            vec_buffer.rewind()?;
            io::copy(&mut vec_buffer, &mut writer)?;
        }
        Rar => {
            #[cfg(feature = "unrar")]
            return Err(archive::rar::no_compression());

            #[cfg(not(feature = "unrar"))]
            return Err(crate::Error::rar_no_support());
        }
        SevenZip => {
            if !formats.is_empty() {
                // Make thread own locks to keep output messages adjacent
                let _locks = lock_and_flush_output_stdio();

                warn_user_about_loading_sevenz_in_memory();
                if !user_wants_to_continue(output_path, question_policy, QuestionAction::Compression)? {
                    return Ok(false);
                }
            }

            let mut vec_buffer = Cursor::new(vec![]);
            archive::sevenz::build_archive(&files, output_path, &mut vec_buffer, file_visibility_policy)?;
            vec_buffer.rewind()?;
            io::copy(&mut vec_buffer, &mut writer)?;
        }
    }

    Ok(true)
}