async-mtzip 0.0.3

A library for making zip archives with multithreaded async compression
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147

#![doc = include_str!("../README.md")]

use std::path::PathBuf;
use std::{
    num::NonZeroUsize,
};

use tokio::io::{AsyncSeek, AsyncWrite};
use tokio::sync::{mpsc, oneshot};
mod zip;
use zip::{
    data::ZipData,
    extra_field::ExtraFields,
    file::ZipFile,
    job::{ZipJob, ZipJobOrigin},
};


use crate::zip::job::ZipCommand;
use crate::zip::level::CompressionLevel;
pub use zip::extra_field;
pub use zip::level;
pub use zip::job;
pub use zip::file::dirs;

#[repr(u16)]
#[derive(Debug, Default, Clone, Copy, PartialEq, Eq)]
pub enum CompressionType {
    Stored = 0,
    #[default]
    Deflate = 8,
}

#[derive(Debug, Default)]
pub struct ZipArchive {
    pub jobs_queue: Vec<ZipJob>,
    data: ZipData,
}

impl ZipArchive {
    fn push_job(&mut self, job: ZipJob) {
        self.jobs_queue.push(job);
    }

    #[inline]
    pub fn new() -> Self {
        Self::default()
    }

    pub fn add_file_from_fs(
        &mut self,
        fs_path: PathBuf,
        archived_path: String,
        compression_level: Option<CompressionLevel>,
        compression_type: Option<CompressionType>,
    ) -> ZipJob {
        // let job = ZipJob {
        //     data_origin: ZipJobOrigin::Filesystem {
        //         path: fs_path.clone(),
        //         compression_level: compression_level.unwrap_or(CompressionLevel::best()),
        //         compression_type: compression_type.unwrap_or(CompressionType::Deflate),
        //     },
        //     archive_path: archived_path.clone(),
        // };
        // self.push_job(job);
        ZipJob {
            data_origin: ZipJobOrigin::Filesystem {
                path: fs_path,
                compression_level: compression_level.unwrap_or(CompressionLevel::best()),
                compression_type: compression_type.unwrap_or(CompressionType::Deflate),
            },
            archive_path: archived_path,
        }
    }

    pub async fn add_file(
        &mut self,
        fs_path: PathBuf,
        archived_path: String,
        compression_level: Option<CompressionLevel>,
        compression_type: Option<CompressionType>,
    ) {
        let job = ZipJob {
            data_origin: ZipJobOrigin::Filesystem {
                path: fs_path,
                compression_level: compression_level.unwrap_or(CompressionLevel::best()),
                compression_type: compression_type.unwrap_or(CompressionType::Deflate),
            },
            archive_path: archived_path,
        };
        self.push_job(job);
    }

    pub fn add_directory(&mut self, archived_path: String, attributes: Option<u16>) -> ZipJob {
        ZipJob {
            data_origin: ZipJobOrigin::Directory {
                extra_fields: ExtraFields::default(),
                external_attributes: attributes.unwrap_or(ZipFile::default_dir_attrs()),
            },
            archive_path: archived_path,
        }
    }

    #[inline]
    pub async fn write<W: AsyncWrite + AsyncSeek + Unpin>(
        &mut self,
        writer: &mut W,
        jobs_provider: mpsc::Sender<ZipCommand>,
        process: Option<mpsc::Sender<u64>>,
    ) -> std::io::Result<()> {
        let threads = Self::get_threads();
        let mut rx = {
            let (tx, rx) = mpsc::channel::<ZipFile>(threads);
            for _ in 0..Self::get_threads() {
                let tx = tx.clone();
                let process = process.clone();
                let jobs_provider = jobs_provider.clone();
                tokio::spawn(async move {
                    loop {
                        let (resp_tx, resp_rx) = oneshot::channel();
                        let resp = jobs_provider.send(ZipCommand::Get { resp: resp_tx }).await;
                        if resp.is_err() {
                            break;
                        }
                        let job = resp_rx.await;
                        let job = if job.is_err() {
                            break;
                        } else {
                            job.unwrap()
                        };
                        let process_new = process.clone();
                        let zip_file = job.into_file(process_new).await.unwrap();
                        tx.send(zip_file).await.unwrap()
                    }
                });
            }
            rx
        };
        self.data.write_with_tokio(writer, &mut rx).await
    }

    fn get_threads() -> usize {
        std::thread::available_parallelism()
            .map(NonZeroUsize::get)
            .unwrap_or(1)
    }
}