seq_here/
process.rs

1use super::error::{e_exit, e_println, ok_println};
2use memmap2::Mmap;
3use rayon::prelude::*;
4use std::fs::File;
5use std::io::Write;
6use std::path::PathBuf;
7use std::sync::{Arc, Mutex};
8
9pub struct ConvertCombine;
10
11const LARGE_FILE_THRESHOLD: u64 = 1_073_741_824;
12const CHUNK_SIZE: usize = 8 * 1024 * 1024;
13
14impl ConvertCombine {
15
16    pub fn combine_all(paths: Vec<PathBuf>, output: PathBuf) {
17
18        let output = match File::create(&output) {
19            Ok(f) => Arc::new(Mutex::new(f)),
20            Err(e) => e_exit("FILE_CREATE", &format!("Failed to create file: {}", e), 1),
21        };
22
23        paths.par_iter().for_each(|path| {
24            // Result handling closure
25            let process_result = || -> std::io::Result<()> {
26                let file = File::open(path)
27                    .map_err(|e| std::io::Error::new(std::io::ErrorKind::NotFound, format!("Failed open: {}", e)))?;
28
29                // Use memmap2
30                let mmap = unsafe { Mmap::map(&file)? };
31
32                if file.metadata()?.len() > LARGE_FILE_THRESHOLD {
33                    Self::process_large(&mmap, &output)
34                } else {
35                    Self::process_small(&mmap, &output)
36                }
37            };
38
39            match process_result() {
40                Ok(_) => ok_println("Merge", &format!("{}", path.display()) ),
41                Err(e) => e_println("PROCESS_ERROR", &format!("Failed to process file [{}]: {}", path.display(), e)),
42            }
43        });
44
45        ok_println("MERGE_COMPLETE", "");
46    }
47
48    /// Small file: Directly write
49    fn process_small(data: &Mmap, output: &Arc<Mutex<File>>) -> std::io::Result<()> {
50        let mut writer = output.lock()
51            .map_err(|e| std::io::Error::new(std::io::ErrorKind::Other, format!("Lock failed: {}", e)))?;
52
53        writer.write_all(data)
54            .map_err(|e| std::io::Error::new(std::io::ErrorKind::WriteZero, format!("Write failed: {}", e)))?;
55
56        Ok(())
57    }
58
59    /// Large file: Write by blocks
60    fn process_large(data: &Mmap, output: &Arc<Mutex<File>>) -> std::io::Result<()> {
61        let mut pos = 0;
62
63        while pos < data.len() {
64            let end = std::cmp::min(pos + CHUNK_SIZE, data.len());
65            let chunk = &data[pos..end];
66
67            let mut writer = output.lock()
68                .map_err(|e| std::io::Error::new(std::io::ErrorKind::Other, format!("Lock failed: {}", e)))?;
69
70            writer.write_all(chunk)
71                .map_err(|e| std::io::Error::new(std::io::ErrorKind::WriteZero, format!("Write by blocks failed: {}", e)))?;
72
73            pos = end;
74            ok_println("PROGRESS", &format!("Wrote already {} MB", pos / 1024 / 1024));
75        }
76
77        Ok(())
78    }
79}
seq_here/process.rs

seq_here/
process.rs
