aruna_file/
lib.rs

1pub mod helpers;
2pub mod notifications;
3pub mod readwrite;
4pub mod streamreadwrite;
5pub mod transformer;
6pub mod transformers;
7
8#[cfg(test)]
9mod tests {
10    use crate::helpers::footer_parser::{FooterParser, Range};
11    use crate::readwrite::ArunaReadWriter;
12    use crate::streamreadwrite::ArunaStreamReadWriter;
13    use crate::transformer::{FileContext, ReadWriter};
14    use crate::transformers::decrypt::ChaCha20Dec;
15    use crate::transformers::encrypt::ChaCha20Enc;
16    use crate::transformers::filter::Filter;
17    use crate::transformers::footer::FooterGenerator;
18    use crate::transformers::gzip_comp::GzipEnc;
19    use crate::transformers::size_probe::SizeProbe;
20    use crate::transformers::tar::TarEnc;
21    //use crate::transformers::zip::ZipEnc;
22    use crate::transformers::zstd_comp::ZstdEnc;
23    use crate::transformers::zstd_decomp::ZstdDec;
24    use bytes::Bytes;
25    use digest::Digest;
26    use futures::{StreamExt, TryStreamExt};
27    use md5::Md5;
28    use tokio::fs::File;
29    use tokio::io::{AsyncReadExt, AsyncSeekExt};
30
31    #[tokio::test]
32    async fn e2e_compressor_test_with_file() {
33        let file = File::open("test.txt").await.unwrap();
34        let file2 = File::create("test.txt.out.1").await.unwrap();
35
36        // Create a new ArunaReadWriter
37        ArunaReadWriter::new_with_writer(file, file2)
38            .add_transformer(ZstdEnc::new(false))
39            .add_transformer(ZstdDec::new())
40            .process()
41            .await
42            .unwrap();
43
44        let mut file = File::open("test.txt").await.unwrap();
45        let mut file2 = File::open("test.txt.out.1").await.unwrap();
46        let mut buf1 = String::new();
47        let mut buf2 = String::new();
48        file.read_to_string(&mut buf1).await.unwrap();
49        file2.read_to_string(&mut buf2).await.unwrap();
50        assert!(buf1 == buf2)
51    }
52
53    #[tokio::test]
54    async fn e2e_encrypt_test_with_vec_no_pad() {
55        let file = b"This is a very very important test".to_vec();
56        let mut file2 = Vec::new();
57
58        // Create a new ArunaReadWriter
59        ArunaReadWriter::new_with_writer(file.as_ref(), &mut file2)
60            .add_transformer(
61                ChaCha20Enc::new(false, b"99wj3485nxgyq5ub9zd3e7jsrq7a92ea".to_vec()).unwrap(),
62            )
63            .add_transformer(
64                ChaCha20Dec::new(Some(b"99wj3485nxgyq5ub9zd3e7jsrq7a92ea".to_vec())).unwrap(),
65            )
66            .process()
67            .await
68            .unwrap();
69
70        assert_eq!(file, file2);
71    }
72
73    #[tokio::test]
74    async fn e2e_encrypt_test_with_vec_with_pad() {
75        let file = b"This is a very very important test".to_vec();
76        let mut file2 = Vec::new();
77
78        // Create a new ArunaReadWriter
79        ArunaReadWriter::new_with_writer(file.as_ref(), &mut file2)
80            .add_transformer(
81                ChaCha20Enc::new(true, b"99wj3485nxgyq5ub9zd3e7jsrq7a92ea".to_vec()).unwrap(),
82            )
83            .add_transformer(
84                ChaCha20Dec::new(Some(b"99wj3485nxgyq5ub9zd3e7jsrq7a92ea".to_vec())).unwrap(),
85            )
86            .process()
87            .await
88            .unwrap();
89
90        assert_eq!(file, file2);
91    }
92
93    #[tokio::test]
94    async fn e2e_encrypt_test_with_file_no_pad() {
95        let file = File::open("test.txt").await.unwrap();
96        let file2 = File::create("test.txt.out.2").await.unwrap();
97
98        // Create a new ArunaReadWriter
99        ArunaReadWriter::new_with_writer(file, file2)
100            .add_transformer(
101                ChaCha20Enc::new(false, b"wvwj3485nxgyq5ub9zd3e7jsrq7a92ea".to_vec()).unwrap(),
102            )
103            .add_transformer(
104                ChaCha20Dec::new(Some(b"wvwj3485nxgyq5ub9zd3e7jsrq7a92ea".to_vec())).unwrap(),
105            )
106            .process()
107            .await
108            .unwrap();
109
110        let mut file = File::open("test.txt").await.unwrap();
111        let mut file2 = File::open("test.txt.out.2").await.unwrap();
112        let mut buf1 = String::new();
113        let mut buf2 = String::new();
114        file.read_to_string(&mut buf1).await.unwrap();
115        file2.read_to_string(&mut buf2).await.unwrap();
116        assert!(buf1 == buf2)
117    }
118
119    #[tokio::test]
120    async fn e2e_encrypt_test_with_file_with_pad() {
121        let file = File::open("test.txt").await.unwrap();
122        let file2 = File::create("test.txt.out.3").await.unwrap();
123
124        // Create a new ArunaReadWriter
125        ArunaReadWriter::new_with_writer(file, file2)
126            .add_transformer(
127                ChaCha20Enc::new(true, b"wvwj3485nxgyq5ub9zd3e7jsrq7a92ea".to_vec()).unwrap(),
128            )
129            .add_transformer(
130                ChaCha20Dec::new(Some(b"wvwj3485nxgyq5ub9zd3e7jsrq7a92ea".to_vec())).unwrap(),
131            )
132            .process()
133            .await
134            .unwrap();
135
136        let mut file = File::open("test.txt").await.unwrap();
137        let mut file2 = File::open("test.txt.out.3").await.unwrap();
138        let mut buf1 = String::new();
139        let mut buf2 = String::new();
140        file.read_to_string(&mut buf1).await.unwrap();
141        file2.read_to_string(&mut buf2).await.unwrap();
142        assert!(buf1 == buf2)
143    }
144
145    #[tokio::test]
146    async fn e2e_test_roundtrip_with_file() {
147        let file = File::open("test.txt").await.unwrap();
148        let file2 = File::create("test.txt.out.4").await.unwrap();
149
150        // Create a new ArunaReadWriter
151        ArunaReadWriter::new_with_writer(file, file2)
152            .add_transformer(ZstdEnc::new(false))
153            .add_transformer(ZstdEnc::new(false))
154            .add_transformer(
155                ChaCha20Enc::new(true, b"wvwj3485nxgyq5ub9zd3e7jsrq7a92ea".to_vec()).unwrap(),
156            )
157            .add_transformer(
158                ChaCha20Enc::new(false, b"99wj3485nxgyq5ub9zd3e7jsrq7a92ea".to_vec()).unwrap(),
159            )
160            .add_transformer(
161                ChaCha20Dec::new(Some(b"99wj3485nxgyq5ub9zd3e7jsrq7a92ea".to_vec())).unwrap(),
162            )
163            .add_transformer(
164                ChaCha20Dec::new(Some(b"wvwj3485nxgyq5ub9zd3e7jsrq7a92ea".to_vec())).unwrap(),
165            )
166            .add_transformer(ZstdDec::new())
167            .add_transformer(ZstdDec::new())
168            .process()
169            .await
170            .unwrap();
171
172        let mut file = File::open("test.txt").await.unwrap();
173        let mut file2 = File::open("test.txt.out.4").await.unwrap();
174        let mut buf1 = String::new();
175        let mut buf2 = String::new();
176        file.read_to_string(&mut buf1).await.unwrap();
177        file2.read_to_string(&mut buf2).await.unwrap();
178        assert!(buf1 == buf2)
179    }
180
181    #[tokio::test]
182    async fn test_with_vec() {
183        let file = b"This is a very very important test".to_vec();
184        let mut file2 = Vec::new();
185
186        // Create a new ArunaReadWriter
187        ArunaReadWriter::new_with_writer(file.as_ref(), &mut file2)
188            .add_transformer(ZstdEnc::new(false))
189            .add_transformer(ZstdEnc::new(false)) // Double compression because we can
190            .add_transformer(
191                ChaCha20Enc::new(false, b"wvwj3485nxgyq5ub9zd3e7jsrq7a92ea".to_vec()).unwrap(),
192            )
193            .add_transformer(
194                ChaCha20Enc::new(false, b"99wj3485nxgyq5ub9zd3e7jsrq7a92ea".to_vec()).unwrap(),
195            )
196            .add_transformer(
197                ChaCha20Dec::new(Some(b"99wj3485nxgyq5ub9zd3e7jsrq7a92ea".to_vec())).unwrap(),
198            )
199            .add_transformer(
200                ChaCha20Dec::new(Some(b"wvwj3485nxgyq5ub9zd3e7jsrq7a92ea".to_vec())).unwrap(),
201            )
202            .add_transformer(ZstdDec::new())
203            .add_transformer(ZstdDec::new()) // Double decompression because we can
204            .process()
205            .await
206            .unwrap();
207        assert!(file == file2)
208    }
209
210    #[tokio::test]
211    async fn test_with_file_footer() {
212        let file = File::open("test.txt").await.unwrap();
213        let file2 = File::create("test.txt.out.5").await.unwrap();
214        ArunaReadWriter::new_with_writer(file, file2)
215            .add_transformer(ZstdEnc::new(false))
216            .add_transformer(
217                ChaCha20Enc::new(false, b"wvwj3485nxgyq5ub9zd3e7jsrq7a92ea".to_vec()).unwrap(),
218            )
219            .add_transformer(
220                ChaCha20Dec::new(Some(b"wvwj3485nxgyq5ub9zd3e7jsrq7a92ea".to_vec())).unwrap(),
221            )
222            .add_transformer(FooterGenerator::new(None))
223            .add_transformer(ZstdDec::new())
224            .process()
225            .await
226            .unwrap();
227
228        let mut file = File::open("test.txt").await.unwrap();
229        let mut file2 = File::open("test.txt.out.5").await.unwrap();
230        let mut buf1 = String::new();
231        let mut buf2 = String::new();
232        file.read_to_string(&mut buf1).await.unwrap();
233        file2.read_to_string(&mut buf2).await.unwrap();
234        assert!(buf1 == buf2)
235    }
236
237    #[tokio::test]
238    async fn test_footer_parsing() {
239        let file = File::open("test.txt").await.unwrap();
240        let file2 = File::create("test.txt.out.6").await.unwrap();
241        ArunaReadWriter::new_with_writer(file, file2)
242            .add_transformer(ZstdEnc::new(false))
243            .add_transformer(FooterGenerator::new(None))
244            .add_transformer(
245                ChaCha20Enc::new(false, b"wvwj3485nxgyq5ub9zd3e7jsrq7a92ea".to_vec()).unwrap(),
246            )
247            .add_transformer(
248                ChaCha20Dec::new(Some(b"wvwj3485nxgyq5ub9zd3e7jsrq7a92ea".to_vec())).unwrap(),
249            )
250            .process()
251            .await
252            .unwrap();
253
254        let mut file2 = File::open("test.txt.out.6").await.unwrap();
255
256        file2
257            .seek(std::io::SeekFrom::End(-65536 * 2))
258            .await
259            .unwrap();
260
261        let buf: &mut [u8; 65536 * 2] = &mut [0; 65536 * 2];
262        file2.read_exact(buf).await.unwrap();
263
264        let mut fp = FooterParser::new(buf);
265
266        fp.parse().unwrap();
267
268        let (a, b) = fp
269            .get_offsets_by_range(Range { from: 0, to: 1000 })
270            .unwrap();
271
272        assert!(a.to % (65536) == 0);
273
274        assert_eq!(
275            a,
276            Range {
277                from: 0,
278                to: 25 * 65536
279            }
280        );
281        assert!(b == Range { from: 0, to: 1000 })
282    }
283
284    #[tokio::test]
285    async fn test_footer_parsing_encrypted() {
286        let file = File::open("test.txt").await.unwrap();
287        let file2 = File::create("test.txt.out.7").await.unwrap();
288        ArunaReadWriter::new_with_writer(file, file2)
289            .add_transformer(ZstdEnc::new(false))
290            .add_transformer(FooterGenerator::new(None))
291            .add_transformer(
292                ChaCha20Enc::new(false, b"wvwj3485nxgyq5ub9zd3e7jsrq7a92ea".to_vec()).unwrap(),
293            )
294            .process()
295            .await
296            .unwrap();
297
298        let mut file2 = File::open("test.txt.out.7").await.unwrap();
299        file2
300            .seek(std::io::SeekFrom::End((-65536 - 28) * 2))
301            .await
302            .unwrap();
303
304        let buf: &mut [u8; (65536 + 28) * 2] = &mut [0; (65536 + 28) * 2];
305        file2.read_exact(buf).await.unwrap();
306
307        let mut fp =
308            FooterParser::from_encrypted(buf, b"wvwj3485nxgyq5ub9zd3e7jsrq7a92ea").unwrap();
309        fp.parse().unwrap();
310
311        let (a, b) = fp
312            .get_offsets_by_range(Range { from: 0, to: 1000 })
313            .unwrap();
314
315        assert!(a.to % (65536 + 28) == 0);
316
317        assert!(
318            a == Range {
319                from: 0,
320                to: 25 * (65536 + 28)
321            }
322        );
323        assert!(b == Range { from: 0, to: 1000 })
324    }
325
326    #[tokio::test]
327    async fn test_with_filter() {
328        let file = b"This is a very very important test".to_vec();
329        let mut file2 = Vec::new();
330
331        // Create a new ArunaReadWriter
332        ArunaReadWriter::new_with_writer(file.as_ref(), &mut file2)
333            .add_transformer(ZstdEnc::new(false))
334            .add_transformer(ZstdEnc::new(false)) // Double compression because we can
335            .add_transformer(
336                ChaCha20Enc::new(false, b"wvwj3485nxgyq5ub9zd3e7jsrq7a92ea".to_vec()).unwrap(),
337            )
338            .add_transformer(
339                ChaCha20Enc::new(false, b"99wj3485nxgyq5ub9zd3e7jsrq7a92ea".to_vec()).unwrap(),
340            )
341            .add_transformer(
342                ChaCha20Dec::new(Some(b"99wj3485nxgyq5ub9zd3e7jsrq7a92ea".to_vec())).unwrap(),
343            )
344            .add_transformer(
345                ChaCha20Dec::new(Some(b"wvwj3485nxgyq5ub9zd3e7jsrq7a92ea".to_vec())).unwrap(),
346            )
347            .add_transformer(ZstdDec::new())
348            .add_transformer(ZstdDec::new())
349            .add_transformer(Filter::new(Range { from: 0, to: 3 }))
350            .process()
351            .await
352            .unwrap();
353
354        println!("{:?}", file2);
355        assert_eq!(file2, b"Thi".to_vec());
356    }
357
358    #[tokio::test]
359    async fn test_read_write_multifile() {
360        let file1 = b"This is a very very important test".to_vec();
361        let file2 = b"This is a very very important test".to_vec();
362        let mut file3: Vec<u8> = Vec::new();
363
364        let combined = Vec::from_iter(file1.clone().into_iter().chain(file2.clone()));
365
366        let (sx, rx) = async_channel::bounded(10);
367        sx.send((
368            FileContext {
369                file_name: "file1.txt".to_string(),
370                input_size: file1.len() as u64,
371                file_size: file1.len() as u64,
372                ..Default::default()
373            },
374            false,
375        ))
376        .await
377        .unwrap();
378
379        sx.send((
380            FileContext {
381                file_name: "file2.txt".to_string(),
382                input_size: file2.len() as u64,
383                file_size: file2.len() as u64,
384                ..Default::default()
385            },
386            true,
387        ))
388        .await
389        .unwrap();
390
391        // Create a new ArunaReadWriter
392        let mut aswr = ArunaReadWriter::new_with_writer(combined.as_ref(), &mut file3)
393            .add_transformer(ZstdEnc::new(false))
394            .add_transformer(ZstdEnc::new(false)) // Double compression because we can
395            .add_transformer(
396                ChaCha20Enc::new(false, b"wvwj3485nxgyq5ub9zd3e7jsrq7a92ea".to_vec()).unwrap(),
397            )
398            .add_transformer(
399                ChaCha20Enc::new(false, b"99wj3485nxgyq5ub9zd3e7jsrq7a92ea".to_vec()).unwrap(),
400            )
401            .add_transformer(
402                ChaCha20Dec::new(Some(b"99wj3485nxgyq5ub9zd3e7jsrq7a92ea".to_vec())).unwrap(),
403            )
404            .add_transformer(
405                ChaCha20Dec::new(Some(b"wvwj3485nxgyq5ub9zd3e7jsrq7a92ea".to_vec())).unwrap(),
406            )
407            .add_transformer(ZstdDec::new())
408            .add_transformer(ZstdDec::new())
409            .add_transformer(Filter::new(Range { from: 0, to: 3 }));
410        aswr.add_file_context_receiver(rx).await.unwrap();
411        aswr.process().await.unwrap();
412        drop(aswr);
413
414        println!("{:?}", file3);
415        assert_eq!(file3, b"Thi".to_vec());
416    }
417
418    #[tokio::test]
419    async fn stream_test() {
420        let mut file2 = Vec::new();
421
422        use futures::stream;
423
424        let stream = stream::iter(vec![
425            Ok(Bytes::from_iter(
426                b"This is a very very important test".to_vec(),
427            )),
428            Ok(Bytes::from(b"This is a very very important test".to_vec())),
429        ]);
430
431        // Create a new ArunaReadWriter
432        ArunaStreamReadWriter::new_with_writer(stream, &mut file2)
433            .add_transformer(ZstdEnc::new(false))
434            .add_transformer(ZstdEnc::new(false)) // Double compression because we can
435            .add_transformer(
436                ChaCha20Enc::new(true, b"wvwj3485nxgyq5ub9zd3e7jsrq7a92ea".to_vec()).unwrap(),
437            )
438            .add_transformer(
439                ChaCha20Enc::new(true, b"99wj3485nxgyq5ub9zd3e7jsrq7a92ea".to_vec()).unwrap(),
440            )
441            .add_transformer(
442                ChaCha20Dec::new(Some(b"99wj3485nxgyq5ub9zd3e7jsrq7a92ea".to_vec())).unwrap(),
443            )
444            .add_transformer(
445                ChaCha20Dec::new(Some(b"wvwj3485nxgyq5ub9zd3e7jsrq7a92ea".to_vec())).unwrap(),
446            )
447            .add_transformer(ZstdDec::new())
448            .add_transformer(ZstdDec::new())
449            .add_transformer(Filter::new(Range { from: 0, to: 3 }))
450            .process()
451            .await
452            .unwrap();
453
454        println!("{:?}", file2);
455        assert_eq!(file2, b"Thi".to_vec());
456    }
457
458    #[tokio::test]
459    async fn e2e_test_read_write_multifile_tar_small() {
460        let file1 = b"This is a very very important test".to_vec();
461        let file2 = b"Another brilliant This is a very very important test1337".to_vec();
462        let mut file3 = File::create("test.txt.out.8").await.unwrap();
463
464        let combined = Vec::from_iter(file1.clone().into_iter().chain(file2.clone()));
465
466        let (sx, rx) = async_channel::bounded(10);
467        sx.send((
468            FileContext {
469                file_name: "file1.txt".to_string(),
470                input_size: file2.len() as u64,
471                file_size: file1.len() as u64,
472                ..Default::default()
473            },
474            false,
475        ))
476        .await
477        .unwrap();
478
479        sx.send((
480            FileContext {
481                file_name: "file2.txt".to_string(),
482                input_size: file2.len() as u64,
483                file_size: file2.len() as u64,
484                ..Default::default()
485            },
486            true,
487        ))
488        .await
489        .unwrap();
490
491        // Create a new ArunaReadWriter
492        let mut aswr = ArunaReadWriter::new_with_writer(combined.as_ref(), &mut file3)
493            .add_transformer(TarEnc::new());
494        aswr.add_file_context_receiver(rx).await.unwrap();
495        aswr.process().await.unwrap();
496    }
497
498    #[tokio::test]
499    async fn e2e_test_read_write_multifile_tar_real() {
500        let mut file1 = File::open("test.txt").await.unwrap();
501        let mut file2 = File::open("test.txt").await.unwrap();
502        let mut file3 = File::create("test.txt.out.9").await.unwrap();
503
504        let mut combined = Vec::new();
505        file1.read_to_end(&mut combined).await.unwrap();
506        file2.read_to_end(&mut combined).await.unwrap();
507
508        let (sx, rx) = async_channel::bounded(10);
509        sx.send((
510            FileContext {
511                file_name: "file1.txt".to_string(),
512                input_size: file1.metadata().await.unwrap().len(),
513                file_size: file1.metadata().await.unwrap().len(),
514                ..Default::default()
515            },
516            false,
517        ))
518        .await
519        .unwrap();
520
521        sx.send((
522            FileContext {
523                file_name: "file2.txt".to_string(),
524                input_size: file2.metadata().await.unwrap().len(),
525                file_size: file2.metadata().await.unwrap().len(),
526                ..Default::default()
527            },
528            true,
529        ))
530        .await
531        .unwrap();
532
533        // Create a new ArunaReadWriter
534        let mut aswr = ArunaReadWriter::new_with_writer(combined.as_ref(), &mut file3)
535            .add_transformer(TarEnc::new());
536        aswr.add_file_context_receiver(rx).await.unwrap();
537        aswr.process().await.unwrap();
538    }
539
540    #[tokio::test]
541    async fn e2e_test_stream_write_multifile_tar_real() {
542        let file1 = File::open("test.txt").await.unwrap();
543        let file2 = File::open("test.txt").await.unwrap();
544
545        let file1_size = file1.metadata().await.unwrap().len();
546        let file2_size = file2.metadata().await.unwrap().len();
547
548        let stream1 = tokio_util::io::ReaderStream::new(file1);
549        let stream2 = tokio_util::io::ReaderStream::new(file2);
550
551        let chained = stream1.chain(stream2);
552        let mapped = chained.map_err(|_| {
553            Box::<(dyn std::error::Error + Send + Sync + 'static)>::from("a_str_error")
554        });
555        let mut file3 = File::create("test.txt.out.10").await.unwrap();
556
557        let (sx, rx) = async_channel::bounded(10);
558        sx.send((
559            FileContext {
560                file_name: "file1.txt".to_string(),
561                input_size: file1_size,
562                file_size: file1_size,
563                ..Default::default()
564            },
565            false,
566        ))
567        .await
568        .unwrap();
569
570        sx.send((
571            FileContext {
572                file_name: "file2.txt".to_string(),
573                input_size: file2_size,
574                file_size: file2_size,
575                ..Default::default()
576            },
577            true,
578        ))
579        .await
580        .unwrap();
581
582        // Create a new ArunaReadWriter
583        let mut aswr = ArunaStreamReadWriter::new_with_writer(mapped, &mut file3)
584            .add_transformer(TarEnc::new());
585        aswr.add_file_context_receiver(rx).await.unwrap();
586        aswr.process().await.unwrap();
587    }
588
589    #[tokio::test]
590    async fn e2e_test_stream_tar_gz() {
591        let file1 = File::open("test.txt").await.unwrap();
592        let file2 = File::open("test.txt").await.unwrap();
593
594        let file1_size = file1.metadata().await.unwrap().len();
595        let file2_size = file2.metadata().await.unwrap().len();
596
597        let stream1 = tokio_util::io::ReaderStream::new(file1);
598        let stream2 = tokio_util::io::ReaderStream::new(file2);
599
600        let chained = stream1.chain(stream2);
601        let mapped = chained.map_err(|_| {
602            Box::<(dyn std::error::Error + Send + Sync + 'static)>::from("a_str_error")
603        });
604        let mut file3 = File::create("test.txt.out.11").await.unwrap();
605
606        let (sx, rx) = async_channel::bounded(10);
607        sx.send((
608            FileContext {
609                file_name: "file1.txt".to_string(),
610                input_size: file1_size,
611                file_size: file1_size,
612                ..Default::default()
613            },
614            false,
615        ))
616        .await
617        .unwrap();
618
619        sx.send((
620            FileContext {
621                file_name: "file2.txt".to_string(),
622                input_size: file2_size,
623                file_size: file2_size,
624                ..Default::default()
625            },
626            true,
627        ))
628        .await
629        .unwrap();
630
631        // Create a new ArunaReadWriter
632        let mut aswr = ArunaStreamReadWriter::new_with_writer(mapped, &mut file3)
633            .add_transformer(TarEnc::new())
634            .add_transformer(GzipEnc::new());
635        aswr.add_file_context_receiver(rx).await.unwrap();
636        aswr.process().await.unwrap();
637    }
638
639    #[tokio::test]
640    async fn hashing_transformer_test() {
641        let file = b"This is a very very important test".to_vec();
642        let mut file2 = Vec::new();
643
644        let (probe, rx) = SizeProbe::new();
645        let (md5_trans, rx2) =
646            crate::transformers::hashing_transformer::HashingTransformer::new(Md5::new());
647
648        // Create a new ArunaReadWriter
649        ArunaReadWriter::new_with_writer(file.as_ref(), &mut file2)
650            .add_transformer(md5_trans)
651            .add_transformer(probe)
652            .process()
653            .await
654            .unwrap();
655
656        let size = rx.try_recv().unwrap();
657        let md5 = rx2.try_recv().unwrap();
658
659        assert_eq!(size, 34);
660        assert_eq!(md5, "4f276870b4b5f84c0b2bbfce30757176".to_string());
661    }
662
663    #[tokio::test]
664    async fn e2e_test_stream_tar_folder() {
665        let file1 = File::open("test.txt").await.unwrap();
666        let file2 = File::open("test.txt").await.unwrap();
667
668        let file1_size = file1.metadata().await.unwrap().len();
669        let file2_size = file2.metadata().await.unwrap().len();
670
671        let stream1 = tokio_util::io::ReaderStream::new(file1);
672        let stream2 = tokio_util::io::ReaderStream::new(file2);
673
674        let chained = stream1.chain(stream2);
675        let mapped = chained.map_err(|_| {
676            Box::<(dyn std::error::Error + Send + Sync + 'static)>::from("a_str_error")
677        });
678        let mut file3 = File::create("test.txt.out.tar").await.unwrap();
679
680        let (sx, rx) = async_channel::bounded(10);
681
682        sx.send((
683            FileContext {
684                file_name: "blup/".to_string(),
685                input_size: 0,
686                file_size: 0,
687                is_dir: true,
688                ..Default::default()
689            },
690            false,
691        ))
692        .await
693        .unwrap();
694
695        sx.send((
696            FileContext {
697                file_name: "blup/file1.txt".to_string(),
698                input_size: file1_size,
699                file_size: file1_size,
700                ..Default::default()
701            },
702            false,
703        ))
704        .await
705        .unwrap();
706
707        sx.send((
708            FileContext {
709                file_name: "blip/".to_string(),
710                input_size: 0,
711                file_size: 0,
712                is_dir: true,
713                ..Default::default()
714            },
715            false,
716        ))
717        .await
718        .unwrap();
719
720        sx.send((
721            FileContext {
722                file_name: "blip/file2.txt".to_string(),
723                input_size: file2_size,
724                file_size: file2_size,
725                ..Default::default()
726            },
727            true,
728        ))
729        .await
730        .unwrap();
731
732        // Create a new ArunaReadWriter
733        let mut aswr = ArunaStreamReadWriter::new_with_writer(mapped, &mut file3)
734            .add_transformer(TarEnc::new());
735        //.add_transformer(GzipEnc::new());
736        aswr.add_file_context_receiver(rx).await.unwrap();
737        aswr.process().await.unwrap();
738    }
739
740    // #[tokio::test]
741    // async fn e2e_test_stream_zip() {
742    //     let file1 = File::open("test.txt").await.unwrap();
743    //     let file2 = File::open("test.txt").await.unwrap();
744
745    //     let file1_size = file1.metadata().await.unwrap().len();
746    //     let file2_size = file2.metadata().await.unwrap().len();
747
748    //     let stream1 = tokio_util::io::ReaderStream::new(file1);
749    //     let stream2 = tokio_util::io::ReaderStream::new(file2);
750
751    //     let chained = stream1.chain(stream2);
752    //     let mapped = chained.map_err(|_| {
753    //         Box::<(dyn std::error::Error + Send + Sync + 'static)>::from("a_str_error")
754    //     });
755    //     let mut file3 = File::create("test.txt.out.zip").await.unwrap();
756
757    //     let (sx, rx) = async_channel::bounded(10);
758
759    //     sx.send((
760    //         FileContext {
761    //             file_name: "file1.txt".to_string(),
762    //             input_size: file1_size,
763    //             file_size: file1_size,
764    //             ..Default::default()
765    //         },
766    //         false,
767    //     ))
768    //     .await
769    //     .unwrap();
770
771    //     sx.send((
772    //         FileContext {
773    //             file_name: "blip/file2.txt".to_string(),
774    //             input_size: file2_size,
775    //             file_size: file2_size,
776    //             ..Default::default()
777    //         },
778    //         true,
779    //     ))
780    //     .await
781    //     .unwrap();
782
783    //     // Create a new ArunaReadWriter
784    //     let mut aswr = ArunaStreamReadWriter::new_with_writer(mapped, &mut file3)
785    //         .add_transformer(ZipEnc::new());
786    //     //.add_transformer(GzipEnc::new());
787    //     aswr.add_file_context_receiver(rx).await.unwrap();
788    //     aswr.process().await.unwrap();
789    // }
790}
aruna_file/lib.rs

aruna_file/
lib.rs
