1mod crypt4gh;
2pub mod helpers;
3pub mod pithos;
4pub mod readwrite;
5pub mod streamreadwrite;
6pub mod transformer;
7pub mod transformers;
8
9#[cfg(test)]
10mod tests {
11 use std::io::SeekFrom;
12
13 use crate::helpers::footer_parser::{Footer, FooterParser, FooterParserState};
14 use crate::helpers::notifications::{DirOrFileIdx, Message};
16 use crate::helpers::structs::{EncryptionKey, FileContext, Range};
17 use crate::pithos::structs::FileContextVariants;
18 use crate::readwrite::GenericReadWriter;
19 use crate::streamreadwrite::GenericStreamReadWriter;
20 use crate::transformer::ReadWriter;
21 use crate::transformers::decrypt::ChaCha20Dec;
22 use crate::transformers::decrypt_resilient::ChaChaResilient;
23 use crate::transformers::decrypt_with_parts::ChaCha20DecParts;
24 use crate::transformers::encrypt::ChaCha20Enc;
25 use crate::transformers::filter::Filter;
26 use crate::transformers::footer::FooterGenerator;
27 use crate::transformers::footer_extractor::FooterExtractor;
28 use crate::transformers::footer_updater::FooterUpdater;
29 use crate::transformers::gzip_comp::GzipEnc;
30 use crate::transformers::pithos_comp_enc::PithosTransformer;
31 use crate::transformers::size_probe::SizeProbe;
32 use crate::transformers::tar::TarEnc;
33 use crate::transformers::zstd_comp::ZstdEnc;
34 use crate::transformers::zstd_decomp::ZstdDec;
35 use base64::prelude::*;
36 use bytes::Bytes;
37 use digest::Digest;
38 use futures::{StreamExt, TryStreamExt};
39 use md5::Md5;
40 use tokio::fs::File;
41 use tokio::io::{AsyncReadExt, AsyncSeekExt};
42
43 #[tokio::test]
44 async fn e2e_compressor_test_with_file() {
45 let file = File::open("test.txt").await.unwrap();
46 let file2 = File::create("test.txt.out.1").await.unwrap();
47
48 GenericReadWriter::new_with_writer(file, file2)
50 .add_transformer(ZstdEnc::new())
51 .add_transformer(ZstdDec::new())
52 .process()
53 .await
54 .unwrap();
55
56 let mut file = File::open("test.txt").await.unwrap();
57 let mut file2 = File::open("test.txt.out.1").await.unwrap();
58 let mut buf1 = String::new();
59 let mut buf2 = String::new();
60 file.read_to_string(&mut buf1).await.unwrap();
61 file2.read_to_string(&mut buf2).await.unwrap();
62 assert!(buf1 == buf2)
63 }
64
65 #[tokio::test]
66 async fn e2e_encrypt_test_with_vec_no_pad() {
67 let file = b"This is a very very important test".to_vec();
68 let mut file2 = Vec::new();
69
70 GenericReadWriter::new_with_writer(file.as_ref(), &mut file2)
72 .add_transformer(
73 ChaCha20Enc::new_with_fixed(
74 b"99wj3485nxgyq5ub9zd3e7jsrq7a92ea"
75 .to_vec()
76 .to_vec()
77 .try_into()
78 .unwrap(),
79 )
80 .unwrap(),
81 )
82 .add_transformer(
83 ChaCha20Dec::new_with_fixed(
84 b"99wj3485nxgyq5ub9zd3e7jsrq7a92ea"
85 .to_vec()
86 .to_vec()
87 .try_into()
88 .unwrap(),
89 )
90 .unwrap(),
91 )
92 .process()
93 .await
94 .unwrap();
95
96 assert_eq!(file, file2);
97 }
98
99 #[tokio::test]
100 async fn e2e_encrypt_test_with_file_no_pad() {
101 let file = File::open("test.txt").await.unwrap();
102 let file2 = File::create("test.txt.out.2").await.unwrap();
103
104 GenericReadWriter::new_with_writer(file, file2)
106 .add_transformer(
107 ChaCha20Enc::new_with_fixed(
108 b"wvwj3485nxgyq5ub9zd3e7jsrq7a92ea"
109 .to_vec()
110 .to_vec()
111 .try_into()
112 .unwrap(),
113 )
114 .unwrap(),
115 )
116 .add_transformer(
117 ChaCha20Dec::new_with_fixed(
118 b"wvwj3485nxgyq5ub9zd3e7jsrq7a92ea"
119 .to_vec()
120 .to_vec()
121 .try_into()
122 .unwrap(),
123 )
124 .unwrap(),
125 )
126 .process()
127 .await
128 .unwrap();
129
130 let mut file = File::open("test.txt").await.unwrap();
131 let mut file2 = File::open("test.txt.out.2").await.unwrap();
132 let mut buf1 = String::new();
133 let mut buf2 = String::new();
134 file.read_to_string(&mut buf1).await.unwrap();
135 file2.read_to_string(&mut buf2).await.unwrap();
136 assert!(buf1 == buf2)
137 }
138
139 #[tokio::test]
140 async fn e2e_test_roundtrip_with_file() {
141 let file = File::open("test.txt").await.unwrap();
142 let file2 = File::create("test.txt.out.4").await.unwrap();
143
144 GenericReadWriter::new_with_writer(file, file2)
146 .add_transformer(ZstdEnc::new())
147 .add_transformer(ZstdEnc::new())
148 .add_transformer(
149 ChaCha20Enc::new_with_fixed(
150 b"wvwj3485nxgyq5ub9zd3e7jsrq7a92ea"
151 .to_vec()
152 .to_vec()
153 .try_into()
154 .unwrap(),
155 )
156 .unwrap(),
157 )
158 .add_transformer(
159 ChaCha20Enc::new_with_fixed(
160 b"99wj3485nxgyq5ub9zd3e7jsrq7a92ea"
161 .to_vec()
162 .to_vec()
163 .try_into()
164 .unwrap(),
165 )
166 .unwrap(),
167 )
168 .add_transformer(
169 ChaCha20Dec::new_with_fixed(
170 b"99wj3485nxgyq5ub9zd3e7jsrq7a92ea"
171 .to_vec()
172 .to_vec()
173 .try_into()
174 .unwrap(),
175 )
176 .unwrap(),
177 )
178 .add_transformer(
179 ChaCha20Dec::new_with_fixed(
180 b"wvwj3485nxgyq5ub9zd3e7jsrq7a92ea"
181 .to_vec()
182 .to_vec()
183 .try_into()
184 .unwrap(),
185 )
186 .unwrap(),
187 )
188 .add_transformer(ZstdDec::new())
189 .add_transformer(ZstdDec::new())
190 .process()
191 .await
192 .unwrap();
193
194 let mut file = File::open("test.txt").await.unwrap();
195 let mut file2 = File::open("test.txt.out.4").await.unwrap();
196 let mut buf1 = String::new();
197 let mut buf2 = String::new();
198 file.read_to_string(&mut buf1).await.unwrap();
199 file2.read_to_string(&mut buf2).await.unwrap();
200 assert!(buf1 == buf2)
201 }
202
203 #[tokio::test]
204 async fn test_with_vec() {
205 let file = b"This is a very very important test".to_vec();
206 let mut file2 = Vec::new();
207
208 GenericReadWriter::new_with_writer(file.as_ref(), &mut file2)
210 .add_transformer(ZstdEnc::new())
211 .add_transformer(ZstdEnc::new()) .add_transformer(
213 ChaCha20Enc::new_with_fixed(
214 b"wvwj3485nxgyq5ub9zd3e7jsrq7a92ea"
215 .to_vec()
216 .to_vec()
217 .try_into()
218 .unwrap(),
219 )
220 .unwrap(),
221 )
222 .add_transformer(
223 ChaCha20Enc::new_with_fixed(
224 b"99wj3485nxgyq5ub9zd3e7jsrq7a92ea"
225 .to_vec()
226 .to_vec()
227 .try_into()
228 .unwrap(),
229 )
230 .unwrap(),
231 )
232 .add_transformer(
233 ChaCha20Dec::new_with_fixed(
234 b"99wj3485nxgyq5ub9zd3e7jsrq7a92ea"
235 .to_vec()
236 .to_vec()
237 .try_into()
238 .unwrap(),
239 )
240 .unwrap(),
241 )
242 .add_transformer(
243 ChaCha20Dec::new_with_fixed(
244 b"wvwj3485nxgyq5ub9zd3e7jsrq7a92ea"
245 .to_vec()
246 .to_vec()
247 .try_into()
248 .unwrap(),
249 )
250 .unwrap(),
251 )
252 .add_transformer(ZstdDec::new())
253 .add_transformer(ZstdDec::new()) .process()
255 .await
256 .unwrap();
257 assert!(file == file2)
258 }
259
260 #[tokio::test]
261 async fn test_footer_parsing() {
262 let file = File::open("test.txt").await.unwrap();
263 let file2 = File::create("test.txt.out.6").await.unwrap();
264 GenericReadWriter::new_with_writer(file, file2)
265 .add_transformer(ZstdEnc::new())
266 .add_transformer(
267 FooterGenerator::new_with_ctx(FileContext {
268 file_path: "test.txt".to_string(),
269 ..Default::default()
270 })
271 .unwrap(),
272 )
273 .process()
274 .await
275 .unwrap();
276
277 let mut file2 = File::open("test.txt.out.6").await.unwrap();
278 file2
279 .seek(std::io::SeekFrom::End(-65536 * 2))
280 .await
281 .unwrap();
282
283 let buf: &mut [u8; 65536 * 2] = &mut [0; 65536 * 2];
284 file2.read_exact(buf).await.unwrap();
285
286 }
305
306 #[tokio::test]
307 async fn test_footer_parsing_encrypted() {
308 let file = File::open("test.txt").await.unwrap();
309 let file2 = File::create("test.txt.out.7").await.unwrap();
310 GenericReadWriter::new_with_writer(file, file2)
311 .add_transformer(ZstdEnc::new())
312 .add_transformer(
313 ChaCha20Enc::new_with_fixed(
314 b"wvwj3485nxgyq5ub9zd3e7jsrq7a92ea"
315 .to_vec()
316 .try_into()
317 .unwrap(),
318 )
319 .unwrap(),
320 )
321 .add_transformer(
322 FooterGenerator::new_with_ctx(FileContext {
323 file_path: "test.txt".to_string(),
324 encryption_key: EncryptionKey::Same(
325 b"wvwj3485nxgyq5ub9zd3e7jsrq7a92ea"
326 .to_vec()
327 .try_into()
328 .unwrap(),
329 ),
330 ..Default::default()
331 })
332 .unwrap(),
333 )
334 .process()
335 .await
336 .unwrap();
337
338 let mut file2 = File::open("test.txt.out.7").await.unwrap();
339 file2
340 .seek(std::io::SeekFrom::End((-65536 - 28) * 2))
341 .await
342 .unwrap();
343
344 let buf: &mut [u8; (65536 + 28) * 2] = &mut [0; (65536 + 28) * 2];
345 file2.read_exact(buf).await.unwrap();
346
347 }
364
365 #[tokio::test]
366 async fn test_with_filter() {
367 let file = b"This is a very very important test".to_vec();
368 let mut file2 = Vec::new();
369
370 GenericReadWriter::new_with_writer(file.as_ref(), &mut file2)
372 .add_transformer(ZstdEnc::new())
373 .add_transformer(ZstdEnc::new()) .add_transformer(
375 ChaCha20Enc::new_with_fixed(
376 b"wvwj3485nxgyq5ub9zd3e7jsrq7a92ea"
377 .to_vec()
378 .to_vec()
379 .try_into()
380 .unwrap(),
381 )
382 .unwrap(),
383 )
384 .add_transformer(
385 ChaCha20Enc::new_with_fixed(
386 b"99wj3485nxgyq5ub9zd3e7jsrq7a92ea"
387 .to_vec()
388 .to_vec()
389 .try_into()
390 .unwrap(),
391 )
392 .unwrap(),
393 )
394 .add_transformer(
395 ChaCha20Dec::new_with_fixed(
396 b"99wj3485nxgyq5ub9zd3e7jsrq7a92ea"
397 .to_vec()
398 .to_vec()
399 .try_into()
400 .unwrap(),
401 )
402 .unwrap(),
403 )
404 .add_transformer(
405 ChaCha20Dec::new_with_fixed(
406 b"wvwj3485nxgyq5ub9zd3e7jsrq7a92ea"
407 .to_vec()
408 .to_vec()
409 .try_into()
410 .unwrap(),
411 )
412 .unwrap(),
413 )
414 .add_transformer(ZstdDec::new())
415 .add_transformer(ZstdDec::new())
416 .add_transformer(Filter::new_with_range(Range { from: 0, to: 3 }))
417 .process()
418 .await
419 .unwrap();
420
421 println!("{:?}", file2);
422 assert_eq!(file2, b"Thi".to_vec());
423 }
424
425 #[tokio::test]
426 async fn test_read_write_multifile() {
427 let file1 = b"This is a very very important test".to_vec();
428 let file2 = b"This is a very very important test".to_vec();
429 let mut file3: Vec<u8> = Vec::new();
430
431 let combined = Vec::from_iter(file1.clone().into_iter().chain(file2.clone()));
432
433 let (sx, rx) = async_channel::bounded(10);
434 sx.send(Message::FileContext(FileContext {
435 file_path: "file1.txt".to_string(),
436 compressed_size: file1.len() as u64,
437 decompressed_size: file1.len() as u64,
438 compression: true,
439 ..Default::default()
440 }))
441 .await
442 .unwrap();
443
444 sx.send(Message::FileContext(FileContext {
445 file_path: "file2.txt".to_string(),
446 compressed_size: file2.len() as u64,
447 decompressed_size: file2.len() as u64,
448 compression: false,
449 ..Default::default()
450 }))
451 .await
452 .unwrap();
453
454 let mut aswr = GenericReadWriter::new_with_writer(combined.as_ref(), &mut file3);
456 aswr.add_message_receiver(rx).await.unwrap();
457 aswr = aswr
458 .add_transformer(ZstdEnc::new())
459 .add_transformer(ZstdEnc::new()) .add_transformer(
461 ChaCha20Enc::new_with_fixed(
462 b"wvwj3485nxgyq5ub9zd3e7jsrq7a92ea"
463 .to_vec()
464 .to_vec()
465 .try_into()
466 .unwrap(),
467 )
468 .unwrap(),
469 )
470 .add_transformer(
471 ChaCha20Enc::new_with_fixed(
472 b"99wj3485nxgyq5ub9zd3e7jsrq7a92ea"
473 .to_vec()
474 .to_vec()
475 .try_into()
476 .unwrap(),
477 )
478 .unwrap(),
479 )
480 .add_transformer(
481 ChaCha20Dec::new_with_fixed(
482 b"99wj3485nxgyq5ub9zd3e7jsrq7a92ea"
483 .to_vec()
484 .to_vec()
485 .try_into()
486 .unwrap(),
487 )
488 .unwrap(),
489 )
490 .add_transformer(
491 ChaCha20Dec::new_with_fixed(
492 b"wvwj3485nxgyq5ub9zd3e7jsrq7a92ea"
493 .to_vec()
494 .to_vec()
495 .try_into()
496 .unwrap(),
497 )
498 .unwrap(),
499 )
500 .add_transformer(ZstdDec::new())
501 .add_transformer(ZstdDec::new())
502 .add_transformer(Filter::new_with_range(Range { from: 0, to: 3 }));
503 aswr.process().await.unwrap();
504 drop(aswr);
505
506 println!("{:?}", file3);
507 assert_eq!(file3, b"Thi".to_vec());
508 }
509
510 #[tokio::test]
511 async fn stream_test() {
512 let mut file2 = Vec::new();
513
514 use futures::stream;
515
516 let stream = stream::iter(vec![
517 Ok(Bytes::from_iter(
518 b"This is a very very important test".to_vec(),
519 )),
520 Ok(Bytes::from(b"This is a very very important test".to_vec())),
521 ]);
522
523 GenericStreamReadWriter::new_with_writer(stream, &mut file2)
525 .add_transformer(ZstdEnc::new())
526 .add_transformer(ZstdEnc::new()) .add_transformer(
528 ChaCha20Enc::new_with_fixed(
529 b"wvwj3485nxgyq5ub9zd3e7jsrq7a92ea"
530 .to_vec()
531 .to_vec()
532 .try_into()
533 .unwrap(),
534 )
535 .unwrap(),
536 )
537 .add_transformer(
538 ChaCha20Enc::new_with_fixed(
539 b"99wj3485nxgyq5ub9zd3e7jsrq7a92ea"
540 .to_vec()
541 .to_vec()
542 .try_into()
543 .unwrap(),
544 )
545 .unwrap(),
546 )
547 .add_transformer(
548 ChaCha20Dec::new_with_fixed(
549 b"99wj3485nxgyq5ub9zd3e7jsrq7a92ea"
550 .to_vec()
551 .to_vec()
552 .try_into()
553 .unwrap(),
554 )
555 .unwrap(),
556 )
557 .add_transformer(
558 ChaCha20Dec::new_with_fixed(
559 b"wvwj3485nxgyq5ub9zd3e7jsrq7a92ea"
560 .to_vec()
561 .to_vec()
562 .try_into()
563 .unwrap(),
564 )
565 .unwrap(),
566 )
567 .add_transformer(ZstdDec::new())
568 .add_transformer(ZstdDec::new())
569 .add_transformer(Filter::new_with_range(Range { from: 0, to: 3 }))
570 .process()
571 .await
572 .unwrap();
573
574 dbg!(format!("{:?}", std::str::from_utf8(&file2)));
575 assert_eq!(file2, b"Thi".to_vec());
576 }
577
578 #[tokio::test]
579 async fn e2e_test_read_write_multifile_tar_small() {
580 let file1 = b"This is a very very important test".to_vec();
581 let file2 = b"Another brilliant This is a very very important test1337".to_vec();
582 let mut file3 = File::create("test.txt.out.8.tar").await.unwrap();
583
584 let combined = Vec::from_iter(file1.clone().into_iter().chain(file2.clone()));
585
586 let (sx, rx) = async_channel::bounded(10);
587 sx.send(Message::FileContext(FileContext {
588 file_path: "file1.txt".to_string(),
589 compressed_size: file1.len() as u64,
590 decompressed_size: file1.len() as u64,
591 compression: true,
592 encryption_key: EncryptionKey::Same(
593 b"wvwj3485nxgyq5ub9zd3e7jsrq7a92ea"
594 .to_vec()
595 .to_vec()
596 .try_into()
597 .unwrap(),
598 ),
599 ..Default::default()
600 }))
601 .await
602 .unwrap();
603
604 sx.send(Message::FileContext(FileContext {
605 file_path: "file2.txt".to_string(),
606 compressed_size: file2.len() as u64,
607 decompressed_size: file2.len() as u64,
608 compression: false,
609 encryption_key: EncryptionKey::Same(
610 b"xxwj3485nxgyq5ub9zd3e7jsrq7a92ea"
611 .to_vec()
612 .to_vec()
613 .try_into()
614 .unwrap(),
615 ),
616 ..Default::default()
617 }))
618 .await
619 .unwrap();
620
621 let mut aswr = GenericReadWriter::new_with_writer(combined.as_ref(), &mut file3)
623 .add_transformer(TarEnc::new());
624 aswr.add_message_receiver(rx).await.unwrap();
625 aswr.process().await.unwrap();
626 }
627
628 #[tokio::test]
629 async fn e2e_test_read_write_multifile_tar_real() {
630 let mut file1 = File::open("test.txt").await.unwrap();
631 let mut file2 = File::open("test.txt").await.unwrap();
632 let mut file3 = File::create("test.txt.out.9.tar").await.unwrap();
633
634 let mut combined = Vec::new();
635 file1.read_to_end(&mut combined).await.unwrap();
636 file2.read_to_end(&mut combined).await.unwrap();
637
638 let (sx, rx) = async_channel::bounded(10);
639 sx.send(Message::FileContext(FileContext {
640 file_path: "file1.txt".to_string(),
641 compressed_size: file1.metadata().await.unwrap().len(),
642 decompressed_size: file1.metadata().await.unwrap().len(),
643 compression: true,
644 encryption_key: EncryptionKey::Same(
645 b"wvwj3485nxgyq5ub9zd3e7jsrq7a92ea"
646 .to_vec()
647 .to_vec()
648 .try_into()
649 .unwrap(),
650 ),
651 ..Default::default()
652 }))
653 .await
654 .unwrap();
655
656 sx.send(Message::FileContext(FileContext {
657 file_path: "file2.txt".to_string(),
658 compressed_size: file2.metadata().await.unwrap().len(),
659 decompressed_size: file2.metadata().await.unwrap().len(),
660 compression: false,
661 encryption_key: EncryptionKey::Same(
662 b"xxwj3485nxgyq5ub9zd3e7jsrq7a92ea"
663 .to_vec()
664 .to_vec()
665 .try_into()
666 .unwrap(),
667 ),
668 ..Default::default()
669 }))
670 .await
671 .unwrap();
672
673 let mut aswr = GenericReadWriter::new_with_writer(combined.as_ref(), &mut file3)
675 .add_transformer(TarEnc::new());
676 aswr.add_message_receiver(rx).await.unwrap();
677 aswr.process().await.unwrap();
678 }
679
680 #[tokio::test]
681 async fn e2e_test_stream_write_multifile_tar_real() {
682 let file1 = File::open("test.txt").await.unwrap();
683 let file2 = File::open("test.txt").await.unwrap();
684
685 let file1_size = file1.metadata().await.unwrap().len();
686 let file2_size = file2.metadata().await.unwrap().len();
687
688 let stream1 = tokio_util::io::ReaderStream::new(file1);
689 let stream2 = tokio_util::io::ReaderStream::new(file2);
690
691 let chained = stream1.chain(stream2);
692 let mapped = chained.map_err(|_| {
693 Box::<(dyn std::error::Error + Send + Sync + 'static)>::from("a_str_error")
694 });
695 let mut file3 = File::create("test.txt.out.10").await.unwrap();
696
697 let (sx, rx) = async_channel::bounded(10);
698 sx.send(Message::FileContext(FileContext {
699 file_path: "file1.txt".to_string(),
700 compressed_size: file1_size,
701 decompressed_size: file1_size,
702 ..Default::default()
703 }))
704 .await
705 .unwrap();
706
707 sx.send(Message::FileContext(FileContext {
708 file_path: "file2.txt".to_string(),
709 compressed_size: file2_size,
710 decompressed_size: file2_size,
711 ..Default::default()
712 }))
713 .await
714 .unwrap();
715
716 let mut aswr = GenericStreamReadWriter::new_with_writer(mapped, &mut file3)
718 .add_transformer(TarEnc::new());
719 aswr.add_message_receiver(rx).await.unwrap();
720 aswr.process().await.unwrap();
721 }
722
723 #[tokio::test]
724 async fn e2e_test_stream_tar_gz() {
725 let file1 = File::open("test.txt").await.unwrap();
726 let file2 = File::open("test.txt").await.unwrap();
727
728 let file1_size = file1.metadata().await.unwrap().len();
729 let file2_size = file2.metadata().await.unwrap().len();
730
731 let stream1 = tokio_util::io::ReaderStream::new(file1);
732 let stream2 = tokio_util::io::ReaderStream::new(file2);
733
734 let chained = stream1.chain(stream2);
735 let mapped = chained.map_err(|_| {
736 Box::<(dyn std::error::Error + Send + Sync + 'static)>::from("a_str_error")
737 });
738 let mut file3 = File::create("test.txt.out.11").await.unwrap();
739
740 let (sx, rx) = async_channel::bounded(10);
741 sx.send(Message::FileContext(FileContext {
742 file_path: "file1.txt".to_string(),
743 compressed_size: file1_size,
744 decompressed_size: file1_size,
745 ..Default::default()
746 }))
747 .await
748 .unwrap();
749
750 sx.send(Message::FileContext(FileContext {
751 file_path: "file2.txt".to_string(),
752 compressed_size: file2_size,
753 decompressed_size: file2_size,
754 ..Default::default()
755 }))
756 .await
757 .unwrap();
758
759 let mut aswr = GenericStreamReadWriter::new_with_writer(mapped, &mut file3)
761 .add_transformer(TarEnc::new())
762 .add_transformer(GzipEnc::new());
763 aswr.add_message_receiver(rx).await.unwrap();
764 aswr.process().await.unwrap();
765 }
766
767 #[tokio::test]
768 async fn hashing_transformer_test() {
769 let file = b"This is a very very important test".to_vec();
770 let mut file2 = Vec::new();
771
772 let (probe, rx) = SizeProbe::new();
773 let md5_trans = crate::transformers::hashing_transformer::HashingTransformer::new(
774 Md5::new(),
775 "md5".to_string(),
776 false,
777 );
778
779 GenericReadWriter::new_with_writer(file.as_ref(), &mut file2)
781 .add_transformer(md5_trans)
782 .add_transformer(probe)
783 .process()
784 .await
785 .unwrap();
786
787 let size = rx.try_recv().unwrap();
788 assert_eq!(size, 34);
792 }
794
795 #[tokio::test]
796 async fn e2e_test_stream_tar_folder() {
797 let file1 = File::open("test.txt").await.unwrap();
798 let file2 = File::open("test.txt").await.unwrap();
799
800 let file1_size = file1.metadata().await.unwrap().len();
801 let file2_size = file2.metadata().await.unwrap().len();
802
803 let stream1 = tokio_util::io::ReaderStream::new(file1);
804 let stream2 = tokio_util::io::ReaderStream::new(file2);
805
806 let chained = stream1.chain(stream2);
807 let mapped = chained.map_err(|_| {
808 Box::<(dyn std::error::Error + Send + Sync + 'static)>::from("a_str_error")
809 });
810 let mut file3 = File::create("test.txt.out.tar").await.unwrap();
811
812 let (sx, rx) = async_channel::bounded(10);
813
814 sx.send(Message::FileContext(FileContext {
815 file_path: "blup/".to_string(),
816 compressed_size: 0,
817 decompressed_size: 0,
818 is_dir: true,
819 ..Default::default()
820 }))
821 .await
822 .unwrap();
823
824 sx.send(Message::FileContext(FileContext {
825 file_path: "blup/file1.txt".to_string(),
826 compressed_size: file1_size,
827 decompressed_size: file1_size,
828 ..Default::default()
829 }))
830 .await
831 .unwrap();
832
833 sx.send(Message::FileContext(FileContext {
834 file_path: "blip/".to_string(),
835 compressed_size: 0,
836 decompressed_size: 0,
837 is_dir: true,
838 ..Default::default()
839 }))
840 .await
841 .unwrap();
842
843 sx.send(Message::FileContext(FileContext {
844 file_path: "blip/file2.txt".to_string(),
845 compressed_size: file2_size,
846 decompressed_size: file2_size,
847 ..Default::default()
848 }))
849 .await
850 .unwrap();
851
852 let mut aswr = GenericStreamReadWriter::new_with_writer(mapped, &mut file3)
854 .add_transformer(TarEnc::new());
855 aswr.add_message_receiver(rx).await.unwrap();
856 aswr.process().await.unwrap();
858 }
859
860 #[tokio::test]
861 async fn e2e_pithos_tar_gz() {
862 let file1 = File::open("test.txt").await.unwrap();
863 let file2 = File::open("test.txt").await.unwrap();
864
865 let file1_size = file1.metadata().await.unwrap().len();
866 let file2_size = file2.metadata().await.unwrap().len();
867
868 let stream1 = tokio_util::io::ReaderStream::new(file1);
869 let stream2 = tokio_util::io::ReaderStream::new(file2);
870
871 let chained = stream1.chain(stream2);
872 let mapped = chained.map_err(|_| {
873 Box::<(dyn std::error::Error + Send + Sync + 'static)>::from("a_str_error")
874 });
875 let mut file3 = File::create("test.txt.out.pto").await.unwrap();
876
877 let (sx, rx) = async_channel::bounded(10);
878
879 let privkey_bytes = BASE64_STANDARD
880 .decode("MC4CAQAwBQYDK2VuBCIEIFDnbf0aEpZxwEdy1qG4xpV8gVNq7zEREtMjLzCE6R5x")
881 .unwrap();
882 let privkey: [u8; 32] = privkey_bytes[privkey_bytes.len() - 32..]
883 .to_vec()
884 .try_into()
885 .unwrap();
886
887 let pubkey_bytes = BASE64_STANDARD
888 .decode("MCowBQYDK2VuAyEA2laqNukb4+2am7QdC6eDANu1DDuKdC5LPtYQM+XE5k8=")
889 .unwrap();
890 let pubkey: [u8; 32] = pubkey_bytes[pubkey_bytes.len() - 32..]
891 .to_vec()
892 .try_into()
893 .unwrap();
894
895 sx.send(Message::FileContext(FileContext {
896 file_path: "file1.txt".to_string(),
897 compressed_size: file1_size,
898 decompressed_size: file1_size,
899 recipients_pubkeys: vec![pubkey],
900 encryption_key: EncryptionKey::Same(
901 b"wvwj3485nxgyq5ub9zd3e7jsrq7a92ea"
902 .to_vec()
903 .to_vec()
904 .try_into()
905 .unwrap(),
906 ),
907 ..Default::default()
908 }))
909 .await
910 .unwrap();
911
912 sx.send(Message::FileContext(FileContext {
913 file_path: "file2.txt".to_string(),
914 compressed_size: file2_size,
915 decompressed_size: file2_size,
916 recipients_pubkeys: vec![pubkey],
917 encryption_key: EncryptionKey::Same(
918 b"wvwj3485nxgyq5ub9zd3e7jsrq7a92ea"
919 .to_vec()
920 .to_vec()
921 .try_into()
922 .unwrap(),
923 ),
924 ..Default::default()
925 }))
926 .await
927 .unwrap();
928
929 let mut aswr = GenericStreamReadWriter::new_with_writer(mapped, &mut file3)
931 .add_transformer(PithosTransformer::new())
932 .add_transformer(FooterGenerator::new(None));
933 aswr.add_message_receiver(rx).await.unwrap();
934 aswr.process().await.unwrap();
935
936 let mut file3 = File::open("test.txt.out.pto").await.unwrap();
937
938 let file_meta = file3.metadata().await.unwrap();
940
941 let footer_prediction = if file_meta.len() < 65536 * 2 {
942 file_meta.len() } else {
944 65536 * 2
945 };
946
947 file3
949 .seek(SeekFrom::End(-(footer_prediction as i64)))
950 .await
951 .unwrap();
952 let buf = &mut vec![0; footer_prediction as usize]; file3.read_exact(buf).await.unwrap();
954
955 let mut parser = FooterParser::new(buf).unwrap();
956 parser = parser.add_recipient(&privkey);
957 parser = parser.parse().unwrap();
958
959 let mut missing_buf;
961 if let FooterParserState::Missing(missing_bytes) = parser.state {
962 let needed_bytes = footer_prediction + missing_bytes as u64;
963 file3
964 .seek(SeekFrom::End(-(needed_bytes as i64)))
965 .await
966 .unwrap();
967 missing_buf = vec![0; missing_bytes as usize]; file3.read_exact(&mut missing_buf).await.unwrap();
969
970 parser = parser.add_bytes(&missing_buf).unwrap();
971 parser = parser.parse().unwrap()
972 }
973
974 let footer: Footer = parser.try_into().unwrap();
976
977 let keys = footer
978 .encryption_keys
979 .map(|keys| {
980 keys.keys
981 .iter()
982 .filter_map(|(k, idx)| {
983 if let DirOrFileIdx::File(i) = idx {
984 Some((k.clone(), *i))
985 } else {
986 None
987 }
988 })
989 .collect::<Vec<_>>()
990 })
991 .unwrap_or_default();
992
993 let (sx2, rx2) = async_channel::bounded(10);
994
995 let file3 = File::open("test.txt.out.pto").await.unwrap();
996
997 let mut out_file1 = File::create("test.txt.out.pto.tar.gz").await.unwrap();
998
999 let read_stream = tokio_util::io::ReaderStream::new(file3).map_err(|_| {
1000 Box::<(dyn std::error::Error + Send + Sync + 'static)>::from("a_str_error")
1001 });
1002
1003 let mut reader = GenericStreamReadWriter::new_with_writer(read_stream, &mut out_file1)
1004 .add_transformer(ChaCha20Dec::new_with_fixed_list(keys).unwrap())
1005 .add_transformer(ZstdDec::new())
1006 .add_transformer(TarEnc::new());
1007 reader.add_message_receiver(rx2).await.unwrap();
1009
1010 for (idx, file) in footer.table_of_contents.files.into_iter().enumerate() {
1011 if let FileContextVariants::FileDecrypted(file) = file {
1012 sx2.send(Message::FileContext(
1013 file.try_into_file_context(idx).unwrap(),
1014 ))
1015 .await
1016 .unwrap();
1017 }
1018 }
1019 reader.process().await.unwrap();
1020 }
1021
1022 #[tokio::test]
1023 async fn e2e_pithos_rewrite_footer() {
1024 let file1 = File::open("test.txt").await.unwrap();
1025
1026 let file1_size = file1.metadata().await.unwrap().len();
1027
1028 let stream1 = tokio_util::io::ReaderStream::new(file1);
1029
1030 let mapped = stream1.map_err(|_| {
1031 Box::<(dyn std::error::Error + Send + Sync + 'static)>::from("a_str_error")
1032 });
1033 let mut file3 = File::create("test.txt.out.2.pto").await.unwrap();
1034
1035 let (sx, rx) = async_channel::bounded(10);
1036
1037 let privkey_bytes = BASE64_STANDARD
1038 .decode("MC4CAQAwBQYDK2VuBCIEIFDnbf0aEpZxwEdy1qG4xpV8gVNq7zEREtMjLzCE6R5x")
1039 .unwrap();
1040 let privkey: [u8; 32] = privkey_bytes[privkey_bytes.len() - 32..]
1041 .to_vec()
1042 .try_into()
1043 .unwrap();
1044
1045 let pubkey_bytes = BASE64_STANDARD
1046 .decode("MCowBQYDK2VuAyEA2laqNukb4+2am7QdC6eDANu1DDuKdC5LPtYQM+XE5k8=")
1047 .unwrap();
1048 let pubkey: [u8; 32] = pubkey_bytes[pubkey_bytes.len() - 32..]
1049 .to_vec()
1050 .try_into()
1051 .unwrap();
1052
1053 sx.send(Message::FileContext(FileContext {
1054 file_path: "file1.txt".to_string(),
1055 compressed_size: file1_size,
1056 decompressed_size: file1_size,
1057 recipients_pubkeys: vec![pubkey],
1058 encryption_key: EncryptionKey::Same(
1059 b"wvwj3485nxgyq5ub9zd3e7jsrq7a92ea"
1060 .to_vec()
1061 .to_vec()
1062 .try_into()
1063 .unwrap(),
1064 ),
1065 ..Default::default()
1066 }))
1067 .await
1068 .unwrap();
1069
1070 let mut aswr = GenericStreamReadWriter::new_with_writer(mapped, &mut file3)
1072 .add_transformer(PithosTransformer::new())
1073 .add_transformer(FooterGenerator::new(None));
1074 aswr.add_message_receiver(rx).await.unwrap();
1075 aswr.process().await.unwrap();
1076
1077 let mut file3 = File::open("test.txt.out.2.pto").await.unwrap();
1078
1079 let file_meta = file3.metadata().await.unwrap();
1081
1082 let footer_prediction = if file_meta.len() < 65536 * 2 {
1083 file_meta.len() } else {
1085 65536 * 2
1086 };
1087
1088 file3
1090 .seek(SeekFrom::End(-(footer_prediction as i64)))
1091 .await
1092 .unwrap();
1093 let buf = &mut vec![0; footer_prediction as usize]; file3.read_exact(buf).await.unwrap();
1095
1096 let mut parser = FooterParser::new(buf).unwrap();
1097 parser = parser.add_recipient(&privkey);
1098 parser = parser.parse().unwrap();
1099
1100 let mut missing_buf;
1102 if let FooterParserState::Missing(missing_bytes) = parser.state {
1103 let needed_bytes = footer_prediction + missing_bytes as u64;
1104 file3
1105 .seek(SeekFrom::End(-(needed_bytes as i64)))
1106 .await
1107 .unwrap();
1108 missing_buf = vec![0; missing_bytes as usize]; file3.read_exact(&mut missing_buf).await.unwrap();
1110
1111 parser = parser.add_bytes(&missing_buf).unwrap();
1112 parser = parser.parse().unwrap()
1113 }
1114
1115 let footer: Footer = parser.try_into().unwrap();
1117
1118 let (_sx2, rx2) = async_channel::bounded(10);
1119
1120 let file3 = File::open("test.txt.out.2.pto").await.unwrap();
1121
1122 let mut out_file1 = File::create("test.txt.out.3.pto").await.unwrap();
1123
1124 let read_stream = tokio_util::io::ReaderStream::new(file3).map_err(|_| {
1125 Box::<(dyn std::error::Error + Send + Sync + 'static)>::from("a_str_error")
1126 });
1127
1128 let privkey_bytes_2 = BASE64_STANDARD
1129 .decode("MC4CAQAwBQYDK2VuBCIEIMhHHRAu72qdkx9I4D08RD3OQniJxGUI420aPlZwAJtX")
1130 .unwrap();
1131 let privkey_2: [u8; 32] = privkey_bytes_2[privkey_bytes_2.len() - 32..]
1132 .to_vec()
1133 .try_into()
1134 .unwrap();
1135
1136 let pubkey_bytes_2 = BASE64_STANDARD
1137 .decode("MCowBQYDK2VuAyEAoqu7pzwam2uks5EseS06jQP6ISX42f613KKWm8cLM1M=")
1138 .unwrap();
1139 let pubkey_2: [u8; 32] = pubkey_bytes_2[pubkey_bytes_2.len() - 32..]
1140 .to_vec()
1141 .try_into()
1142 .unwrap();
1143
1144 let mut reader = GenericStreamReadWriter::new_with_writer(read_stream, &mut out_file1)
1145 .add_transformer(FooterUpdater::new(vec![pubkey_2], footer));
1146 reader.add_message_receiver(rx2).await.unwrap();
1147 reader.process().await.unwrap();
1148
1149 let mut file3 = File::open("test.txt.out.3.pto").await.unwrap();
1150
1151 let file_meta = file3.metadata().await.unwrap();
1153
1154 let footer_prediction = if file_meta.len() < 65536 * 2 {
1155 file_meta.len() } else {
1157 65536 * 2
1158 };
1159
1160 file3
1162 .seek(SeekFrom::End(-(footer_prediction as i64)))
1163 .await
1164 .unwrap();
1165 let buf = &mut vec![0; footer_prediction as usize]; file3.read_exact(buf).await.unwrap();
1167
1168 let mut parser = FooterParser::new(buf).unwrap();
1169 parser = parser.add_recipient(&privkey_2);
1170 parser = parser.parse().unwrap();
1171
1172 let footer: Footer = parser.try_into().unwrap();
1173
1174 assert!(footer.encryption_keys.unwrap().keys.len() > 0)
1175 }
1176
1177 #[tokio::test]
1178 async fn e2e_pithos_extractor() {
1179 let file1 = File::open("test.txt").await.unwrap();
1180
1181 let file1_size = file1.metadata().await.unwrap().len();
1182
1183 let stream1 = tokio_util::io::ReaderStream::new(file1);
1184
1185 let mapped = stream1.map_err(|_| {
1186 Box::<(dyn std::error::Error + Send + Sync + 'static)>::from("a_str_error")
1187 });
1188 let mut file3 = File::create("test.txt.out.4.pto").await.unwrap();
1189
1190 let (sx, rx) = async_channel::bounded(10);
1191
1192 let privkey_bytes = BASE64_STANDARD
1193 .decode("MC4CAQAwBQYDK2VuBCIEIFDnbf0aEpZxwEdy1qG4xpV8gVNq7zEREtMjLzCE6R5x")
1194 .unwrap();
1195 let privkey: [u8; 32] = privkey_bytes[privkey_bytes.len() - 32..]
1196 .to_vec()
1197 .try_into()
1198 .unwrap();
1199
1200 let pubkey_bytes = BASE64_STANDARD
1201 .decode("MCowBQYDK2VuAyEA2laqNukb4+2am7QdC6eDANu1DDuKdC5LPtYQM+XE5k8=")
1202 .unwrap();
1203 let pubkey: [u8; 32] = pubkey_bytes[pubkey_bytes.len() - 32..]
1204 .to_vec()
1205 .try_into()
1206 .unwrap();
1207
1208 sx.send(Message::FileContext(FileContext {
1209 file_path: "file1.txt".to_string(),
1210 compressed_size: file1_size,
1211 decompressed_size: file1_size,
1212 recipients_pubkeys: vec![pubkey],
1213 encryption_key: EncryptionKey::Same(
1214 b"wvwj3485nxgyq5ub9zd3e7jsrq7a92ea"
1215 .to_vec()
1216 .to_vec()
1217 .try_into()
1218 .unwrap(),
1219 ),
1220 ..Default::default()
1221 }))
1222 .await
1223 .unwrap();
1224
1225 let mut aswr = GenericStreamReadWriter::new_with_writer(mapped, &mut file3)
1227 .add_transformer(PithosTransformer::new())
1228 .add_transformer(FooterGenerator::new(None));
1229 aswr.add_message_receiver(rx).await.unwrap();
1230 aswr.process().await.unwrap();
1231
1232 let mut file3 = File::open("test.txt.out.4.pto").await.unwrap();
1233
1234 let file_meta = file3.metadata().await.unwrap();
1236
1237 let footer_prediction = if file_meta.len() < 65536 * 2 {
1238 file_meta.len() } else {
1240 65536 * 2
1241 };
1242
1243 file3
1245 .seek(SeekFrom::End(-(footer_prediction as i64)))
1246 .await
1247 .unwrap();
1248 let buf = &mut vec![0; footer_prediction as usize]; file3.read_exact(buf).await.unwrap();
1250
1251 let mut parser = FooterParser::new(buf).unwrap();
1252 parser = parser.add_recipient(&privkey);
1253 parser = parser.parse().unwrap();
1254
1255 let footer: Footer = parser.try_into().unwrap();
1257
1258 let file3 = File::open("test.txt.out.4.pto").await.unwrap();
1259
1260 let mut vec = Vec::new();
1261
1262 let stream1 = tokio_util::io::ReaderStream::new(file3);
1263
1264 let mapped = stream1.map_err(|_| {
1265 Box::<(dyn std::error::Error + Send + Sync + 'static)>::from("a_str_error")
1266 });
1267
1268 let (extractor, rcv) = FooterExtractor::new(Some(privkey));
1269
1270 GenericStreamReadWriter::new_with_writer(mapped, &mut vec)
1271 .add_transformer(extractor)
1272 .process()
1273 .await
1274 .unwrap();
1275
1276 let extracted_footer = rcv.recv_blocking().unwrap();
1277 assert_eq!(extracted_footer, footer);
1278 }
1279
1280 #[tokio::test]
1281 async fn e2e_test_parts_decryptor() {
1282 let file = File::open("test.txt").await.unwrap();
1283 let file2 = vec![];
1284
1285 let repeated: Vec<u64> = vec![65564u64 * 60, 65564 * 16, 65564, 50860];
1286
1287 GenericReadWriter::new_with_writer(file, file2)
1289 .add_transformer(
1290 ChaCha20Enc::new_with_fixed(
1291 b"wvwj3485nxgyq5ub9zd3e7jsrq7a92ea"
1292 .to_vec()
1293 .try_into()
1294 .unwrap(),
1295 )
1296 .unwrap(),
1297 )
1298 .add_transformer(ChaCha20DecParts::new_with_lengths(
1299 b"wvwj3485nxgyq5ub9zd3e7jsrq7a92ea"
1300 .to_vec()
1301 .try_into()
1302 .unwrap(),
1303 repeated,
1304 ))
1305 .process()
1306 .await
1307 .unwrap();
1308 }
1309
1310 #[tokio::test]
1311 async fn e2e_test_resilient_parts_decryptor() {
1312 let file = File::open("test.txt").await.unwrap();
1313 let file2 = vec![];
1314
1315 let repeated: Vec<u64> = vec![50860];
1316
1317 GenericReadWriter::new_with_writer(file, file2)
1319 .add_transformer(
1320 ChaCha20Enc::new_with_fixed(
1321 b"wvwj3485nxgyq5ub9zd3e7jsrq7a92ea"
1322 .to_vec()
1323 .try_into()
1324 .unwrap(),
1325 )
1326 .unwrap(),
1327 )
1328 .add_transformer(ChaChaResilient::new_with_lengths(
1329 b"wvwj3485nxgyq5ub9zd3e7jsrq7a92ea"
1330 .to_vec()
1331 .try_into()
1332 .unwrap(),
1333 repeated,
1334 ))
1335 .process()
1336 .await
1337 .unwrap();
1338 }
1339}