1use crate::error::Result;
2use async_trait::async_trait;
3use std::path::{Path, PathBuf};
4use std::sync::Arc;
5use tracing::debug;
6
7use super::disk_cache::WrDiskCache;
8use super::mmap_disk_writer::MmapDiskWriter;
9use super::positioned_disk_writer::PositionedDiskWriter;
10
11#[async_trait]
12pub trait DiskWriter: Send + Sync {
13 async fn write(&mut self, data: &[u8]) -> Result<()>;
14 async fn finalize(&mut self) -> Result<Vec<u8>>;
15}
16
17pub struct DefaultDiskWriter {
18 path: std::path::PathBuf,
19 file: Option<tokio::fs::File>,
20}
21
22impl DefaultDiskWriter {
23 pub fn new(path: &Path) -> Self {
24 DefaultDiskWriter {
25 path: path.to_path_buf(),
26 file: None,
27 }
28 }
29
30 pub fn path(&self) -> &std::path::Path {
31 &self.path
32 }
33}
34
35#[async_trait]
36impl DiskWriter for DefaultDiskWriter {
37 async fn write(&mut self, data: &[u8]) -> Result<()> {
38 if self.file.is_none() {
39 let f = tokio::fs::File::create(&self.path)
40 .await
41 .map_err(|e| crate::error::Aria2Error::Io(e.to_string()))?;
42 self.file = Some(f);
43 }
44 if let Some(ref mut file) = self.file {
45 use tokio::io::AsyncWriteExt;
46 file.write_all(data)
47 .await
48 .map_err(|e| crate::error::Aria2Error::Io(e.to_string()))?;
49 }
50 Ok(())
51 }
52
53 async fn finalize(&mut self) -> Result<Vec<u8>> {
54 if let Some(mut file) = self.file.take() {
55 use tokio::io::AsyncWriteExt;
56 file.flush()
57 .await
58 .map_err(|e| crate::error::Aria2Error::Io(e.to_string()))?;
59 drop(file.into_std().await);
64 }
65 Ok(vec![])
66 }
67}
68
69pub struct ByteArrayDiskWriter {
70 buffer: Vec<u8>,
71}
72
73impl ByteArrayDiskWriter {
74 pub fn new() -> Self {
75 ByteArrayDiskWriter { buffer: Vec::new() }
76 }
77
78 pub fn with_capacity(capacity: usize) -> Self {
79 ByteArrayDiskWriter {
80 buffer: Vec::with_capacity(capacity),
81 }
82 }
83
84 pub fn len(&self) -> usize {
85 self.buffer.len()
86 }
87
88 pub fn is_empty(&self) -> bool {
89 self.buffer.is_empty()
90 }
91}
92
93impl Default for ByteArrayDiskWriter {
94 fn default() -> Self {
95 Self::new()
96 }
97}
98
99#[async_trait]
100impl DiskWriter for ByteArrayDiskWriter {
101 async fn write(&mut self, data: &[u8]) -> Result<()> {
102 self.buffer.extend_from_slice(data);
103 Ok(())
104 }
105
106 async fn finalize(&mut self) -> Result<Vec<u8>> {
107 let buffer = self.buffer.clone();
108 Ok(buffer)
109 }
110}
111
112const DIRECT_WRITE_THRESHOLD: usize = 1024 * 1024;
114
115#[async_trait]
116#[allow(clippy::len_without_is_empty)]
117pub trait SeekableDiskWriter: Send + Sync {
118 async fn open(&mut self) -> Result<()>;
119 async fn write_at(&mut self, offset: u64, data: &[u8]) -> Result<()>;
120 async fn write_bytes_at(&mut self, offset: u64, data: bytes::Bytes) -> Result<()> {
123 self.write_at(offset, &data).await
125 }
126 async fn read_at(&mut self, offset: u64, buf: &mut [u8]) -> Result<usize>;
127 async fn truncate(&mut self, length: u64) -> Result<()>;
128 async fn flush(&mut self) -> Result<()>;
129 async fn len(&self) -> Result<u64>;
130 fn path(&self) -> &Path;
131 async fn close(&mut self) -> Result<()> {
137 Ok(())
138 }
139}
140
141pub struct CachedDiskWriter {
142 writer: Box<dyn SeekableDiskWriter>,
148 cache: Option<Arc<WrDiskCache>>,
149 path: PathBuf,
150 total_size: Option<u64>,
151 opened: bool,
152 rate_limiter: Option<Arc<crate::rate_limiter::RateLimiter>>,
154}
155
156impl CachedDiskWriter {
157 pub fn new(path: &Path, total_size: Option<u64>, cache_size_mb: Option<usize>) -> Self {
163 Self::new_with_mmap(path, total_size, cache_size_mb, false)
164 }
165
166 pub fn new_with_mmap(
175 path: &Path,
176 total_size: Option<u64>,
177 cache_size_mb: Option<usize>,
178 use_mmap: bool,
179 ) -> Self {
180 let writer: Box<dyn SeekableDiskWriter> = if use_mmap {
181 Box::new(MmapDiskWriter::new(path, total_size))
182 } else {
183 Box::new(PositionedDiskWriter::new(path, total_size))
184 };
185 let cache = cache_size_mb.map(|mb| Arc::new(WrDiskCache::new(mb)));
186 Self {
187 writer,
188 cache,
189 path: path.to_path_buf(),
190 total_size,
191 opened: false,
192 rate_limiter: None,
193 }
194 }
195
196 pub fn open_existing(path: &Path) -> Result<Self> {
197 let mut writer = Self::new(path, None, None);
198 writer.opened = true;
199 Ok(writer)
200 }
201
202 pub fn with_rate_limiter(mut self, limiter: Arc<crate::rate_limiter::RateLimiter>) -> Self {
205 self.rate_limiter = Some(limiter);
206 self
207 }
208
209 pub fn is_opened(&self) -> bool {
210 self.opened
211 }
212}
213
214#[async_trait]
215impl SeekableDiskWriter for CachedDiskWriter {
216 async fn open(&mut self) -> Result<()> {
217 if self.opened {
218 return Ok(());
219 }
220 self.writer.open().await?;
224 self.opened = true;
225 Ok(())
226 }
227
228 async fn write_at(&mut self, offset: u64, data: &[u8]) -> Result<()> {
229 self.open().await?;
230
231 if let Some(ref limiter) = self.rate_limiter
233 && !limiter.try_acquire_download(data.len() as u64).await
234 {
235 debug!(
236 "Rate limit exceeded for {} bytes at offset {}, writing without throttling",
237 data.len(),
238 offset
239 );
240 }
241
242 if data.len() >= DIRECT_WRITE_THRESHOLD {
243 self.writer.write_at(offset, data).await?;
245 } else if let Some(ref cache) = self.cache {
246 cache
250 .write(offset, bytes::Bytes::copy_from_slice(data))
251 .await?;
252 } else {
253 self.writer.write_at(offset, data).await?;
255 }
256
257 Ok(())
258 }
259
260 async fn write_bytes_at(&mut self, offset: u64, data: bytes::Bytes) -> Result<()> {
264 self.open().await?;
265
266 if let Some(ref limiter) = self.rate_limiter
268 && !limiter.try_acquire_download(data.len() as u64).await
269 {
270 debug!(
271 "Rate limit exceeded for {} bytes at offset {}, writing without throttling",
272 data.len(),
273 offset
274 );
275 }
276
277 if data.len() >= DIRECT_WRITE_THRESHOLD {
278 self.writer.write_bytes_at(offset, data).await?;
280 } else if let Some(ref cache) = self.cache {
281 cache.write(offset, data).await?;
283 } else {
284 self.writer.write_bytes_at(offset, data).await?;
286 }
287
288 Ok(())
289 }
290
291 async fn read_at(&mut self, offset: u64, buf: &mut [u8]) -> Result<usize> {
292 self.flush_cache().await?;
295 self.writer.read_at(offset, buf).await
298 }
299
300 async fn truncate(&mut self, length: u64) -> Result<()> {
301 self.flush_cache().await?;
302 self.writer.truncate(length).await
303 }
304
305 async fn flush(&mut self) -> Result<()> {
306 self.flush_cache().await?;
307 self.writer.flush().await
308 }
309
310 async fn len(&self) -> Result<u64> {
311 if !self.opened {
312 if let Some(size) = self.total_size {
313 return Ok(size);
314 }
315 return Ok(0);
316 }
317 self.writer.len().await
318 }
319
320 fn path(&self) -> &Path {
321 &self.path
322 }
323
324 async fn close(&mut self) -> Result<()> {
325 self.flush().await?;
326 self.writer.close().await?;
327 self.opened = false;
328 Ok(())
329 }
330}
331
332impl CachedDiskWriter {
333 async fn flush_cache(&mut self) -> Result<()> {
339 if let Some(ref cache) = self.cache {
340 let entries = cache.flush().await?;
341 if !entries.is_empty() {
342 for entry in entries {
343 let offset = entry.offset();
344 let data = entry.into_data();
345 if !data.is_empty() {
346 self.writer.write_bytes_at(offset, data).await?;
347 }
348 }
349 self.writer.flush().await?;
350 }
351 }
352 Ok(())
353 }
354
355 pub async fn read_all(&mut self) -> Result<Vec<u8>> {
356 let len = self.len().await? as usize;
357 if len == 0 {
358 return Ok(Vec::new());
359 }
360 let mut buf = vec![0u8; len];
361 self.read_at(0, &mut buf).await?;
362 Ok(buf)
363 }
364}
365
366#[cfg(test)]
367mod tests {
368 use super::*;
369
370 #[tokio::test]
371 async fn test_default_disk_writer_write_and_finalize() {
372 let dir = tempfile::tempdir().unwrap();
373 let path = dir.path().join("test_default.bin");
374
375 let mut writer = DefaultDiskWriter::new(&path);
376 writer.write(b"hello").await.unwrap();
377 writer.write(b" world").await.unwrap();
378 writer.finalize().await.unwrap();
379
380 let content = tokio::fs::read_to_string(&path).await.unwrap();
381 assert_eq!(content, "hello world");
382 }
383
384 #[tokio::test]
385 async fn test_byte_array_disk_writer() {
386 let mut writer = ByteArrayDiskWriter::with_capacity(10);
387 writer.write(b"abc").await.unwrap();
388 writer.write(b"def").await.unwrap();
389 let result = writer.finalize().await.unwrap();
390 assert_eq!(result, b"abcdef");
391 assert_eq!(writer.len(), 6);
392 }
393
394 #[tokio::test]
395 async fn test_seekable_writer_basic() {
396 let dir = tempfile::tempdir().unwrap();
397 let path = dir.path().join("test_seekable.bin");
398
399 let mut writer = CachedDiskWriter::new(&path, Some(1024), None);
400 writer.open().await.unwrap();
401 assert!(writer.is_opened());
402
403 writer.write_at(0, b"hello").await.unwrap();
404 writer.write_at(5, b" world").await.unwrap();
405 writer.flush().await.unwrap();
406
407 let content = tokio::fs::read(&path).await.unwrap();
408 assert_eq!(&content[..11], b"hello world");
409 }
410
411 #[tokio::test]
412 async fn test_seekable_writer_random_access() {
413 let dir = tempfile::tempdir().unwrap();
414 let path = dir.path().join("test_random.bin");
415
416 let mut writer = CachedDiskWriter::new(&path, None, None);
417 writer.open().await.unwrap();
418
419 writer.write_at(200, b"SEG2").await.unwrap();
420 writer.write_at(0, b"SEG0").await.unwrap();
421 writer.write_at(100, b"SEG1").await.unwrap();
422 writer.flush().await.unwrap();
423
424 let content = tokio::fs::read(&path).await.unwrap();
425 assert_eq!(content.len(), 204);
426 assert_eq!(&content[0..4], b"SEG0");
427 assert_eq!(&content[100..104], b"SEG1");
428 assert_eq!(&content[200..204], b"SEG2");
429 }
430
431 #[tokio::test]
432 async fn test_seekable_writer_read_at() {
433 let dir = tempfile::tempdir().unwrap();
434 let path = dir.path().join("test_read.bin");
435
436 let mut writer = CachedDiskWriter::new(&path, Some(100), None);
437 writer.open().await.unwrap();
438 writer.write_at(50, b"offset-50-data").await.unwrap();
439 writer.flush().await.unwrap();
440
441 let mut buf = [0u8; 14];
442 let n = writer.read_at(50, &mut buf).await.unwrap();
443 assert_eq!(n, 14);
444 assert_eq!(&buf, b"offset-50-data");
445 }
446
447 #[tokio::test]
448 async fn test_cached_writer_with_cache() {
449 let dir = tempfile::tempdir().unwrap();
450 let path = dir.path().join("test_cached.bin");
451
452 let mut writer = CachedDiskWriter::new(&path, Some(4096), Some(1));
453 writer.open().await.unwrap();
454
455 for i in 0..100 {
456 let data = vec![i as u8; 64];
457 writer.write_at((i * 64) as u64, &data).await.unwrap();
458 }
459
460 writer.flush().await.unwrap();
461
462 let content = tokio::fs::read(&path).await.unwrap();
463 assert_eq!(content.len(), 6400);
464
465 for i in 0..100 {
466 let start = i * 64;
467 assert_eq!(content[start], i as u8, "mismatch at byte {}", start);
468 }
469 }
470
471 #[tokio::test]
472 async fn test_cached_writer_large_write_bypasses_cache() {
473 let dir = tempfile::tempdir().unwrap();
474 let path = dir.path().join("test_large.bin");
475
476 let large_data = vec![0xAB; 128 * 1024]; let mut writer = CachedDiskWriter::new(&path, None, Some(1));
480 writer.open().await.unwrap();
481 writer.write_at(0, &large_data).await.unwrap();
482 writer.flush().await.unwrap();
483
484 let content = tokio::fs::read(&path).await.unwrap();
485 assert_eq!(content.len(), large_data.len());
486 assert!(content.iter().all(|&b| b == 0xAB));
487 }
488
489 #[tokio::test]
490 async fn test_seekable_writer_truncate() {
491 let dir = tempfile::tempdir().unwrap();
492 let path = dir.path().join("test_trunc.bin");
493
494 let mut writer = CachedDiskWriter::new(&path, Some(1000), None);
495 writer.open().await.unwrap();
496 writer
497 .write_at(0, b"hello world - this is longer than 20 bytes of data")
498 .await
499 .unwrap();
500 writer.flush().await.unwrap();
501
502 writer.truncate(20).await.unwrap();
503 writer.flush().await.unwrap();
504
505 let len = writer.len().await.unwrap();
506 assert!(len <= 21);
507
508 let content = tokio::fs::read(&path).await.unwrap();
509 assert!(content.len() <= 21);
510 assert_eq!(&content[..4], b"hell");
511 }
512
513 #[tokio::test]
514 async fn test_seekable_writer_len_before_open() {
515 let dir = tempfile::tempdir().unwrap();
516 let path = dir.path().join("test_len.bin");
517
518 let writer = CachedDiskWriter::new(&path, Some(9999), None);
519 let len = writer.len().await.unwrap();
520 assert_eq!(len, 9999);
521 }
522
523 #[tokio::test]
524 async fn test_close_reopens_cleanly() {
525 let dir = tempfile::tempdir().unwrap();
526 let path = dir.path().join("test_close.bin");
527
528 let mut writer = CachedDiskWriter::new(&path, None, None);
529 writer.open().await.unwrap();
530 writer.write_at(0, b"before close").await.unwrap();
531 writer.close().await.unwrap();
532 assert!(!writer.is_opened());
533
534 writer.open().await.unwrap();
535 writer.write_at(12, b" after reopen").await.unwrap();
536 writer.close().await.unwrap();
537
538 let content = tokio::fs::read_to_string(&path).await.unwrap();
539 assert_eq!(content, "before close after reopen");
540 }
541
542 #[tokio::test]
545 async fn test_cached_writer_with_rate_limiter() {
546 use crate::rate_limiter::{RateLimiter, RateLimiterConfig};
547
548 let dir = tempfile::tempdir().unwrap();
549 let path = dir.path().join("test_ratelimited.bin");
550
551 let cfg = RateLimiterConfig::new(Some(10), None).with_burst(Some(20), None);
553 let rl = Arc::new(RateLimiter::new(&cfg));
554
555 let mut writer =
556 CachedDiskWriter::new(&path, Some(4096), None).with_rate_limiter(rl.clone());
557 writer.open().await.unwrap();
558
559 let data = vec![0x42u8; 512];
561 writer.write_at(0, &data).await.unwrap();
562 writer.flush().await.unwrap();
563
564 let content = tokio::fs::read(&path).await.unwrap();
565 assert!(content.len() >= 512, "file should be at least 512 bytes");
566 assert_eq!(&content[..512], &vec![0x42u8; 512][..]);
567 assert!(content.iter().take(512).all(|&b| b == 0x42));
568 }
569
570 #[tokio::test]
571 async fn test_cached_writer_without_rate_limiter_no_effect() {
572 let dir = tempfile::tempdir().unwrap();
573 let path = dir.path().join("test_nolimiter.bin");
574
575 let mut writer = CachedDiskWriter::new(&path, Some(1024), None);
577 writer.open().await.unwrap();
578 writer.write_at(0, b"no limiter").await.unwrap();
579 writer.flush().await.unwrap();
580
581 let content = tokio::fs::read(&path).await.unwrap();
582 assert!(
583 content.starts_with(b"no limiter"),
584 "should contain written data"
585 );
586 }
587
588 #[tokio::test]
591 async fn test_concurrent_writes_different_offsets() {
592 let dir = tempfile::tempdir().unwrap();
593 let path = dir.path().join("test_concurrent.bin");
594
595 let mut writer = CachedDiskWriter::new(&path, Some(16 * 1024 * 1024), None);
596 writer.open().await.unwrap();
597
598 let mut handles = vec![];
599 for i in 0..16 {
600 let offset = (i as u64) * 1024 * 1024;
601 let data = vec![i as u8; 4096];
602 let path_clone = path.clone();
603
604 handles.push(tokio::spawn(async move {
605 let mut w = CachedDiskWriter::new(&path_clone, None, None);
606 w.open().await.unwrap();
607 w.write_at(offset, &data).await.unwrap();
608 w.flush().await.unwrap();
609 w.close().await.unwrap();
610 }));
611 }
612
613 for handle in handles {
614 handle.await.unwrap();
615 }
616
617 let content = tokio::fs::read(&path).await.unwrap();
618 for i in 0..16 {
619 let offset = (i as usize) * 1024 * 1024;
620 let expected = vec![i as u8; 4096];
621 assert_eq!(
622 &content[offset..offset + 4096],
623 &expected[..],
624 "Data mismatch at offset {}",
625 i
626 );
627 }
628 }
629
630 #[tokio::test]
631 async fn test_concurrent_writes_serialized() {
632 use std::sync::Arc;
633 use std::sync::atomic::{AtomicUsize, Ordering};
634
635 let dir = tempfile::tempdir().unwrap();
636 let path = dir.path().join("test_same_offset.bin");
637
638 let mut writer = CachedDiskWriter::new(&path, Some(1024 * 1024), None);
639 writer.open().await.unwrap();
640 writer.close().await.unwrap();
641
642 let write_count = Arc::new(AtomicUsize::new(0));
643 let mut handles = vec![];
644
645 for i in 0..10 {
646 let offset = (i as u64) * 1024;
647 let data = vec![i as u8; 1024];
648 let path_clone = path.clone();
649 let counter = write_count.clone();
650
651 handles.push(tokio::spawn(async move {
652 let mut w = CachedDiskWriter::new(&path_clone, None, None);
653 w.open().await.unwrap();
654 w.write_at(offset, &data).await.unwrap();
655 counter.fetch_add(1, Ordering::SeqCst);
656 w.flush().await.unwrap();
657 w.close().await.unwrap();
658 }));
659 }
660
661 for handle in handles {
662 handle.await.unwrap();
663 }
664
665 assert_eq!(write_count.load(Ordering::SeqCst), 10);
666
667 let content = tokio::fs::read(&path).await.unwrap();
668 for i in 0..10 {
669 let offset = i * 1024;
670 let expected = vec![i as u8; 1024];
671 assert_eq!(
672 &content[offset..offset + 1024],
673 &expected[..],
674 "Data mismatch at offset {}",
675 offset
676 );
677 }
678 }
679
680 #[tokio::test]
681 async fn test_high_concurrency_stress() {
682 let dir = tempfile::tempdir().unwrap();
683 let path = dir.path().join("test_stress.bin");
684
685 let mut writer = CachedDiskWriter::new(&path, Some(32 * 1024 * 1024), None);
686 writer.open().await.unwrap();
687 writer.close().await.unwrap();
688
689 let num_threads = 32;
690 let writes_per_thread = 100;
691 let mut handles = vec![];
692
693 for thread_id in 0..num_threads {
694 let path_clone = path.clone();
695
696 handles.push(tokio::spawn(async move {
697 let mut w = CachedDiskWriter::new(&path_clone, None, None);
698 w.open().await.unwrap();
699
700 for write_id in 0..writes_per_thread {
701 let offset = ((thread_id * writes_per_thread + write_id) as u64) * 8192;
702 let data = vec![(thread_id + write_id) as u8; 8192];
703 w.write_at(offset, &data).await.unwrap();
704 }
705
706 w.flush().await.unwrap();
707 w.close().await.unwrap();
708 }));
709 }
710
711 for handle in handles {
712 handle.await.unwrap();
713 }
714
715 let content = tokio::fs::read(&path).await.unwrap();
716 for thread_id in 0..num_threads {
717 for write_id in 0..writes_per_thread {
718 let offset = ((thread_id * writes_per_thread + write_id) as usize) * 8192;
719 let expected = vec![(thread_id + write_id) as u8; 8192];
720 if offset + 8192 <= content.len() {
721 assert_eq!(
722 &content[offset..offset + 8192],
723 &expected[..],
724 "Data mismatch at thread {} write {}",
725 thread_id,
726 write_id
727 );
728 }
729 }
730 }
731 }
732
733 #[tokio::test]
743 async fn test_concurrent_writes_no_mutex_contention() {
744 let dir = tempfile::tempdir().unwrap();
745 let path = dir.path().join("test_no_contention.bin");
746
747 let chunk_size: usize = 64 * 1024;
748 let num_tasks: usize = 8;
749 let total_size = (chunk_size * num_tasks) as u64;
750
751 let mut writer = CachedDiskWriter::new(&path, Some(total_size), None);
752 writer.open().await.unwrap();
753 let writer = Arc::new(tokio::sync::Mutex::new(writer));
754
755 let mut handles = Vec::with_capacity(num_tasks);
756 for i in 0..num_tasks {
757 let offset = (i as u64) * chunk_size as u64;
758 let fill = (i as u8) + 1;
759 let data = bytes::Bytes::from(vec![fill; chunk_size]);
760 let w = writer.clone();
761 handles.push(tokio::spawn(async move {
762 let mut guard = w.lock().await;
763 guard.write_bytes_at(offset, data).await.unwrap();
764 }));
765 }
766
767 for handle in handles {
769 handle.await.unwrap();
770 }
771
772 {
773 let mut guard = writer.lock().await;
774 guard.flush().await.unwrap();
775 }
776
777 let content = tokio::fs::read(&path).await.unwrap();
779 assert_eq!(content.len(), total_size as usize);
780 for i in 0..num_tasks {
781 let start = i * chunk_size;
782 let expected = (i as u8) + 1;
783 let chunk = &content[start..start + chunk_size];
784 assert!(
785 chunk.iter().all(|&b| b == expected),
786 "data mismatch in task {} chunk",
787 i
788 );
789 }
790 }
791}