1use std::fs::{File, OpenOptions};
26use std::io;
27#[cfg(windows)]
28use std::io::{Read, Seek, SeekFrom, Write};
29#[cfg(unix)]
30use std::os::unix::io::{AsRawFd, RawFd};
31#[cfg(windows)]
32use std::os::windows::io::AsRawHandle;
33use std::path::Path;
34#[cfg(windows)]
35use std::ptr;
36
37#[cfg(windows)]
38use crate::extent::mark_sparse;
39#[cfg(windows)]
40use windows_sys::Win32::Foundation::HANDLE;
41#[cfg(windows)]
42use windows_sys::Win32::Storage::FileSystem::GetVolumeInformationByHandleW;
43#[cfg(windows)]
44use windows_sys::Win32::System::IO::DeviceIoControl;
45#[cfg(windows)]
46use windows_sys::Win32::System::Ioctl::{
47 DUPLICATE_EXTENTS_DATA, FSCTL_DUPLICATE_EXTENTS_TO_FILE, FSCTL_GET_INTEGRITY_INFORMATION,
48 FSCTL_GET_INTEGRITY_INFORMATION_BUFFER, FSCTL_SET_INTEGRITY_INFORMATION,
49 FSCTL_SET_INTEGRITY_INFORMATION_BUFFER,
50};
51#[cfg(windows)]
52use windows_sys::Win32::System::SystemServices::FILE_SUPPORTS_BLOCK_REFCOUNTING;
53
54#[cfg(windows)]
60const WINDOWS_CLONE_ALIGNMENT: u64 = 64 * 1024;
61
62#[cfg(windows)]
64const WINDOWS_MAX_CLONE_CHUNK: u64 =
65 (u32::MAX as u64 / WINDOWS_CLONE_ALIGNMENT) * WINDOWS_CLONE_ALIGNMENT;
66
67#[derive(Debug, Clone, Copy, PartialEq, Eq)]
73pub enum FastCopyStrategy {
74 Reflink,
76 SparseCopy,
78}
79
80pub fn fast_copy(src: &Path, dst: &Path) -> io::Result<u64> {
94 fast_copy_with_strategy(src, dst).map(|(len, _)| len)
95}
96
97pub fn fast_copy_with_strategy(src: &Path, dst: &Path) -> io::Result<(u64, FastCopyStrategy)> {
99 let src_len = std::fs::metadata(src)?.len();
105
106 match reflink_impl(src, dst) {
110 Ok(()) => return Ok((src_len, FastCopyStrategy::Reflink)),
111 Err(e) if is_reflink_unsupported(&e) => {
112 }
114 Err(e) => return Err(e),
115 }
116
117 sparse_copy(src, dst).map(|len| (len, FastCopyStrategy::SparseCopy))
118}
119
120pub fn reflink(src: &Path, dst: &Path) -> io::Result<u64> {
122 let src_len = std::fs::metadata(src)?.len();
123 reflink_impl(src, dst)?;
124 Ok(src_len)
125}
126
127pub fn sparse_copy(src: &Path, dst: &Path) -> io::Result<u64> {
133 sparse_copy_impl(src, dst)
134}
135
136#[cfg(unix)]
137fn sparse_copy_impl(src: &Path, dst: &Path) -> io::Result<u64> {
138 let src_file = File::open(src)?;
139 let len = src_file.metadata()?.len();
140
141 let dst_file = OpenOptions::new()
142 .read(true)
143 .write(true)
144 .create(true)
145 .truncate(true)
146 .open(dst)?;
147 dst_file.set_len(len)?;
150
151 let src_fd = src_file.as_raw_fd();
152 let dst_fd = dst_file.as_raw_fd();
153
154 let mut off: i64 = 0;
155 while (off as u64) < len {
156 let data_start = unsafe { libc::lseek(src_fd, off, libc::SEEK_DATA) };
158 if data_start < 0 {
159 let err = io::Error::last_os_error();
160 if err.raw_os_error() == Some(libc::ENXIO) {
162 break;
163 }
164 return Err(err);
165 }
166 let data_end = unsafe { libc::lseek(src_fd, data_start, libc::SEEK_HOLE) };
168 if data_end < 0 {
169 return Err(io::Error::last_os_error());
170 }
171 let data_end = (data_end as u64).min(len);
172 let data_start = data_start as u64;
173 if data_end <= data_start {
174 break;
175 }
176
177 copy_extent(src_fd, dst_fd, data_start, data_end - data_start)?;
178 off = data_end as i64;
179 }
180
181 dst_file.sync_all()?;
182 Ok(len)
183}
184
185#[cfg(unix)]
187fn reflink_impl(src: &Path, dst: &Path) -> io::Result<()> {
188 reflink_copy::reflink(src, dst)
189}
190
191#[cfg(windows)]
193fn reflink_impl(src: &Path, dst: &Path) -> io::Result<()> {
194 let mut src_file = File::open(src)?;
195 let mut dst_file = OpenOptions::new()
196 .read(true)
197 .write(true)
198 .create_new(true)
199 .open(dst)?;
200
201 let result = reflink_windows_files(&mut src_file, &mut dst_file);
202 drop(dst_file);
203 drop(src_file);
204 if result.is_err() {
205 let _ = std::fs::remove_file(dst);
206 }
207 result
208}
209
210#[cfg(not(any(unix, windows)))]
211fn reflink_impl(_src: &Path, _dst: &Path) -> io::Result<()> {
212 Err(io::Error::new(
213 io::ErrorKind::Unsupported,
214 "filesystem reflinks are unsupported on this platform",
215 ))
216}
217
218#[cfg(windows)]
219fn sparse_copy_impl(src: &Path, dst: &Path) -> io::Result<u64> {
220 const BUF_SIZE: usize = 1024 * 1024;
221
222 let mut src_file = File::open(src)?;
223 let len = src_file.metadata()?.len();
224
225 let mut dst_file = OpenOptions::new()
226 .read(true)
227 .write(true)
228 .create(true)
229 .truncate(true)
230 .open(dst)?;
231 dst_file.set_len(len)?;
232 mark_sparse(&dst_file)?;
233
234 let mut offset = 0u64;
235 let mut buf = vec![0u8; BUF_SIZE];
236 loop {
237 let n = src_file.read(&mut buf)?;
238 if n == 0 {
239 break;
240 }
241
242 write_nonzero_runs(&mut dst_file, offset, &buf[..n])?;
243 offset += n as u64;
244 }
245
246 dst_file.sync_all()?;
247 Ok(len)
248}
249
250#[cfg(windows)]
255fn reflink_windows_files(src: &mut File, dst: &mut File) -> io::Result<()> {
256 let src_volume = windows_volume_identity(src)?;
257 let dst_volume = windows_volume_identity(dst)?;
258 if src_volume.0 != dst_volume.0 {
259 return Err(io::Error::new(
260 io::ErrorKind::Unsupported,
261 "Windows block cloning requires source and destination on the same volume",
262 ));
263 }
264 if src_volume.1 & FILE_SUPPORTS_BLOCK_REFCOUNTING == 0
265 || dst_volume.1 & FILE_SUPPORTS_BLOCK_REFCOUNTING == 0
266 {
267 return Err(io::Error::new(
268 io::ErrorKind::Unsupported,
269 "destination volume does not advertise block-refcounting support",
270 ));
271 }
272
273 mark_sparse(dst)?;
274 match_windows_integrity(src, dst)?;
275
276 let len = src.metadata()?.len();
277 dst.set_len(len)?;
278 let clone_len = len / WINDOWS_CLONE_ALIGNMENT * WINDOWS_CLONE_ALIGNMENT;
279 let mut offset = 0u64;
280 while offset < clone_len {
281 let chunk = (clone_len - offset).min(WINDOWS_MAX_CLONE_CHUNK);
282 duplicate_windows_extents(src, dst, offset, chunk)?;
283 offset += chunk;
284 }
285 if clone_len < len {
286 copy_windows_tail(src, dst, clone_len, len - clone_len)?;
287 }
288 Ok(())
289}
290
291#[cfg(windows)]
292fn windows_volume_identity(file: &File) -> io::Result<(u32, u32)> {
293 let mut serial = 0u32;
294 let mut flags = 0u32;
295 let ok = unsafe {
296 GetVolumeInformationByHandleW(
297 file.as_raw_handle() as HANDLE,
298 ptr::null_mut(),
299 0,
300 &mut serial,
301 ptr::null_mut(),
302 &mut flags,
303 ptr::null_mut(),
304 0,
305 )
306 };
307 if ok == 0 {
308 return Err(io::Error::last_os_error());
309 }
310 Ok((serial, flags))
311}
312
313#[cfg(windows)]
314fn match_windows_integrity(src: &File, dst: &File) -> io::Result<()> {
315 let Some(src_info) = get_windows_integrity(src)? else {
316 return Ok(());
317 };
318 let Some(dst_info) = get_windows_integrity(dst)? else {
319 return Ok(());
320 };
321 if src_info.ChecksumAlgorithm == dst_info.ChecksumAlgorithm && src_info.Flags == dst_info.Flags
322 {
323 return Ok(());
324 }
325
326 let info = FSCTL_SET_INTEGRITY_INFORMATION_BUFFER {
327 ChecksumAlgorithm: src_info.ChecksumAlgorithm,
328 Reserved: 0,
329 Flags: src_info.Flags,
330 };
331 let mut returned = 0u32;
332 let ok = unsafe {
333 DeviceIoControl(
334 dst.as_raw_handle() as HANDLE,
335 FSCTL_SET_INTEGRITY_INFORMATION,
336 &info as *const _ as *const _,
337 size_of::<FSCTL_SET_INTEGRITY_INFORMATION_BUFFER>() as u32,
338 ptr::null_mut(),
339 0,
340 &mut returned,
341 ptr::null_mut(),
342 )
343 };
344 if ok == 0 {
345 return Err(io::Error::last_os_error());
346 }
347 Ok(())
348}
349
350#[cfg(windows)]
351fn get_windows_integrity(
352 file: &File,
353) -> io::Result<Option<FSCTL_GET_INTEGRITY_INFORMATION_BUFFER>> {
354 let mut info = FSCTL_GET_INTEGRITY_INFORMATION_BUFFER::default();
355 let mut returned = 0u32;
356 let ok = unsafe {
357 DeviceIoControl(
358 file.as_raw_handle() as HANDLE,
359 FSCTL_GET_INTEGRITY_INFORMATION,
360 ptr::null(),
361 0,
362 &mut info as *mut _ as *mut _,
363 size_of::<FSCTL_GET_INTEGRITY_INFORMATION_BUFFER>() as u32,
364 &mut returned,
365 ptr::null_mut(),
366 )
367 };
368 if ok != 0 {
369 return Ok(Some(info));
370 }
371 let error = io::Error::last_os_error();
372 if is_reflink_unsupported(&error) {
373 Ok(None)
374 } else {
375 Err(error)
376 }
377}
378
379#[cfg(windows)]
380fn duplicate_windows_extents(src: &File, dst: &File, offset: u64, len: u64) -> io::Result<()> {
381 let request = DUPLICATE_EXTENTS_DATA {
382 FileHandle: src.as_raw_handle() as HANDLE,
383 SourceFileOffset: offset as i64,
384 TargetFileOffset: offset as i64,
385 ByteCount: len as i64,
386 };
387 let mut returned = 0u32;
388 let ok = unsafe {
389 DeviceIoControl(
390 dst.as_raw_handle() as HANDLE,
391 FSCTL_DUPLICATE_EXTENTS_TO_FILE,
392 &request as *const _ as *const _,
393 size_of::<DUPLICATE_EXTENTS_DATA>() as u32,
394 ptr::null_mut(),
395 0,
396 &mut returned,
397 ptr::null_mut(),
398 )
399 };
400 if ok == 0 {
401 return Err(io::Error::last_os_error());
402 }
403 Ok(())
404}
405
406#[cfg(windows)]
407fn copy_windows_tail(src: &mut File, dst: &mut File, offset: u64, len: u64) -> io::Result<()> {
408 src.seek(SeekFrom::Start(offset))?;
409 dst.seek(SeekFrom::Start(offset))?;
410 let copied = io::copy(&mut src.take(len), dst)?;
411 if copied != len {
412 return Err(io::Error::new(
413 io::ErrorKind::UnexpectedEof,
414 format!("Windows reflink tail copied {copied} of {len} bytes"),
415 ));
416 }
417 Ok(())
418}
419
420fn is_reflink_unsupported(e: &io::Error) -> bool {
427 if matches!(e.kind(), io::ErrorKind::Unsupported) {
428 return true;
429 }
430
431 let Some(code) = e.raw_os_error() else {
432 return false;
433 };
434
435 #[cfg(target_os = "linux")]
436 let aliases: &[i32] = &[libc::ENOTSUP, libc::EXDEV, libc::EINVAL];
437 #[cfg(all(unix, not(target_os = "linux")))]
438 let aliases: &[i32] = &[libc::ENOTSUP, libc::EOPNOTSUPP, libc::EXDEV, libc::EINVAL];
439 #[cfg(windows)]
440 let aliases: &[i32] = &[
441 1, 17, 50, 87, 124, 775, ];
448
449 #[cfg(windows)]
450 {
451 let win32_code = (code as u32 & 0xffff) as i32;
452 aliases.contains(&code) || aliases.contains(&win32_code)
453 }
454
455 #[cfg(unix)]
456 aliases.contains(&code)
457}
458
459#[cfg(unix)]
460fn copy_extent(src_fd: RawFd, dst_fd: RawFd, off: u64, len: u64) -> io::Result<()> {
461 read_write_extent(src_fd, dst_fd, off, len)
463}
464
465#[cfg(unix)]
472fn read_write_extent(src_fd: RawFd, dst_fd: RawFd, off: u64, len: u64) -> io::Result<()> {
473 const BUF_SIZE: usize = 1024 * 1024;
474 let mut buf = vec![0u8; BUF_SIZE];
475 let mut copied: u64 = 0;
476
477 while copied < len {
478 let to_read = (len - copied).min(BUF_SIZE as u64) as usize;
479 let read_off = (off + copied) as i64;
480 let n = unsafe {
481 libc::pread(
482 src_fd,
483 buf.as_mut_ptr() as *mut libc::c_void,
484 to_read,
485 read_off,
486 )
487 };
488 if n < 0 {
489 return Err(io::Error::last_os_error());
490 }
491 if n == 0 {
492 return Err(io::Error::new(
493 io::ErrorKind::UnexpectedEof,
494 "unexpected EOF mid-extent",
495 ));
496 }
497 let n = n as usize;
498
499 let mut written: usize = 0;
500 while written < n {
501 let w_off = (off + copied + written as u64) as i64;
502 let w = unsafe {
503 libc::pwrite(
504 dst_fd,
505 buf[written..n].as_ptr() as *const libc::c_void,
506 n - written,
507 w_off,
508 )
509 };
510 if w < 0 {
511 return Err(io::Error::last_os_error());
512 }
513 if w == 0 {
514 return Err(io::Error::new(
515 io::ErrorKind::WriteZero,
516 "pwrite returned 0",
517 ));
518 }
519 written += w as usize;
520 }
521 copied += n as u64;
522 }
523 Ok(())
524}
525
526#[cfg(windows)]
527fn write_nonzero_runs(dst: &mut File, base_offset: u64, bytes: &[u8]) -> io::Result<()> {
528 let mut cursor = 0;
529 while cursor < bytes.len() {
530 while cursor < bytes.len() && bytes[cursor] == 0 {
531 cursor += 1;
532 }
533 if cursor == bytes.len() {
534 break;
535 }
536
537 let start = cursor;
538 while cursor < bytes.len() && bytes[cursor] != 0 {
539 cursor += 1;
540 }
541
542 dst.seek(SeekFrom::Start(base_offset + start as u64))?;
543 dst.write_all(&bytes[start..cursor])?;
544 }
545
546 Ok(())
547}
548
549#[cfg(test)]
554mod tests {
555 use super::*;
556 use std::io::{Read, Seek, SeekFrom, Write};
557 #[cfg(unix)]
558 use std::os::unix::fs::MetadataExt;
559
560 fn make_sparse(path: &Path, len: u64, data_offsets: &[u64]) -> io::Result<()> {
563 let mut f = OpenOptions::new()
564 .read(true)
565 .write(true)
566 .create(true)
567 .truncate(true)
568 .open(path)?;
569 f.set_len(len)?;
570 for &off in data_offsets {
571 let buf = vec![0xAB_u8; 64 * 1024];
572 f.seek(SeekFrom::Start(off))?;
573 f.write_all(&buf)?;
574 }
575 f.sync_all()?;
576 Ok(())
577 }
578
579 #[test]
580 fn round_trip_small() {
581 let dir = tempfile::tempdir().unwrap();
582 let src = dir.path().join("src.bin");
583 let dst = dir.path().join("dst.bin");
584
585 std::fs::write(&src, b"hello world").unwrap();
586 let n = fast_copy(&src, &dst).unwrap();
587 assert_eq!(n, 11);
588 assert_eq!(std::fs::read(&dst).unwrap(), b"hello world");
589 }
590
591 #[test]
592 fn sparse_copy_preserves_holes_and_data() {
593 let dir = tempfile::tempdir().unwrap();
597 let src = dir.path().join("src.bin");
598 let dst = dir.path().join("dst.bin");
599
600 let len: u64 = 16 * 1024 * 1024;
601 let offsets = [0u64, 4 * 1024 * 1024, 8 * 1024 * 1024, 12 * 1024 * 1024];
602 make_sparse(&src, len, &offsets).unwrap();
603
604 let n = sparse_copy(&src, &dst).unwrap();
605 assert_eq!(n, len);
606
607 let dst_meta = std::fs::metadata(&dst).unwrap();
609 assert_eq!(dst_meta.len(), len);
610
611 let mut buf = [0u8; 64 * 1024];
613 let mut dst_file = File::open(&dst).unwrap();
614 for &off in &offsets {
615 dst_file.seek(SeekFrom::Start(off)).unwrap();
616 dst_file.read_exact(&mut buf).unwrap();
617 assert!(buf.iter().all(|&b| b == 0xAB));
618 }
619
620 #[cfg(unix)]
627 {
628 let src_bytes_on_disk = std::fs::metadata(&src).unwrap().blocks() * 512;
629 let dst_bytes_on_disk = dst_meta.blocks() * 512;
630 if src_bytes_on_disk < len / 2 {
631 assert!(
635 dst_bytes_on_disk < len / 2,
636 "source is sparse ({src_bytes_on_disk} bytes on disk) but destination densified to {dst_bytes_on_disk} bytes for an apparent size of {len}",
637 );
638 assert!(
639 dst_bytes_on_disk <= src_bytes_on_disk * 4 + 1024 * 1024,
640 "destination allocated significantly more than source: src={src_bytes_on_disk} dst={dst_bytes_on_disk}",
641 );
642 } else {
643 eprintln!(
644 "filesystem did not sparsify the source (src_bytes_on_disk={src_bytes_on_disk}, apparent={len}); sparseness preservation not exercised in this run",
645 );
646 assert!(
649 dst_bytes_on_disk <= src_bytes_on_disk + 1024 * 1024,
650 "destination grew beyond source footprint: src={src_bytes_on_disk} dst={dst_bytes_on_disk}",
651 );
652 }
653 }
654 }
655
656 #[test]
657 fn fast_copy_matches_source_size() {
658 let dir = tempfile::tempdir().unwrap();
659 let src = dir.path().join("src.bin");
660 let dst = dir.path().join("dst.bin");
661
662 let len: u64 = 4 * 1024 * 1024;
663 make_sparse(&src, len, &[0, 2 * 1024 * 1024]).unwrap();
664
665 let n = fast_copy(&src, &dst).unwrap();
666 assert_eq!(n, len);
667 assert_eq!(std::fs::metadata(&dst).unwrap().len(), len);
668 }
669
670 #[test]
671 fn missing_source_errors() {
672 let dir = tempfile::tempdir().unwrap();
673 let err = fast_copy(&dir.path().join("nope.bin"), &dir.path().join("dst.bin")).unwrap_err();
674 assert_eq!(err.kind(), io::ErrorKind::NotFound);
675 }
676}