Skip to main content

shared_target_core/
seed.rs

1use std::cell::Cell;
2use std::ffi::OsStr;
3use std::fs;
4use std::path::{Path, PathBuf};
5
6use filetime::FileTime;
7use walkdir::WalkDir;
8
9use crate::error::{Error, Result};
10use crate::{lock, probe};
11
12/// Files this size and above under a `deps/` directory keep a shared inode
13/// rather than a copy of their own.
14///
15/// What makes sharing safe there is not the size but what Cargo does with the
16/// file: a compiled artifact under `deps/` is written once under a hashed name
17/// and never modified, so two trees pointing at one inode never disagree. The
18/// size is what makes it worth doing.
19pub const DEFAULT_MIN_SHARED_SIZE: u64 = 1024 * 1024;
20
21#[derive(Debug, Clone, Copy, PartialEq, Eq)]
22pub enum Strategy {
23    /// The filesystem clones blocks, so every file is shared and none of the
24    /// tree costs disk until something writes to it.
25    Clone,
26    /// The filesystem does not clone, so the large write-once artifacts are
27    /// shared by hard link and everything else is given a copy of its own.
28    LinkAndCopy,
29}
30
31#[derive(Debug, Clone)]
32pub struct Options {
33    /// The target directory to seed from.
34    pub src: PathBuf,
35    /// The target directory to create.
36    pub dest: PathBuf,
37    /// Where to build the tree before it is renamed into place. Defaults to a
38    /// sibling of `dest`.
39    pub staging: Option<PathBuf>,
40    /// See [`DEFAULT_MIN_SHARED_SIZE`].
41    pub min_shared_size: u64,
42}
43
44impl Options {
45    pub fn new(src: impl Into<PathBuf>, dest: impl Into<PathBuf>) -> Self {
46        Self {
47            src: src.into(),
48            dest: dest.into(),
49            staging: None,
50            min_shared_size: DEFAULT_MIN_SHARED_SIZE,
51        }
52    }
53}
54
55#[derive(Debug, Clone)]
56pub struct Report {
57    pub strategy: Strategy,
58    pub dest: PathBuf,
59    pub dirs: u64,
60    pub shared: u64,
61    pub copied: u64,
62    pub symlinks: u64,
63    pub incremental_dropped: u64,
64    pub shared_bytes: u64,
65    pub copied_bytes: u64,
66    /// How many of Cargo's build locks were held while the source was read.
67    /// Zero is a fact about this run and not a guarantee about it — see
68    /// [`crate::lock`].
69    pub build_locks_held: usize,
70}
71
72/// Builds `dest` from `src`, sharing whatever the filesystem under `dest` lets
73/// it share.
74///
75/// The tree is built under a staging name and renamed into place only once it
76/// is whole. Cargo reads freshness from a fingerprint beside each artifact, so
77/// a tree that is half there is worse than no tree at all: the fingerprints
78/// that did land say fresh about artifacts that did not. Under the staging name
79/// an unfinished tree is not a `target/` at all, and is inert rather than wrong.
80pub fn seed(opts: &Options) -> Result<Report> {
81    if !opts.src.is_dir() {
82        return Err(Error::SourceMissing(opts.src.clone()));
83    }
84    if opts.dest.exists() {
85        return Err(Error::DestinationExists(opts.dest.clone()));
86    }
87
88    let staging = opts
89        .staging
90        .clone()
91        .unwrap_or_else(|| default_staging(&opts.dest));
92    if staging.exists() {
93        return Err(Error::StagingExists(staging));
94    }
95
96    // Before anything is created, including the parents. A refusal that has
97    // already made a directory inside the tree it is refusing to write into is
98    // not the promise this makes.
99    refuse_nesting(&opts.src, &opts.dest, &staging)?;
100
101    let dest_parent = parent_of(&opts.dest)?;
102    fs::create_dir_all(dest_parent).map_err(Error::io("creating", dest_parent))?;
103    let staging_parent = parent_of(&staging)?.to_path_buf();
104    fs::create_dir_all(&staging_parent).map_err(Error::io("creating", &staging_parent))?;
105
106    // The staged tree is renamed into place at the end, and a rename does not
107    // cross a filesystem. This one holds whichever way the blocks get there,
108    // and finding it out at the last line would mean discovering it after the
109    // whole tree had been built.
110    same_filesystem(&staging_parent, dest_parent)?;
111
112    let strategy = if probe::supports_reflink(&opts.src, &staging_parent)? {
113        Strategy::Clone
114    } else {
115        Strategy::LinkAndCopy
116    };
117
118    // Asked only of the strategy that needs it. A hard link cannot cross a
119    // device, but a clone can: btrfs gives every subvolume a device number of
120    // its own while `FICLONE` works across all of them, so refusing on that
121    // number would turn away the arrangement with the most to gain — a worktree
122    // in its own subvolume, where the seeding would have cost nothing at all.
123    if strategy == Strategy::LinkAndCopy {
124        same_filesystem(&opts.src, &staging_parent)?;
125    }
126
127    // Held until the reading is done. Taken after the probe, which writes only
128    // into the staging parent and reads one file that a build would not be
129    // rewriting under it.
130    let locks = lock::acquire(&opts.src)?;
131
132    // A failure leaves the staged tree where it is. It is named so that nothing
133    // reads it, and deleting on the way out of an error is how the one run that
134    // could have been inspected stops being inspectable.
135    let mut report = fill(&opts.src, &staging, strategy, opts.min_shared_size)?;
136    report.build_locks_held = locks.held().len();
137    drop(locks);
138
139    fs::rename(&staging, &opts.dest)
140        .map_err(Error::io("renaming the staged tree to", &opts.dest))?;
141    report.dest = opts.dest.clone();
142    Ok(report)
143}
144
145fn fill(src: &Path, staging: &Path, strategy: Strategy, min_shared_size: u64) -> Result<Report> {
146    fs::create_dir_all(staging).map_err(Error::io("creating", staging))?;
147
148    let mut report = Report {
149        strategy,
150        dest: staging.to_path_buf(),
151        dirs: 0,
152        shared: 0,
153        copied: 0,
154        symlinks: 0,
155        incremental_dropped: 0,
156        shared_bytes: 0,
157        copied_bytes: 0,
158        build_locks_held: 0,
159    };
160
161    // Dropped rather than carried when carrying it would mean real bytes.
162    // Cargo regenerates `incremental/`, it is the one large thing here that
163    // nothing needs brought over, and under `LinkAndCopy` almost none of it is
164    // large enough to be shared — so it would arrive as a copy and cost its
165    // full size. Where blocks clone it costs nothing, so it comes along.
166    let drop_incremental = strategy == Strategy::LinkAndCopy;
167    let dropped = Cell::new(0u64);
168
169    let walker = WalkDir::new(src)
170        .min_depth(1)
171        .into_iter()
172        .filter_entry(|entry| {
173            let prune = drop_incremental && is_profile_incremental(entry.path(), entry.file_type());
174            if prune {
175                dropped.set(dropped.get() + 1);
176            }
177            !prune
178        });
179
180    for entry in walker {
181        let entry = entry.map_err(|source| Error::Walk {
182            path: src.to_path_buf(),
183            source,
184        })?;
185        let relative = entry
186            .path()
187            .strip_prefix(src)
188            .expect("walkdir yields paths under its root");
189        let destination = staging.join(relative);
190        let file_type = entry.file_type();
191
192        if file_type.is_dir() {
193            fs::create_dir_all(&destination).map_err(Error::io("creating", &destination))?;
194            report.dirs += 1;
195            continue;
196        }
197
198        if file_type.is_symlink() {
199            let points_to = fs::read_link(entry.path())
200                .map_err(Error::io("reading the symlink", entry.path()))?;
201            symlink(entry.path(), &points_to, &destination)?;
202            report.symlinks += 1;
203            continue;
204        }
205
206        let metadata = entry.metadata().map_err(|source| Error::Walk {
207            path: entry.path().to_path_buf(),
208            source,
209        })?;
210        let size = metadata.len();
211
212        let share = match strategy {
213            Strategy::Clone => true,
214            Strategy::LinkAndCopy => {
215                under_deps(relative) && !is_dep_info(relative) && size >= min_shared_size
216            }
217        };
218
219        if share {
220            match strategy {
221                Strategy::Clone => {
222                    reflink_copy::reflink(entry.path(), &destination)
223                        .map_err(Error::io("cloning to", &destination))?;
224                    carry_metadata(&metadata, &destination)?;
225                }
226                // A hard link is the same inode, so it arrives with the times
227                // and the mode already on it.
228                Strategy::LinkAndCopy => {
229                    fs::hard_link(entry.path(), &destination)
230                        .map_err(Error::io("linking to", &destination))?;
231                }
232            }
233            report.shared += 1;
234            report.shared_bytes += size;
235        } else {
236            fs::copy(entry.path(), &destination).map_err(Error::io("copying to", &destination))?;
237            carry_metadata(&metadata, &destination)?;
238            report.copied += 1;
239            report.copied_bytes += size;
240        }
241    }
242
243    report.incremental_dropped = dropped.get();
244    Ok(report)
245}
246
247/// Carries the modification time and the mode across.
248///
249/// What the times are for is that the seeded tree should be the same tree. They
250/// are not load-bearing for freshness in the direction it is tempting to
251/// assume: Cargo compares each source file against the dep-info recorded beside
252/// the artifact, so a copy landing with today's date is *newer* than what it was
253/// built from and reads as fresh either way. Carrying them is what keeps the two
254/// trees answering the same question the same way — and what keeps a hard link,
255/// which has no choice but to bring its times along, from being the odd one out
256/// beside the files that were copied.
257fn carry_metadata(source: &fs::Metadata, destination: &Path) -> Result<()> {
258    fs::set_permissions(destination, source.permissions())
259        .map_err(Error::io("setting the mode on", destination))?;
260    filetime::set_file_times(
261        destination,
262        FileTime::from_last_access_time(source),
263        FileTime::from_last_modification_time(source),
264    )
265    .map_err(Error::io("setting the timestamps on", destination))?;
266    Ok(())
267}
268
269/// Cargo's own incremental directory, told apart from any other directory that
270/// happens to carry the name.
271///
272/// Matching on the name alone reaches into `OUT_DIR`, where a build script is
273/// free to emit an `incremental/` of its own. Dropping that while carrying the
274/// `run-build-script` fingerprint beside it produces exactly the tree the
275/// staging rename exists to prevent — one whose fingerprints say fresh about
276/// files that are not there — only arrived at from the other side, and Cargo
277/// will not re-run the script that would put them back.
278///
279/// The one Cargo writes sits in a profile directory, and what distinguishes a
280/// profile directory is that `deps/` is beside it.
281fn is_profile_incremental(path: &Path, file_type: fs::FileType) -> bool {
282    file_type.is_dir()
283        && path.file_name() == Some(OsStr::new("incremental"))
284        && path
285            .parent()
286            .is_some_and(|profile| profile.join("deps").is_dir())
287}
288
289/// Dep-info, which is never shared however large it grows.
290///
291/// Cargo rewrites these on every build, and what they contain is a list of
292/// absolute source paths. Two checkouts sitting at two paths need two of them
293/// saying two different things, so one inode between them is wrong on both
294/// counts. The size rule below happens to keep them apart at the default
295/// threshold, which is not the same as deciding to.
296fn is_dep_info(relative: &Path) -> bool {
297    relative
298        .extension()
299        .is_some_and(|extension| extension == "d")
300}
301
302/// `deps` as an ancestor rather than as the file's own name: what is being
303/// asked is which directory Cargo writes the file into.
304fn under_deps(relative: &Path) -> bool {
305    relative.parent().is_some_and(|parent| {
306        parent
307            .components()
308            .any(|component| component.as_os_str() == "deps")
309    })
310}
311
312/// Refuses a destination that lives under the directory being read.
313///
314/// The walk and the writing would then be the same tree: every file written
315/// becomes a file to walk, which becomes a file to write. Nothing about it
316/// converges, and what it consumes is the disk. Both the destination and the
317/// staged tree are checked, since the staged one is where everything is written
318/// first.
319///
320/// Both paths are resolved through whatever symlinks their existing parts
321/// contain before being compared — the directory a worktree is cut into is
322/// free to be a symlink, and
323/// comparing the paths as they were written would let a destination that
324/// reaches the source by another name straight through.
325fn refuse_nesting(src: &Path, dest: &Path, staging: &Path) -> Result<()> {
326    let source = src.canonicalize().map_err(Error::io("resolving", src))?;
327
328    for candidate in [dest, staging] {
329        let resolved = resolve(candidate)?;
330        if resolved.starts_with(&source) {
331            return Err(Error::DestinationInsideSource {
332                src: source,
333                dest: resolved,
334            });
335        }
336    }
337
338    Ok(())
339}
340
341/// Resolves a path that does not exist yet, by resolving as much of it as does
342/// and keeping the rest as written.
343///
344/// `canonicalize` needs the whole path to be there, and this runs before
345/// anything has been created on purpose.
346fn resolve(path: &Path) -> Result<PathBuf> {
347    let mut unresolved = Vec::new();
348    let mut existing = path.to_path_buf();
349
350    loop {
351        if let Ok(mut base) = existing.canonicalize() {
352            base.extend(unresolved.iter().rev());
353            return Ok(base);
354        }
355
356        let name = existing
357            .file_name()
358            .ok_or_else(|| Error::DestinationHasNoParent(path.to_path_buf()))?
359            .to_os_string();
360        unresolved.push(name);
361        existing = parent_of(&existing)?.to_path_buf();
362    }
363}
364
365fn parent_of(path: &Path) -> Result<&Path> {
366    match path.parent() {
367        // A relative single-component path has a parent, and it is empty.
368        Some(parent) if parent.as_os_str().is_empty() => Ok(Path::new(".")),
369        Some(parent) => Ok(parent),
370        None => Err(Error::DestinationHasNoParent(path.to_path_buf())),
371    }
372}
373
374fn default_staging(dest: &Path) -> PathBuf {
375    let name = dest.file_name().unwrap_or_else(|| OsStr::new("target"));
376    let mut staged = name.to_os_string();
377    staged.push(".partial");
378    dest.with_file_name(staged)
379}
380
381#[cfg(unix)]
382fn same_filesystem(a: &Path, b: &Path) -> Result<()> {
383    use std::os::unix::fs::MetadataExt;
384
385    let left = fs::metadata(a).map_err(Error::io("reading", a))?;
386    let right = fs::metadata(b).map_err(Error::io("reading", b))?;
387    if left.dev() == right.dev() {
388        return Ok(());
389    }
390    Err(Error::CrossDevice {
391        src: a.to_path_buf(),
392        dest: b.to_path_buf(),
393    })
394}
395
396#[cfg(not(unix))]
397fn same_filesystem(_a: &Path, _b: &Path) -> Result<()> {
398    // Left to the clone and the link to report in their own words rather than
399    // guessed at from a device number this platform does not hand out.
400    Ok(())
401}
402
403#[cfg(unix)]
404fn symlink(_link: &Path, points_to: &Path, at: &Path) -> Result<()> {
405    std::os::unix::fs::symlink(points_to, at).map_err(Error::io("creating the symlink", at))
406}
407
408#[cfg(not(unix))]
409fn symlink(link: &Path, _points_to: &Path, at: &Path) -> Result<()> {
410    // Followed rather than recreated: making a symlink is a privileged
411    // operation on Windows, and a copy of what it pointed at is the same tree
412    // to a compiler.
413    //
414    // Copied from the link itself rather than from what `read_link` returned.
415    // That is normally relative to the link's own directory, and handing it to
416    // `fs::copy` would resolve it against the working directory instead —
417    // failing outright, or finding some unrelated file that happens to sit
418    // there and writing its bytes into the tree as if they belonged.
419    fs::copy(link, at)
420        .map(|_| ())
421        .map_err(Error::io("copying what the symlink pointed at, to", at))
422}