shared_target_core/seed.rs
1use std::cell::Cell;
2use std::ffi::OsStr;
3use std::fs;
4use std::path::{Path, PathBuf};
5
6use filetime::FileTime;
7use walkdir::WalkDir;
8
9use crate::error::{Error, Result};
10use crate::{lock, probe};
11
12/// Files this size and above under a `deps/` directory keep a shared inode
13/// rather than a copy of their own.
14///
15/// What makes sharing safe there is not the size but what Cargo does with the
16/// file: a compiled artifact under `deps/` is written once under a hashed name
17/// and never modified, so two trees pointing at one inode never disagree. The
18/// size is what makes it worth doing.
19pub const DEFAULT_MIN_SHARED_SIZE: u64 = 1024 * 1024;
20
21#[derive(Debug, Clone, Copy, PartialEq, Eq)]
22pub enum Strategy {
23 /// The filesystem clones blocks, so every file is shared and none of the
24 /// tree costs disk until something writes to it.
25 Clone,
26 /// The filesystem does not clone, so the large write-once artifacts are
27 /// shared by hard link and everything else is given a copy of its own.
28 LinkAndCopy,
29}
30
31#[derive(Debug, Clone)]
32pub struct Options {
33 /// The target directory to seed from.
34 pub src: PathBuf,
35 /// The target directory to create.
36 pub dest: PathBuf,
37 /// Where to build the tree before it is renamed into place. Defaults to a
38 /// sibling of `dest`.
39 pub staging: Option<PathBuf>,
40 /// See [`DEFAULT_MIN_SHARED_SIZE`].
41 pub min_shared_size: u64,
42}
43
44impl Options {
45 pub fn new(src: impl Into<PathBuf>, dest: impl Into<PathBuf>) -> Self {
46 Self {
47 src: src.into(),
48 dest: dest.into(),
49 staging: None,
50 min_shared_size: DEFAULT_MIN_SHARED_SIZE,
51 }
52 }
53}
54
55#[derive(Debug, Clone)]
56pub struct Report {
57 pub strategy: Strategy,
58 pub dest: PathBuf,
59 pub dirs: u64,
60 pub shared: u64,
61 pub copied: u64,
62 pub symlinks: u64,
63 pub incremental_dropped: u64,
64 pub shared_bytes: u64,
65 pub copied_bytes: u64,
66 /// How many of Cargo's build locks were held while the source was read.
67 /// Zero is a fact about this run and not a guarantee about it — see
68 /// [`crate::lock`].
69 pub build_locks_held: usize,
70}
71
72/// Builds `dest` from `src`, sharing whatever the filesystem under `dest` lets
73/// it share.
74///
75/// The tree is built under a staging name and renamed into place only once it
76/// is whole. Cargo reads freshness from a fingerprint beside each artifact, so
77/// a tree that is half there is worse than no tree at all: the fingerprints
78/// that did land say fresh about artifacts that did not. Under the staging name
79/// an unfinished tree is not a `target/` at all, and is inert rather than wrong.
80pub fn seed(opts: &Options) -> Result<Report> {
81 if !opts.src.is_dir() {
82 return Err(Error::SourceMissing(opts.src.clone()));
83 }
84 if opts.dest.exists() {
85 return Err(Error::DestinationExists(opts.dest.clone()));
86 }
87
88 let staging = opts
89 .staging
90 .clone()
91 .unwrap_or_else(|| default_staging(&opts.dest));
92 if staging.exists() {
93 return Err(Error::StagingExists(staging));
94 }
95
96 // Before anything is created, including the parents. A refusal that has
97 // already made a directory inside the tree it is refusing to write into is
98 // not the promise this makes.
99 refuse_nesting(&opts.src, &opts.dest, &staging)?;
100
101 let dest_parent = parent_of(&opts.dest)?;
102 fs::create_dir_all(dest_parent).map_err(Error::io("creating", dest_parent))?;
103 let staging_parent = parent_of(&staging)?.to_path_buf();
104 fs::create_dir_all(&staging_parent).map_err(Error::io("creating", &staging_parent))?;
105
106 // The staged tree is renamed into place at the end, and a rename does not
107 // cross a filesystem. This one holds whichever way the blocks get there,
108 // and finding it out at the last line would mean discovering it after the
109 // whole tree had been built.
110 same_filesystem(&staging_parent, dest_parent)?;
111
112 let strategy = if probe::supports_reflink(&opts.src, &staging_parent)? {
113 Strategy::Clone
114 } else {
115 Strategy::LinkAndCopy
116 };
117
118 // Asked only of the strategy that needs it. A hard link cannot cross a
119 // device, but a clone can: btrfs gives every subvolume a device number of
120 // its own while `FICLONE` works across all of them, so refusing on that
121 // number would turn away the arrangement with the most to gain — a worktree
122 // in its own subvolume, where the seeding would have cost nothing at all.
123 if strategy == Strategy::LinkAndCopy {
124 same_filesystem(&opts.src, &staging_parent)?;
125 }
126
127 // Held until the reading is done. Taken after the probe, which writes only
128 // into the staging parent and reads one file that a build would not be
129 // rewriting under it.
130 let locks = lock::acquire(&opts.src)?;
131
132 // A failure leaves the staged tree where it is. It is named so that nothing
133 // reads it, and deleting on the way out of an error is how the one run that
134 // could have been inspected stops being inspectable.
135 let mut report = fill(&opts.src, &staging, strategy, opts.min_shared_size)?;
136 report.build_locks_held = locks.held().len();
137 drop(locks);
138
139 fs::rename(&staging, &opts.dest)
140 .map_err(Error::io("renaming the staged tree to", &opts.dest))?;
141 report.dest = opts.dest.clone();
142 Ok(report)
143}
144
145fn fill(src: &Path, staging: &Path, strategy: Strategy, min_shared_size: u64) -> Result<Report> {
146 fs::create_dir_all(staging).map_err(Error::io("creating", staging))?;
147
148 let mut report = Report {
149 strategy,
150 dest: staging.to_path_buf(),
151 dirs: 0,
152 shared: 0,
153 copied: 0,
154 symlinks: 0,
155 incremental_dropped: 0,
156 shared_bytes: 0,
157 copied_bytes: 0,
158 build_locks_held: 0,
159 };
160
161 // Dropped rather than carried when carrying it would mean real bytes.
162 // Cargo regenerates `incremental/`, it is the one large thing here that
163 // nothing needs brought over, and under `LinkAndCopy` almost none of it is
164 // large enough to be shared — so it would arrive as a copy and cost its
165 // full size. Where blocks clone it costs nothing, so it comes along.
166 let drop_incremental = strategy == Strategy::LinkAndCopy;
167 let dropped = Cell::new(0u64);
168
169 let walker = WalkDir::new(src)
170 .min_depth(1)
171 .into_iter()
172 .filter_entry(|entry| {
173 let prune = drop_incremental && is_profile_incremental(entry.path(), entry.file_type());
174 if prune {
175 dropped.set(dropped.get() + 1);
176 }
177 !prune
178 });
179
180 for entry in walker {
181 let entry = entry.map_err(|source| Error::Walk {
182 path: src.to_path_buf(),
183 source,
184 })?;
185 let relative = entry
186 .path()
187 .strip_prefix(src)
188 .expect("walkdir yields paths under its root");
189 let destination = staging.join(relative);
190 let file_type = entry.file_type();
191
192 if file_type.is_dir() {
193 fs::create_dir_all(&destination).map_err(Error::io("creating", &destination))?;
194 report.dirs += 1;
195 continue;
196 }
197
198 if file_type.is_symlink() {
199 let points_to = fs::read_link(entry.path())
200 .map_err(Error::io("reading the symlink", entry.path()))?;
201 symlink(entry.path(), &points_to, &destination)?;
202 report.symlinks += 1;
203 continue;
204 }
205
206 let metadata = entry.metadata().map_err(|source| Error::Walk {
207 path: entry.path().to_path_buf(),
208 source,
209 })?;
210 let size = metadata.len();
211
212 let share = match strategy {
213 Strategy::Clone => true,
214 Strategy::LinkAndCopy => {
215 under_deps(relative) && !is_dep_info(relative) && size >= min_shared_size
216 }
217 };
218
219 if share {
220 match strategy {
221 Strategy::Clone => {
222 reflink_copy::reflink(entry.path(), &destination)
223 .map_err(Error::io("cloning to", &destination))?;
224 carry_metadata(&metadata, &destination)?;
225 }
226 // A hard link is the same inode, so it arrives with the times
227 // and the mode already on it.
228 Strategy::LinkAndCopy => {
229 fs::hard_link(entry.path(), &destination)
230 .map_err(Error::io("linking to", &destination))?;
231 }
232 }
233 report.shared += 1;
234 report.shared_bytes += size;
235 } else {
236 fs::copy(entry.path(), &destination).map_err(Error::io("copying to", &destination))?;
237 carry_metadata(&metadata, &destination)?;
238 report.copied += 1;
239 report.copied_bytes += size;
240 }
241 }
242
243 report.incremental_dropped = dropped.get();
244 Ok(report)
245}
246
247/// Carries the modification time and the mode across.
248///
249/// What the times are for is that the seeded tree should be the same tree. They
250/// are not load-bearing for freshness in the direction it is tempting to
251/// assume: Cargo compares each source file against the dep-info recorded beside
252/// the artifact, so a copy landing with today's date is *newer* than what it was
253/// built from and reads as fresh either way. Carrying them is what keeps the two
254/// trees answering the same question the same way — and what keeps a hard link,
255/// which has no choice but to bring its times along, from being the odd one out
256/// beside the files that were copied.
257fn carry_metadata(source: &fs::Metadata, destination: &Path) -> Result<()> {
258 fs::set_permissions(destination, source.permissions())
259 .map_err(Error::io("setting the mode on", destination))?;
260 filetime::set_file_times(
261 destination,
262 FileTime::from_last_access_time(source),
263 FileTime::from_last_modification_time(source),
264 )
265 .map_err(Error::io("setting the timestamps on", destination))?;
266 Ok(())
267}
268
269/// Cargo's own incremental directory, told apart from any other directory that
270/// happens to carry the name.
271///
272/// Matching on the name alone reaches into `OUT_DIR`, where a build script is
273/// free to emit an `incremental/` of its own. Dropping that while carrying the
274/// `run-build-script` fingerprint beside it produces exactly the tree the
275/// staging rename exists to prevent — one whose fingerprints say fresh about
276/// files that are not there — only arrived at from the other side, and Cargo
277/// will not re-run the script that would put them back.
278///
279/// The one Cargo writes sits in a profile directory, and what distinguishes a
280/// profile directory is that `deps/` is beside it.
281fn is_profile_incremental(path: &Path, file_type: fs::FileType) -> bool {
282 file_type.is_dir()
283 && path.file_name() == Some(OsStr::new("incremental"))
284 && path
285 .parent()
286 .is_some_and(|profile| profile.join("deps").is_dir())
287}
288
289/// Dep-info, which is never shared however large it grows.
290///
291/// Cargo rewrites these on every build, and what they contain is a list of
292/// absolute source paths. Two checkouts sitting at two paths need two of them
293/// saying two different things, so one inode between them is wrong on both
294/// counts. The size rule below happens to keep them apart at the default
295/// threshold, which is not the same as deciding to.
296fn is_dep_info(relative: &Path) -> bool {
297 relative
298 .extension()
299 .is_some_and(|extension| extension == "d")
300}
301
302/// `deps` as an ancestor rather than as the file's own name: what is being
303/// asked is which directory Cargo writes the file into.
304fn under_deps(relative: &Path) -> bool {
305 relative.parent().is_some_and(|parent| {
306 parent
307 .components()
308 .any(|component| component.as_os_str() == "deps")
309 })
310}
311
312/// Refuses a destination that lives under the directory being read.
313///
314/// The walk and the writing would then be the same tree: every file written
315/// becomes a file to walk, which becomes a file to write. Nothing about it
316/// converges, and what it consumes is the disk. Both the destination and the
317/// staged tree are checked, since the staged one is where everything is written
318/// first.
319///
320/// Both paths are resolved through whatever symlinks their existing parts
321/// contain before being compared — the directory a worktree is cut into is
322/// free to be a symlink, and
323/// comparing the paths as they were written would let a destination that
324/// reaches the source by another name straight through.
325fn refuse_nesting(src: &Path, dest: &Path, staging: &Path) -> Result<()> {
326 let source = src.canonicalize().map_err(Error::io("resolving", src))?;
327
328 for candidate in [dest, staging] {
329 let resolved = resolve(candidate)?;
330 if resolved.starts_with(&source) {
331 return Err(Error::DestinationInsideSource {
332 src: source,
333 dest: resolved,
334 });
335 }
336 }
337
338 Ok(())
339}
340
341/// Resolves a path that does not exist yet, by resolving as much of it as does
342/// and keeping the rest as written.
343///
344/// `canonicalize` needs the whole path to be there, and this runs before
345/// anything has been created on purpose.
346fn resolve(path: &Path) -> Result<PathBuf> {
347 let mut unresolved = Vec::new();
348 let mut existing = path.to_path_buf();
349
350 loop {
351 if let Ok(mut base) = existing.canonicalize() {
352 base.extend(unresolved.iter().rev());
353 return Ok(base);
354 }
355
356 let name = existing
357 .file_name()
358 .ok_or_else(|| Error::DestinationHasNoParent(path.to_path_buf()))?
359 .to_os_string();
360 unresolved.push(name);
361 existing = parent_of(&existing)?.to_path_buf();
362 }
363}
364
365fn parent_of(path: &Path) -> Result<&Path> {
366 match path.parent() {
367 // A relative single-component path has a parent, and it is empty.
368 Some(parent) if parent.as_os_str().is_empty() => Ok(Path::new(".")),
369 Some(parent) => Ok(parent),
370 None => Err(Error::DestinationHasNoParent(path.to_path_buf())),
371 }
372}
373
374fn default_staging(dest: &Path) -> PathBuf {
375 let name = dest.file_name().unwrap_or_else(|| OsStr::new("target"));
376 let mut staged = name.to_os_string();
377 staged.push(".partial");
378 dest.with_file_name(staged)
379}
380
381#[cfg(unix)]
382fn same_filesystem(a: &Path, b: &Path) -> Result<()> {
383 use std::os::unix::fs::MetadataExt;
384
385 let left = fs::metadata(a).map_err(Error::io("reading", a))?;
386 let right = fs::metadata(b).map_err(Error::io("reading", b))?;
387 if left.dev() == right.dev() {
388 return Ok(());
389 }
390 Err(Error::CrossDevice {
391 src: a.to_path_buf(),
392 dest: b.to_path_buf(),
393 })
394}
395
396#[cfg(not(unix))]
397fn same_filesystem(_a: &Path, _b: &Path) -> Result<()> {
398 // Left to the clone and the link to report in their own words rather than
399 // guessed at from a device number this platform does not hand out.
400 Ok(())
401}
402
403#[cfg(unix)]
404fn symlink(_link: &Path, points_to: &Path, at: &Path) -> Result<()> {
405 std::os::unix::fs::symlink(points_to, at).map_err(Error::io("creating the symlink", at))
406}
407
408#[cfg(not(unix))]
409fn symlink(link: &Path, _points_to: &Path, at: &Path) -> Result<()> {
410 // Followed rather than recreated: making a symlink is a privileged
411 // operation on Windows, and a copy of what it pointed at is the same tree
412 // to a compiler.
413 //
414 // Copied from the link itself rather than from what `read_link` returned.
415 // That is normally relative to the link's own directory, and handing it to
416 // `fs::copy` would resolve it against the working directory instead —
417 // failing outright, or finding some unrelated file that happens to sit
418 // there and writing its bytes into the tree as if they belonged.
419 fs::copy(link, at)
420 .map(|_| ())
421 .map_err(Error::io("copying what the symlink pointed at, to", at))
422}