1use sha2::{Digest, Sha256};
2use std::{
3 borrow::Cow,
4 collections::BTreeSet,
5 ffi::OsStr,
6 fmt::Write as _,
7 fs::{self, File, OpenOptions},
8 io::{self, Read as _, Write as _},
9 path::{Path, PathBuf},
10 sync::atomic::{AtomicU64, Ordering},
11};
12
13static TEMP_FILE_SEQUENCE: AtomicU64 = AtomicU64::new(0);
14
15#[derive(Debug)]
16struct AtomicCopyErrorContext {
17 source_path: PathBuf,
18 destination_path: PathBuf,
19 source: io::Error,
20}
21
22impl std::fmt::Display for AtomicCopyErrorContext {
23 fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
24 write!(
25 formatter,
26 "failed to atomically copy {} to {}: {}",
27 self.source_path.display(),
28 self.destination_path.display(),
29 self.source
30 )
31 }
32}
33
34impl std::error::Error for AtomicCopyErrorContext {
35 fn source(&self) -> Option<&(dyn std::error::Error + 'static)> {
36 Some(&self.source)
37 }
38}
39
40#[derive(Clone, Copy, Debug, Eq, Hash, Ord, PartialEq, PartialOrd)]
42pub struct InputDigest([u8; 32]);
43
44impl InputDigest {
45 #[must_use]
47 pub const fn as_bytes(&self) -> &[u8; 32] {
48 &self.0
49 }
50
51 #[must_use]
53 pub fn to_hex(self) -> String {
54 let mut hex = String::with_capacity(64);
55 for byte in self.0 {
56 write!(hex, "{byte:02x}").expect("writing to a String cannot fail");
57 }
58 hex
59 }
60}
61
62impl std::fmt::Display for InputDigest {
63 fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
64 formatter.write_str(&self.to_hex())
65 }
66}
67
68pub(super) struct InputHasher(Sha256);
69
70impl InputHasher {
71 pub(super) fn new(domain: &str) -> Self {
72 let mut hasher = Self(Sha256::new());
73 hasher.field("domain", domain.as_bytes());
74 hasher
75 }
76
77 pub(super) fn field(&mut self, label: &str, value: &[u8]) {
78 self.field_header(
79 label,
80 u64::try_from(value.len()).expect("input value length must fit in u64"),
81 );
82 self.0.update(value);
83 }
84
85 fn field_header(&mut self, label: &str, value_len: u64) {
86 self.0.update(
87 u64::try_from(label.len())
88 .expect("input label length must fit in u64")
89 .to_le_bytes(),
90 );
91 self.0.update(label.as_bytes());
92 self.0.update(value_len.to_le_bytes());
93 }
94
95 fn file_field(&mut self, label: &str, path: &Path) -> io::Result<u64> {
96 let mut file = File::open(path)?;
97 let expected_len = file.metadata()?.len();
98 self.field_header(label, expected_len);
99
100 let mut actual_len = 0_u64;
101 let mut buffer = [0_u8; 16 * 1024];
102 loop {
103 let read = file.read(&mut buffer)?;
104 if read == 0 {
105 break;
106 }
107 actual_len = actual_len
108 .saturating_add(u64::try_from(read).expect("artifact read length must fit in u64"));
109 self.0.update(&buffer[..read]);
110 }
111 if actual_len != expected_len {
112 return Err(io::Error::new(
113 io::ErrorKind::InvalidData,
114 format!(
115 "file changed size while hashing: expected {expected_len} bytes, read {actual_len}"
116 ),
117 ));
118 }
119 Ok(actual_len)
120 }
121
122 pub(super) fn finish(self) -> InputDigest {
123 InputDigest(self.0.finalize().into())
124 }
125}
126
127pub(super) fn digest_bytes(domain: &str, value: &[u8]) -> InputDigest {
128 let mut hasher = InputHasher::new(domain);
129 hasher.field("content", value);
130 hasher.finish()
131}
132
133pub(super) fn digest_file(domain: &str, path: &Path) -> io::Result<(u64, InputDigest)> {
134 let mut hasher = InputHasher::new(domain);
135 let bytes = hasher.file_field("content", path)?;
136 Ok((bytes, hasher.finish()))
137}
138
139pub(super) fn digest_labeled_paths(
140 domain: &str,
141 paths: &[(PathBuf, PathBuf)],
142 excluded_roots: &[PathBuf],
143) -> io::Result<InputDigest> {
144 let mut paths = paths.iter().collect::<Vec<_>>();
145 paths.sort_by(|(left, _), (right, _)| {
146 os_bytes(left.as_os_str()).cmp(&os_bytes(right.as_os_str()))
147 });
148
149 let excluded_roots = excluded_roots
150 .iter()
151 .filter_map(|path| path.canonicalize().ok())
152 .collect::<Vec<_>>();
153 let mut visited_directories = BTreeSet::new();
154 let mut hasher = InputHasher::new(domain);
155 for (label, path) in paths {
156 hash_path(
157 &mut hasher,
158 label,
159 path,
160 &excluded_roots,
161 &mut visited_directories,
162 true,
163 None,
164 )?;
165 }
166 Ok(hasher.finish())
167}
168
169#[derive(Default)]
170pub(super) struct LabeledPathDigestCache {
171 entries: Vec<LabeledPathDigestCacheEntry>,
172}
173
174struct LabeledPathDigestCacheEntry {
175 domain: String,
176 label: PathBuf,
177 path: PathBuf,
178 canonical_root: PathBuf,
179 excluded_roots: Vec<PathBuf>,
180 traversed_external_path: bool,
181 digest: InputDigest,
182}
183
184struct HashPathTrace {
185 canonical_root: PathBuf,
186 traversed_external_path: bool,
187}
188
189pub(super) fn digest_labeled_paths_composable(
190 domain: &str,
191 paths: &[(PathBuf, PathBuf)],
192 excluded_roots: &[PathBuf],
193 cache: &mut LabeledPathDigestCache,
194) -> io::Result<InputDigest> {
195 let mut paths = paths.iter().collect::<Vec<_>>();
196 paths.sort_by(|(left, _), (right, _)| {
197 os_bytes(left.as_os_str()).cmp(&os_bytes(right.as_os_str()))
198 });
199 let excluded_roots = excluded_roots
200 .iter()
201 .filter_map(|path| path.canonicalize().ok())
202 .collect::<Vec<_>>();
203 let mut hasher = InputHasher::new(&format!("{domain}/composable-v1"));
204 for (label, path) in paths {
205 let digest = cache.digest_root(domain, label, path, &excluded_roots)?;
206 hasher.field("input-label", &os_bytes(label.as_os_str()));
207 hasher.field("input-digest", digest.as_bytes());
208 }
209 Ok(hasher.finish())
210}
211
212impl LabeledPathDigestCache {
213 fn digest_root(
214 &mut self,
215 domain: &str,
216 label: &Path,
217 path: &Path,
218 excluded_roots: &[PathBuf],
219 ) -> io::Result<InputDigest> {
220 let canonical_root = path.canonicalize()?;
221 if let Some(entry) = self.entries.iter().find(|entry| {
222 entry.domain == domain
223 && entry.label == label
224 && entry.path == path
225 && entry.excluded_roots.iter().eq(effective_root_exclusions(
226 &entry.canonical_root,
227 excluded_roots,
228 entry.traversed_external_path,
229 ))
230 }) {
231 return Ok(entry.digest);
232 }
233 let mut hasher = InputHasher::new(&format!("{domain}/root-v1"));
234 let mut trace = HashPathTrace {
235 canonical_root: canonical_root.clone(),
236 traversed_external_path: false,
237 };
238 hash_path(
239 &mut hasher,
240 label,
241 path,
242 excluded_roots,
243 &mut BTreeSet::new(),
244 true,
245 Some(&mut trace),
246 )?;
247 let digest = hasher.finish();
248 self.entries.push(LabeledPathDigestCacheEntry {
249 domain: domain.to_owned(),
250 label: label.to_owned(),
251 path: path.to_owned(),
252 canonical_root,
253 excluded_roots: effective_root_exclusions(
254 &trace.canonical_root,
255 excluded_roots,
256 trace.traversed_external_path,
257 )
258 .cloned()
259 .collect(),
260 traversed_external_path: trace.traversed_external_path,
261 digest,
262 });
263 Ok(digest)
264 }
265}
266
267fn effective_root_exclusions<'a>(
268 canonical_root: &'a Path,
269 excluded_roots: &'a [PathBuf],
270 traversed_external_path: bool,
271) -> impl Iterator<Item = &'a PathBuf> {
272 excluded_roots.iter().filter(move |excluded| {
273 traversed_external_path
274 || excluded.starts_with(canonical_root)
275 || canonical_root.starts_with(excluded)
276 })
277}
278
279fn hash_path(
280 hasher: &mut InputHasher,
281 label: &Path,
282 path: &Path,
283 excluded_roots: &[PathBuf],
284 visited_directories: &mut BTreeSet<PathBuf>,
285 declared_root: bool,
286 mut trace: Option<&mut HashPathTrace>,
287) -> io::Result<()> {
288 let context =
289 |error: io::Error| io::Error::new(error.kind(), format!("{}: {error}", path.display()));
290 let canonical = path.canonicalize().map_err(context)?;
291 if let Some(trace) = &mut trace
292 && !canonical.starts_with(&trace.canonical_root)
293 {
294 trace.traversed_external_path = true;
295 }
296 if excluded_roots
297 .iter()
298 .any(|excluded| canonical.starts_with(excluded))
299 {
300 if declared_root {
301 return Err(io::Error::new(
302 io::ErrorKind::InvalidInput,
303 format!(
304 "declared input is located inside an excluded cache root: {}",
305 path.display()
306 ),
307 ));
308 }
309 return Ok(());
310 }
311
312 let metadata = fs::metadata(path).map_err(context)?;
313 let label_bytes = os_bytes(label.as_os_str());
314 if metadata.is_file() {
315 hasher.field("file-path", &label_bytes);
316 hasher.file_field("file-content", path).map_err(context)?;
317 return Ok(());
318 }
319 if !metadata.is_dir() {
320 return Err(io::Error::new(
321 io::ErrorKind::InvalidInput,
322 format!(
323 "watched input is not a regular file or directory: {}",
324 path.display()
325 ),
326 ));
327 }
328
329 hasher.field("directory", &label_bytes);
330 if !visited_directories.insert(canonical) {
331 hasher.field("directory-already-visited", &label_bytes);
332 return Ok(());
333 }
334
335 let mut entries = fs::read_dir(path)
336 .map_err(context)?
337 .collect::<Result<Vec<_>, _>>()
338 .map_err(context)?;
339 entries.sort_by_cached_key(|entry| os_bytes(&entry.file_name()).into_owned());
340 for entry in entries {
341 hash_path(
342 hasher,
343 &label.join(entry.file_name()),
344 &entry.path(),
345 excluded_roots,
346 visited_directories,
347 false,
348 trace.as_deref_mut(),
349 )?;
350 }
351 Ok(())
352}
353
354pub(super) fn write_atomic(path: &Path, contents: &[u8]) -> io::Result<()> {
355 write_file_atomic(path, |file| file.write_all(contents))
356}
357
358pub(super) fn copy_file_atomic(source: &Path, destination: &Path) -> io::Result<u64> {
359 let result = (|| {
360 let mut source_file = File::open(source)?;
361 write_file_atomic(destination, |destination_file| {
362 io::copy(&mut source_file, destination_file)
363 })
364 })();
365 result.map_err(|source_error| {
366 io::Error::new(
367 source_error.kind(),
368 AtomicCopyErrorContext {
369 source_path: source.to_owned(),
370 destination_path: destination.to_owned(),
371 source: source_error,
372 },
373 )
374 })
375}
376
377fn write_file_atomic<T>(
378 path: &Path,
379 write: impl FnOnce(&mut File) -> io::Result<T>,
380) -> io::Result<T> {
381 let parent = path.parent().ok_or_else(|| {
382 io::Error::new(
383 io::ErrorKind::InvalidInput,
384 format!("atomic output path has no parent: {}", path.display()),
385 )
386 })?;
387 fs::create_dir_all(parent)?;
388
389 let file_name = path.file_name().ok_or_else(|| {
390 io::Error::new(
391 io::ErrorKind::InvalidInput,
392 format!("atomic output path has no file name: {}", path.display()),
393 )
394 })?;
395 let sequence = TEMP_FILE_SEQUENCE.fetch_add(1, Ordering::Relaxed);
396 let mut temp_name = file_name.to_os_string();
397 temp_name.push(format!(".tmp-{}-{sequence}", std::process::id()));
398 let temp_path = parent.join(temp_name);
399
400 let result = (|| {
401 let mut file = OpenOptions::new()
402 .create_new(true)
403 .write(true)
404 .open(&temp_path)?;
405 let value = write(&mut file)?;
406 file.sync_all()?;
407 fs::rename(&temp_path, path)?;
408 Ok(value)
409 })();
410 if result.is_err() {
411 let _ = fs::remove_file(&temp_path);
412 }
413 result
414}
415
416#[cfg(unix)]
417pub(super) fn os_bytes(value: &OsStr) -> Cow<'_, [u8]> {
418 use std::os::unix::ffi::OsStrExt as _;
419 Cow::Borrowed(value.as_bytes())
420}
421
422#[cfg(windows)]
423pub(super) fn os_bytes(value: &OsStr) -> Cow<'_, [u8]> {
424 use std::os::windows::ffi::OsStrExt as _;
425 Cow::Owned(value.encode_wide().flat_map(u16::to_le_bytes).collect())
426}
427
428#[cfg(not(any(unix, windows)))]
429pub(super) fn os_bytes(value: &OsStr) -> Cow<'_, [u8]> {
430 Cow::Owned(value.to_string_lossy().as_bytes().to_vec())
431}
432
433#[cfg(test)]
434mod tests {
435 use super::{
436 LabeledPathDigestCache, copy_file_atomic, digest_bytes, digest_file,
437 digest_labeled_paths_composable, write_atomic,
438 };
439 use crate::artifacts::test_support::unique_temp_directory;
440 use std::{fs, path::PathBuf};
441
442 #[test]
443 #[cfg(unix)]
444 fn labeled_path_digests_preserve_native_names_and_sorted_order() {
445 use super::{InputHasher, digest_labeled_paths};
446 use std::{ffi::OsStr, os::unix::ffi::OsStrExt as _};
447
448 let root = unique_temp_directory("native-path-digest");
449 let tree = root.join("tree");
450 fs::create_dir_all(tree.join("nested")).unwrap();
451 fs::write(tree.join(OsStr::from_bytes(b"\xff")), b"native").unwrap();
452 fs::write(tree.join("nested/z"), b"last").unwrap();
453 fs::write(tree.join("a"), b"first").unwrap();
454 fs::write(root.join("top"), b"top").unwrap();
455 let mut paths = [
456 (PathBuf::from("tree"), tree),
457 (PathBuf::from("aaa"), root.join("top")),
458 ];
459
460 let tree_fields = |hasher: &mut InputHasher| {
461 hasher.field("directory", b"tree");
462 hasher.field("file-path", b"tree/a");
463 hasher.field("file-content", b"first");
464 hasher.field("directory", b"tree/nested");
465 hasher.field("file-path", b"tree/nested/z");
466 hasher.field("file-content", b"last");
467 hasher.field("file-path", b"tree/\xff");
468 hasher.field("file-content", b"native");
469 };
470 let mut expected = InputHasher::new("native-path-test-v1");
471 expected.field("file-path", b"aaa");
472 expected.field("file-content", b"top");
473 tree_fields(&mut expected);
474 let expected = expected.finish();
475
476 let mut top = InputHasher::new("native-path-test-v1/root-v1");
477 top.field("file-path", b"aaa");
478 top.field("file-content", b"top");
479 let mut tree = InputHasher::new("native-path-test-v1/root-v1");
480 tree_fields(&mut tree);
481 let mut composable = InputHasher::new("native-path-test-v1/composable-v1");
482 composable.field("input-label", b"aaa");
483 composable.field("input-digest", top.finish().as_bytes());
484 composable.field("input-label", b"tree");
485 composable.field("input-digest", tree.finish().as_bytes());
486 let composable = composable.finish();
487
488 for _ in 0..2 {
489 assert_eq!(
490 digest_labeled_paths("native-path-test-v1", &paths, &[]).unwrap(),
491 expected,
492 );
493 assert_eq!(
494 digest_labeled_paths_composable(
495 "native-path-test-v1",
496 &paths,
497 &[],
498 &mut LabeledPathDigestCache::default(),
499 )
500 .unwrap(),
501 composable,
502 );
503 paths.reverse();
504 }
505 fs::remove_dir_all(root).unwrap();
506 }
507
508 #[test]
509 #[cfg(windows)]
510 fn native_names_preserve_utf16_little_endian_encoding() {
511 use std::{ffi::OsString, os::windows::ffi::OsStringExt as _};
512 let value = OsString::from_wide(&[0x0061, 0xd800, 0x0100]);
513 assert_eq!(super::os_bytes(&value).as_ref(), &[0x61, 0, 0, 0xd8, 0, 1]);
514 }
515
516 #[test]
517 fn streaming_digest_and_atomic_copy_preserve_exact_bytes() {
518 let root = unique_temp_directory("streaming-digest");
519 let source = root.join("source");
520 let destination = root.join("destination");
521 let mut contents = vec![0_u8; 192 * 1024];
522 for (index, byte) in contents.iter_mut().enumerate() {
523 *byte = u8::try_from(index % 251).expect("test byte must fit");
524 }
525 fs::write(&source, &contents).expect("write source");
526
527 let (bytes, streamed) = digest_file("streaming-test-v1", &source).expect("digest file");
528 assert_eq!(
529 bytes,
530 u64::try_from(contents.len()).expect("fixture length must fit in u64")
531 );
532 assert_eq!(streamed, digest_bytes("streaming-test-v1", &contents));
533
534 write_atomic(&destination, b"old").expect("write original destination");
535 assert_eq!(
536 copy_file_atomic(&source, &destination).expect("copy source atomically"),
537 bytes
538 );
539 assert_eq!(
540 fs::read(&destination).expect("read copied destination"),
541 contents
542 );
543
544 let missing = root.join("missing");
545 let error = copy_file_atomic(&missing, &destination).expect_err("missing source must fail");
546 let message = error.to_string();
547 assert!(message.contains(&missing.display().to_string()));
548 assert!(message.contains(&destination.display().to_string()));
549 fs::remove_dir_all(root).expect("remove streaming-digest test directory");
550 }
551
552 #[test]
553 fn composable_digest_reuses_roots_across_irrelevant_exclusion_changes() {
554 let root = unique_temp_directory("composable-digest-cache");
555 let input = root.join("input");
556 fs::create_dir_all(&input).expect("create composable input");
557 fs::create_dir_all(root.join("generated-a")).expect("create first generated root");
558 fs::create_dir_all(root.join("generated-b")).expect("create second generated root");
559 fs::write(input.join("source"), b"source").expect("write composable input");
560 let paths = [(PathBuf::from("shared"), input)];
561 let mut cache = LabeledPathDigestCache::default();
562
563 let first = digest_labeled_paths_composable(
564 "composable-test-v1",
565 &paths,
566 &[root.join("generated-a")],
567 &mut cache,
568 )
569 .expect("hash first composable input");
570 let second = digest_labeled_paths_composable(
571 "composable-test-v1",
572 &paths,
573 &[root.join("generated-b")],
574 &mut cache,
575 )
576 .expect("reuse composable input root");
577
578 assert_eq!(first, second);
579 assert_eq!(cache.entries.len(), 1);
580 fs::remove_dir_all(root).expect("remove composable digest fixture");
581 }
582
583 #[test]
584 fn composable_digest_rehashes_changed_descendant_exclusions_and_rejects_ancestors() {
585 let root = unique_temp_directory("composable-relevant-exclusions");
586 let input = root.join("input");
587 let generated = input.join("generated");
588 fs::create_dir_all(&generated).unwrap();
589 fs::write(input.join("source"), b"source").unwrap();
590 fs::write(generated.join("artifact"), b"generated").unwrap();
591 let paths = [(PathBuf::from("input"), input.clone())];
592 let digest = |exclusions: &[PathBuf], cache: &mut LabeledPathDigestCache| {
593 digest_labeled_paths_composable("exclusions-test-v1", &paths, exclusions, cache)
594 };
595 let mut cache = LabeledPathDigestCache::default();
596 let excluded = digest(std::slice::from_ref(&generated), &mut cache).unwrap();
597 let included = digest(&[], &mut cache).unwrap();
598 assert_ne!(included, excluded);
599 assert_eq!(
600 included,
601 digest(&[], &mut LabeledPathDigestCache::default()).unwrap(),
602 );
603 for ancestor in [&input, &root] {
604 assert_eq!(
605 digest(std::slice::from_ref(ancestor), &mut cache)
606 .unwrap_err()
607 .kind(),
608 std::io::ErrorKind::InvalidInput,
609 );
610 }
611 assert_eq!(
612 digest(std::slice::from_ref(&generated), &mut cache).unwrap(),
613 excluded,
614 );
615 fs::remove_dir_all(root).unwrap();
616 }
617
618 #[test]
619 #[cfg(unix)]
620 fn composable_digest_tracks_exclusions_beyond_an_external_symlink() {
621 let root = unique_temp_directory("composable-external-exclusions");
622 let input = root.join("input");
623 let external = root.join("external");
624 fs::create_dir_all(&input).unwrap();
625 fs::create_dir_all(external.join("first")).unwrap();
626 fs::create_dir_all(external.join("second")).unwrap();
627 fs::write(input.join("source"), b"source").unwrap();
628 fs::write(external.join("first/file"), b"first").unwrap();
629 fs::write(external.join("second/file"), b"second").unwrap();
630 std::os::unix::fs::symlink(&external, input.join("linked")).unwrap();
631 let paths = [(PathBuf::from("input"), input)];
632 let digest = |exclusion: &PathBuf, cache: &mut LabeledPathDigestCache| {
633 digest_labeled_paths_composable(
634 "external-exclusions-test-v1",
635 &paths,
636 std::slice::from_ref(exclusion),
637 cache,
638 )
639 };
640 let mut cache = LabeledPathDigestCache::default();
641 let first = digest(&external.join("first"), &mut cache).unwrap();
642 let second = digest(&external.join("second"), &mut cache).unwrap();
643 assert_ne!(first, second);
644 assert_eq!(
645 second,
646 digest(
647 &external.join("second"),
648 &mut LabeledPathDigestCache::default(),
649 )
650 .unwrap(),
651 );
652 assert_eq!(digest(&external.join("first"), &mut cache).unwrap(), first);
653 fs::remove_dir_all(root).unwrap();
654 }
655}