1use sha2::{Digest, Sha256};
2use std::{
3 borrow::Cow,
4 collections::BTreeSet,
5 ffi::OsStr,
6 fmt::Write as _,
7 fs::{self, File, OpenOptions},
8 io::{self, Read as _, Write as _},
9 path::{Path, PathBuf},
10 sync::atomic::{AtomicU64, Ordering},
11};
12
13static TEMP_FILE_SEQUENCE: AtomicU64 = AtomicU64::new(0);
14
15#[derive(Debug)]
16struct AtomicCopyErrorContext {
17 source_path: PathBuf,
18 destination_path: PathBuf,
19 source: io::Error,
20}
21
22impl std::fmt::Display for AtomicCopyErrorContext {
23 fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
24 write!(
25 formatter,
26 "failed to atomically copy {} to {}: {}",
27 self.source_path.display(),
28 self.destination_path.display(),
29 self.source
30 )
31 }
32}
33
34impl std::error::Error for AtomicCopyErrorContext {
35 fn source(&self) -> Option<&(dyn std::error::Error + 'static)> {
36 Some(&self.source)
37 }
38}
39
40#[derive(Clone, Copy, Debug, Eq, Hash, Ord, PartialEq, PartialOrd)]
42pub struct InputDigest([u8; 32]);
43
44impl InputDigest {
45 #[must_use]
47 pub const fn as_bytes(&self) -> &[u8; 32] {
48 &self.0
49 }
50
51 #[must_use]
53 pub fn to_hex(self) -> String {
54 let mut hex = String::with_capacity(64);
55 write!(hex, "{self}").expect("writing to a String cannot fail");
56 hex
57 }
58}
59
60impl std::fmt::Display for InputDigest {
61 fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
62 for byte in self.0 {
63 write!(formatter, "{byte:02x}")?;
64 }
65 Ok(())
66 }
67}
68
69pub(super) struct InputHasher(Sha256);
70
71impl InputHasher {
72 pub(super) fn new(domain: &str) -> Self {
73 let mut hasher = Self(Sha256::new());
74 hasher.field("domain", domain.as_bytes());
75 hasher
76 }
77
78 pub(super) fn field(&mut self, label: &str, value: &[u8]) {
79 self.field_header(
80 label,
81 u64::try_from(value.len()).expect("input value length must fit in u64"),
82 );
83 self.0.update(value);
84 }
85
86 fn field_header(&mut self, label: &str, value_len: u64) {
87 self.0.update(
88 u64::try_from(label.len())
89 .expect("input label length must fit in u64")
90 .to_le_bytes(),
91 );
92 self.0.update(label.as_bytes());
93 self.0.update(value_len.to_le_bytes());
94 }
95
96 fn file_field(&mut self, label: &str, path: &Path) -> io::Result<u64> {
97 let mut file = File::open(path)?;
98 let expected_len = file.metadata()?.len();
99 self.field_header(label, expected_len);
100
101 let mut actual_len = 0_u64;
102 let mut buffer = [0_u8; 16 * 1024];
103 loop {
104 let read = file.read(&mut buffer)?;
105 if read == 0 {
106 break;
107 }
108 actual_len = actual_len
109 .saturating_add(u64::try_from(read).expect("artifact read length must fit in u64"));
110 self.0.update(&buffer[..read]);
111 }
112 if actual_len != expected_len {
113 return Err(io::Error::new(
114 io::ErrorKind::InvalidData,
115 format!(
116 "file changed size while hashing: expected {expected_len} bytes, read {actual_len}"
117 ),
118 ));
119 }
120 Ok(actual_len)
121 }
122
123 pub(super) fn finish(self) -> InputDigest {
124 InputDigest(self.0.finalize().into())
125 }
126}
127
128pub(super) fn digest_bytes(domain: &str, value: &[u8]) -> InputDigest {
129 let mut hasher = InputHasher::new(domain);
130 hasher.field("content", value);
131 hasher.finish()
132}
133
134pub(super) fn digest_file(domain: &str, path: &Path) -> io::Result<(u64, InputDigest)> {
135 let mut hasher = InputHasher::new(domain);
136 let bytes = hasher.file_field("content", path)?;
137 Ok((bytes, hasher.finish()))
138}
139
140pub(super) fn digest_labeled_paths(
141 domain: &str,
142 paths: &[(PathBuf, PathBuf)],
143 excluded_roots: &[PathBuf],
144) -> io::Result<InputDigest> {
145 let mut paths = paths.iter().collect::<Vec<_>>();
146 paths.sort_by(|(left, _), (right, _)| {
147 os_bytes(left.as_os_str()).cmp(&os_bytes(right.as_os_str()))
148 });
149
150 let excluded_roots = excluded_roots
151 .iter()
152 .filter_map(|path| path.canonicalize().ok())
153 .collect::<Vec<_>>();
154 let mut visited_directories = BTreeSet::new();
155 let mut hasher = InputHasher::new(domain);
156 for (label, path) in paths {
157 hash_path(
158 &mut hasher,
159 label,
160 path,
161 &excluded_roots,
162 &mut visited_directories,
163 true,
164 None,
165 )?;
166 }
167 Ok(hasher.finish())
168}
169
170#[derive(Default)]
171pub(super) struct LabeledPathDigestCache {
172 entries: Vec<LabeledPathDigestCacheEntry>,
173}
174
175struct LabeledPathDigestCacheEntry {
176 domain: String,
177 label: PathBuf,
178 path: PathBuf,
179 canonical_root: PathBuf,
180 excluded_roots: Vec<PathBuf>,
181 traversed_external_path: bool,
182 digest: InputDigest,
183}
184
185struct HashPathTrace {
186 canonical_root: PathBuf,
187 traversed_external_path: bool,
188}
189
190pub(super) fn digest_labeled_paths_composable<'a>(
191 domain: &str,
192 paths: impl IntoIterator<Item = (&'a Path, &'a Path)>,
193 excluded_roots: &[PathBuf],
194 cache: &mut LabeledPathDigestCache,
195) -> io::Result<InputDigest> {
196 let mut paths = paths.into_iter().collect::<Vec<_>>();
197 paths.sort_by(|(left, _), (right, _)| {
198 os_bytes(left.as_os_str()).cmp(&os_bytes(right.as_os_str()))
199 });
200 let excluded_roots = excluded_roots
201 .iter()
202 .filter_map(|path| path.canonicalize().ok())
203 .collect::<Vec<_>>();
204 let mut hasher = InputHasher::new(&format!("{domain}/composable-v1"));
205 for (label, path) in paths {
206 let digest = cache.digest_root(domain, label, path, &excluded_roots)?;
207 hasher.field("input-label", &os_bytes(label.as_os_str()));
208 hasher.field("input-digest", digest.as_bytes());
209 }
210 Ok(hasher.finish())
211}
212
213impl LabeledPathDigestCache {
214 fn digest_root(
215 &mut self,
216 domain: &str,
217 label: &Path,
218 path: &Path,
219 excluded_roots: &[PathBuf],
220 ) -> io::Result<InputDigest> {
221 let canonical_root = path.canonicalize()?;
222 if let Some(entry) = self.entries.iter().find(|entry| {
223 entry.domain == domain
224 && entry.label == label
225 && entry.path == path
226 && entry.excluded_roots.iter().eq(effective_root_exclusions(
227 &entry.canonical_root,
228 excluded_roots,
229 entry.traversed_external_path,
230 ))
231 }) {
232 return Ok(entry.digest);
233 }
234 let mut hasher = InputHasher::new(&format!("{domain}/root-v1"));
235 let mut trace = HashPathTrace {
236 canonical_root: canonical_root.clone(),
237 traversed_external_path: false,
238 };
239 hash_path(
240 &mut hasher,
241 label,
242 path,
243 excluded_roots,
244 &mut BTreeSet::new(),
245 true,
246 Some(&mut trace),
247 )?;
248 let digest = hasher.finish();
249 self.entries.push(LabeledPathDigestCacheEntry {
250 domain: domain.to_owned(),
251 label: label.to_owned(),
252 path: path.to_owned(),
253 canonical_root,
254 excluded_roots: effective_root_exclusions(
255 &trace.canonical_root,
256 excluded_roots,
257 trace.traversed_external_path,
258 )
259 .cloned()
260 .collect(),
261 traversed_external_path: trace.traversed_external_path,
262 digest,
263 });
264 Ok(digest)
265 }
266}
267
268fn effective_root_exclusions<'a>(
269 canonical_root: &'a Path,
270 excluded_roots: &'a [PathBuf],
271 traversed_external_path: bool,
272) -> impl Iterator<Item = &'a PathBuf> {
273 excluded_roots.iter().filter(move |excluded| {
274 traversed_external_path
275 || excluded.starts_with(canonical_root)
276 || canonical_root.starts_with(excluded)
277 })
278}
279
280fn hash_path(
281 hasher: &mut InputHasher,
282 label: &Path,
283 path: &Path,
284 excluded_roots: &[PathBuf],
285 visited_directories: &mut BTreeSet<PathBuf>,
286 declared_root: bool,
287 mut trace: Option<&mut HashPathTrace>,
288) -> io::Result<()> {
289 let context =
290 |error: io::Error| io::Error::new(error.kind(), format!("{}: {error}", path.display()));
291 let canonical = path.canonicalize().map_err(context)?;
292 if let Some(trace) = &mut trace
293 && !canonical.starts_with(&trace.canonical_root)
294 {
295 trace.traversed_external_path = true;
296 }
297 if excluded_roots
298 .iter()
299 .any(|excluded| canonical.starts_with(excluded))
300 {
301 if declared_root {
302 return Err(io::Error::new(
303 io::ErrorKind::InvalidInput,
304 format!(
305 "declared input is located inside an excluded cache root: {}",
306 path.display()
307 ),
308 ));
309 }
310 return Ok(());
311 }
312
313 let metadata = fs::metadata(path).map_err(context)?;
314 let label_bytes = os_bytes(label.as_os_str());
315 if metadata.is_file() {
316 hasher.field("file-path", &label_bytes);
317 hasher.file_field("file-content", path).map_err(context)?;
318 return Ok(());
319 }
320 if !metadata.is_dir() {
321 return Err(io::Error::new(
322 io::ErrorKind::InvalidInput,
323 format!(
324 "watched input is not a regular file or directory: {}",
325 path.display()
326 ),
327 ));
328 }
329
330 hasher.field("directory", &label_bytes);
331 if !visited_directories.insert(canonical) {
332 hasher.field("directory-already-visited", &label_bytes);
333 return Ok(());
334 }
335
336 let mut entries = fs::read_dir(path)
337 .map_err(context)?
338 .collect::<Result<Vec<_>, _>>()
339 .map_err(context)?;
340 entries.sort_by_cached_key(|entry| os_bytes(&entry.file_name()).into_owned());
341 for entry in entries {
342 hash_path(
343 hasher,
344 &label.join(entry.file_name()),
345 &entry.path(),
346 excluded_roots,
347 visited_directories,
348 false,
349 trace.as_deref_mut(),
350 )?;
351 }
352 Ok(())
353}
354
355pub(super) fn write_atomic(path: &Path, contents: &[u8]) -> io::Result<()> {
356 write_file_atomic(path, |file| file.write_all(contents))
357}
358
359pub(super) fn copy_file_atomic(source: &Path, destination: &Path) -> io::Result<u64> {
360 let result = (|| {
361 let mut source_file = File::open(source)?;
362 write_file_atomic(destination, |destination_file| {
363 io::copy(&mut source_file, destination_file)
364 })
365 })();
366 result.map_err(|source_error| {
367 io::Error::new(
368 source_error.kind(),
369 AtomicCopyErrorContext {
370 source_path: source.to_owned(),
371 destination_path: destination.to_owned(),
372 source: source_error,
373 },
374 )
375 })
376}
377
378fn write_file_atomic<T>(
379 path: &Path,
380 write: impl FnOnce(&mut File) -> io::Result<T>,
381) -> io::Result<T> {
382 let parent = path.parent().ok_or_else(|| {
383 io::Error::new(
384 io::ErrorKind::InvalidInput,
385 format!("atomic output path has no parent: {}", path.display()),
386 )
387 })?;
388 fs::create_dir_all(parent)?;
389
390 let file_name = path.file_name().ok_or_else(|| {
391 io::Error::new(
392 io::ErrorKind::InvalidInput,
393 format!("atomic output path has no file name: {}", path.display()),
394 )
395 })?;
396 let sequence = TEMP_FILE_SEQUENCE.fetch_add(1, Ordering::Relaxed);
397 let mut temp_name = file_name.to_os_string();
398 temp_name.push(format!(".tmp-{}-{sequence}", std::process::id()));
399 let temp_path = parent.join(temp_name);
400
401 let result = (|| {
402 let mut file = OpenOptions::new()
403 .create_new(true)
404 .write(true)
405 .open(&temp_path)?;
406 let value = write(&mut file)?;
407 file.sync_all()?;
408 fs::rename(&temp_path, path)?;
409 Ok(value)
410 })();
411 if result.is_err() {
412 let _ = fs::remove_file(&temp_path);
413 }
414 result
415}
416
417#[cfg(unix)]
418pub(super) fn os_bytes(value: &OsStr) -> Cow<'_, [u8]> {
419 use std::os::unix::ffi::OsStrExt as _;
420 Cow::Borrowed(value.as_bytes())
421}
422
423#[cfg(windows)]
424pub(super) fn os_bytes(value: &OsStr) -> Cow<'_, [u8]> {
425 use std::os::windows::ffi::OsStrExt as _;
426 Cow::Owned(value.encode_wide().flat_map(u16::to_le_bytes).collect())
427}
428
429#[cfg(not(any(unix, windows)))]
430pub(super) fn os_bytes(value: &OsStr) -> Cow<'_, [u8]> {
431 Cow::Owned(value.to_string_lossy().as_bytes().to_vec())
432}
433
434#[cfg(test)]
435mod tests {
436 use super::{
437 LabeledPathDigestCache, copy_file_atomic, digest_bytes, digest_file,
438 digest_labeled_paths_composable, write_atomic,
439 };
440 use crate::artifacts::test_support::unique_temp_directory;
441 use std::{fs, path::PathBuf};
442
443 #[test]
444 fn digest_text_preserves_lowercase_hex_and_leading_zeroes() {
445 let digest = super::InputDigest(std::array::from_fn(|index| {
446 u8::try_from(index).expect("digest byte index must fit")
447 }));
448 let expected = "000102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f";
449 assert_eq!(digest.to_hex(), expected);
450 assert_eq!(digest.to_string(), expected);
451 assert_eq!(super::InputDigest([0xff; 32]).to_string(), "ff".repeat(32));
452 }
453
454 #[test]
455 #[cfg(unix)]
456 fn labeled_path_digests_preserve_native_names_and_sorted_order() {
457 use super::{InputHasher, digest_labeled_paths};
458 use std::{ffi::OsStr, os::unix::ffi::OsStrExt as _};
459
460 let root = unique_temp_directory("native-path-digest");
461 let tree = root.join("tree");
462 fs::create_dir_all(tree.join("nested")).unwrap();
463 fs::write(tree.join(OsStr::from_bytes(b"\xff")), b"native").unwrap();
464 fs::write(tree.join("nested/z"), b"last").unwrap();
465 fs::write(tree.join("a"), b"first").unwrap();
466 fs::write(root.join("top"), b"top").unwrap();
467 let mut paths = [
468 (PathBuf::from("tree"), tree),
469 (PathBuf::from("aaa"), root.join("top")),
470 ];
471
472 let tree_fields = |hasher: &mut InputHasher| {
473 hasher.field("directory", b"tree");
474 hasher.field("file-path", b"tree/a");
475 hasher.field("file-content", b"first");
476 hasher.field("directory", b"tree/nested");
477 hasher.field("file-path", b"tree/nested/z");
478 hasher.field("file-content", b"last");
479 hasher.field("file-path", b"tree/\xff");
480 hasher.field("file-content", b"native");
481 };
482 let mut expected = InputHasher::new("native-path-test-v1");
483 expected.field("file-path", b"aaa");
484 expected.field("file-content", b"top");
485 tree_fields(&mut expected);
486 let expected = expected.finish();
487
488 let mut top = InputHasher::new("native-path-test-v1/root-v1");
489 top.field("file-path", b"aaa");
490 top.field("file-content", b"top");
491 let mut tree = InputHasher::new("native-path-test-v1/root-v1");
492 tree_fields(&mut tree);
493 let mut composable = InputHasher::new("native-path-test-v1/composable-v1");
494 composable.field("input-label", b"aaa");
495 composable.field("input-digest", top.finish().as_bytes());
496 composable.field("input-label", b"tree");
497 composable.field("input-digest", tree.finish().as_bytes());
498 let composable = composable.finish();
499
500 for _ in 0..2 {
501 assert_eq!(
502 digest_labeled_paths("native-path-test-v1", &paths, &[]).unwrap(),
503 expected,
504 );
505 assert_eq!(
506 digest_labeled_paths_composable(
507 "native-path-test-v1",
508 paths
509 .iter()
510 .map(|(label, path)| (label.as_path(), path.as_path())),
511 &[],
512 &mut LabeledPathDigestCache::default(),
513 )
514 .unwrap(),
515 composable,
516 );
517 paths.reverse();
518 }
519 fs::remove_dir_all(root).unwrap();
520 }
521
522 #[test]
523 #[cfg(windows)]
524 fn native_names_preserve_utf16_little_endian_encoding() {
525 use std::{ffi::OsString, os::windows::ffi::OsStringExt as _};
526 let value = OsString::from_wide(&[0x0061, 0xd800, 0x0100]);
527 assert_eq!(super::os_bytes(&value).as_ref(), &[0x61, 0, 0, 0xd8, 0, 1]);
528 }
529
530 #[test]
531 fn streaming_digest_and_atomic_copy_preserve_exact_bytes() {
532 let root = unique_temp_directory("streaming-digest");
533 let source = root.join("source");
534 let destination = root.join("destination");
535 let mut contents = vec![0_u8; 192 * 1024];
536 for (index, byte) in contents.iter_mut().enumerate() {
537 *byte = u8::try_from(index % 251).expect("test byte must fit");
538 }
539 fs::write(&source, &contents).expect("write source");
540
541 let (bytes, streamed) = digest_file("streaming-test-v1", &source).expect("digest file");
542 assert_eq!(
543 bytes,
544 u64::try_from(contents.len()).expect("fixture length must fit in u64")
545 );
546 assert_eq!(streamed, digest_bytes("streaming-test-v1", &contents));
547
548 write_atomic(&destination, b"old").expect("write original destination");
549 assert_eq!(
550 copy_file_atomic(&source, &destination).expect("copy source atomically"),
551 bytes
552 );
553 assert_eq!(
554 fs::read(&destination).expect("read copied destination"),
555 contents
556 );
557
558 let missing = root.join("missing");
559 let error = copy_file_atomic(&missing, &destination).expect_err("missing source must fail");
560 let message = error.to_string();
561 assert!(message.contains(&missing.display().to_string()));
562 assert!(message.contains(&destination.display().to_string()));
563 fs::remove_dir_all(root).expect("remove streaming-digest test directory");
564 }
565
566 #[test]
567 fn composable_digest_reuses_roots_across_irrelevant_exclusion_changes() {
568 let root = unique_temp_directory("composable-digest-cache");
569 let input = root.join("input");
570 fs::create_dir_all(&input).expect("create composable input");
571 fs::create_dir_all(root.join("generated-a")).expect("create first generated root");
572 fs::create_dir_all(root.join("generated-b")).expect("create second generated root");
573 fs::write(input.join("source"), b"source").expect("write composable input");
574 let paths = [(PathBuf::from("shared"), input)];
575 let mut cache = LabeledPathDigestCache::default();
576
577 let first = digest_labeled_paths_composable(
578 "composable-test-v1",
579 paths
580 .iter()
581 .map(|(label, path)| (label.as_path(), path.as_path())),
582 &[root.join("generated-a")],
583 &mut cache,
584 )
585 .expect("hash first composable input");
586 let second = digest_labeled_paths_composable(
587 "composable-test-v1",
588 paths
589 .iter()
590 .map(|(label, path)| (label.as_path(), path.as_path())),
591 &[root.join("generated-b")],
592 &mut cache,
593 )
594 .expect("reuse composable input root");
595
596 assert_eq!(first, second);
597 assert_eq!(cache.entries.len(), 1);
598 fs::remove_dir_all(root).expect("remove composable digest fixture");
599 }
600
601 #[test]
602 fn composable_digest_rehashes_changed_descendant_exclusions_and_rejects_ancestors() {
603 let root = unique_temp_directory("composable-relevant-exclusions");
604 let input = root.join("input");
605 let generated = input.join("generated");
606 fs::create_dir_all(&generated).unwrap();
607 fs::write(input.join("source"), b"source").unwrap();
608 fs::write(generated.join("artifact"), b"generated").unwrap();
609 let paths = [(PathBuf::from("input"), input.clone())];
610 let digest = |exclusions: &[PathBuf], cache: &mut LabeledPathDigestCache| {
611 digest_labeled_paths_composable(
612 "exclusions-test-v1",
613 paths
614 .iter()
615 .map(|(label, path)| (label.as_path(), path.as_path())),
616 exclusions,
617 cache,
618 )
619 };
620 let mut cache = LabeledPathDigestCache::default();
621 let excluded = digest(std::slice::from_ref(&generated), &mut cache).unwrap();
622 let included = digest(&[], &mut cache).unwrap();
623 assert_ne!(included, excluded);
624 assert_eq!(
625 included,
626 digest(&[], &mut LabeledPathDigestCache::default()).unwrap(),
627 );
628 for ancestor in [&input, &root] {
629 assert_eq!(
630 digest(std::slice::from_ref(ancestor), &mut cache)
631 .unwrap_err()
632 .kind(),
633 std::io::ErrorKind::InvalidInput,
634 );
635 }
636 assert_eq!(
637 digest(std::slice::from_ref(&generated), &mut cache).unwrap(),
638 excluded,
639 );
640 fs::remove_dir_all(root).unwrap();
641 }
642
643 #[test]
644 #[cfg(unix)]
645 fn composable_digest_tracks_exclusions_beyond_an_external_symlink() {
646 let root = unique_temp_directory("composable-external-exclusions");
647 let input = root.join("input");
648 let external = root.join("external");
649 fs::create_dir_all(&input).unwrap();
650 fs::create_dir_all(external.join("first")).unwrap();
651 fs::create_dir_all(external.join("second")).unwrap();
652 fs::write(input.join("source"), b"source").unwrap();
653 fs::write(external.join("first/file"), b"first").unwrap();
654 fs::write(external.join("second/file"), b"second").unwrap();
655 std::os::unix::fs::symlink(&external, input.join("linked")).unwrap();
656 let paths = [(PathBuf::from("input"), input)];
657 let digest = |exclusion: &PathBuf, cache: &mut LabeledPathDigestCache| {
658 digest_labeled_paths_composable(
659 "external-exclusions-test-v1",
660 paths
661 .iter()
662 .map(|(label, path)| (label.as_path(), path.as_path())),
663 std::slice::from_ref(exclusion),
664 cache,
665 )
666 };
667 let mut cache = LabeledPathDigestCache::default();
668 let first = digest(&external.join("first"), &mut cache).unwrap();
669 let second = digest(&external.join("second"), &mut cache).unwrap();
670 assert_ne!(first, second);
671 assert_eq!(
672 second,
673 digest(
674 &external.join("second"),
675 &mut LabeledPathDigestCache::default(),
676 )
677 .unwrap(),
678 );
679 assert_eq!(digest(&external.join("first"), &mut cache).unwrap(), first);
680 fs::remove_dir_all(root).unwrap();
681 }
682}