1use std::{
2 collections::{BTreeMap, BTreeSet},
3 ffi::{OsStr, OsString},
4 fs::{self, File},
5 io,
6 path::{Component, Path, PathBuf},
7 sync::atomic::{AtomicU64, Ordering},
8 time::{Duration, Instant},
9};
10
11use super::{
12 cache_fs::{
13 ArtifactCacheMaintenance, ArtifactCachePrunePolicy, ArtifactCachePruneReport, CacheFsError,
14 LAST_USED_FILE, directory_logical_size, ensure_cache_directory_tag, is_sha256_directory,
15 lock_cache_file, prune_direct_child_directories, record_cache_entry_use,
16 remove_path_if_present, try_lock_cache_file,
17 },
18 digest::{
19 InputDigest, InputHasher, copy_file_atomic, digest_bytes, digest_file,
20 digest_labeled_paths, os_bytes, write_atomic,
21 },
22};
23
24const ARTIFACT_CACHE_FORMAT: &str = "ic-testkit-artifact-set-v1";
25const MANIFEST_FILE: &str = "manifest.ic-testkit";
26const MAX_PREPARATION_RETRIES: usize = 3;
27static STAGING_SEQUENCE: AtomicU64 = AtomicU64::new(0);
28
29#[derive(Clone, Copy, Debug, Default, Eq, Ord, PartialEq, PartialOrd)]
31pub enum ArtifactOutputValidation {
32 RegularFile,
34 #[default]
36 NonEmptyFile,
37}
38
39#[derive(Clone, Eq, PartialEq)]
45pub struct ArtifactCacheSpec {
46 cache_root: PathBuf,
47 namespace: String,
48 recipe_id: String,
49 coordination_scope: String,
50 inputs: Vec<LabeledPath>,
51 tools: Vec<LabeledPath>,
52 arguments: Vec<OsString>,
53 environment: BTreeMap<OsString, Option<OsString>>,
54 identities: Vec<LabeledIdentity>,
55 outputs: Vec<OutputSpec>,
56 prune_policy: Option<ArtifactCachePrunePolicy>,
57}
58
59pub enum ArtifactCachePreparation {
61 Reused(ArtifactCacheRecord),
63 Build(ArtifactBuildTransaction),
65}
66
67#[derive(Clone, Debug, Eq, PartialEq)]
69pub enum ArtifactCacheOutcome {
70 Built(ArtifactCacheRecord),
72 Reused(ArtifactCacheRecord),
74}
75
76#[derive(Clone, Debug, Eq, PartialEq)]
78pub struct ArtifactCacheRecord {
79 key: InputDigest,
80 input_digest: InputDigest,
81 artifacts: Vec<ArtifactCacheArtifact>,
82 timings: ArtifactCacheTimings,
83 maintenance: Option<ArtifactCacheMaintenance>,
84}
85
86#[derive(Clone, Debug, Eq, PartialEq)]
88pub struct ArtifactCacheArtifact {
89 name: String,
90 path: PathBuf,
91}
92
93#[derive(Clone, Copy, Debug, Default, Eq, PartialEq)]
95pub struct ArtifactCacheTimings {
96 coordination_lock_wait: Duration,
97 content_lock_wait: Duration,
98 namespace_lock_wait: Duration,
99 input_capture: Duration,
100 cache_lookup: Duration,
101 caller_build: Option<Duration>,
102 output_validation: Duration,
103 publication: Duration,
104 materialization: Duration,
105 maintenance: Option<Duration>,
106 total: Duration,
107}
108
109pub struct ArtifactBuildTransaction {
111 spec: Box<ArtifactCacheSpec>,
112 resolved: ResolvedKey,
113 staging_directory: PathBuf,
114 entry_directory: PathBuf,
115 namespace_directory: PathBuf,
116 _coordination_lock: File,
117 _content_lock: File,
118 timings: ArtifactCacheTimings,
119 total_started: Instant,
120 caller_build_started: Instant,
121 staging_armed: bool,
122}
123
124#[non_exhaustive]
126#[derive(Debug)]
127pub enum ArtifactCacheError {
128 InvalidSpec { message: String },
130 Io {
132 operation: &'static str,
133 path: PathBuf,
134 source: io::Error,
135 },
136 InputsChangedDuringPreparation {
138 before: InputDigest,
139 after: InputDigest,
140 },
141 InputsChangedDuringBuild {
143 before: InputDigest,
144 after: InputDigest,
145 },
146 InvalidOutputs { outputs: Vec<(String, PathBuf)> },
148 UnknownOutput { name: String },
150 FailedTransactionCleanup {
152 transaction_error: Box<Self>,
153 path: PathBuf,
154 source: io::Error,
155 },
156}
157
158#[derive(Clone, Debug, Eq, PartialEq)]
159struct LabeledPath {
160 label: String,
161 path: PathBuf,
162}
163
164#[derive(Clone, Eq, PartialEq)]
165struct LabeledIdentity {
166 label: String,
167 value: Vec<u8>,
168}
169
170#[derive(Clone, Debug, Eq, PartialEq)]
171struct OutputSpec {
172 name: String,
173 destination: PathBuf,
174 validation: ArtifactOutputValidation,
175}
176
177#[derive(Clone, Copy)]
178struct ResolvedKey {
179 key: InputDigest,
180 input_digest: InputDigest,
181}
182
183struct ArtifactInfo {
184 bytes: u64,
185 digest: InputDigest,
186}
187
188impl ArtifactCacheSpec {
189 #[must_use]
195 pub fn new(cache_root: &Path, namespace: &str, recipe_id: &str) -> Self {
196 Self {
197 cache_root: cache_root.to_owned(),
198 namespace: namespace.to_owned(),
199 recipe_id: recipe_id.to_owned(),
200 coordination_scope: namespace.to_owned(),
201 inputs: Vec::new(),
202 tools: Vec::new(),
203 arguments: Vec::new(),
204 environment: BTreeMap::new(),
205 identities: Vec::new(),
206 outputs: Vec::new(),
207 prune_policy: None,
208 }
209 }
210
211 #[must_use]
213 pub fn with_coordination_scope(mut self, coordination_scope: &str) -> Self {
214 coordination_scope.clone_into(&mut self.coordination_scope);
215 self
216 }
217
218 #[must_use]
220 pub fn with_input(mut self, label: &str, path: &Path) -> Self {
221 self.inputs.push(LabeledPath {
222 label: label.to_owned(),
223 path: path.to_owned(),
224 });
225 self
226 }
227
228 #[must_use]
230 pub fn with_tool(mut self, label: &str, path: &Path) -> Self {
231 self.tools.push(LabeledPath {
232 label: label.to_owned(),
233 path: path.to_owned(),
234 });
235 self
236 }
237
238 #[must_use]
240 pub fn with_arguments(mut self, arguments: &[&str]) -> Self {
241 self.arguments = arguments.iter().map(OsString::from).collect();
242 self
243 }
244
245 #[must_use]
247 pub fn with_environment(mut self, environment: &[(&str, &str)]) -> Self {
248 self.environment.extend(
249 environment
250 .iter()
251 .map(|(name, value)| (OsString::from(name), Some(OsString::from(value)))),
252 );
253 self
254 }
255
256 #[must_use]
258 pub fn with_unset_environment(mut self, names: &[&str]) -> Self {
259 self.environment
260 .extend(names.iter().map(|name| (OsString::from(name), None)));
261 self
262 }
263
264 #[must_use]
266 pub fn with_identity_bytes(mut self, label: &str, value: &[u8]) -> Self {
267 self.identities.push(LabeledIdentity {
268 label: label.to_owned(),
269 value: value.to_vec(),
270 });
271 self
272 }
273
274 #[must_use]
276 pub fn with_output(self, name: &str, destination: &Path) -> Self {
277 self.with_output_validation(name, destination, ArtifactOutputValidation::NonEmptyFile)
278 }
279
280 #[must_use]
282 pub fn with_output_validation(
283 mut self,
284 name: &str,
285 destination: &Path,
286 validation: ArtifactOutputValidation,
287 ) -> Self {
288 self.outputs.push(OutputSpec {
289 name: name.to_owned(),
290 destination: destination.to_owned(),
291 validation,
292 });
293 self.outputs
294 .sort_by(|left, right| left.name.cmp(&right.name));
295 self
296 }
297
298 #[must_use]
300 pub const fn with_prune_policy(mut self, policy: ArtifactCachePrunePolicy) -> Self {
301 self.prune_policy = Some(policy);
302 self
303 }
304
305 #[must_use]
307 pub fn cache_root(&self) -> &Path {
308 &self.cache_root
309 }
310
311 #[must_use]
313 pub fn namespace(&self) -> &str {
314 &self.namespace
315 }
316
317 #[must_use]
319 pub fn recipe_id(&self) -> &str {
320 &self.recipe_id
321 }
322}
323
324impl std::fmt::Debug for ArtifactCacheSpec {
325 fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
326 formatter
327 .debug_struct("ArtifactCacheSpec")
328 .field("cache_root", &self.cache_root)
329 .field("namespace", &self.namespace)
330 .field("recipe_id", &self.recipe_id)
331 .field("coordination_scope", &self.coordination_scope)
332 .field("inputs", &self.inputs)
333 .field("tools", &self.tools)
334 .field("argument_count", &self.arguments.len())
335 .field(
336 "environment_names",
337 &self.environment.keys().collect::<Vec<_>>(),
338 )
339 .field(
340 "identity_labels",
341 &self
342 .identities
343 .iter()
344 .map(|identity| identity.label.as_str())
345 .collect::<Vec<_>>(),
346 )
347 .field("outputs", &self.outputs)
348 .field("prune_policy", &self.prune_policy)
349 .finish()
350 }
351}
352
353impl ArtifactCachePreparation {
354 #[must_use]
356 pub const fn reused_record(&self) -> Option<&ArtifactCacheRecord> {
357 match self {
358 Self::Reused(record) => Some(record),
359 Self::Build(_) => None,
360 }
361 }
362}
363
364impl ArtifactCacheOutcome {
365 #[must_use]
367 pub const fn record(&self) -> &ArtifactCacheRecord {
368 match self {
369 Self::Built(record) | Self::Reused(record) => record,
370 }
371 }
372
373 #[must_use]
375 pub const fn is_reused(&self) -> bool {
376 matches!(self, Self::Reused(_))
377 }
378}
379
380impl ArtifactCacheRecord {
381 #[must_use]
383 pub const fn key(&self) -> InputDigest {
384 self.key
385 }
386
387 #[must_use]
389 pub const fn input_digest(&self) -> InputDigest {
390 self.input_digest
391 }
392
393 #[must_use]
395 pub fn artifacts(&self) -> &[ArtifactCacheArtifact] {
396 &self.artifacts
397 }
398
399 #[must_use]
401 pub const fn timings(&self) -> ArtifactCacheTimings {
402 self.timings
403 }
404
405 #[must_use]
407 pub const fn maintenance(&self) -> Option<&ArtifactCacheMaintenance> {
408 self.maintenance.as_ref()
409 }
410}
411
412impl ArtifactCacheArtifact {
413 #[must_use]
415 pub fn name(&self) -> &str {
416 &self.name
417 }
418
419 #[must_use]
421 pub fn path(&self) -> &Path {
422 &self.path
423 }
424}
425
426impl ArtifactCacheTimings {
427 #[must_use]
429 pub const fn coordination_lock_wait(self) -> Duration {
430 self.coordination_lock_wait
431 }
432
433 #[must_use]
435 pub const fn content_lock_wait(self) -> Duration {
436 self.content_lock_wait
437 }
438
439 #[must_use]
441 pub const fn namespace_lock_wait(self) -> Duration {
442 self.namespace_lock_wait
443 }
444
445 #[must_use]
447 pub const fn input_capture(self) -> Duration {
448 self.input_capture
449 }
450
451 #[must_use]
453 pub const fn cache_lookup(self) -> Duration {
454 self.cache_lookup
455 }
456
457 #[must_use]
459 pub const fn caller_build(self) -> Option<Duration> {
460 self.caller_build
461 }
462
463 #[must_use]
465 pub const fn output_validation(self) -> Duration {
466 self.output_validation
467 }
468
469 #[must_use]
471 pub const fn publication(self) -> Duration {
472 self.publication
473 }
474
475 #[must_use]
477 pub const fn materialization(self) -> Duration {
478 self.materialization
479 }
480
481 #[must_use]
483 pub const fn maintenance(self) -> Option<Duration> {
484 self.maintenance
485 }
486
487 #[must_use]
489 pub const fn total(self) -> Duration {
490 self.total
491 }
492}
493
494impl ArtifactBuildTransaction {
495 #[must_use]
500 pub fn staging_directory(&self) -> &Path {
501 &self.staging_directory
502 }
503
504 pub fn output_path(&self, name: &str) -> Result<PathBuf, ArtifactCacheError> {
506 self.output_index(name)
507 .map(|index| staged_output_path(&self.staging_directory, index))
508 .ok_or_else(|| ArtifactCacheError::UnknownOutput {
509 name: name.to_owned(),
510 })
511 }
512
513 pub fn import_output(&self, name: &str, source: &Path) -> Result<(), ArtifactCacheError> {
515 let destination = self.output_path(name)?;
516 copy_file_atomic(source, &destination).map_err(|source_error| ArtifactCacheError::Io {
517 operation: "import artifact output into staging",
518 path: destination,
519 source: source_error,
520 })?;
521 Ok(())
522 }
523
524 pub fn commit(mut self) -> Result<ArtifactCacheOutcome, ArtifactCacheError> {
526 let result = self.commit_inner();
527 match result {
528 Ok(outcome) => Ok(outcome),
529 Err(transaction_error) if self.staging_armed => {
530 let path = self.staging_directory.clone();
531 match remove_path_if_present(&path) {
532 Ok(()) => {
533 self.staging_armed = false;
534 Err(transaction_error)
535 }
536 Err(source) => Err(ArtifactCacheError::FailedTransactionCleanup {
537 transaction_error: Box::new(transaction_error),
538 path,
539 source,
540 }),
541 }
542 }
543 Err(transaction_error) => Err(transaction_error),
544 }
545 }
546
547 pub fn abort(mut self) -> Result<(), ArtifactCacheError> {
549 remove_path_if_present(&self.staging_directory).map_err(|source| {
550 ArtifactCacheError::Io {
551 operation: "abort artifact cache transaction",
552 path: self.staging_directory.clone(),
553 source,
554 }
555 })?;
556 self.staging_armed = false;
557 Ok(())
558 }
559
560 fn output_index(&self, name: &str) -> Option<usize> {
561 self.spec
562 .outputs
563 .iter()
564 .position(|output| output.name == name)
565 }
566
567 fn commit_inner(&mut self) -> Result<ArtifactCacheOutcome, ArtifactCacheError> {
568 self.timings.caller_build = Some(self.caller_build_started.elapsed());
569
570 let validation_started = Instant::now();
571 let output_info = inspect_complete_output_set(&self.spec, &self.staging_directory)?;
572 self.timings.output_validation = validation_started.elapsed();
573
574 let capture_started = Instant::now();
575 let verified = resolve_key(&self.spec)?;
576 self.timings.input_capture = self
577 .timings
578 .input_capture
579 .saturating_add(capture_started.elapsed());
580 if verified.input_digest != self.resolved.input_digest {
581 return Err(ArtifactCacheError::InputsChangedDuringBuild {
582 before: self.resolved.input_digest,
583 after: verified.input_digest,
584 });
585 }
586
587 let publication_started = Instant::now();
588 let manifest = manifest_contents(self.resolved.key, &self.spec, &output_info);
589 write_atomic(
590 &self.staging_directory.join(MANIFEST_FILE),
591 manifest.as_bytes(),
592 )
593 .map_err(|source| ArtifactCacheError::Io {
594 operation: "write artifact cache manifest",
595 path: self.staging_directory.join(MANIFEST_FILE),
596 source,
597 })?;
598 let namespace_lock_path = namespace_lock_path(&self.spec);
599 let (_namespace_lock, namespace_wait) =
600 lock_cache_file(&namespace_lock_path).map_err(artifact_cache_fs_error)?;
601 self.timings.namespace_lock_wait = self
602 .timings
603 .namespace_lock_wait
604 .saturating_add(namespace_wait);
605 remove_path_if_present(&self.entry_directory).map_err(|source| ArtifactCacheError::Io {
606 operation: "remove conflicting artifact cache entry",
607 path: self.entry_directory.clone(),
608 source,
609 })?;
610 fs::rename(&self.staging_directory, &self.entry_directory).map_err(|source| {
611 ArtifactCacheError::Io {
612 operation: "publish artifact cache entry",
613 path: self.entry_directory.clone(),
614 source,
615 }
616 })?;
617 self.staging_armed = false;
618 self.timings.publication = publication_started.elapsed();
619
620 let materialization_started = Instant::now();
621 materialize_outputs(&self.spec, &self.entry_directory)?;
622 self.timings.materialization = materialization_started.elapsed();
623 record_cache_entry_use(&self.entry_directory).map_err(artifact_cache_fs_error)?;
624 let (maintenance, maintenance_timing) = perform_maintenance_locked(
625 &self.spec,
626 &self.namespace_directory,
627 &self.entry_directory,
628 );
629 self.timings.maintenance = maintenance_timing;
630 self.timings.total = self.total_started.elapsed();
631
632 Ok(ArtifactCacheOutcome::Built(cache_record(
633 &self.spec,
634 self.resolved,
635 self.timings,
636 maintenance,
637 )))
638 }
639}
640
641impl Drop for ArtifactBuildTransaction {
642 fn drop(&mut self) {
643 if self.staging_armed {
644 let _ = remove_path_if_present(&self.staging_directory);
645 }
646 }
647}
648
649pub fn prepare_artifact_cache(
651 spec: &ArtifactCacheSpec,
652) -> Result<ArtifactCachePreparation, ArtifactCacheError> {
653 let total_started = Instant::now();
654 validate_spec(spec)?;
655 let namespace_directory = initialize_cache(spec)?;
656 validate_filesystem_boundaries(spec)?;
657
658 let coordination_lock_path = coordination_lock_path(spec);
659 let (coordination_lock, coordination_wait) =
660 lock_cache_file(&coordination_lock_path).map_err(artifact_cache_fs_error)?;
661 let mut timings = ArtifactCacheTimings {
662 coordination_lock_wait: coordination_wait,
663 ..ArtifactCacheTimings::default()
664 };
665 let mut last_change = None;
666
667 for _ in 0..MAX_PREPARATION_RETRIES {
668 let capture_started = Instant::now();
669 let resolved = resolve_key(spec)?;
670 timings.input_capture = timings
671 .input_capture
672 .saturating_add(capture_started.elapsed());
673
674 let content_lock_path = content_lock_path(spec, resolved.key);
675 let (content_lock, content_wait) =
676 lock_cache_file(&content_lock_path).map_err(artifact_cache_fs_error)?;
677 timings.content_lock_wait = timings.content_lock_wait.saturating_add(content_wait);
678
679 let verification_started = Instant::now();
680 let verified = resolve_key(spec)?;
681 timings.input_capture = timings
682 .input_capture
683 .saturating_add(verification_started.elapsed());
684 if resolved.input_digest != verified.input_digest {
685 last_change = Some((resolved.input_digest, verified.input_digest));
686 drop(content_lock);
687 continue;
688 }
689
690 let entry_directory = entry_directory(&namespace_directory, resolved.key);
691 let namespace_lock_path = namespace_lock_path(spec);
692 let (namespace_lock, namespace_wait) =
693 lock_cache_file(&namespace_lock_path).map_err(artifact_cache_fs_error)?;
694 timings.namespace_lock_wait = timings.namespace_lock_wait.saturating_add(namespace_wait);
695 let lookup_started = Instant::now();
696 let reusable = cache_entry_is_valid(spec, resolved.key, &entry_directory)?;
697 timings.cache_lookup = timings
698 .cache_lookup
699 .saturating_add(lookup_started.elapsed());
700
701 if reusable {
702 let materialization_started = Instant::now();
703 materialize_outputs(spec, &entry_directory)?;
704 timings.materialization = timings
705 .materialization
706 .saturating_add(materialization_started.elapsed());
707 let after_started = Instant::now();
708 let after = resolve_key(spec)?;
709 timings.input_capture = timings
710 .input_capture
711 .saturating_add(after_started.elapsed());
712 if after.input_digest != resolved.input_digest {
713 last_change = Some((resolved.input_digest, after.input_digest));
714 drop(namespace_lock);
715 drop(content_lock);
716 continue;
717 }
718 record_cache_entry_use(&entry_directory).map_err(artifact_cache_fs_error)?;
719 let (maintenance, maintenance_timing) =
720 perform_maintenance_locked(spec, &namespace_directory, &entry_directory);
721 timings.maintenance = maintenance_timing;
722 timings.total = total_started.elapsed();
723 return Ok(ArtifactCachePreparation::Reused(cache_record(
724 spec,
725 resolved,
726 timings,
727 maintenance,
728 )));
729 }
730
731 remove_path_if_present(&entry_directory).map_err(|source| ArtifactCacheError::Io {
732 operation: "remove invalid artifact cache entry",
733 path: entry_directory.clone(),
734 source,
735 })?;
736 let staging_directory = create_staging_directory(&namespace_directory, resolved.key)?;
737 drop(namespace_lock);
738 return Ok(ArtifactCachePreparation::Build(ArtifactBuildTransaction {
739 spec: Box::new(spec.clone()),
740 resolved,
741 staging_directory,
742 entry_directory,
743 namespace_directory,
744 _coordination_lock: coordination_lock,
745 _content_lock: content_lock,
746 timings,
747 total_started,
748 caller_build_started: Instant::now(),
749 staging_armed: true,
750 }));
751 }
752
753 let (before, after) = last_change.expect("preparation retries require a recorded input change");
754 Err(ArtifactCacheError::InputsChangedDuringPreparation { before, after })
755}
756
757pub fn prune_artifact_cache(
765 cache_root: &Path,
766 namespace: &str,
767 policy: ArtifactCachePrunePolicy,
768) -> Result<ArtifactCachePruneReport, ArtifactCacheError> {
769 validate_identifier("namespace", namespace)?;
770 if cache_root.as_os_str().is_empty() {
771 return invalid_spec("cache root must not be empty");
772 }
773 ensure_cache_directory_tag(cache_root).map_err(artifact_cache_fs_error)?;
774 let namespace_directory = namespace_directory_for(cache_root, namespace);
775 let lock_path = namespace_lock_path_for(cache_root, namespace);
776 let (_lock, _) = lock_cache_file(&lock_path).map_err(artifact_cache_fs_error)?;
777 prune_artifact_namespace_locked(cache_root, &namespace_directory, policy, None)
778}
779
780fn initialize_cache(spec: &ArtifactCacheSpec) -> Result<PathBuf, ArtifactCacheError> {
781 ensure_cache_directory_tag(&spec.cache_root).map_err(artifact_cache_fs_error)?;
782 let namespace = namespace_directory(spec);
783 fs::create_dir_all(entries_directory(&namespace)).map_err(|source| ArtifactCacheError::Io {
784 operation: "create artifact cache namespace",
785 path: namespace.clone(),
786 source,
787 })?;
788 Ok(namespace)
789}
790
791fn validate_spec(spec: &ArtifactCacheSpec) -> Result<(), ArtifactCacheError> {
792 validate_identifier("namespace", &spec.namespace)?;
793 validate_identifier("recipe identity", &spec.recipe_id)?;
794 validate_identifier("coordination scope", &spec.coordination_scope)?;
795 if spec.cache_root.as_os_str().is_empty() {
796 return invalid_spec("cache root must not be empty");
797 }
798 if spec.outputs.is_empty() {
799 return invalid_spec("at least one artifact output is required");
800 }
801
802 let mut labels = BTreeSet::new();
803 for input in &spec.inputs {
804 validate_path_label("input", &input.label)?;
805 if !labels.insert(format!("input/{}", input.label)) {
806 return invalid_spec(&format!("duplicate input label `{}`", input.label));
807 }
808 }
809 for tool in &spec.tools {
810 validate_path_label("tool", &tool.label)?;
811 if !labels.insert(format!("tool/{}", tool.label)) {
812 return invalid_spec(&format!("duplicate tool label `{}`", tool.label));
813 }
814 }
815 let mut identity_labels = BTreeSet::new();
816 for identity in &spec.identities {
817 validate_label("identity", &identity.label)?;
818 if !identity_labels.insert(&identity.label) {
819 return invalid_spec(&format!("duplicate identity label `{}`", identity.label));
820 }
821 }
822 if spec
823 .environment
824 .keys()
825 .any(|name| name.as_os_str().is_empty())
826 {
827 return invalid_spec("environment names must not be empty");
828 }
829 let mut output_names = BTreeSet::new();
830 let mut destinations = BTreeSet::new();
831 for output in &spec.outputs {
832 validate_output_name(&output.name)?;
833 if !output_names.insert(&output.name) {
834 return invalid_spec(&format!("duplicate output name `{}`", output.name));
835 }
836 if output.destination.as_os_str().is_empty() {
837 return invalid_spec(&format!(
838 "output `{}` destination must not be empty",
839 output.name
840 ));
841 }
842 if !destinations.insert(&output.destination) {
843 return invalid_spec(&format!(
844 "output `{}` shares a destination with another output",
845 output.name
846 ));
847 }
848 }
849 Ok(())
850}
851
852fn validate_filesystem_boundaries(spec: &ArtifactCacheSpec) -> Result<(), ArtifactCacheError> {
853 let cache_root = canonicalize_path(&spec.cache_root, "canonicalize artifact cache root")?;
854 let mut declared_paths = Vec::with_capacity(spec.inputs.len() + spec.tools.len());
855 for (kind, labeled_paths) in [("input", &spec.inputs), ("tool", &spec.tools)] {
856 for labeled in labeled_paths {
857 let canonical =
858 canonicalize_path(&labeled.path, "canonicalize declared artifact cache path")?;
859 if canonical.starts_with(&cache_root) {
860 return invalid_spec(&format!(
861 "{kind} `{}` must not be located inside the artifact cache root",
862 labeled.label
863 ));
864 }
865 let is_directory = fs::metadata(&canonical)
866 .map_err(|source| ArtifactCacheError::Io {
867 operation: "inspect declared artifact cache path",
868 path: canonical.clone(),
869 source,
870 })?
871 .is_dir();
872 declared_paths.push((kind, labeled.label.as_str(), canonical, is_directory));
873 }
874 }
875
876 let mut destinations = BTreeSet::new();
877 for output in &spec.outputs {
878 let destination = canonicalize_allow_missing(&output.destination).map_err(|source| {
879 ArtifactCacheError::Io {
880 operation: "resolve artifact output destination",
881 path: output.destination.clone(),
882 source,
883 }
884 })?;
885 if destination.starts_with(&cache_root) {
886 return invalid_spec(&format!(
887 "output `{}` destination must be outside the artifact cache root",
888 output.name
889 ));
890 }
891 if !destinations.insert(destination.clone()) {
892 return invalid_spec(&format!(
893 "output `{}` resolves to the same destination as another output",
894 output.name
895 ));
896 }
897 for (kind, label, declared, is_directory) in &declared_paths {
898 if destination == *declared || (*is_directory && destination.starts_with(declared)) {
899 return invalid_spec(&format!(
900 "output `{}` destination overlaps declared {kind} `{label}`",
901 output.name
902 ));
903 }
904 }
905 match fs::metadata(&destination) {
906 Ok(metadata) if metadata.is_dir() => {
907 return invalid_spec(&format!(
908 "output `{}` destination must not be an existing directory",
909 output.name
910 ));
911 }
912 Ok(_) => {}
913 Err(error) if error.kind() == io::ErrorKind::NotFound => {}
914 Err(source) => {
915 return Err(ArtifactCacheError::Io {
916 operation: "inspect artifact output destination",
917 path: destination,
918 source,
919 });
920 }
921 }
922 }
923 Ok(())
924}
925
926fn canonicalize_path(path: &Path, operation: &'static str) -> Result<PathBuf, ArtifactCacheError> {
927 path.canonicalize()
928 .map_err(|source| ArtifactCacheError::Io {
929 operation,
930 path: path.to_owned(),
931 source,
932 })
933}
934
935fn canonicalize_allow_missing(path: &Path) -> io::Result<PathBuf> {
936 let absolute = if path.is_absolute() {
937 path.to_owned()
938 } else {
939 std::env::current_dir()?.join(path)
940 };
941 let mut resolved = PathBuf::new();
942 let mut missing_depth = 0_usize;
943 for component in absolute.components() {
944 match component {
945 Component::Prefix(_) | Component::RootDir | Component::Normal(_) => {
946 let candidate = resolved.join(component.as_os_str());
947 if missing_depth == 0 && matches!(component, Component::Normal(_)) {
948 match candidate.canonicalize() {
949 Ok(canonical) => resolved = canonical,
950 Err(error) if error.kind() == io::ErrorKind::NotFound => {
951 resolved = candidate;
952 missing_depth = 1;
953 }
954 Err(error) => return Err(error),
955 }
956 } else {
957 resolved = candidate;
958 if matches!(component, Component::Normal(_)) && missing_depth > 0 {
959 missing_depth += 1;
960 }
961 }
962 }
963 Component::CurDir => {}
964 Component::ParentDir => {
965 resolved.pop();
966 missing_depth = missing_depth.saturating_sub(1);
967 }
968 }
969 }
970 Ok(resolved)
971}
972
973fn validate_identifier(kind: &str, value: &str) -> Result<(), ArtifactCacheError> {
974 if value.is_empty() {
975 return invalid_spec(&format!("{kind} must not be empty"));
976 }
977 if value.len() > 256 {
978 return invalid_spec(&format!("{kind} must not exceed 256 bytes"));
979 }
980 Ok(())
981}
982
983fn validate_label(kind: &str, value: &str) -> Result<(), ArtifactCacheError> {
984 if value.is_empty() {
985 return invalid_spec(&format!("{kind} label must not be empty"));
986 }
987 if value.len() > 256 {
988 return invalid_spec(&format!("{kind} label must not exceed 256 bytes"));
989 }
990 Ok(())
991}
992
993fn validate_path_label(kind: &str, value: &str) -> Result<(), ArtifactCacheError> {
994 validate_label(kind, value)?;
995 if value
996 .bytes()
997 .any(|byte| !(byte.is_ascii_alphanumeric() || matches!(byte, b'-' | b'_' | b'.' | b'/')))
998 || value
999 .split('/')
1000 .any(|component| component.is_empty() || matches!(component, "." | ".."))
1001 {
1002 return invalid_spec(&format!(
1003 "{kind} label `{value}` must be a portable relative logical path"
1004 ));
1005 }
1006 Ok(())
1007}
1008
1009fn validate_output_name(name: &str) -> Result<(), ArtifactCacheError> {
1010 if name.is_empty() || name.len() > 128 {
1011 return invalid_spec("output names must contain 1 to 128 bytes");
1012 }
1013 if !name
1014 .bytes()
1015 .all(|byte| byte.is_ascii_alphanumeric() || matches!(byte, b'-' | b'_' | b'.'))
1016 {
1017 return invalid_spec(&format!(
1018 "output name `{name}` must use only ASCII letters, digits, dot, dash, or underscore"
1019 ));
1020 }
1021 if matches!(name, "." | "..") {
1022 return invalid_spec("output names must not be dot path components");
1023 }
1024 Ok(())
1025}
1026
1027fn invalid_spec<T>(message: &str) -> Result<T, ArtifactCacheError> {
1028 Err(ArtifactCacheError::InvalidSpec {
1029 message: message.to_owned(),
1030 })
1031}
1032
1033fn resolve_key(spec: &ArtifactCacheSpec) -> Result<ResolvedKey, ArtifactCacheError> {
1034 let paths = spec
1035 .inputs
1036 .iter()
1037 .map(|input| {
1038 (
1039 PathBuf::from("input").join(&input.label),
1040 input.path.clone(),
1041 )
1042 })
1043 .chain(
1044 spec.tools
1045 .iter()
1046 .map(|tool| (PathBuf::from("tool").join(&tool.label), tool.path.clone())),
1047 )
1048 .collect::<Vec<_>>();
1049 let input_digest = digest_labeled_paths(
1050 "artifact-set-inputs-v1",
1051 &paths,
1052 std::slice::from_ref(&spec.cache_root),
1053 )
1054 .map_err(|source| ArtifactCacheError::Io {
1055 operation: "hash artifact cache inputs",
1056 path: spec.cache_root.clone(),
1057 source,
1058 })?;
1059
1060 let mut hasher = InputHasher::new(ARTIFACT_CACHE_FORMAT);
1061 hasher.field("namespace", spec.namespace.as_bytes());
1062 hasher.field("recipe-id", spec.recipe_id.as_bytes());
1063 hasher.field("input-digest", input_digest.as_bytes());
1064 for argument in &spec.arguments {
1065 hasher.field("argument", &os_bytes(argument));
1066 }
1067 for (name, value) in &spec.environment {
1068 hasher.field("environment-name", &os_bytes(name));
1069 match value {
1070 Some(value) => hasher.field("environment-value", &os_bytes(value)),
1071 None => hasher.field("environment-unset", b""),
1072 }
1073 }
1074 let mut identities = spec.identities.iter().collect::<Vec<_>>();
1075 identities.sort_by(|left, right| left.label.cmp(&right.label));
1076 for identity in identities {
1077 hasher.field("identity-label", identity.label.as_bytes());
1078 hasher.field("identity-value", &identity.value);
1079 }
1080 for output in &spec.outputs {
1081 hasher.field("output-name", output.name.as_bytes());
1082 hasher.field(
1083 "output-validation",
1084 output.validation.cache_token().as_bytes(),
1085 );
1086 }
1087 Ok(ResolvedKey {
1088 key: hasher.finish(),
1089 input_digest,
1090 })
1091}
1092
1093fn cache_entry_is_valid(
1094 spec: &ArtifactCacheSpec,
1095 key: InputDigest,
1096 entry: &Path,
1097) -> Result<bool, ArtifactCacheError> {
1098 let entry_metadata = match fs::symlink_metadata(entry) {
1099 Ok(metadata) => metadata,
1100 Err(error) if error.kind() == io::ErrorKind::NotFound => return Ok(false),
1101 Err(source) => {
1102 return Err(ArtifactCacheError::Io {
1103 operation: "inspect artifact cache entry",
1104 path: entry.to_owned(),
1105 source,
1106 });
1107 }
1108 };
1109 if !entry_metadata.file_type().is_dir() || !cache_entry_root_is_valid(entry)? {
1110 return Ok(false);
1111 }
1112 let manifest_path = entry.join(MANIFEST_FILE);
1113 let manifest_metadata = match fs::symlink_metadata(&manifest_path) {
1114 Ok(metadata) => metadata,
1115 Err(error) if error.kind() == io::ErrorKind::NotFound => return Ok(false),
1116 Err(source) => {
1117 return Err(ArtifactCacheError::Io {
1118 operation: "inspect artifact cache manifest",
1119 path: manifest_path,
1120 source,
1121 });
1122 }
1123 };
1124 if !manifest_metadata.file_type().is_file() {
1125 return Ok(false);
1126 }
1127 let manifest = fs::read(&manifest_path).map_err(|source| ArtifactCacheError::Io {
1128 operation: "read artifact cache manifest",
1129 path: manifest_path,
1130 source,
1131 })?;
1132 let Some(output_info) = inspect_cached_output_set(spec, entry)? else {
1133 return Ok(false);
1134 };
1135 Ok(manifest == manifest_contents(key, spec, &output_info).as_bytes())
1136}
1137
1138fn inspect_complete_output_set(
1139 spec: &ArtifactCacheSpec,
1140 root: &Path,
1141) -> Result<Vec<ArtifactInfo>, ArtifactCacheError> {
1142 let mut info = Vec::new();
1143 let mut invalid = Vec::new();
1144 let output_directory = root.join("outputs");
1145 if !is_plain_directory(
1146 &output_directory,
1147 "inspect artifact staging output directory",
1148 )? {
1149 invalid.push(("<outputs>".to_owned(), output_directory));
1150 return Err(ArtifactCacheError::InvalidOutputs { outputs: invalid });
1151 }
1152 let outputs = &spec.outputs;
1153 for (index, output) in outputs.iter().enumerate() {
1154 let path = staged_output_path(root, index);
1155 match inspect_artifact(&path, output.validation) {
1156 Ok(Some(artifact)) => info.push(artifact),
1157 Ok(None) => invalid.push((output.name.clone(), path)),
1158 Err(source) => {
1159 return Err(ArtifactCacheError::Io {
1160 operation: "inspect staged artifact output",
1161 path,
1162 source,
1163 });
1164 }
1165 }
1166 }
1167 invalid.extend(
1168 undeclared_output_paths(root, outputs.len())?
1169 .into_iter()
1170 .map(|path| ("<undeclared>".to_owned(), path)),
1171 );
1172 invalid.extend(
1173 undeclared_child_paths(
1174 root,
1175 &BTreeSet::from([OsString::from("outputs")]),
1176 "read artifact staging directory",
1177 )?
1178 .into_iter()
1179 .map(|path| ("<undeclared>".to_owned(), path)),
1180 );
1181 if invalid.is_empty() {
1182 Ok(info)
1183 } else {
1184 Err(ArtifactCacheError::InvalidOutputs { outputs: invalid })
1185 }
1186}
1187
1188fn inspect_cached_output_set(
1189 spec: &ArtifactCacheSpec,
1190 root: &Path,
1191) -> Result<Option<Vec<ArtifactInfo>>, ArtifactCacheError> {
1192 let mut info = Vec::new();
1193 let outputs = &spec.outputs;
1194 for (index, output) in outputs.iter().enumerate() {
1195 let path = staged_output_path(root, index);
1196 match inspect_artifact(&path, output.validation) {
1197 Ok(Some(artifact)) => info.push(artifact),
1198 Ok(None) => return Ok(None),
1199 Err(source) => {
1200 return Err(ArtifactCacheError::Io {
1201 operation: "inspect cached artifact output",
1202 path,
1203 source,
1204 });
1205 }
1206 }
1207 }
1208 if !undeclared_output_paths(root, outputs.len())?.is_empty() {
1209 return Ok(None);
1210 }
1211 Ok(Some(info))
1212}
1213
1214fn undeclared_output_paths(
1215 root: &Path,
1216 output_count: usize,
1217) -> Result<Vec<PathBuf>, ArtifactCacheError> {
1218 let output_directory = root.join("outputs");
1219 let expected = (0..output_count)
1220 .map(format_output_index)
1221 .map(OsString::from)
1222 .collect::<BTreeSet<_>>();
1223 undeclared_child_paths(
1224 &output_directory,
1225 &expected,
1226 "read artifact output directory",
1227 )
1228}
1229
1230fn undeclared_child_paths(
1231 directory: &Path,
1232 expected: &BTreeSet<OsString>,
1233 operation: &'static str,
1234) -> Result<Vec<PathBuf>, ArtifactCacheError> {
1235 let entries = fs::read_dir(directory).map_err(|source| ArtifactCacheError::Io {
1236 operation,
1237 path: directory.to_owned(),
1238 source,
1239 })?;
1240 let mut undeclared = Vec::new();
1241 for entry in entries {
1242 let entry = entry.map_err(|source| ArtifactCacheError::Io {
1243 operation,
1244 path: directory.to_owned(),
1245 source,
1246 })?;
1247 if !expected.contains(&entry.file_name()) {
1248 undeclared.push(entry.path());
1249 }
1250 }
1251 Ok(undeclared)
1252}
1253
1254fn cache_entry_root_is_valid(root: &Path) -> Result<bool, ArtifactCacheError> {
1255 let expected = BTreeSet::from([
1256 OsString::from("outputs"),
1257 OsString::from(MANIFEST_FILE),
1258 OsString::from(LAST_USED_FILE),
1259 ]);
1260 if !undeclared_child_paths(root, &expected, "read artifact cache entry")?.is_empty() {
1261 return Ok(false);
1262 }
1263 if !is_plain_directory(
1264 &root.join("outputs"),
1265 "inspect artifact cache output directory",
1266 )? {
1267 return Ok(false);
1268 }
1269 let last_used = root.join(LAST_USED_FILE);
1270 match fs::symlink_metadata(&last_used) {
1271 Ok(metadata) => Ok(metadata.file_type().is_file()),
1272 Err(error) if error.kind() == io::ErrorKind::NotFound => Ok(true),
1273 Err(source) => Err(ArtifactCacheError::Io {
1274 operation: "inspect artifact cache use marker",
1275 path: last_used,
1276 source,
1277 }),
1278 }
1279}
1280
1281fn is_plain_directory(path: &Path, operation: &'static str) -> Result<bool, ArtifactCacheError> {
1282 match fs::symlink_metadata(path) {
1283 Ok(metadata) => Ok(metadata.file_type().is_dir()),
1284 Err(error) if error.kind() == io::ErrorKind::NotFound => Ok(false),
1285 Err(source) => Err(ArtifactCacheError::Io {
1286 operation,
1287 path: path.to_owned(),
1288 source,
1289 }),
1290 }
1291}
1292
1293fn inspect_artifact(
1294 path: &Path,
1295 validation: ArtifactOutputValidation,
1296) -> io::Result<Option<ArtifactInfo>> {
1297 let metadata = match fs::symlink_metadata(path) {
1298 Ok(metadata) => metadata,
1299 Err(error) if error.kind() == io::ErrorKind::NotFound => return Ok(None),
1300 Err(error) => return Err(error),
1301 };
1302 if !metadata.file_type().is_file() {
1303 return Ok(None);
1304 }
1305 if validation == ArtifactOutputValidation::NonEmptyFile && metadata.len() == 0 {
1306 return Ok(None);
1307 }
1308 let (bytes, digest) = digest_file("artifact-set-output-v1", path)?;
1309 Ok(Some(ArtifactInfo { bytes, digest }))
1310}
1311
1312fn manifest_contents(
1313 key: InputDigest,
1314 spec: &ArtifactCacheSpec,
1315 output_info: &[ArtifactInfo],
1316) -> String {
1317 let mut manifest = format!("{ARTIFACT_CACHE_FORMAT}\nkey:{key}\n");
1318 for ((index, output), info) in spec.outputs.iter().enumerate().zip(output_info) {
1319 use std::fmt::Write as _;
1320 writeln!(
1321 manifest,
1322 "output:{index}:{}:{}:{}:{}",
1323 output.name,
1324 output.validation.cache_token(),
1325 info.bytes,
1326 info.digest,
1327 )
1328 .expect("writing an artifact manifest to a String cannot fail");
1329 }
1330 manifest
1331}
1332
1333fn materialize_outputs(spec: &ArtifactCacheSpec, entry: &Path) -> Result<(), ArtifactCacheError> {
1334 for (index, output) in spec.outputs.iter().enumerate() {
1335 let cached = staged_output_path(entry, index);
1336 copy_file_atomic(&cached, &output.destination).map_err(|source| {
1337 ArtifactCacheError::Io {
1338 operation: "materialize artifact output",
1339 path: output.destination.clone(),
1340 source,
1341 }
1342 })?;
1343 }
1344 Ok(())
1345}
1346
1347fn perform_maintenance_locked(
1348 spec: &ArtifactCacheSpec,
1349 namespace: &Path,
1350 protected_entry: &Path,
1351) -> (Option<ArtifactCacheMaintenance>, Option<Duration>) {
1352 spec.prune_policy.map_or((None, None), |policy| {
1353 let started = Instant::now();
1354 let result = prune_artifact_namespace_locked(
1355 &spec.cache_root,
1356 namespace,
1357 policy,
1358 Some(protected_entry),
1359 );
1360 let maintenance = match result {
1361 Ok(report) => ArtifactCacheMaintenance::Pruned(report),
1362 Err(error) => ArtifactCacheMaintenance::PruneFailed {
1363 message: error.to_string(),
1364 },
1365 };
1366 (Some(maintenance), Some(started.elapsed()))
1367 })
1368}
1369
1370fn prune_artifact_namespace_locked(
1371 cache_root: &Path,
1372 namespace: &Path,
1373 policy: ArtifactCachePrunePolicy,
1374 protected_entry: Option<&Path>,
1375) -> Result<ArtifactCachePruneReport, ArtifactCacheError> {
1376 let mut report = prune_direct_child_directories(
1377 &entries_directory(namespace),
1378 policy,
1379 protected_entry,
1380 is_sha256_directory,
1381 )
1382 .map_err(artifact_cache_fs_error)?;
1383 remove_abandoned_staging(cache_root, namespace, &mut report)?;
1384 Ok(report)
1385}
1386
1387fn remove_abandoned_staging(
1388 cache_root: &Path,
1389 namespace: &Path,
1390 report: &mut ArtifactCachePruneReport,
1391) -> Result<(), ArtifactCacheError> {
1392 let staging_root = namespace.join("staging");
1393 let entries = match fs::read_dir(&staging_root) {
1394 Ok(entries) => entries,
1395 Err(error) if error.kind() == io::ErrorKind::NotFound => return Ok(()),
1396 Err(source) => {
1397 return Err(ArtifactCacheError::Io {
1398 operation: "read artifact staging root during pruning",
1399 path: staging_root,
1400 source,
1401 });
1402 }
1403 };
1404 for entry in entries {
1405 let entry = entry.map_err(|source| ArtifactCacheError::Io {
1406 operation: "read artifact staging entry during pruning",
1407 path: staging_root.clone(),
1408 source,
1409 })?;
1410 let file_type = entry.file_type().map_err(|source| ArtifactCacheError::Io {
1411 operation: "inspect artifact staging entry during pruning",
1412 path: entry.path(),
1413 source,
1414 })?;
1415 if !file_type.is_dir() {
1416 continue;
1417 }
1418 let file_name = entry.file_name();
1419 let Some(key) = staging_content_key(&file_name) else {
1420 continue;
1421 };
1422 let lock_path = content_lock_path_for_key(cache_root, key);
1423 let Some(_content_lock) =
1424 try_lock_cache_file(&lock_path).map_err(artifact_cache_fs_error)?
1425 else {
1426 continue;
1427 };
1428 let path = entry.path();
1429 let bytes = directory_logical_size(&path).map_err(|source| ArtifactCacheError::Io {
1430 operation: "measure abandoned artifact staging directory",
1431 path: path.clone(),
1432 source,
1433 })?;
1434 remove_path_if_present(&path).map_err(|source| ArtifactCacheError::Io {
1435 operation: "remove abandoned artifact staging directory during pruning",
1436 path,
1437 source,
1438 })?;
1439 report.record_uncommitted_removal(bytes);
1440 }
1441 Ok(())
1442}
1443
1444fn staging_content_key(name: &OsStr) -> Option<&str> {
1445 let name = name.to_str()?;
1446 let (key, suffix) = name.split_once('-')?;
1447 (!suffix.is_empty() && key.len() == 64 && key.as_bytes().iter().all(u8::is_ascii_hexdigit))
1448 .then_some(key)
1449}
1450
1451fn cache_record(
1452 spec: &ArtifactCacheSpec,
1453 resolved: ResolvedKey,
1454 timings: ArtifactCacheTimings,
1455 maintenance: Option<ArtifactCacheMaintenance>,
1456) -> ArtifactCacheRecord {
1457 ArtifactCacheRecord {
1458 key: resolved.key,
1459 input_digest: resolved.input_digest,
1460 artifacts: spec
1461 .outputs
1462 .iter()
1463 .map(|output| ArtifactCacheArtifact {
1464 name: output.name.clone(),
1465 path: output.destination.clone(),
1466 })
1467 .collect(),
1468 timings,
1469 maintenance,
1470 }
1471}
1472
1473fn create_staging_directory(
1474 namespace: &Path,
1475 key: InputDigest,
1476) -> Result<PathBuf, ArtifactCacheError> {
1477 let staging_root = namespace.join("staging");
1478 fs::create_dir_all(&staging_root).map_err(|source| ArtifactCacheError::Io {
1479 operation: "create artifact staging root",
1480 path: staging_root.clone(),
1481 source,
1482 })?;
1483 remove_same_key_staging(&staging_root, key)?;
1484 let sequence = STAGING_SEQUENCE.fetch_add(1, Ordering::Relaxed);
1485 let staging = staging_root.join(format!("{key}-{}-{sequence}", std::process::id()));
1486 fs::create_dir_all(staging.join("outputs")).map_err(|source| ArtifactCacheError::Io {
1487 operation: "create artifact transaction staging directory",
1488 path: staging.clone(),
1489 source,
1490 })?;
1491 Ok(staging)
1492}
1493
1494fn remove_same_key_staging(root: &Path, key: InputDigest) -> Result<(), ArtifactCacheError> {
1495 let prefix = format!("{key}-");
1496 let entries = fs::read_dir(root).map_err(|source| ArtifactCacheError::Io {
1497 operation: "read artifact staging root",
1498 path: root.to_owned(),
1499 source,
1500 })?;
1501 for entry in entries {
1502 let entry = entry.map_err(|source| ArtifactCacheError::Io {
1503 operation: "read artifact staging entry",
1504 path: root.to_owned(),
1505 source,
1506 })?;
1507 if entry.file_name().to_string_lossy().starts_with(&prefix) {
1508 remove_path_if_present(&entry.path()).map_err(|source| ArtifactCacheError::Io {
1509 operation: "remove abandoned artifact staging directory",
1510 path: entry.path(),
1511 source,
1512 })?;
1513 }
1514 }
1515 Ok(())
1516}
1517
1518fn staged_output_path(root: &Path, index: usize) -> PathBuf {
1519 root.join("outputs").join(format_output_index(index))
1520}
1521
1522fn format_output_index(index: usize) -> String {
1523 format!("{index:04}.artifact")
1524}
1525
1526fn namespace_directory(spec: &ArtifactCacheSpec) -> PathBuf {
1527 namespace_directory_for(&spec.cache_root, &spec.namespace)
1528}
1529
1530fn namespace_directory_for(cache_root: &Path, namespace: &str) -> PathBuf {
1531 cache_root
1532 .join(".ic-testkit/artifact-sets/namespaces")
1533 .join(identifier_digest("artifact-cache-namespace-v1", namespace))
1534}
1535
1536fn entries_directory(namespace: &Path) -> PathBuf {
1537 namespace.join("entries")
1538}
1539
1540fn entry_directory(namespace: &Path, key: InputDigest) -> PathBuf {
1541 entries_directory(namespace).join(key.to_hex())
1542}
1543
1544fn coordination_lock_path(spec: &ArtifactCacheSpec) -> PathBuf {
1545 spec.cache_root
1546 .join(".ic-testkit/artifact-sets/locks/coordination")
1547 .join(format!(
1548 "{}.lock",
1549 identifier_digest("artifact-cache-coordination-v1", &spec.coordination_scope,)
1550 ))
1551}
1552
1553fn content_lock_path(spec: &ArtifactCacheSpec, key: InputDigest) -> PathBuf {
1554 content_lock_path_for_key(&spec.cache_root, &key.to_hex())
1555}
1556
1557fn content_lock_path_for_key(cache_root: &Path, key: &str) -> PathBuf {
1558 cache_root
1559 .join(".ic-testkit/artifact-sets/locks/content")
1560 .join(format!("{key}.lock"))
1561}
1562
1563fn namespace_lock_path(spec: &ArtifactCacheSpec) -> PathBuf {
1564 namespace_lock_path_for(&spec.cache_root, &spec.namespace)
1565}
1566
1567fn namespace_lock_path_for(cache_root: &Path, namespace: &str) -> PathBuf {
1568 cache_root
1569 .join(".ic-testkit/artifact-sets/locks/namespaces")
1570 .join(format!(
1571 "{}.lock",
1572 identifier_digest("artifact-cache-namespace-v1", namespace)
1573 ))
1574}
1575
1576fn identifier_digest(domain: &str, identifier: &str) -> String {
1577 digest_bytes(domain, identifier.as_bytes()).to_hex()
1578}
1579
1580fn artifact_cache_fs_error(error: CacheFsError) -> ArtifactCacheError {
1581 ArtifactCacheError::Io {
1582 operation: error.operation,
1583 path: error.path,
1584 source: error.source,
1585 }
1586}
1587
1588impl ArtifactOutputValidation {
1589 const fn cache_token(self) -> &'static str {
1590 match self {
1591 Self::RegularFile => "regular-file",
1592 Self::NonEmptyFile => "nonempty-file",
1593 }
1594 }
1595}
1596
1597impl std::fmt::Display for ArtifactCacheError {
1598 fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
1599 match self {
1600 Self::InvalidSpec { message } => {
1601 write!(formatter, "invalid artifact cache spec: {message}")
1602 }
1603 Self::Io {
1604 operation,
1605 path,
1606 source,
1607 } => write!(
1608 formatter,
1609 "failed to {operation} at {}: {source}",
1610 path.display()
1611 ),
1612 Self::InputsChangedDuringPreparation { before, after } => write!(
1613 formatter,
1614 "artifact inputs repeatedly changed during cache preparation: {before} -> {after}",
1615 ),
1616 Self::InputsChangedDuringBuild { before, after } => write!(
1617 formatter,
1618 "artifact inputs changed while the caller was building: {before} -> {after}",
1619 ),
1620 Self::InvalidOutputs { outputs } => write!(
1621 formatter,
1622 "artifact transaction has missing or invalid outputs: {}",
1623 outputs
1624 .iter()
1625 .map(|(name, path)| format!("{name} ({})", path.display()))
1626 .collect::<Vec<_>>()
1627 .join(", "),
1628 ),
1629 Self::UnknownOutput { name } => {
1630 write!(
1631 formatter,
1632 "artifact transaction has no output named `{name}`"
1633 )
1634 }
1635 Self::FailedTransactionCleanup {
1636 transaction_error,
1637 path,
1638 source,
1639 } => write!(
1640 formatter,
1641 "artifact transaction failed ({transaction_error}) and staging cleanup at {} also failed: {source}",
1642 path.display(),
1643 ),
1644 }
1645 }
1646}
1647
1648impl std::error::Error for ArtifactCacheError {
1649 fn source(&self) -> Option<&(dyn std::error::Error + 'static)> {
1650 match self {
1651 Self::Io { source, .. } | Self::FailedTransactionCleanup { source, .. } => Some(source),
1652 _ => None,
1653 }
1654 }
1655}
1656
1657#[cfg(test)]
1658mod tests {
1659 use super::{
1660 ArtifactCacheError, ArtifactCacheOutcome, ArtifactCachePreparation, ArtifactCacheSpec,
1661 ArtifactOutputValidation, entry_directory, namespace_directory, prepare_artifact_cache,
1662 prune_artifact_cache, resolve_key,
1663 };
1664 use crate::artifacts::{
1665 ArtifactCacheMaintenance, ArtifactCachePrunePolicy, test_support::unique_temp_directory,
1666 };
1667 use std::{
1668 fs,
1669 panic::{AssertUnwindSafe, catch_unwind},
1670 path::Path,
1671 sync::{
1672 Arc, Barrier,
1673 atomic::{AtomicUsize, Ordering},
1674 },
1675 thread,
1676 };
1677
1678 #[test]
1679 fn one_output_is_built_materialized_repaired_and_reused() {
1680 let root = unique_temp_directory("one-output");
1681 let input = root.join("input.wasm");
1682 let destination = root.join("public/optimized.wasm");
1683 fs::write(&input, b"raw-wasm").expect("write input");
1684 let spec = ArtifactCacheSpec::new(&root.join("cache"), "optimizer", "pipeline/v1")
1685 .with_input("raw-wasm", &input)
1686 .with_arguments(&["-O3", "--strip-debug"])
1687 .with_output("optimized.wasm", &destination);
1688 assert!(!destination.starts_with(spec.cache_root()));
1689
1690 let transaction = expect_build(prepare_artifact_cache(&spec).expect("prepare miss"));
1691 fs::write(
1692 transaction
1693 .output_path("optimized.wasm")
1694 .expect("declared output path"),
1695 b"optimized-wasm",
1696 )
1697 .expect("write staged output");
1698 let built = transaction.commit().expect("commit artifact transaction");
1699
1700 assert!(matches!(built, ArtifactCacheOutcome::Built(_)));
1701 assert_eq!(fs::read(&destination).unwrap(), b"optimized-wasm");
1702 fs::write(&destination, b"tampered-public-output").expect("tamper public output");
1703
1704 let reused = prepare_artifact_cache(&spec).expect("prepare exact reuse");
1705 let record = reused.reused_record().expect("expected exact reuse");
1706 assert_eq!(record.key(), built.record().key());
1707 assert_eq!(record.artifacts()[0].name(), "optimized.wasm");
1708 assert_eq!(fs::read(&destination).unwrap(), b"optimized-wasm");
1709 assert!(record.timings().caller_build().is_none());
1710 fs::remove_dir_all(root).expect("remove one-output test directory");
1711 }
1712
1713 #[test]
1714 fn multi_output_commit_is_complete_and_name_order_independent() {
1715 let root = unique_temp_directory("multi-output");
1716 let input = root.join("source");
1717 fs::write(&input, b"source").expect("write input");
1718 let spec = ArtifactCacheSpec::new(&root.join("cache"), "release-set", "recipe/v2")
1719 .with_input("source", &input)
1720 .with_output("role-b.wasm", &root.join("public/role-b.wasm"))
1721 .with_output("metadata.json", &root.join("public/metadata.json"))
1722 .with_output("root.wasm", &root.join("public/root.wasm"));
1723 let reordered = ArtifactCacheSpec::new(&root.join("cache"), "release-set", "recipe/v2")
1724 .with_input("source", &input)
1725 .with_output("root.wasm", &root.join("public/root.wasm"))
1726 .with_output("role-b.wasm", &root.join("public/role-b.wasm"))
1727 .with_output("metadata.json", &root.join("public/metadata.json"));
1728 assert_eq!(spec, reordered);
1729 assert_eq!(
1730 resolve_key(&spec).unwrap().key,
1731 resolve_key(&reordered).unwrap().key
1732 );
1733 let transaction = expect_build(prepare_artifact_cache(&spec).expect("prepare miss"));
1734 for (name, contents) in [
1735 ("root.wasm", b"root".as_slice()),
1736 ("role-b.wasm", b"role-b".as_slice()),
1737 ("metadata.json", b"{}".as_slice()),
1738 ] {
1739 fs::write(transaction.output_path(name).unwrap(), contents)
1740 .expect("write staged output");
1741 }
1742
1743 let outcome = transaction.commit().expect("commit complete output set");
1744
1745 assert_eq!(
1746 outcome
1747 .record()
1748 .artifacts()
1749 .iter()
1750 .map(super::ArtifactCacheArtifact::name)
1751 .collect::<Vec<_>>(),
1752 ["metadata.json", "role-b.wasm", "root.wasm"],
1753 );
1754 assert!(matches!(
1755 prepare_artifact_cache(&spec).unwrap(),
1756 ArtifactCachePreparation::Reused(_)
1757 ));
1758 fs::remove_dir_all(root).expect("remove multi-output test directory");
1759 }
1760
1761 #[test]
1762 fn incomplete_output_set_fails_and_removes_staging() {
1763 let root = unique_temp_directory("incomplete-output");
1764 let input = root.join("input");
1765 fs::write(&input, b"input").expect("write input");
1766 let spec = ArtifactCacheSpec::new(&root.join("cache"), "batch", "recipe/v1")
1767 .with_input("input", &input)
1768 .with_output("first", &root.join("first"))
1769 .with_output("second", &root.join("second"));
1770 let transaction = expect_build(prepare_artifact_cache(&spec).expect("prepare miss"));
1771 assert!(matches!(
1772 transaction.output_path("undeclared"),
1773 Err(ArtifactCacheError::UnknownOutput { .. })
1774 ));
1775 fs::write(transaction.output_path("first").unwrap(), b"first")
1776 .expect("write only first output");
1777 let staging = transaction.staging_directory().to_owned();
1778
1779 let error = transaction
1780 .commit()
1781 .expect_err("partial transaction must fail");
1782
1783 assert!(matches!(error, ArtifactCacheError::InvalidOutputs { .. }));
1784 assert!(!staging.exists());
1785 assert!(matches!(
1786 prepare_artifact_cache(&spec).unwrap(),
1787 ArtifactCachePreparation::Build(_)
1788 ));
1789 fs::remove_dir_all(root).expect("remove incomplete-output test directory");
1790 }
1791
1792 #[test]
1793 fn changed_inputs_reject_commit_and_remove_staging() {
1794 let root = unique_temp_directory("changed-inputs");
1795 let input = root.join("input");
1796 fs::write(&input, b"before").expect("write original input");
1797 let spec = ArtifactCacheSpec::new(&root.join("cache"), "transform", "recipe/v1")
1798 .with_input("input", &input)
1799 .with_output("output", &root.join("output"));
1800 let transaction = expect_build(prepare_artifact_cache(&spec).expect("prepare miss"));
1801 fs::write(transaction.output_path("output").unwrap(), b"output")
1802 .expect("write staged output");
1803 let staging = transaction.staging_directory().to_owned();
1804 fs::write(&input, b"after").expect("change input during transaction");
1805
1806 let error = transaction
1807 .commit()
1808 .expect_err("input race must reject commit");
1809
1810 assert!(matches!(
1811 error,
1812 ArtifactCacheError::InputsChangedDuringBuild { .. }
1813 ));
1814 assert!(!staging.exists());
1815 fs::remove_dir_all(root).expect("remove changed-inputs test directory");
1816 }
1817
1818 #[test]
1819 fn dropped_and_panicked_transactions_remove_staging() {
1820 let root = unique_temp_directory("dropped-transactions");
1821 let input = root.join("input");
1822 fs::write(&input, b"input").expect("write input");
1823 let spec = ArtifactCacheSpec::new(&root.join("cache"), "drop", "recipe/v1")
1824 .with_input("input", &input)
1825 .with_output("output", &root.join("output"));
1826
1827 let transaction = expect_build(prepare_artifact_cache(&spec).expect("prepare miss"));
1828 let dropped_staging = transaction.staging_directory().to_owned();
1829 drop(transaction);
1830 assert!(!dropped_staging.exists());
1831
1832 let panicked_staging = Arc::new(std::sync::Mutex::new(None));
1833 let captured_staging = Arc::clone(&panicked_staging);
1834 let result = catch_unwind(AssertUnwindSafe(|| {
1835 let transaction = expect_build(prepare_artifact_cache(&spec).expect("prepare miss"));
1836 *captured_staging.lock().unwrap() = Some(transaction.staging_directory().to_owned());
1837 panic!("synthetic caller panic");
1838 }));
1839 assert!(result.is_err());
1840 assert!(!panicked_staging.lock().unwrap().as_ref().unwrap().exists());
1841 fs::remove_dir_all(root).expect("remove dropped-transactions test directory");
1842 }
1843
1844 #[test]
1845 fn every_declared_identity_dimension_changes_the_content_key() {
1846 let root = unique_temp_directory("identity-dimensions");
1847 let input = root.join("input");
1848 let tool = root.join("tool");
1849 fs::write(&input, b"input-v1").expect("write input");
1850 fs::write(&tool, b"tool-v1").expect("write tool");
1851 let base = ArtifactCacheSpec::new(&root.join("cache"), "identity", "recipe/v1")
1852 .with_input("input", &input)
1853 .with_tool("optimizer", &tool)
1854 .with_arguments(&["-O2"])
1855 .with_environment(&[("MODE", "release")])
1856 .with_identity_bytes("pipeline", b"one")
1857 .with_output("output", &root.join("output"));
1858 let original = resolve_key(&base).unwrap().key;
1859 let mut changed_namespace_spec = base.clone();
1860 changed_namespace_spec.namespace = "identity-other".to_owned();
1861 let changed_namespace = resolve_key(&changed_namespace_spec).unwrap().key;
1862 let changed_argument = resolve_key(&base.clone().with_arguments(&["-O3"]))
1863 .unwrap()
1864 .key;
1865 let changed_environment =
1866 resolve_key(&base.clone().with_environment(&[("MODE", "size-optimized")]))
1867 .unwrap()
1868 .key;
1869 let changed_unset_environment =
1870 resolve_key(&base.clone().with_unset_environment(&["MODE"]))
1871 .unwrap()
1872 .key;
1873 let changed_recipe = resolve_key(&ArtifactCacheSpec {
1874 recipe_id: "recipe/v2".to_owned(),
1875 ..base.clone()
1876 })
1877 .unwrap()
1878 .key;
1879 let mut changed_identity_spec = base.clone();
1880 changed_identity_spec.identities[0].value = b"two".to_vec();
1881 let changed_identity = resolve_key(&changed_identity_spec).unwrap().key;
1882 let mut changed_input_label_spec = base.clone();
1883 changed_input_label_spec.inputs[0].label = "renamed-input".to_owned();
1884 let changed_input_label = resolve_key(&changed_input_label_spec).unwrap().key;
1885 let mut changed_tool_label_spec = base.clone();
1886 changed_tool_label_spec.tools[0].label = "renamed-optimizer".to_owned();
1887 let changed_tool_label = resolve_key(&changed_tool_label_spec).unwrap().key;
1888 let mut changed_output_schema_spec = base.clone();
1889 changed_output_schema_spec.outputs[0].validation = ArtifactOutputValidation::RegularFile;
1890 let changed_output_schema = resolve_key(&changed_output_schema_spec).unwrap().key;
1891 let mut changed_output_name_spec = base.clone();
1892 changed_output_name_spec.outputs[0].name = "renamed-output".to_owned();
1893 let changed_output_name = resolve_key(&changed_output_name_spec).unwrap().key;
1894 fs::write(&tool, b"tool-v2").expect("change tool bytes");
1895 let changed_tool = resolve_key(&base).unwrap().key;
1896 fs::write(&tool, b"tool-v1").expect("restore tool bytes");
1897 fs::write(&input, b"input-v2").expect("change input bytes");
1898 let changed_input = resolve_key(&base).unwrap().key;
1899
1900 for changed in [
1901 changed_namespace,
1902 changed_argument,
1903 changed_environment,
1904 changed_unset_environment,
1905 changed_recipe,
1906 changed_identity,
1907 changed_input_label,
1908 changed_tool_label,
1909 changed_output_schema,
1910 changed_output_name,
1911 changed_tool,
1912 changed_input,
1913 ] {
1914 assert_ne!(original, changed);
1915 }
1916
1917 fs::write(&input, b"input-v1").expect("restore input bytes");
1918 let mut unkeyed_changes = base;
1919 unkeyed_changes.coordination_scope = "another-lock".to_owned();
1920 unkeyed_changes.outputs[0].destination = root.join("another-output");
1921 assert_eq!(original, resolve_key(&unkeyed_changes).unwrap().key);
1922 fs::remove_dir_all(root).expect("remove identity-dimensions test directory");
1923 }
1924
1925 #[test]
1926 fn tampered_cache_entry_is_rebuilt_instead_of_reused() {
1927 let root = unique_temp_directory("tampered-entry");
1928 let input = root.join("input");
1929 fs::write(&input, b"input").expect("write input");
1930 let spec = ArtifactCacheSpec::new(&root.join("cache"), "tamper", "recipe/v1")
1931 .with_input("input", &input)
1932 .with_output("output", &root.join("output"));
1933 let transaction = expect_build(prepare_artifact_cache(&spec).expect("prepare miss"));
1934 fs::write(transaction.output_path("output").unwrap(), b"valid")
1935 .expect("write staged output");
1936 let outcome = transaction.commit().expect("commit valid entry");
1937 let entry = entry_directory(&namespace_directory(&spec), outcome.record().key());
1938 fs::write(entry.join("outputs/0000.artifact"), b"tampered").expect("tamper cached output");
1939
1940 let rebuilt = prepare_artifact_cache(&spec).expect("prepare after corruption");
1941
1942 assert!(matches!(rebuilt, ArtifactCachePreparation::Build(_)));
1943 fs::remove_dir_all(root).expect("remove tampered-entry test directory");
1944 }
1945
1946 #[test]
1947 fn malformed_manifests_and_nondirectory_entries_are_rebuilt() {
1948 let root = unique_temp_directory("malformed-entry");
1949 let input = root.join("input");
1950 fs::write(&input, b"input").expect("write input");
1951 let spec = ArtifactCacheSpec::new(&root.join("cache"), "malformed", "recipe/v1")
1952 .with_input("input", &input)
1953 .with_output("output", &root.join("output"));
1954 let outcome = build_output(&spec, b"valid");
1955 let entry = entry_directory(&namespace_directory(&spec), outcome.record().key());
1956 fs::write(entry.join(super::MANIFEST_FILE), [0xff, 0xfe])
1957 .expect("write invalid UTF-8 manifest");
1958
1959 let transaction =
1960 expect_build(prepare_artifact_cache(&spec).expect("prepare after malformed manifest"));
1961 assert!(!entry.exists());
1962 transaction.abort().expect("abort manifest recovery");
1963
1964 fs::write(&entry, b"not a directory").expect("write nondirectory cache entry");
1965 let transaction =
1966 expect_build(prepare_artifact_cache(&spec).expect("prepare after nondirectory entry"));
1967 assert!(!entry.exists());
1968 transaction.abort().expect("abort nondirectory recovery");
1969 fs::remove_dir_all(root).expect("remove malformed-entry test directory");
1970 }
1971
1972 #[test]
1973 fn undeclared_entry_root_files_are_never_published_or_reused() {
1974 let root = unique_temp_directory("undeclared-entry-root");
1975 let input = root.join("input");
1976 fs::write(&input, b"input").expect("write input");
1977 let spec = ArtifactCacheSpec::new(&root.join("cache"), "root-schema", "recipe/v1")
1978 .with_input("input", &input)
1979 .with_output("output", &root.join("output"));
1980 let transaction = expect_build(prepare_artifact_cache(&spec).expect("prepare miss"));
1981 fs::write(transaction.output_path("output").unwrap(), b"output")
1982 .expect("write staged output");
1983 fs::write(transaction.staging_directory().join("build.log"), b"log")
1984 .expect("write undeclared root file");
1985 assert!(matches!(
1986 transaction.commit(),
1987 Err(ArtifactCacheError::InvalidOutputs { .. })
1988 ));
1989
1990 let transaction = expect_build(prepare_artifact_cache(&spec).expect("prepare next miss"));
1991 fs::remove_dir_all(transaction.staging_directory().join("outputs"))
1992 .expect("remove staging output directory");
1993 fs::write(
1994 transaction.staging_directory().join("outputs"),
1995 b"not a directory",
1996 )
1997 .expect("replace staging output directory");
1998 assert!(matches!(
1999 transaction.commit(),
2000 Err(ArtifactCacheError::InvalidOutputs { .. })
2001 ));
2002
2003 let outcome = build_output(&spec, b"valid");
2004 let entry = entry_directory(&namespace_directory(&spec), outcome.record().key());
2005 fs::write(entry.join("unexpected"), b"extra").expect("write corrupt root file");
2006 let transaction = expect_build(
2007 prepare_artifact_cache(&spec).expect("prepare after root-schema corruption"),
2008 );
2009 assert!(!entry.exists());
2010 transaction.abort().expect("abort root-schema recovery");
2011 fs::remove_dir_all(root).expect("remove undeclared-entry-root test directory");
2012 }
2013
2014 #[test]
2015 fn pruning_protects_active_entry_and_removes_older_key() {
2016 let root = unique_temp_directory("transaction-pruning");
2017 let input = root.join("input");
2018 fs::write(&input, b"input").expect("write input");
2019 let base = ArtifactCacheSpec::new(&root.join("cache"), "prune", "recipe/v1")
2020 .with_input("input", &input)
2021 .with_output("output", &root.join("output"));
2022 build_output(&base.clone().with_arguments(&["old"]), b"old");
2023 let active = base
2024 .clone()
2025 .with_arguments(&["active"])
2026 .with_prune_policy(ArtifactCachePrunePolicy::new().with_max_size_bytes(0));
2027
2028 let outcome = build_output(&active, b"active");
2029 let report = outcome
2030 .record()
2031 .maintenance()
2032 .and_then(ArtifactCacheMaintenance::prune_report)
2033 .expect("successful configured pruning");
2034
2035 assert_eq!(report.entries_scanned(), 2);
2036 assert_eq!(report.entries_removed(), 1);
2037 assert_eq!(report.entries_retained(), 1);
2038 assert!(matches!(
2039 prepare_artifact_cache(&base.with_arguments(&["old"])).unwrap(),
2040 ArtifactCachePreparation::Build(_)
2041 ));
2042 assert!(matches!(
2043 prepare_artifact_cache(&active).unwrap(),
2044 ArtifactCachePreparation::Reused(_)
2045 ));
2046 let strict = prune_artifact_cache(
2047 active.cache_root(),
2048 active.namespace(),
2049 ArtifactCachePrunePolicy::new().with_max_size_bytes(0),
2050 )
2051 .expect("strict namespace pruning");
2052 assert_eq!(strict.entries_removed(), 1);
2053 assert!(matches!(
2054 prepare_artifact_cache(&active).unwrap(),
2055 ArtifactCachePreparation::Build(_)
2056 ));
2057 fs::remove_dir_all(root).expect("remove transaction-pruning test directory");
2058 }
2059
2060 #[test]
2061 fn pruning_removes_abandoned_staging_without_touching_active_transactions() {
2062 let root = unique_temp_directory("staging-pruning");
2063 let input = root.join("input");
2064 fs::write(&input, b"input").expect("write input");
2065 let spec = ArtifactCacheSpec::new(&root.join("cache"), "staging-prune", "recipe/v1")
2066 .with_input("input", &input)
2067 .with_output("output", &root.join("output"));
2068 let transaction = expect_build(prepare_artifact_cache(&spec).expect("prepare miss"));
2069 let active_staging = transaction.staging_directory().to_owned();
2070
2071 let active_report = prune_artifact_cache(
2072 spec.cache_root(),
2073 spec.namespace(),
2074 ArtifactCachePrunePolicy::new(),
2075 )
2076 .expect("prune around active transaction");
2077 assert_eq!(active_report.uncommitted_directories_removed(), 0);
2078 assert!(active_staging.exists());
2079 transaction.abort().expect("abort active transaction");
2080
2081 let key = resolve_key(&spec).unwrap().key;
2082 let orphan = namespace_directory(&spec)
2083 .join("staging")
2084 .join(format!("{key}-terminated-0"));
2085 fs::create_dir_all(orphan.join("outputs")).expect("create orphan staging");
2086 fs::write(orphan.join("outputs/payload"), b"abandoned").expect("write orphan payload");
2087
2088 let report = prune_artifact_cache(
2089 spec.cache_root(),
2090 spec.namespace(),
2091 ArtifactCachePrunePolicy::new(),
2092 )
2093 .expect("prune abandoned staging");
2094 assert_eq!(report.uncommitted_directories_removed(), 1);
2095 assert!(report.uncommitted_bytes_removed() >= 9);
2096 assert!(!orphan.exists());
2097 fs::remove_dir_all(root).expect("remove staging-pruning test directory");
2098 }
2099
2100 #[test]
2101 fn overlapping_exact_acquisitions_build_once() {
2102 let root = unique_temp_directory("overlapping-acquisitions");
2103 let input = root.join("input");
2104 fs::write(&input, b"input").expect("write input");
2105 let spec = Arc::new(
2106 ArtifactCacheSpec::new(&root.join("cache"), "concurrent", "recipe/v1")
2107 .with_input("input", &input)
2108 .with_output("output", &root.join("output")),
2109 );
2110 let start = Arc::new(Barrier::new(3));
2111 let builds = Arc::new(AtomicUsize::new(0));
2112 let workers = std::array::from_fn::<_, 2, _>(|_| {
2113 let spec = Arc::clone(&spec);
2114 let start = Arc::clone(&start);
2115 let builds = Arc::clone(&builds);
2116 thread::spawn(move || {
2117 start.wait();
2118 match prepare_artifact_cache(&spec).expect("prepare overlapping acquisition") {
2119 ArtifactCachePreparation::Reused(record) => record.key(),
2120 ArtifactCachePreparation::Build(transaction) => {
2121 builds.fetch_add(1, Ordering::SeqCst);
2122 fs::write(transaction.output_path("output").unwrap(), b"built")
2123 .expect("write concurrent staged output");
2124 transaction
2125 .commit()
2126 .expect("commit concurrent output")
2127 .record()
2128 .key()
2129 }
2130 }
2131 })
2132 });
2133 start.wait();
2134 let keys = workers.map(|worker| worker.join().expect("worker must not panic"));
2135
2136 assert_eq!(keys[0], keys[1]);
2137 assert_eq!(builds.load(Ordering::SeqCst), 1);
2138 fs::remove_dir_all(root).expect("remove overlapping-acquisitions test directory");
2139 }
2140
2141 #[test]
2142 fn different_keys_sharing_a_coordination_scope_do_not_overlap() {
2143 let root = unique_temp_directory("shared-coordination");
2144 let input = root.join("input");
2145 fs::write(&input, b"input").expect("write input");
2146 let base = ArtifactCacheSpec::new(&root.join("cache"), "coordinated", "recipe/v1")
2147 .with_coordination_scope("shared-external-tree")
2148 .with_input("input", &input)
2149 .with_output("output", &root.join("output"));
2150 let start = Arc::new(Barrier::new(3));
2151 let active = Arc::new(AtomicUsize::new(0));
2152 let maximum = Arc::new(AtomicUsize::new(0));
2153 let workers = ["first", "second"].map(|argument| {
2154 let spec = base.clone().with_arguments(&[argument]);
2155 let start = Arc::clone(&start);
2156 let active = Arc::clone(&active);
2157 let maximum = Arc::clone(&maximum);
2158 thread::spawn(move || {
2159 start.wait();
2160 let transaction = expect_build(
2161 prepare_artifact_cache(&spec).expect("prepare coordinated transaction"),
2162 );
2163 let current = active.fetch_add(1, Ordering::SeqCst) + 1;
2164 maximum.fetch_max(current, Ordering::SeqCst);
2165 thread::sleep(std::time::Duration::from_millis(20));
2166 fs::write(transaction.output_path("output").unwrap(), argument)
2167 .expect("write coordinated output");
2168 active.fetch_sub(1, Ordering::SeqCst);
2169 transaction.commit().expect("commit coordinated output");
2170 })
2171 });
2172 start.wait();
2173 for worker in workers {
2174 worker.join().expect("coordinated worker must not panic");
2175 }
2176
2177 assert_eq!(maximum.load(Ordering::SeqCst), 1);
2178 fs::remove_dir_all(root).expect("remove shared-coordination test directory");
2179 }
2180
2181 #[test]
2182 fn content_identity_is_independent_of_checkout_and_destination_paths() {
2183 let first = unique_temp_directory("checkout-first");
2184 let second = unique_temp_directory("checkout-second");
2185 for root in [&first, &second] {
2186 fs::create_dir_all(root.join("source")).expect("create source directory");
2187 fs::write(root.join("source/input"), b"same input").expect("write input");
2188 fs::write(root.join("tool"), b"same tool").expect("write tool");
2189 }
2190 let spec = |root: &Path| {
2191 ArtifactCacheSpec::new(&root.join("cache"), "portable", "recipe/v1")
2192 .with_input("source", &root.join("source"))
2193 .with_tool("optimizer", &root.join("tool"))
2194 .with_arguments(&["--exact"])
2195 .with_output("output", &root.join("different/public/output"))
2196 };
2197
2198 let first_key = resolve_key(&spec(&first)).unwrap();
2199 let second_key = resolve_key(&spec(&second)).unwrap();
2200
2201 assert_eq!(first_key.key, second_key.key);
2202 assert_eq!(first_key.input_digest, second_key.input_digest);
2203 fs::remove_dir_all(first).expect("remove first checkout");
2204 fs::remove_dir_all(second).expect("remove second checkout");
2205 }
2206
2207 #[test]
2208 fn import_helper_and_debug_output_do_not_expose_identity_values() {
2209 let root = unique_temp_directory("import-output");
2210 let input = root.join("input");
2211 let external = root.join("fixed-build-location/output");
2212 fs::create_dir_all(external.parent().unwrap()).expect("create fixed output directory");
2213 fs::write(&input, b"input").expect("write input");
2214 fs::write(&external, b"external-output").expect("write external output");
2215 let spec = ArtifactCacheSpec::new(&root.join("cache"), "import", "recipe/v1")
2216 .with_input("input", &input)
2217 .with_environment(&[("SECRET_TOKEN", "do-not-render")])
2218 .with_identity_bytes("private-ish", b"also-do-not-render")
2219 .with_output("output", &root.join("public/output"));
2220 let debug = format!("{spec:?}");
2221 assert!(debug.contains("SECRET_TOKEN"));
2222 assert!(!debug.contains("do-not-render"));
2223 assert!(!debug.contains("also-do-not-render"));
2224 let transaction = expect_build(prepare_artifact_cache(&spec).unwrap());
2225
2226 transaction
2227 .import_output("output", &external)
2228 .expect("import external output");
2229 let outcome = transaction.commit().expect("commit imported output");
2230
2231 assert_eq!(
2232 fs::read(outcome.record().artifacts()[0].path()).unwrap(),
2233 b"external-output"
2234 );
2235 fs::remove_dir_all(root).expect("remove import-output test directory");
2236 }
2237
2238 #[test]
2239 fn undeclared_staging_output_rejects_the_transaction() {
2240 let root = unique_temp_directory("undeclared-output");
2241 let input = root.join("input");
2242 fs::write(&input, b"input").expect("write input");
2243 let spec = ArtifactCacheSpec::new(&root.join("cache"), "extra", "recipe/v1")
2244 .with_input("input", &input)
2245 .with_output("output", &root.join("output"));
2246 let transaction = expect_build(prepare_artifact_cache(&spec).unwrap());
2247 fs::write(transaction.output_path("output").unwrap(), b"declared")
2248 .expect("write declared output");
2249 fs::write(
2250 transaction.staging_directory().join("outputs/extra"),
2251 b"undeclared",
2252 )
2253 .expect("write undeclared output");
2254
2255 assert!(matches!(
2256 transaction.commit(),
2257 Err(ArtifactCacheError::InvalidOutputs { .. })
2258 ));
2259 fs::remove_dir_all(root).expect("remove undeclared-output test directory");
2260 }
2261
2262 #[test]
2263 fn empty_output_requires_explicit_regular_file_validation() {
2264 let root = unique_temp_directory("empty-output");
2265 let input = root.join("input");
2266 fs::write(&input, b"input").expect("write input");
2267 let default_spec = ArtifactCacheSpec::new(&root.join("cache"), "empty", "recipe/v1")
2268 .with_input("input", &input)
2269 .with_output("output", &root.join("output"));
2270 let transaction = expect_build(prepare_artifact_cache(&default_spec).unwrap());
2271 fs::write(transaction.output_path("output").unwrap(), b"").expect("write empty output");
2272 assert!(matches!(
2273 transaction.commit(),
2274 Err(ArtifactCacheError::InvalidOutputs { .. })
2275 ));
2276
2277 let regular_spec = ArtifactCacheSpec::new(&root.join("cache"), "empty", "recipe/v1")
2278 .with_input("input", &input)
2279 .with_output_validation(
2280 "output",
2281 &root.join("output"),
2282 ArtifactOutputValidation::RegularFile,
2283 );
2284 let transaction = expect_build(prepare_artifact_cache(®ular_spec).unwrap());
2285 fs::write(transaction.output_path("output").unwrap(), b"").expect("write empty output");
2286 transaction
2287 .commit()
2288 .expect("commit explicitly valid empty file");
2289 fs::remove_dir_all(root).expect("remove empty-output test directory");
2290 }
2291
2292 #[test]
2293 fn invalid_specifications_are_rejected_before_acquisition() {
2294 let root = unique_temp_directory("invalid-specifications");
2295 let input = root.join("input");
2296 let tool = root.join("tool");
2297 let output = root.join("output");
2298 fs::write(&input, b"input").expect("write input");
2299 fs::write(&tool, b"tool").expect("write tool");
2300 let cache = root.join("cache");
2301 let specs = [
2302 ArtifactCacheSpec::new(&cache, "namespace", "recipe/v1").with_input("input", &input),
2303 ArtifactCacheSpec::new(&cache, "namespace", "recipe/v1")
2304 .with_input("bad label", &input)
2305 .with_output("output", &output),
2306 ArtifactCacheSpec::new(&cache, "namespace", "recipe/v1")
2307 .with_input("input", &input)
2308 .with_input("input", &input)
2309 .with_output("output", &output),
2310 ArtifactCacheSpec::new(&cache, "namespace", "recipe/v1")
2311 .with_tool("tool", &tool)
2312 .with_tool("tool", &tool)
2313 .with_output("output", &output),
2314 ArtifactCacheSpec::new(&cache, "namespace", "recipe/v1")
2315 .with_identity_bytes("identity", b"one")
2316 .with_identity_bytes("identity", b"two")
2317 .with_output("output", &output),
2318 ArtifactCacheSpec::new(&cache, "namespace", "recipe/v1")
2319 .with_environment(&[("", "value")])
2320 .with_output("output", &output),
2321 ArtifactCacheSpec::new(&cache, "namespace", "recipe/v1")
2322 .with_output("same", &output)
2323 .with_output("same", &root.join("other")),
2324 ArtifactCacheSpec::new(&cache, "namespace", "recipe/v1")
2325 .with_output("first", &output)
2326 .with_output("second", &output),
2327 ArtifactCacheSpec::new(&cache, "namespace", "recipe/v1").with_output(".", &output),
2328 ArtifactCacheSpec::new(&cache, "namespace", "recipe/v1")
2329 .with_output("output", Path::new("")),
2330 ArtifactCacheSpec::new(&cache, "", "recipe/v1").with_output("output", &output),
2331 ArtifactCacheSpec::new(&cache, "namespace", "").with_output("output", &output),
2332 ArtifactCacheSpec::new(&cache, "namespace", "recipe/v1")
2333 .with_coordination_scope("")
2334 .with_output("output", &output),
2335 ArtifactCacheSpec::new(Path::new(""), "namespace", "recipe/v1")
2336 .with_output("output", &output),
2337 ];
2338
2339 for spec in specs {
2340 expect_invalid_spec(prepare_artifact_cache(&spec));
2341 }
2342 fs::remove_dir_all(root).expect("remove invalid-specification test directory");
2343 }
2344
2345 #[test]
2346 fn filesystem_boundaries_reject_cache_inputs_and_destination_aliases() {
2347 let root = unique_temp_directory("filesystem-boundaries");
2348 let cache = root.join("cache");
2349 fs::create_dir_all(&cache).expect("create cache root");
2350 let cache_input = cache.join("input");
2351 let cache_tool = cache.join("tool");
2352 let input = root.join("input");
2353 let input_directory = root.join("source");
2354 fs::write(&cache_input, b"cache input").expect("write cache input");
2355 fs::write(&cache_tool, b"cache tool").expect("write cache tool");
2356 fs::write(&input, b"input").expect("write input");
2357 fs::create_dir_all(&input_directory).expect("create input directory");
2358 fs::write(input_directory.join("source"), b"source").expect("write source input");
2359
2360 let invalid = [
2361 ArtifactCacheSpec::new(&cache, "cache-input", "recipe/v1")
2362 .with_input("input", &cache_input)
2363 .with_output("output", &root.join("public/cache-input")),
2364 ArtifactCacheSpec::new(&cache, "cache-tool", "recipe/v1")
2365 .with_tool("tool", &cache_tool)
2366 .with_output("output", &root.join("public/cache-tool")),
2367 ArtifactCacheSpec::new(&cache, "cache-output", "recipe/v1")
2368 .with_input("input", &input)
2369 .with_output("output", &cache.join("public-output")),
2370 ArtifactCacheSpec::new(&cache, "input-output", "recipe/v1")
2371 .with_input("input", &input)
2372 .with_output("output", &input),
2373 ArtifactCacheSpec::new(&cache, "directory-output", "recipe/v1")
2374 .with_input("source", &input_directory)
2375 .with_output("output", &input_directory.join("generated")),
2376 ArtifactCacheSpec::new(&cache, "alias-output", "recipe/v1")
2377 .with_input("input", &input)
2378 .with_output("first", &root.join("public/result"))
2379 .with_output("second", &root.join("public/nested/../result")),
2380 ArtifactCacheSpec::new(&cache, "directory-destination", "recipe/v1")
2381 .with_input("input", &input)
2382 .with_output("output", &input_directory),
2383 ];
2384 for spec in invalid {
2385 expect_invalid_spec(prepare_artifact_cache(&spec));
2386 }
2387
2388 #[cfg(unix)]
2389 {
2390 use std::os::unix::fs::symlink;
2391
2392 let public = root.join("symlink-public");
2393 fs::create_dir_all(&public).expect("create symlink destination directory");
2394 let alias = root.join("symlink-alias");
2395 symlink(&public, &alias).expect("create destination-directory symlink");
2396 let spec = ArtifactCacheSpec::new(&cache, "symlink-output", "recipe/v1")
2397 .with_input("input", &input)
2398 .with_output("first", &public.join("result"))
2399 .with_output("second", &alias.join("result"));
2400 expect_invalid_spec(prepare_artifact_cache(&spec));
2401 }
2402
2403 let source = root.join("ancestor-source");
2404 fs::create_dir_all(&source).expect("create ancestor source");
2405 fs::write(source.join("source"), b"source").expect("write ancestor source");
2406 let allowed =
2407 ArtifactCacheSpec::new(&source.join("nested-cache"), "ancestor-input", "recipe/v1")
2408 .with_input("source", &source)
2409 .with_output("output", &root.join("outside-source/output"));
2410 expect_build(prepare_artifact_cache(&allowed).expect("prepare ancestor input"))
2411 .abort()
2412 .expect("abort ancestor input transaction");
2413
2414 fs::remove_dir_all(root).expect("remove filesystem-boundary test directory");
2415 }
2416
2417 fn expect_build(preparation: ArtifactCachePreparation) -> super::ArtifactBuildTransaction {
2418 match preparation {
2419 ArtifactCachePreparation::Build(transaction) => transaction,
2420 ArtifactCachePreparation::Reused(_) => panic!("expected a cache miss transaction"),
2421 }
2422 }
2423
2424 fn expect_invalid_spec(result: Result<ArtifactCachePreparation, ArtifactCacheError>) {
2425 assert!(matches!(
2426 result,
2427 Err(ArtifactCacheError::InvalidSpec { .. })
2428 ));
2429 }
2430
2431 fn build_output(spec: &ArtifactCacheSpec, contents: &[u8]) -> ArtifactCacheOutcome {
2432 let transaction = expect_build(prepare_artifact_cache(spec).expect("prepare build"));
2433 fs::write(transaction.output_path("output").unwrap(), contents)
2434 .expect("write staged output");
2435 transaction.commit().expect("commit output")
2436 }
2437}