1use serde_json::Value;
2use std::{
3 collections::{BTreeMap, BTreeSet, HashMap, HashSet, VecDeque},
4 ffi::{OsStr, OsString},
5 fs::{self, File},
6 io,
7 path::{Path, PathBuf},
8 process::{Child, Command, ExitStatus, Output, Stdio},
9 sync::{
10 Arc, RwLock,
11 atomic::{AtomicUsize, Ordering},
12 mpsc::{self, RecvTimeoutError},
13 },
14 thread,
15 time::{Duration, Instant, SystemTime},
16};
17use toml::Value as TomlValue;
18
19use crate::timing::saturating_add_optional_duration;
20
21use super::{
22 cache_fs::{
23 ArtifactCacheMaintenance, ArtifactCachePrunePolicy, ArtifactCachePruneReport, CacheFsError,
24 RetainedCacheEntry, cache_entry_last_used, cache_maintenance_due,
25 canonicalize_allow_missing, directory_logical_size,
26 ensure_cache_directory_tag as ensure_cache_tag, is_sha256_directory, lock_cache_file,
27 lock_cache_file_with_wait_observer, perform_scheduled_cache_maintenance,
28 prune_direct_child_directories, record_cache_entry_use as record_entry_use,
29 record_cache_maintenance, remove_path_if_present, remove_unretained_entry,
30 },
31 digest::{
32 InputDigest, InputHasher, LabeledPathDigestCache, copy_file_atomic, digest_bytes,
33 digest_file, digest_labeled_paths_composable, os_bytes, write_atomic,
34 },
35 wasm::wasm_path,
36};
37
38const CACHE_FORMAT_VERSION: &str = "ic-testkit-wasm-build-v1";
39const DEFAULT_TARGET: &str = "wasm32-unknown-unknown";
40const AUTOMATIC_ENVIRONMENT: &[&str] = &[
41 "CARGO_BUILD_RUSTC",
42 "CARGO_ENCODED_RUSTFLAGS",
43 "RUSTC",
44 "RUSTC_WRAPPER",
45 "RUSTC_WORKSPACE_WRAPPER",
46 "RUSTFLAGS",
47 "RUSTUP_TOOLCHAIN",
48];
49
50#[derive(Clone, Debug, Eq, PartialEq)]
58pub struct WasmBuildSpec {
59 workspace_root: PathBuf,
60 target_dir: PathBuf,
61 packages: Vec<String>,
62 profile_target_dir: String,
63 cargo_profile_args: Vec<OsString>,
64 extra_env: BTreeMap<OsString, OsString>,
65 inherited_env: BTreeSet<OsString>,
66 additional_inputs: Vec<PathBuf>,
67 target: String,
68 cargo_program: OsString,
69 rustc_program: OsString,
70 cache_mode: WasmBuildCacheMode,
71 prune_policy: Option<ArtifactCachePrunePolicy>,
72 prune_interval: Option<Duration>,
73 shared_incremental_maintenance_config: Option<SharedIncrementalTargetMaintenanceConfig>,
74}
75
76#[derive(Clone, Copy, Debug, Default, Eq, PartialEq)]
78pub enum SharedIncrementalTargetMaintenanceFailureMode {
79 #[default]
81 Strict,
82 BestEffort,
84}
85
86#[derive(Clone, Copy, Debug, Eq, PartialEq)]
88pub struct SharedIncrementalTargetMaintenanceConfig {
89 policy: SharedIncrementalTargetPrunePolicy,
90 minimum_interval: Duration,
91 failure_mode: SharedIncrementalTargetMaintenanceFailureMode,
92}
93
94#[non_exhaustive]
96#[derive(Clone, Debug, Eq, PartialEq)]
97pub enum WasmBuildCacheMode {
98 Isolated,
100 SharedIncremental {
102 target_dir: PathBuf,
104 },
105}
106
107#[derive(Clone, Debug, Eq, PartialEq)]
109pub enum WasmBuildOutcome {
110 Built(WasmBuildRecord),
112 Reused(WasmBuildRecord),
114}
115
116#[derive(Clone, Debug, Eq, PartialEq)]
118pub struct WasmBuildRecord {
119 fingerprint: InputDigest,
120 input_digest: InputDigest,
121 retention: RetainedCacheEntry,
122 artifacts: Vec<PathBuf>,
123 timings: WasmBuildTimings,
124 maintenance: Option<ArtifactCacheMaintenance>,
125 shared_incremental_maintenance: Option<SharedIncrementalTargetMaintenanceOutcome>,
126}
127
128#[derive(Clone, Copy, Debug, Default, Eq, PartialEq)]
130pub struct WasmBuildTimings {
131 lock_wait: Duration,
132 shared_incremental_lock_wait: Option<Duration>,
133 input_resolution: WasmInputResolutionTimings,
134 cargo_build: Option<Duration>,
135 cache_maintenance: Option<Duration>,
136 total: Duration,
137}
138
139#[derive(Clone, Copy, Debug, Default, Eq, PartialEq)]
141pub struct WasmInputResolutionTimings {
142 tool_identity: Duration,
143 cargo_metadata: Duration,
144 input_discovery: Duration,
145 content_hashing: Duration,
146 total: Duration,
147}
148
149#[non_exhaustive]
151#[derive(Clone, Copy, Debug, Eq, PartialEq)]
152pub enum WasmBuildFailurePhase {
153 Specification,
155 ExactCacheCoordination,
157 ToolIdentity,
159 CargoMetadata,
161 InputDiscovery,
163 ContentHashing,
165 SharedTargetCoordination,
167 SharedTargetMaintenance,
169 CargoBuild,
171 ArtifactPublication,
173 ExactCacheMaintenance,
175 Cleanup,
177}
178
179#[derive(Clone, Copy, Debug, Default, Eq, PartialEq)]
181pub struct WasmBuildFailureTimings {
182 exact_cache_coordination: Duration,
183 shared_target_coordination: Option<Duration>,
184 input_resolution: WasmInputResolutionTimings,
185 shared_target_maintenance: Option<Duration>,
186 cargo_build: Option<Duration>,
187 artifact_publication: Option<Duration>,
188 exact_cache_maintenance: Option<Duration>,
189 cleanup: Option<Duration>,
190 total: Duration,
191}
192
193#[derive(Clone, Copy, Debug, Eq, PartialEq)]
195pub struct WasmBuildFailureDetails {
196 phase: WasmBuildFailurePhase,
197 timings: WasmBuildFailureTimings,
198}
199
200#[derive(Clone, Debug, Eq, PartialEq)]
202pub struct CargoBuildInput {
203 label: PathBuf,
204 path: PathBuf,
205}
206
207#[derive(Clone, Debug, Eq, PartialEq)]
212pub struct ResolvedCargoBuildInputs {
213 fingerprint: InputDigest,
214 input_digest: InputDigest,
215 validation_digest: InputDigest,
216 inputs: Vec<CargoBuildInput>,
217 exclusions: Vec<PathBuf>,
218 timings: WasmInputResolutionTimings,
219}
220
221pub(super) enum WasmBuildInputReuse<'reuse> {
222 Session(&'reuse mut WasmBuildSessionState),
223 Snapshot(&'reuse WasmBuildInputSnapshotState),
224}
225
226pub(super) struct WasmBuildBatchInputResolver<'a, 'session> {
227 specs: Vec<&'a WasmBuildSpec>,
228 groups: Vec<BatchResolutionGroup>,
229 group_by_index: Vec<usize>,
230 resolved:
231 Vec<Option<Result<ResolvedCargoBuildInputs, (WasmBuildFailurePhase, WasmBuildError)>>>,
232 reuse: Option<WasmBuildInputReuse<'session>>,
233 metrics: WasmBuildBatchInputMetrics,
234}
235
236pub(super) struct WasmBuildSessionState {
237 snapshots: Vec<(WasmBuildSpec, ResolvedCargoBuildInputs)>,
238 digest_cache: LabeledPathDigestCache,
239 snapshot_reuses: usize,
240 invalidated: bool,
241}
242
243pub(super) struct WasmBuildInputSnapshotState {
244 snapshots: Vec<(WasmBuildSpec, ResolvedCargoBuildInputs)>,
245 preparation_metrics: WasmBuildBatchInputMetrics,
246 preparation_timings: WasmInputResolutionTimings,
247 reader_reuses: AtomicUsize,
248 invalidation: Arc<RwLock<bool>>,
249}
250
251pub(super) struct WasmBuildBatchAttempt {
253 pub(super) result: Result<WasmBuildOutcome, (WasmBuildError, WasmBuildFailureDetails)>,
254}
255
256struct BatchResolutionGroup {
257 indexes: Vec<usize>,
258}
259
260struct ResolvedLocalInputs {
261 validation_inputs: Vec<(PathBuf, PathBuf)>,
262 workspace_projection: Option<InputDigest>,
263}
264
265#[derive(Clone, Copy, Debug, Default, Eq, PartialEq)]
266pub(super) struct WasmBuildBatchInputMetrics {
267 pub(super) runs: usize,
268 pub(super) reuses: usize,
269 pub(super) session_reuses: usize,
270 pub(super) prepared_reuses: usize,
271}
272
273#[derive(Eq, PartialEq)]
274struct BatchResolutionKey<'a> {
275 workspace_root: &'a Path,
276 cargo_program: &'a OsStr,
277 rustc_program: &'a OsStr,
278 metadata_arguments: Vec<OsString>,
279 environment: BTreeMap<OsString, Option<OsString>>,
280}
281
282#[derive(Clone, Debug, Eq, PartialEq)]
284pub struct SharedIncrementalTargetInspection {
285 target_dir: PathBuf,
286 logical_size_bytes: u64,
287 last_used: SystemTime,
288 lock_wait: Duration,
289}
290
291#[derive(Clone, Copy, Debug, Default, Eq, PartialEq)]
299pub struct SharedIncrementalTargetPrunePolicy {
300 max_age: Option<Duration>,
301 max_size_bytes: Option<u64>,
302}
303
304#[derive(Clone, Debug, Eq, PartialEq)]
306pub struct SharedIncrementalTargetMaintenance {
307 target_dir: PathBuf,
308 logical_size_bytes_before: u64,
309 logical_size_bytes_after: u64,
310 last_used_before: SystemTime,
311 cleared: bool,
312 lock_wait: Duration,
313 maintenance: Duration,
314}
315
316#[non_exhaustive]
318#[derive(Clone, Debug, Eq, PartialEq)]
319pub enum SharedIncrementalTargetMaintenanceOutcome {
320 Missing {
322 target_dir: PathBuf,
324 },
325 Skipped {
327 target_dir: PathBuf,
329 lock_wait: Duration,
331 schedule_check: Duration,
333 },
334 Performed {
336 maintenance: SharedIncrementalTargetMaintenance,
338 schedule_check: Duration,
340 },
341 Failed {
343 target_dir: PathBuf,
345 lock_wait: Duration,
347 message: String,
349 },
350}
351
352#[derive(Clone, Copy, Debug, Eq, PartialEq)]
354pub struct WasmBuildProgressConfig {
355 heartbeat_interval: Option<Duration>,
356 emit_cargo_output: bool,
357}
358
359#[derive(Clone, Copy, Debug, Eq, PartialEq)]
361pub enum WasmBuildOutputStream {
362 Stdout,
364 Stderr,
366}
367
368#[derive(Clone, Copy, Debug, Eq, PartialEq)]
370pub enum WasmBuildProgressOutcome {
371 Built,
373 Reused,
375}
376
377#[non_exhaustive]
379#[derive(Clone, Copy, Debug, Eq, PartialEq)]
380pub enum WasmBuildProgressPhase {
381 ExactCacheLock,
383 CargoIdentity,
385 RustcIdentity,
387 CargoMetadata,
389 InputDiscovery,
391 ContentHashing,
393 SharedTargetLock,
395 SharedTargetMaintenance,
397 CargoBuild,
399 ArtifactPublication,
401 ExactCacheMaintenance,
403}
404
405#[non_exhaustive]
407#[derive(Clone, Debug, Eq, PartialEq)]
408pub enum WasmBuildProgressEvent {
409 Started,
411 InputsResolved {
413 fingerprint: InputDigest,
415 input_digest: InputDigest,
417 elapsed: Duration,
419 },
420 CacheMiss {
422 fingerprint: InputDigest,
424 },
425 CacheHit {
427 fingerprint: InputDigest,
429 },
430 SharedTargetLockStarted {
432 target_dir: PathBuf,
434 },
435 SharedTargetLockAcquired {
437 target_dir: PathBuf,
439 wait: Duration,
441 },
442 SharedTargetMaintenanceStarted {
444 target_dir: PathBuf,
446 },
447 SharedTargetMaintenanceFinished {
449 outcome: SharedIncrementalTargetMaintenanceOutcome,
451 },
452 CargoStarted {
454 target_dir: PathBuf,
456 },
457 CargoOutput {
459 stream: WasmBuildOutputStream,
461 bytes: Vec<u8>,
463 },
464 Heartbeat {
466 phase: WasmBuildProgressPhase,
468 elapsed: Duration,
470 },
471 CargoFinished {
473 success: bool,
475 code: Option<i32>,
477 elapsed: Duration,
479 },
480 Finished {
482 outcome: WasmBuildProgressOutcome,
484 fingerprint: InputDigest,
486 elapsed: Duration,
488 },
489}
490
491impl Default for WasmBuildProgressConfig {
492 fn default() -> Self {
493 Self {
494 heartbeat_interval: Some(Duration::from_secs(10)),
495 emit_cargo_output: true,
496 }
497 }
498}
499
500impl WasmBuildProgressConfig {
501 #[must_use]
503 pub fn new() -> Self {
504 Self::default()
505 }
506
507 #[must_use]
511 pub const fn with_heartbeat_interval(mut self, interval: Duration) -> Self {
512 self.heartbeat_interval = Some(interval);
513 self
514 }
515
516 #[must_use]
518 pub const fn without_heartbeats(mut self) -> Self {
519 self.heartbeat_interval = None;
520 self
521 }
522
523 #[must_use]
527 pub const fn with_cargo_output(mut self, emit: bool) -> Self {
528 self.emit_cargo_output = emit;
529 self
530 }
531
532 #[must_use]
534 pub const fn heartbeat_interval(self) -> Option<Duration> {
535 self.heartbeat_interval
536 }
537
538 #[must_use]
540 pub const fn emits_cargo_output(self) -> bool {
541 self.emit_cargo_output
542 }
543}
544
545struct ProgressReporter<'a> {
546 config: WasmBuildProgressConfig,
547 observer: Option<&'a mut dyn FnMut(WasmBuildProgressEvent)>,
548 last_event: Instant,
549 failure_phase: Option<WasmBuildFailurePhase>,
550 failure_timings: WasmBuildFailureTimings,
551}
552
553impl ProgressReporter<'_> {
554 fn silent() -> Self {
555 Self {
556 config: WasmBuildProgressConfig {
557 heartbeat_interval: None,
558 emit_cargo_output: false,
559 },
560 observer: None,
561 last_event: Instant::now(),
562 failure_phase: None,
563 failure_timings: WasmBuildFailureTimings::default(),
564 }
565 }
566
567 fn observed(
568 config: WasmBuildProgressConfig,
569 observer: &'_ mut dyn FnMut(WasmBuildProgressEvent),
570 ) -> ProgressReporter<'_> {
571 ProgressReporter {
572 config,
573 observer: Some(observer),
574 last_event: Instant::now(),
575 failure_phase: None,
576 failure_timings: WasmBuildFailureTimings::default(),
577 }
578 }
579
580 fn emit(&mut self, event: WasmBuildProgressEvent) {
581 if let Some(observer) = &mut self.observer {
582 observer(event);
583 self.last_event = Instant::now();
584 }
585 }
586
587 const fn is_observed(&self) -> bool {
588 self.observer.is_some()
589 }
590
591 fn heartbeat_due_in(&self) -> Option<Duration> {
592 self.config
593 .heartbeat_interval
594 .map(|interval| interval.saturating_sub(self.last_event.elapsed()))
595 }
596
597 fn emit_heartbeat(&mut self, phase: WasmBuildProgressPhase, elapsed: Duration) {
598 self.emit(WasmBuildProgressEvent::Heartbeat { phase, elapsed });
599 }
600
601 fn emit_heartbeat_if_due(&mut self, phase: WasmBuildProgressPhase, elapsed: Duration) {
602 if self.heartbeat_due_in() == Some(Duration::ZERO) {
603 self.emit_heartbeat(phase, elapsed);
604 }
605 }
606
607 fn run_phase<T, F>(&mut self, phase: WasmBuildProgressPhase, operation: F) -> T
608 where
609 T: Send,
610 F: FnOnce() -> T + Send,
611 {
612 let started = Instant::now();
613 self.begin_phase(progress_failure_phase(phase));
614 let result = if !self.is_observed() || self.config.heartbeat_interval.is_none() {
615 operation()
616 } else {
617 thread::scope(|scope| {
618 let (finished, completion) = mpsc::sync_channel(0);
619 let worker = scope.spawn(move || {
620 let result = operation();
621 let _ = finished.send(());
622 result
623 });
624 loop {
625 let wait = self
626 .heartbeat_due_in()
627 .expect("observed phase must have a heartbeat interval");
628 match completion.recv_timeout(wait) {
629 Ok(()) | Err(RecvTimeoutError::Disconnected) => {
630 return worker
631 .join()
632 .unwrap_or_else(|panic| std::panic::resume_unwind(panic));
633 }
634 Err(RecvTimeoutError::Timeout) => {
635 self.emit_heartbeat(phase, started.elapsed());
636 }
637 }
638 }
639 })
640 };
641 self.record_phase(progress_failure_phase(phase), started.elapsed());
642 result
643 }
644
645 const fn begin_phase(&mut self, phase: WasmBuildFailurePhase) {
646 self.failure_phase = Some(phase);
647 }
648
649 fn record_phase(&mut self, phase: WasmBuildFailurePhase, elapsed: Duration) {
650 self.failure_phase = Some(phase);
651 let timings = &mut self.failure_timings;
652 match phase {
653 WasmBuildFailurePhase::Specification => {}
654 WasmBuildFailurePhase::ExactCacheCoordination => {
655 timings.exact_cache_coordination =
656 timings.exact_cache_coordination.saturating_add(elapsed);
657 }
658 WasmBuildFailurePhase::ToolIdentity => {
659 timings.input_resolution.tool_identity = timings
660 .input_resolution
661 .tool_identity
662 .saturating_add(elapsed);
663 timings.input_resolution.total =
664 timings.input_resolution.total.saturating_add(elapsed);
665 }
666 WasmBuildFailurePhase::CargoMetadata => {
667 timings.input_resolution.cargo_metadata = timings
668 .input_resolution
669 .cargo_metadata
670 .saturating_add(elapsed);
671 timings.input_resolution.total =
672 timings.input_resolution.total.saturating_add(elapsed);
673 }
674 WasmBuildFailurePhase::InputDiscovery => {
675 timings.input_resolution.input_discovery = timings
676 .input_resolution
677 .input_discovery
678 .saturating_add(elapsed);
679 timings.input_resolution.total =
680 timings.input_resolution.total.saturating_add(elapsed);
681 }
682 WasmBuildFailurePhase::ContentHashing => {
683 timings.input_resolution.content_hashing = timings
684 .input_resolution
685 .content_hashing
686 .saturating_add(elapsed);
687 timings.input_resolution.total =
688 timings.input_resolution.total.saturating_add(elapsed);
689 }
690 WasmBuildFailurePhase::SharedTargetCoordination => {
691 timings.shared_target_coordination = Some(
692 timings
693 .shared_target_coordination
694 .unwrap_or_default()
695 .saturating_add(elapsed),
696 );
697 }
698 WasmBuildFailurePhase::SharedTargetMaintenance => {
699 timings.shared_target_maintenance = Some(
700 timings
701 .shared_target_maintenance
702 .unwrap_or_default()
703 .saturating_add(elapsed),
704 );
705 }
706 WasmBuildFailurePhase::CargoBuild => {
707 timings.cargo_build = Some(
708 timings
709 .cargo_build
710 .unwrap_or_default()
711 .saturating_add(elapsed),
712 );
713 }
714 WasmBuildFailurePhase::ArtifactPublication => {
715 timings.artifact_publication = Some(
716 timings
717 .artifact_publication
718 .unwrap_or_default()
719 .saturating_add(elapsed),
720 );
721 }
722 WasmBuildFailurePhase::ExactCacheMaintenance => {
723 timings.exact_cache_maintenance = Some(
724 timings
725 .exact_cache_maintenance
726 .unwrap_or_default()
727 .saturating_add(elapsed),
728 );
729 }
730 WasmBuildFailurePhase::Cleanup => {
731 timings.cleanup = Some(timings.cleanup.unwrap_or_default().saturating_add(elapsed));
732 }
733 }
734 }
735
736 fn failure_details(&self, error: &WasmBuildError, total: Duration) -> WasmBuildFailureDetails {
737 let phase = self
738 .failure_phase
739 .unwrap_or_else(|| classify_unobserved_failure(error));
740 let mut timings = self.failure_timings;
741 timings.total = total;
742 WasmBuildFailureDetails { phase, timings }
743 }
744}
745
746const fn progress_failure_phase(phase: WasmBuildProgressPhase) -> WasmBuildFailurePhase {
747 match phase {
748 WasmBuildProgressPhase::ExactCacheLock => WasmBuildFailurePhase::ExactCacheCoordination,
749 WasmBuildProgressPhase::CargoIdentity | WasmBuildProgressPhase::RustcIdentity => {
750 WasmBuildFailurePhase::ToolIdentity
751 }
752 WasmBuildProgressPhase::CargoMetadata => WasmBuildFailurePhase::CargoMetadata,
753 WasmBuildProgressPhase::InputDiscovery => WasmBuildFailurePhase::InputDiscovery,
754 WasmBuildProgressPhase::ContentHashing => WasmBuildFailurePhase::ContentHashing,
755 WasmBuildProgressPhase::SharedTargetLock => WasmBuildFailurePhase::SharedTargetCoordination,
756 WasmBuildProgressPhase::SharedTargetMaintenance => {
757 WasmBuildFailurePhase::SharedTargetMaintenance
758 }
759 WasmBuildProgressPhase::CargoBuild => WasmBuildFailurePhase::CargoBuild,
760 WasmBuildProgressPhase::ArtifactPublication => WasmBuildFailurePhase::ArtifactPublication,
761 WasmBuildProgressPhase::ExactCacheMaintenance => {
762 WasmBuildFailurePhase::ExactCacheMaintenance
763 }
764 }
765}
766
767const fn classify_unobserved_failure(error: &WasmBuildError) -> WasmBuildFailurePhase {
768 match error {
769 WasmBuildError::InvalidSpec { .. } => WasmBuildFailurePhase::Specification,
770 WasmBuildError::CommandSpawn { phase, .. }
771 | WasmBuildError::CommandFailed { phase, .. } => match phase {
772 WasmBuildPhase::CargoIdentity | WasmBuildPhase::RustcIdentity => {
773 WasmBuildFailurePhase::ToolIdentity
774 }
775 WasmBuildPhase::CargoMetadata => WasmBuildFailurePhase::CargoMetadata,
776 WasmBuildPhase::CargoBuild => WasmBuildFailurePhase::CargoBuild,
777 },
778 WasmBuildError::InvalidMetadata { .. } => WasmBuildFailurePhase::CargoMetadata,
779 WasmBuildError::InvalidCargoConfiguration { .. } => WasmBuildFailurePhase::InputDiscovery,
780 WasmBuildError::MissingArtifacts { .. } => WasmBuildFailurePhase::ArtifactPublication,
781 WasmBuildError::InputsChangedDuringAcquisition { .. } => {
782 WasmBuildFailurePhase::ContentHashing
783 }
784 WasmBuildError::PreparedInputSnapshotInvalidated => {
785 WasmBuildFailurePhase::ArtifactPublication
786 }
787 WasmBuildError::FailedBuildCleanup { .. } => WasmBuildFailurePhase::Cleanup,
788 WasmBuildError::Io { .. } => WasmBuildFailurePhase::ExactCacheCoordination,
789 }
790}
791
792#[non_exhaustive]
794#[derive(Clone, Copy, Debug, Eq, PartialEq)]
795pub enum WasmBuildPhase {
796 CargoMetadata,
798 CargoIdentity,
800 RustcIdentity,
802 CargoBuild,
804}
805
806#[non_exhaustive]
808#[derive(Debug)]
809pub enum WasmBuildError {
810 InvalidSpec { message: String },
812 Io {
814 operation: &'static str,
815 path: PathBuf,
816 source: io::Error,
817 },
818 CommandSpawn {
820 phase: WasmBuildPhase,
821 program: OsString,
822 source: io::Error,
823 },
824 CommandFailed {
826 phase: WasmBuildPhase,
827 status: ExitStatus,
828 stdout: String,
829 stderr: String,
830 },
831 InvalidMetadata { message: String },
833 InvalidCargoConfiguration { path: PathBuf, message: String },
835 MissingArtifacts { paths: Vec<PathBuf> },
837 InputsChangedDuringAcquisition {
839 before: InputDigest,
840 after: InputDigest,
841 },
842 PreparedInputSnapshotInvalidated,
844 FailedBuildCleanup {
846 build_error: Box<Self>,
847 path: PathBuf,
848 source: io::Error,
849 },
850}
851
852impl WasmBuildSpec {
853 #[must_use]
860 pub fn new(
861 workspace_root: &Path,
862 target_dir: &Path,
863 packages: &[&str],
864 profile_target_dir: &str,
865 ) -> Self {
866 Self {
867 workspace_root: workspace_root.to_owned(),
868 target_dir: target_dir.to_owned(),
869 packages: packages
870 .iter()
871 .map(|package| (*package).to_owned())
872 .collect(),
873 profile_target_dir: profile_target_dir.to_owned(),
874 cargo_profile_args: Vec::new(),
875 extra_env: BTreeMap::new(),
876 inherited_env: BTreeSet::new(),
877 additional_inputs: Vec::new(),
878 target: DEFAULT_TARGET.to_owned(),
879 cargo_program: std::env::var_os("CARGO").unwrap_or_else(|| "cargo".into()),
880 rustc_program: std::env::var_os("RUSTC").unwrap_or_else(|| "rustc".into()),
881 cache_mode: WasmBuildCacheMode::Isolated,
882 prune_policy: None,
883 prune_interval: None,
884 shared_incremental_maintenance_config: None,
885 }
886 }
887
888 #[must_use]
893 pub fn with_cargo_profile_args<I, S>(mut self, arguments: I) -> Self
894 where
895 I: IntoIterator<Item = S>,
896 S: AsRef<OsStr>,
897 {
898 self.cargo_profile_args = arguments
899 .into_iter()
900 .map(|argument| argument.as_ref().to_owned())
901 .collect();
902 self
903 }
904
905 #[must_use]
910 pub fn with_extra_env<I, K, V>(mut self, environment: I) -> Self
911 where
912 I: IntoIterator<Item = (K, V)>,
913 K: Into<OsString>,
914 V: Into<OsString>,
915 {
916 self.extra_env = environment
917 .into_iter()
918 .map(|(key, value)| (key.into(), value.into()))
919 .collect();
920 self
921 }
922
923 #[must_use]
928 pub fn with_inherited_env<I, S>(mut self, names: I) -> Self
929 where
930 I: IntoIterator<Item = S>,
931 S: Into<OsString>,
932 {
933 self.inherited_env.extend(names.into_iter().map(Into::into));
934 self
935 }
936
937 #[must_use]
942 pub fn with_additional_inputs<I, P>(mut self, paths: I) -> Self
943 where
944 I: IntoIterator<Item = P>,
945 P: Into<PathBuf>,
946 {
947 self.additional_inputs
948 .extend(paths.into_iter().map(Into::into));
949 self
950 }
951
952 #[must_use]
954 pub fn with_target(mut self, target: &str) -> Self {
955 target.clone_into(&mut self.target);
956 self
957 }
958
959 #[must_use]
961 pub fn with_cargo_program(mut self, program: impl Into<OsString>) -> Self {
962 self.cargo_program = program.into();
963 self
964 }
965
966 #[must_use]
968 pub fn with_rustc_program(mut self, program: impl Into<OsString>) -> Self {
969 self.rustc_program = program.into();
970 self
971 }
972
973 #[must_use]
979 pub fn with_shared_incremental_target(mut self, target_dir: impl Into<PathBuf>) -> Self {
980 self.cache_mode = WasmBuildCacheMode::SharedIncremental {
981 target_dir: target_dir.into(),
982 };
983 self
984 }
985
986 #[must_use]
997 pub const fn with_shared_incremental_target_maintenance_at_most_every(
998 mut self,
999 policy: SharedIncrementalTargetPrunePolicy,
1000 minimum_interval: Duration,
1001 ) -> Self {
1002 self.shared_incremental_maintenance_config = Some(
1003 SharedIncrementalTargetMaintenanceConfig::new(policy, minimum_interval),
1004 );
1005 self
1006 }
1007
1008 #[must_use]
1014 pub const fn with_shared_incremental_target_maintenance(
1015 mut self,
1016 config: SharedIncrementalTargetMaintenanceConfig,
1017 ) -> Self {
1018 self.shared_incremental_maintenance_config = Some(config);
1019 self
1020 }
1021
1022 #[must_use]
1028 pub const fn with_prune_policy(mut self, policy: ArtifactCachePrunePolicy) -> Self {
1029 self.prune_policy = Some(policy);
1030 self.prune_interval = None;
1031 self
1032 }
1033
1034 #[must_use]
1041 pub const fn with_prune_policy_at_most_every(
1042 mut self,
1043 policy: ArtifactCachePrunePolicy,
1044 minimum_interval: Duration,
1045 ) -> Self {
1046 self.prune_policy = Some(policy);
1047 self.prune_interval = Some(minimum_interval);
1048 self
1049 }
1050
1051 #[must_use]
1053 pub fn workspace_root(&self) -> &Path {
1054 &self.workspace_root
1055 }
1056
1057 #[must_use]
1059 pub fn target_dir(&self) -> &Path {
1060 &self.target_dir
1061 }
1062
1063 #[must_use]
1065 pub fn packages(&self) -> &[String] {
1066 &self.packages
1067 }
1068
1069 #[must_use]
1071 pub const fn cache_mode(&self) -> &WasmBuildCacheMode {
1072 &self.cache_mode
1073 }
1074
1075 #[must_use]
1077 pub const fn prune_policy(&self) -> Option<ArtifactCachePrunePolicy> {
1078 self.prune_policy
1079 }
1080
1081 #[must_use]
1083 pub const fn prune_interval(&self) -> Option<Duration> {
1084 self.prune_interval
1085 }
1086
1087 #[must_use]
1089 pub const fn shared_incremental_target_maintenance(
1090 &self,
1091 ) -> Option<SharedIncrementalTargetMaintenanceConfig> {
1092 self.shared_incremental_maintenance_config
1093 }
1094}
1095
1096impl WasmBuildOutcome {
1097 #[must_use]
1099 pub const fn record(&self) -> &WasmBuildRecord {
1100 match self {
1101 Self::Built(record) | Self::Reused(record) => record,
1102 }
1103 }
1104
1105 #[must_use]
1107 pub const fn is_reused(&self) -> bool {
1108 matches!(self, Self::Reused(_))
1109 }
1110}
1111
1112impl WasmBuildRecord {
1113 #[must_use]
1115 pub const fn fingerprint(&self) -> InputDigest {
1116 self.fingerprint
1117 }
1118
1119 #[must_use]
1121 pub const fn input_digest(&self) -> InputDigest {
1122 self.input_digest
1123 }
1124
1125 #[must_use]
1132 pub fn exact_cache_path(&self) -> &Path {
1133 self.retention.path()
1134 }
1135
1136 #[must_use]
1141 pub fn artifacts(&self) -> &[PathBuf] {
1142 &self.artifacts
1143 }
1144
1145 #[must_use]
1147 pub const fn timings(&self) -> WasmBuildTimings {
1148 self.timings
1149 }
1150
1151 #[must_use]
1153 pub const fn maintenance(&self) -> Option<&ArtifactCacheMaintenance> {
1154 self.maintenance.as_ref()
1155 }
1156
1157 #[must_use]
1159 pub const fn shared_incremental_maintenance(
1160 &self,
1161 ) -> Option<&SharedIncrementalTargetMaintenanceOutcome> {
1162 self.shared_incremental_maintenance.as_ref()
1163 }
1164}
1165
1166impl WasmBuildTimings {
1167 #[must_use]
1169 pub const fn lock_wait(self) -> Duration {
1170 self.lock_wait
1171 }
1172
1173 #[must_use]
1175 pub const fn shared_incremental_lock_wait(self) -> Option<Duration> {
1176 self.shared_incremental_lock_wait
1177 }
1178
1179 #[must_use]
1181 pub const fn input_resolution(self) -> WasmInputResolutionTimings {
1182 self.input_resolution
1183 }
1184
1185 #[must_use]
1187 pub const fn cargo_build(self) -> Option<Duration> {
1188 self.cargo_build
1189 }
1190
1191 #[must_use]
1193 pub const fn cache_maintenance(self) -> Option<Duration> {
1194 self.cache_maintenance
1195 }
1196
1197 #[must_use]
1199 pub const fn total(self) -> Duration {
1200 self.total
1201 }
1202
1203 pub(super) const fn saturating_add(self, other: Self) -> Self {
1204 let mut input_resolution = self.input_resolution;
1205 input_resolution.include(other.input_resolution);
1206 Self {
1207 lock_wait: self.lock_wait.saturating_add(other.lock_wait),
1208 shared_incremental_lock_wait: saturating_add_optional_duration(
1209 self.shared_incremental_lock_wait,
1210 other.shared_incremental_lock_wait,
1211 ),
1212 input_resolution,
1213 cargo_build: saturating_add_optional_duration(self.cargo_build, other.cargo_build),
1214 cache_maintenance: saturating_add_optional_duration(
1215 self.cache_maintenance,
1216 other.cache_maintenance,
1217 ),
1218 total: self.total.saturating_add(other.total),
1219 }
1220 }
1221}
1222
1223impl WasmInputResolutionTimings {
1224 #[must_use]
1226 pub const fn tool_identity(self) -> Duration {
1227 self.tool_identity
1228 }
1229
1230 #[must_use]
1232 pub const fn cargo_metadata(self) -> Duration {
1233 self.cargo_metadata
1234 }
1235
1236 #[must_use]
1238 pub const fn input_discovery(self) -> Duration {
1239 self.input_discovery
1240 }
1241
1242 #[must_use]
1244 pub const fn content_hashing(self) -> Duration {
1245 self.content_hashing
1246 }
1247
1248 #[must_use]
1250 pub const fn total(self) -> Duration {
1251 self.total
1252 }
1253
1254 const fn include(&mut self, other: Self) {
1255 self.tool_identity = self.tool_identity.saturating_add(other.tool_identity);
1256 self.cargo_metadata = self.cargo_metadata.saturating_add(other.cargo_metadata);
1257 self.input_discovery = self.input_discovery.saturating_add(other.input_discovery);
1258 self.content_hashing = self.content_hashing.saturating_add(other.content_hashing);
1259 self.total = self.total.saturating_add(other.total);
1260 }
1261}
1262
1263impl WasmBuildFailureDetails {
1264 pub(super) fn specification(total: Duration) -> Self {
1265 Self {
1266 phase: WasmBuildFailurePhase::Specification,
1267 timings: WasmBuildFailureTimings {
1268 total,
1269 ..WasmBuildFailureTimings::default()
1270 },
1271 }
1272 }
1273
1274 #[must_use]
1276 pub const fn phase(self) -> WasmBuildFailurePhase {
1277 self.phase
1278 }
1279
1280 #[must_use]
1282 pub const fn timings(self) -> WasmBuildFailureTimings {
1283 self.timings
1284 }
1285}
1286
1287impl WasmBuildFailureTimings {
1288 #[must_use]
1290 pub const fn exact_cache_coordination(self) -> Duration {
1291 self.exact_cache_coordination
1292 }
1293
1294 #[must_use]
1296 pub const fn shared_target_coordination(self) -> Option<Duration> {
1297 self.shared_target_coordination
1298 }
1299
1300 #[must_use]
1302 pub const fn input_resolution(self) -> WasmInputResolutionTimings {
1303 self.input_resolution
1304 }
1305
1306 #[must_use]
1308 pub const fn shared_target_maintenance(self) -> Option<Duration> {
1309 self.shared_target_maintenance
1310 }
1311
1312 #[must_use]
1314 pub const fn cargo_build(self) -> Option<Duration> {
1315 self.cargo_build
1316 }
1317
1318 #[must_use]
1320 pub const fn artifact_publication(self) -> Option<Duration> {
1321 self.artifact_publication
1322 }
1323
1324 #[must_use]
1326 pub const fn exact_cache_maintenance(self) -> Option<Duration> {
1327 self.exact_cache_maintenance
1328 }
1329
1330 #[must_use]
1332 pub const fn cleanup(self) -> Option<Duration> {
1333 self.cleanup
1334 }
1335
1336 #[must_use]
1338 pub const fn total(self) -> Duration {
1339 self.total
1340 }
1341}
1342
1343impl CargoBuildInput {
1344 #[must_use]
1346 pub fn label(&self) -> &Path {
1347 &self.label
1348 }
1349
1350 #[must_use]
1352 pub fn path(&self) -> &Path {
1353 &self.path
1354 }
1355}
1356
1357impl ResolvedCargoBuildInputs {
1358 #[must_use]
1360 pub const fn fingerprint(&self) -> InputDigest {
1361 self.fingerprint
1362 }
1363
1364 #[must_use]
1369 pub const fn input_digest(&self) -> InputDigest {
1370 self.input_digest
1371 }
1372
1373 #[must_use]
1378 pub const fn validation_digest(&self) -> InputDigest {
1379 self.validation_digest
1380 }
1381
1382 #[must_use]
1384 pub fn inputs(&self) -> &[CargoBuildInput] {
1385 &self.inputs
1386 }
1387
1388 #[must_use]
1393 pub fn exclusions(&self) -> &[PathBuf] {
1394 &self.exclusions
1395 }
1396
1397 #[must_use]
1399 pub const fn timings(&self) -> WasmInputResolutionTimings {
1400 self.timings
1401 }
1402
1403 pub fn is_current(&self, spec: &WasmBuildSpec) -> Result<bool, WasmBuildError> {
1405 resolve_cargo_build_inputs(spec).map(|current| current.fingerprint == self.fingerprint)
1406 }
1407
1408 pub fn is_content_current(&self) -> Result<bool, WasmBuildError> {
1415 self.current_validation_digest()
1416 .map(|current| current == self.validation_digest)
1417 }
1418
1419 pub(super) fn current_validation_digest(&self) -> Result<InputDigest, WasmBuildError> {
1420 digest_labeled_paths_composable(
1421 "wasm-source-inputs-v1",
1422 self.inputs
1423 .iter()
1424 .map(|input| (input.label.as_path(), input.path.as_path())),
1425 &self.exclusions,
1426 &mut LabeledPathDigestCache::default(),
1427 )
1428 .map_err(|source| WasmBuildError::Io {
1429 operation: "rehash resolved Cargo build inputs",
1430 path: self
1431 .inputs
1432 .first()
1433 .map_or_else(PathBuf::new, |input| input.path.clone()),
1434 source,
1435 })
1436 }
1437}
1438
1439impl WasmBuildSessionState {
1440 pub(super) fn new() -> Self {
1441 Self {
1442 snapshots: Vec::new(),
1443 digest_cache: LabeledPathDigestCache::default(),
1444 snapshot_reuses: 0,
1445 invalidated: false,
1446 }
1447 }
1448
1449 pub(super) const fn snapshot_count(&self) -> usize {
1450 self.snapshots.len()
1451 }
1452
1453 pub(super) const fn snapshot_reuses(&self) -> usize {
1454 self.snapshot_reuses
1455 }
1456
1457 pub(super) const fn is_invalidated(&self) -> bool {
1458 self.invalidated
1459 }
1460
1461 fn reuse(&mut self, spec: &WasmBuildSpec) -> Option<ResolvedCargoBuildInputs> {
1462 let (_, snapshot) = self
1463 .snapshots
1464 .iter()
1465 .find(|(candidate, _)| candidate == spec)?;
1466 self.snapshot_reuses = self.snapshot_reuses.saturating_add(1);
1467 let mut snapshot = snapshot.clone();
1468 snapshot.timings = WasmInputResolutionTimings::default();
1469 Some(snapshot)
1470 }
1471
1472 fn remember(&mut self, spec: &WasmBuildSpec, resolved: &ResolvedCargoBuildInputs) {
1473 if self
1474 .snapshots
1475 .iter()
1476 .any(|(candidate, _)| candidate == spec)
1477 {
1478 return;
1479 }
1480 let mut snapshot = resolved.clone();
1481 snapshot.timings = WasmInputResolutionTimings::default();
1482 self.snapshots.push((spec.clone(), snapshot));
1483 }
1484
1485 fn invalidate(&mut self) {
1486 self.snapshots.clear();
1487 self.digest_cache = LabeledPathDigestCache::default();
1488 self.invalidated = true;
1489 }
1490}
1491
1492impl WasmBuildInputSnapshotState {
1493 pub(super) fn prepare(specs: &[WasmBuildSpec]) -> Result<Self, WasmBuildError> {
1494 for spec in specs {
1495 validate_spec(spec)?;
1496 }
1497 let mut resolver = WasmBuildBatchInputResolver::new(specs, None);
1498 let mut snapshots = Vec::with_capacity(specs.len());
1499 let mut preparation_timings = WasmInputResolutionTimings::default();
1500 let mut progress = ProgressReporter::silent();
1501 for (index, spec) in specs.iter().enumerate() {
1502 let mut prepared_input = resolver.resolve(index, &mut progress)?;
1503 preparation_timings.include(prepared_input.timings);
1504 prepared_input.timings = WasmInputResolutionTimings::default();
1505 snapshots.push((spec.clone(), prepared_input));
1506 }
1507 Ok(Self {
1508 snapshots,
1509 preparation_metrics: resolver.metrics(),
1510 preparation_timings,
1511 reader_reuses: AtomicUsize::new(0),
1512 invalidation: Arc::new(RwLock::new(false)),
1513 })
1514 }
1515
1516 pub(super) fn contains(&self, spec: &WasmBuildSpec) -> bool {
1517 self.snapshots
1518 .iter()
1519 .any(|(candidate, _)| candidate == spec)
1520 }
1521
1522 fn reuse(&self, spec: &WasmBuildSpec) -> Option<ResolvedCargoBuildInputs> {
1523 let (_, snapshot) = self
1524 .snapshots
1525 .iter()
1526 .find(|(candidate, _)| candidate == spec)?;
1527 let mut current = self.reader_reuses.load(Ordering::Relaxed);
1528 loop {
1529 match self.reader_reuses.compare_exchange_weak(
1530 current,
1531 current.saturating_add(1),
1532 Ordering::Relaxed,
1533 Ordering::Relaxed,
1534 ) {
1535 Ok(_) => break,
1536 Err(observed) => current = observed,
1537 }
1538 }
1539 Some(snapshot.clone())
1540 }
1541
1542 pub(super) const fn specification_count(&self) -> usize {
1543 self.snapshots.len()
1544 }
1545
1546 pub(super) const fn preparation_metrics(&self) -> WasmBuildBatchInputMetrics {
1547 self.preparation_metrics
1548 }
1549
1550 pub(super) const fn preparation_timings(&self) -> WasmInputResolutionTimings {
1551 self.preparation_timings
1552 }
1553
1554 pub(super) fn reader_reuses(&self) -> usize {
1555 self.reader_reuses.load(Ordering::Relaxed)
1556 }
1557
1558 pub(super) fn is_invalidated(&self) -> bool {
1559 *self
1560 .invalidation
1561 .read()
1562 .unwrap_or_else(std::sync::PoisonError::into_inner)
1563 }
1564
1565 fn invalidate(&self) {
1566 *self
1567 .invalidation
1568 .write()
1569 .unwrap_or_else(std::sync::PoisonError::into_inner) = true;
1570 }
1571
1572 fn invalidation(&self) -> Arc<RwLock<bool>> {
1573 Arc::clone(&self.invalidation)
1574 }
1575}
1576
1577impl<'a, 'session> WasmBuildBatchInputResolver<'a, 'session> {
1578 pub(super) fn new(
1579 specs: impl IntoIterator<Item = &'a WasmBuildSpec>,
1580 mut reuse: Option<WasmBuildInputReuse<'session>>,
1581 ) -> Self {
1582 let specs = specs.into_iter().collect::<Vec<_>>();
1583 let mut keys = Vec::<BatchResolutionKey>::new();
1584 let mut groups = Vec::<BatchResolutionGroup>::new();
1585 let mut group_by_index = Vec::with_capacity(specs.len());
1586 for (index, spec) in specs.iter().copied().enumerate() {
1587 let key = BatchResolutionKey::for_spec(spec);
1588 let group = keys
1589 .iter()
1590 .position(|candidate| *candidate == key)
1591 .unwrap_or_else(|| {
1592 keys.push(key);
1593 groups.push(BatchResolutionGroup {
1594 indexes: Vec::new(),
1595 });
1596 groups.len() - 1
1597 });
1598 groups[group].indexes.push(index);
1599 group_by_index.push(group);
1600 }
1601 let mut metrics = WasmBuildBatchInputMetrics::default();
1602 let resolved = specs
1603 .iter()
1604 .map(|spec| match reuse.as_mut() {
1605 Some(WasmBuildInputReuse::Session(session)) => {
1606 let reused = session.reuse(spec);
1607 if reused.is_some() {
1608 metrics.session_reuses = metrics.session_reuses.saturating_add(1);
1609 }
1610 reused.map(Ok)
1611 }
1612 Some(WasmBuildInputReuse::Snapshot(snapshot)) => {
1613 let reused = snapshot
1614 .reuse(spec)
1615 .expect("prepared input snapshot must contain every reader specification");
1616 metrics.prepared_reuses = metrics.prepared_reuses.saturating_add(1);
1617 Some(Ok(reused))
1618 }
1619 None => None,
1620 })
1621 .collect();
1622 Self {
1623 specs,
1624 groups,
1625 group_by_index,
1626 resolved,
1627 reuse,
1628 metrics,
1629 }
1630 }
1631
1632 pub(super) const fn metrics(&self) -> WasmBuildBatchInputMetrics {
1633 self.metrics
1634 }
1635
1636 pub(super) fn invalidate_source_lease(&mut self) {
1637 match self.reuse.as_mut() {
1638 Some(WasmBuildInputReuse::Session(session)) => {
1639 session.invalidate();
1640 for resolved in &mut self.resolved {
1644 *resolved = None;
1645 }
1646 self.reuse = None;
1647 }
1648 Some(WasmBuildInputReuse::Snapshot(snapshot)) => snapshot.invalidate(),
1649 None => {}
1650 }
1651 }
1652
1653 pub(super) const fn assumes_sources_immutable(&self) -> bool {
1654 self.reuse.is_some()
1655 }
1656
1657 pub(super) fn prepared_invalidation(&self) -> Option<Arc<RwLock<bool>>> {
1658 match self.reuse.as_ref() {
1659 Some(WasmBuildInputReuse::Snapshot(snapshot)) => Some(snapshot.invalidation()),
1660 _ => None,
1661 }
1662 }
1663
1664 fn resolve(
1665 &mut self,
1666 index: usize,
1667 progress: &mut ProgressReporter<'_>,
1668 ) -> Result<ResolvedCargoBuildInputs, WasmBuildError> {
1669 if self.resolved[index].is_none() {
1670 self.resolve_group(index, progress)?;
1671 }
1672 self.resolved[index]
1673 .take()
1674 .expect("resolved batch input must be populated")
1675 .map_err(|(phase, error)| {
1676 progress.begin_phase(phase);
1677 error
1678 })
1679 }
1680
1681 fn resolve_group(
1682 &mut self,
1683 active_index: usize,
1684 progress: &mut ProgressReporter<'_>,
1685 ) -> Result<(), WasmBuildError> {
1686 let total_started = Instant::now();
1687 let indexes = self.groups[self.group_by_index[active_index]]
1688 .indexes
1689 .clone();
1690 let active = self.specs[active_index];
1691
1692 let (cargo_identity, rustc_identity, tool_identity) =
1693 resolve_tool_identity(active, progress)?;
1694
1695 let metadata_started = Instant::now();
1696 let metadata = progress.run_phase(WasmBuildProgressPhase::CargoMetadata, || {
1697 cargo_metadata(active)
1698 })?;
1699 let cargo_metadata = metadata_started.elapsed();
1700
1701 let (discovered, input_discovery) =
1702 self.discover_group_inputs(indexes, &metadata, progress);
1703
1704 let hashing_started = Instant::now();
1705 let mut batch_digest_cache = LabeledPathDigestCache::default();
1706 let digest_cache = match self.reuse.as_mut() {
1707 Some(WasmBuildInputReuse::Session(session)) => &mut session.digest_cache,
1708 _ => &mut batch_digest_cache,
1709 };
1710 let workspace_root = active.workspace_root.clone();
1711 let resolved_inputs = progress.run_phase(WasmBuildProgressPhase::ContentHashing, || {
1712 discovered
1713 .into_iter()
1714 .map(|(index, inputs, exclusions)| {
1715 let result = digest_resolved_local_inputs(
1716 &inputs,
1717 &exclusions,
1718 digest_cache,
1719 &workspace_root,
1720 "hash batched Wasm build inputs",
1721 "hash batched semantic Wasm build inputs",
1722 )
1723 .map(|(input_digest, validation_digest)| {
1724 (
1725 inputs.validation_inputs,
1726 exclusions,
1727 input_digest,
1728 validation_digest,
1729 )
1730 });
1731 (index, result)
1732 })
1733 .collect::<Vec<_>>()
1734 });
1735 let content_hashing = hashing_started.elapsed();
1736 let timings = WasmInputResolutionTimings {
1737 tool_identity,
1738 cargo_metadata,
1739 input_discovery,
1740 content_hashing,
1741 total: total_started.elapsed(),
1742 };
1743 let resolved_count = resolved_inputs
1744 .iter()
1745 .filter(|(_, result)| result.is_ok())
1746 .count();
1747 self.metrics.runs += usize::from(resolved_count > 0);
1748 self.metrics.reuses += resolved_count.saturating_sub(1);
1749 let timing_index = resolved_inputs
1750 .iter()
1751 .find(|(index, result)| *index == active_index && result.is_ok())
1752 .or_else(|| resolved_inputs.iter().find(|(_, result)| result.is_ok()))
1753 .map(|(index, _)| *index);
1754 for (index, result) in resolved_inputs {
1755 let (inputs, exclusions, input_digest, validation_digest) = match result {
1756 Ok(resolved) => resolved,
1757 Err(error) => {
1758 self.resolved[index] =
1759 Some(Err((WasmBuildFailurePhase::ContentHashing, error)));
1760 continue;
1761 }
1762 };
1763 let spec = self.specs[index];
1764 let resolved = ResolvedCargoBuildInputs {
1765 fingerprint: finish_build_fingerprint(
1766 spec,
1767 &cargo_identity,
1768 &rustc_identity,
1769 input_digest,
1770 ),
1771 input_digest,
1772 validation_digest,
1773 inputs: inputs
1774 .into_iter()
1775 .map(|(label, path)| CargoBuildInput { label, path })
1776 .collect(),
1777 exclusions,
1778 timings: if Some(index) == timing_index {
1779 timings
1780 } else {
1781 WasmInputResolutionTimings::default()
1782 },
1783 };
1784 if let Some(WasmBuildInputReuse::Session(session)) = self.reuse.as_mut() {
1785 session.remember(spec, &resolved);
1786 }
1787 self.resolved[index] = Some(Ok(resolved));
1788 }
1789 Ok(())
1790 }
1791
1792 fn discover_group_inputs(
1793 &mut self,
1794 indexes: Vec<usize>,
1795 metadata: &Value,
1796 progress: &mut ProgressReporter<'_>,
1797 ) -> (Vec<(usize, ResolvedLocalInputs, Vec<PathBuf>)>, Duration) {
1798 let started = Instant::now();
1799 let pending = indexes
1800 .into_iter()
1801 .filter(|index| {
1802 self.resolved[*index].is_none() && validate_spec(self.specs[*index]).is_ok()
1803 })
1804 .collect::<Vec<_>>();
1805 let results = progress.run_phase(WasmBuildProgressPhase::InputDiscovery, || {
1806 let parsed = ParsedCargoMetadata::parse(metadata);
1807 pending
1808 .into_iter()
1809 .map(|index| {
1810 let spec = self.specs[index];
1811 let result = (|| {
1812 let parsed = parsed
1813 .as_ref()
1814 .map_err(|message| invalid_metadata(message))?;
1815 resolve_local_inputs(spec, parsed)
1816 })();
1817 (index, result)
1818 })
1819 .collect::<Vec<_>>()
1820 });
1821 let mut discovered = Vec::new();
1822 for (index, result) in results {
1823 match result {
1824 Ok((inputs, exclusions)) => discovered.push((index, inputs, exclusions)),
1825 Err(error) => {
1826 self.resolved[index] =
1827 Some(Err((WasmBuildFailurePhase::InputDiscovery, error)));
1828 }
1829 }
1830 }
1831 (discovered, started.elapsed())
1832 }
1833}
1834
1835fn resolve_tool_identity(
1836 spec: &WasmBuildSpec,
1837 progress: &mut ProgressReporter<'_>,
1838) -> Result<(Vec<u8>, Vec<u8>, Duration), WasmBuildError> {
1839 let started = Instant::now();
1840 let cargo_identity = progress.run_phase(WasmBuildProgressPhase::CargoIdentity, || {
1841 command_identity(
1842 spec,
1843 WasmBuildPhase::CargoIdentity,
1844 &spec.cargo_program,
1845 &["--version", "--verbose"],
1846 )
1847 })?;
1848 let rustc_program = spec
1849 .extra_env
1850 .get(OsStr::new("RUSTC"))
1851 .unwrap_or(&spec.rustc_program);
1852 let rustc_identity = progress.run_phase(WasmBuildProgressPhase::RustcIdentity, || {
1853 command_identity(spec, WasmBuildPhase::RustcIdentity, rustc_program, &["-vV"])
1854 })?;
1855 Ok((cargo_identity, rustc_identity, started.elapsed()))
1856}
1857
1858impl<'a> BatchResolutionKey<'a> {
1859 fn for_spec(spec: &'a WasmBuildSpec) -> Self {
1860 Self {
1861 workspace_root: &spec.workspace_root,
1862 cargo_program: &spec.cargo_program,
1863 rustc_program: spec
1864 .extra_env
1865 .get(OsStr::new("RUSTC"))
1866 .unwrap_or(&spec.rustc_program),
1867 metadata_arguments: metadata_arguments(&spec.cargo_profile_args),
1868 environment: effective_environment(spec),
1869 }
1870 }
1871}
1872
1873impl SharedIncrementalTargetInspection {
1874 #[must_use]
1876 pub fn target_dir(&self) -> &Path {
1877 &self.target_dir
1878 }
1879
1880 #[must_use]
1882 pub const fn logical_size_bytes(&self) -> u64 {
1883 self.logical_size_bytes
1884 }
1885
1886 #[must_use]
1888 pub const fn last_used(&self) -> SystemTime {
1889 self.last_used
1890 }
1891
1892 #[must_use]
1894 pub const fn lock_wait(&self) -> Duration {
1895 self.lock_wait
1896 }
1897}
1898
1899impl SharedIncrementalTargetPrunePolicy {
1900 #[must_use]
1902 pub const fn new() -> Self {
1903 Self {
1904 max_age: None,
1905 max_size_bytes: None,
1906 }
1907 }
1908
1909 #[must_use]
1911 pub const fn with_max_age(mut self, max_age: Duration) -> Self {
1912 self.max_age = Some(max_age);
1913 self
1914 }
1915
1916 #[must_use]
1918 pub const fn with_max_size_bytes(mut self, bytes: u64) -> Self {
1919 self.max_size_bytes = Some(bytes);
1920 self
1921 }
1922
1923 #[must_use]
1925 pub const fn max_age(self) -> Option<Duration> {
1926 self.max_age
1927 }
1928
1929 #[must_use]
1931 pub const fn max_size_bytes(self) -> Option<u64> {
1932 self.max_size_bytes
1933 }
1934
1935 fn maintenance_identity(self) -> String {
1936 format!(
1937 "age={:?};size={:?}",
1938 self.max_age.map(|duration| duration.as_nanos()),
1939 self.max_size_bytes
1940 )
1941 }
1942}
1943
1944impl SharedIncrementalTargetMaintenanceConfig {
1945 #[must_use]
1947 pub const fn new(
1948 policy: SharedIncrementalTargetPrunePolicy,
1949 minimum_interval: Duration,
1950 ) -> Self {
1951 Self {
1952 policy,
1953 minimum_interval,
1954 failure_mode: SharedIncrementalTargetMaintenanceFailureMode::Strict,
1955 }
1956 }
1957
1958 #[must_use]
1960 pub const fn with_failure_mode(
1961 mut self,
1962 failure_mode: SharedIncrementalTargetMaintenanceFailureMode,
1963 ) -> Self {
1964 self.failure_mode = failure_mode;
1965 self
1966 }
1967
1968 #[must_use]
1970 pub const fn policy(self) -> SharedIncrementalTargetPrunePolicy {
1971 self.policy
1972 }
1973
1974 #[must_use]
1976 pub const fn minimum_interval(self) -> Duration {
1977 self.minimum_interval
1978 }
1979
1980 #[must_use]
1982 pub const fn failure_mode(self) -> SharedIncrementalTargetMaintenanceFailureMode {
1983 self.failure_mode
1984 }
1985}
1986
1987impl SharedIncrementalTargetMaintenance {
1988 #[must_use]
1990 pub fn target_dir(&self) -> &Path {
1991 &self.target_dir
1992 }
1993
1994 #[must_use]
1996 pub const fn logical_size_bytes_before(&self) -> u64 {
1997 self.logical_size_bytes_before
1998 }
1999
2000 #[must_use]
2002 pub const fn logical_size_bytes_after(&self) -> u64 {
2003 self.logical_size_bytes_after
2004 }
2005
2006 #[must_use]
2008 pub const fn last_used_before(&self) -> SystemTime {
2009 self.last_used_before
2010 }
2011
2012 #[must_use]
2014 pub const fn was_cleared(&self) -> bool {
2015 self.cleared
2016 }
2017
2018 #[must_use]
2020 pub const fn lock_wait(&self) -> Duration {
2021 self.lock_wait
2022 }
2023
2024 #[must_use]
2026 pub const fn maintenance(&self) -> Duration {
2027 self.maintenance
2028 }
2029}
2030
2031impl std::fmt::Display for SharedIncrementalTargetMaintenance {
2032 fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
2033 write!(
2034 formatter,
2035 "target={} action={} bytes={}=>{} lock={:?} maintenance={:?}",
2036 self.target_dir.display(),
2037 if self.cleared { "cleared" } else { "retained" },
2038 self.logical_size_bytes_before,
2039 self.logical_size_bytes_after,
2040 self.lock_wait,
2041 self.maintenance,
2042 )
2043 }
2044}
2045
2046impl SharedIncrementalTargetMaintenanceOutcome {
2047 #[must_use]
2049 pub fn target_dir(&self) -> &Path {
2050 match self {
2051 Self::Missing { target_dir }
2052 | Self::Skipped { target_dir, .. }
2053 | Self::Failed { target_dir, .. } => target_dir,
2054 Self::Performed { maintenance, .. } => maintenance.target_dir(),
2055 }
2056 }
2057
2058 #[must_use]
2060 pub const fn maintenance(&self) -> Option<&SharedIncrementalTargetMaintenance> {
2061 match self {
2062 Self::Performed { maintenance, .. } => Some(maintenance),
2063 Self::Missing { .. } | Self::Skipped { .. } | Self::Failed { .. } => None,
2064 }
2065 }
2066
2067 #[must_use]
2069 pub const fn was_performed(&self) -> bool {
2070 matches!(self, Self::Performed { .. })
2071 }
2072
2073 #[must_use]
2075 pub const fn lock_wait(&self) -> Option<Duration> {
2076 match self {
2077 Self::Missing { .. } => None,
2078 Self::Skipped { lock_wait, .. } | Self::Failed { lock_wait, .. } => Some(*lock_wait),
2079 Self::Performed { maintenance, .. } => Some(maintenance.lock_wait()),
2080 }
2081 }
2082
2083 #[must_use]
2085 pub const fn schedule_check(&self) -> Option<Duration> {
2086 match self {
2087 Self::Missing { .. } | Self::Failed { .. } => None,
2088 Self::Skipped { schedule_check, .. } | Self::Performed { schedule_check, .. } => {
2089 Some(*schedule_check)
2090 }
2091 }
2092 }
2093
2094 #[must_use]
2096 pub fn failure_message(&self) -> Option<&str> {
2097 match self {
2098 Self::Failed { message, .. } => Some(message),
2099 Self::Missing { .. } | Self::Skipped { .. } | Self::Performed { .. } => None,
2100 }
2101 }
2102}
2103
2104impl std::fmt::Display for SharedIncrementalTargetMaintenanceOutcome {
2105 fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
2106 match self {
2107 Self::Missing { target_dir } => {
2108 write!(formatter, "target={} action=missing", target_dir.display())
2109 }
2110 Self::Skipped {
2111 target_dir,
2112 lock_wait,
2113 schedule_check,
2114 } => write!(
2115 formatter,
2116 "target={} action=skipped lock={lock_wait:?} schedule={schedule_check:?}",
2117 target_dir.display(),
2118 ),
2119 Self::Performed {
2120 maintenance,
2121 schedule_check,
2122 } => write!(formatter, "{maintenance} schedule={schedule_check:?}"),
2123 Self::Failed {
2124 target_dir,
2125 lock_wait,
2126 message,
2127 } => write!(
2128 formatter,
2129 "target={} action=failed lock={lock_wait:?} error={message}",
2130 target_dir.display(),
2131 ),
2132 }
2133 }
2134}
2135
2136impl std::fmt::Display for WasmBuildTimings {
2137 fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
2138 write!(
2139 formatter,
2140 "total={:?} lock={:?} shared_lock={:?} inputs={:?} cargo={:?} maintenance={:?}",
2141 self.total,
2142 self.lock_wait,
2143 self.shared_incremental_lock_wait,
2144 self.input_resolution.total,
2145 self.cargo_build,
2146 self.cache_maintenance,
2147 )
2148 }
2149}
2150
2151impl std::fmt::Display for WasmBuildOutcome {
2152 fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
2153 let state = if self.is_reused() { "reused" } else { "built" };
2154 write!(
2155 formatter,
2156 "{state} fingerprint={} artifacts={} {}",
2157 self.record().fingerprint,
2158 self.record().artifacts.len(),
2159 self.record().timings,
2160 )?;
2161 if let Some(maintenance) = self.record().shared_incremental_maintenance() {
2162 write!(formatter, " shared_maintenance=({maintenance})")?;
2163 }
2164 Ok(())
2165 }
2166}
2167
2168pub fn resolve_cargo_build_inputs(
2173 spec: &WasmBuildSpec,
2174) -> Result<ResolvedCargoBuildInputs, WasmBuildError> {
2175 validate_spec(spec)?;
2176 build_fingerprint(spec)
2177}
2178
2179pub fn inspect_shared_incremental_target(
2184 spec: &WasmBuildSpec,
2185) -> Result<Option<SharedIncrementalTargetInspection>, WasmBuildError> {
2186 if !shared_incremental_target_exists(spec, "inspect shared incremental Cargo target")? {
2187 return Ok(None);
2188 }
2189
2190 let (_lock, lock_wait, canonical) = lock_shared_incremental_target(spec)?;
2191 let logical_size_bytes =
2192 directory_logical_size(&canonical).map_err(|source| WasmBuildError::Io {
2193 operation: "measure shared incremental Cargo target",
2194 path: canonical.clone(),
2195 source,
2196 })?;
2197 let last_used = cache_entry_last_used(&canonical).map_err(|source| WasmBuildError::Io {
2198 operation: "read shared incremental Cargo target use time",
2199 path: canonical.clone(),
2200 source,
2201 })?;
2202 Ok(Some(SharedIncrementalTargetInspection {
2203 target_dir: canonical,
2204 logical_size_bytes,
2205 last_used,
2206 lock_wait,
2207 }))
2208}
2209
2210pub fn maintain_shared_incremental_target(
2224 spec: &WasmBuildSpec,
2225 policy: SharedIncrementalTargetPrunePolicy,
2226) -> Result<Option<SharedIncrementalTargetMaintenance>, WasmBuildError> {
2227 if !shared_incremental_target_exists(
2228 spec,
2229 "inspect shared incremental Cargo target before maintenance",
2230 )? {
2231 return Ok(None);
2232 }
2233
2234 let _ = resolve_cargo_build_inputs(spec)?;
2238 let (_lock, lock_wait, canonical) = lock_shared_incremental_target(spec)?;
2239 maintain_shared_incremental_target_locked(&canonical, policy, lock_wait).map(Some)
2240}
2241
2242pub fn maintain_shared_incremental_target_at_most_every(
2256 spec: &WasmBuildSpec,
2257 policy: SharedIncrementalTargetPrunePolicy,
2258 minimum_interval: Duration,
2259) -> Result<SharedIncrementalTargetMaintenanceOutcome, WasmBuildError> {
2260 let target_dir =
2261 shared_incremental_target(spec).ok_or_else(|| WasmBuildError::InvalidSpec {
2262 message: "shared incremental target is not configured".to_owned(),
2263 })?;
2264 if !shared_incremental_target_exists(
2265 spec,
2266 "inspect shared incremental Cargo target before scheduled maintenance",
2267 )? {
2268 return Ok(SharedIncrementalTargetMaintenanceOutcome::Missing { target_dir });
2269 }
2270
2271 let (_lock, lock_wait, canonical) = lock_shared_incremental_target(spec)?;
2272 let schedule = schedule_shared_incremental_target_maintenance(
2273 &canonical,
2274 policy,
2275 minimum_interval,
2276 lock_wait,
2277 )?;
2278 let schedule = match schedule {
2279 SharedIncrementalTargetMaintenanceSchedule::Skipped(outcome) => return Ok(outcome),
2280 SharedIncrementalTargetMaintenanceSchedule::Due(due) => due,
2281 };
2282
2283 let _ = resolve_cargo_build_inputs(spec)?;
2286 perform_due_shared_incremental_target_maintenance(&canonical, policy, lock_wait, schedule)
2287}
2288
2289enum SharedIncrementalTargetMaintenanceSchedule {
2290 Skipped(SharedIncrementalTargetMaintenanceOutcome),
2291 Due(DueSharedIncrementalTargetMaintenance),
2292}
2293
2294struct DueSharedIncrementalTargetMaintenance {
2295 schedule_root: PathBuf,
2296 maintenance_identity: String,
2297 schedule_check: Duration,
2298}
2299
2300fn schedule_shared_incremental_target_maintenance(
2301 canonical: &Path,
2302 policy: SharedIncrementalTargetPrunePolicy,
2303 minimum_interval: Duration,
2304 lock_wait: Duration,
2305) -> Result<SharedIncrementalTargetMaintenanceSchedule, WasmBuildError> {
2306 let schedule_root = canonical.join(".ic-testkit");
2307 let maintenance_identity = policy.maintenance_identity();
2308 let schedule_started = Instant::now();
2309 let due = cache_maintenance_due(
2310 &schedule_root,
2311 Some(minimum_interval),
2312 &maintenance_identity,
2313 )
2314 .map_err(wasm_cache_fs_error)?;
2315 let schedule_check = schedule_started.elapsed();
2316 if !due {
2317 return Ok(SharedIncrementalTargetMaintenanceSchedule::Skipped(
2318 SharedIncrementalTargetMaintenanceOutcome::Skipped {
2319 target_dir: canonical.to_owned(),
2320 lock_wait,
2321 schedule_check,
2322 },
2323 ));
2324 }
2325 Ok(SharedIncrementalTargetMaintenanceSchedule::Due(
2326 DueSharedIncrementalTargetMaintenance {
2327 schedule_root,
2328 maintenance_identity,
2329 schedule_check,
2330 },
2331 ))
2332}
2333
2334fn perform_due_shared_incremental_target_maintenance(
2335 canonical: &Path,
2336 policy: SharedIncrementalTargetPrunePolicy,
2337 lock_wait: Duration,
2338 due: DueSharedIncrementalTargetMaintenance,
2339) -> Result<SharedIncrementalTargetMaintenanceOutcome, WasmBuildError> {
2340 let DueSharedIncrementalTargetMaintenance {
2341 schedule_root,
2342 maintenance_identity,
2343 schedule_check,
2344 } = due;
2345 let maintenance = maintain_shared_incremental_target_locked(canonical, policy, lock_wait)?;
2346 record_cache_maintenance(&schedule_root, &maintenance_identity).map_err(wasm_cache_fs_error)?;
2347 Ok(SharedIncrementalTargetMaintenanceOutcome::Performed {
2348 maintenance,
2349 schedule_check,
2350 })
2351}
2352
2353fn maintain_shared_incremental_target_locked(
2354 canonical: &Path,
2355 policy: SharedIncrementalTargetPrunePolicy,
2356 lock_wait: Duration,
2357) -> Result<SharedIncrementalTargetMaintenance, WasmBuildError> {
2358 let started = Instant::now();
2359 let logical_size_bytes_before =
2360 directory_logical_size(canonical).map_err(|source| WasmBuildError::Io {
2361 operation: "measure shared incremental Cargo target before maintenance",
2362 path: canonical.to_owned(),
2363 source,
2364 })?;
2365 let last_used_before =
2366 cache_entry_last_used(canonical).map_err(|source| WasmBuildError::Io {
2367 operation: "read shared incremental Cargo target use time before maintenance",
2368 path: canonical.to_owned(),
2369 source,
2370 })?;
2371 let expired = policy.max_age.is_some_and(|max_age| {
2372 SystemTime::now()
2373 .duration_since(last_used_before)
2374 .is_ok_and(|age| age > max_age)
2375 });
2376 let oversized = policy
2377 .max_size_bytes
2378 .is_some_and(|max_size_bytes| logical_size_bytes_before > max_size_bytes);
2379 let cleared = expired || oversized;
2380 if cleared {
2381 clear_shared_incremental_target_contents(canonical)?;
2382 record_cache_entry_use(canonical)?;
2383 }
2384 let logical_size_bytes_after = if cleared {
2385 directory_logical_size(canonical).map_err(|source| WasmBuildError::Io {
2386 operation: "measure shared incremental Cargo target after maintenance",
2387 path: canonical.to_owned(),
2388 source,
2389 })?
2390 } else {
2391 logical_size_bytes_before
2392 };
2393 Ok(SharedIncrementalTargetMaintenance {
2394 target_dir: canonical.to_owned(),
2395 logical_size_bytes_before,
2396 logical_size_bytes_after,
2397 last_used_before,
2398 cleared,
2399 lock_wait,
2400 maintenance: started.elapsed(),
2401 })
2402}
2403
2404fn clear_shared_incremental_target_contents(target_dir: &Path) -> Result<(), WasmBuildError> {
2405 let entries = fs::read_dir(target_dir).map_err(|source| WasmBuildError::Io {
2406 operation: "read shared incremental Cargo target for maintenance",
2407 path: target_dir.to_owned(),
2408 source,
2409 })?;
2410 for entry in entries {
2411 let path = entry
2412 .map_err(|source| WasmBuildError::Io {
2413 operation: "read shared incremental Cargo target entry for maintenance",
2414 path: target_dir.to_owned(),
2415 source,
2416 })?
2417 .path();
2418 let preserved = path
2419 .file_name()
2420 .is_some_and(|name| name == ".ic-testkit" || name == "CACHEDIR.TAG");
2421 if !preserved {
2422 remove_path_if_present(&path).map_err(|source| WasmBuildError::Io {
2423 operation: "clear shared incremental Cargo target entry",
2424 path,
2425 source,
2426 })?;
2427 }
2428 }
2429 Ok(())
2430}
2431
2432pub fn build_wasm_canisters_cached(
2442 spec: &WasmBuildSpec,
2443) -> Result<WasmBuildOutcome, WasmBuildError> {
2444 build_wasm_canisters_cached_internal(spec, &mut ProgressReporter::silent(), None)
2445}
2446
2447pub(super) fn build_wasm_canisters_cached_in_batch(
2448 spec: &WasmBuildSpec,
2449 index: usize,
2450 resolver: &mut WasmBuildBatchInputResolver<'_, '_>,
2451) -> WasmBuildBatchAttempt {
2452 let started = Instant::now();
2453 let mut progress = ProgressReporter::silent();
2454 let result = build_wasm_canisters_cached_internal(spec, &mut progress, Some((resolver, index)));
2455 if result
2456 .as_ref()
2457 .is_err_and(WasmBuildError::indicates_input_change)
2458 {
2459 resolver.invalidate_source_lease();
2460 }
2461 batch_result(result, &progress, started.elapsed())
2462}
2463
2464pub fn build_wasm_canisters_cached_with_progress<F>(
2473 spec: &WasmBuildSpec,
2474 config: WasmBuildProgressConfig,
2475 mut observer: F,
2476) -> Result<WasmBuildOutcome, WasmBuildError>
2477where
2478 F: FnMut(WasmBuildProgressEvent),
2479{
2480 if config.heartbeat_interval == Some(Duration::ZERO) {
2481 return Err(WasmBuildError::InvalidSpec {
2482 message: "Wasm build progress heartbeat interval must be greater than zero".to_owned(),
2483 });
2484 }
2485 build_wasm_canisters_cached_internal(
2486 spec,
2487 &mut ProgressReporter::observed(config, &mut observer),
2488 None,
2489 )
2490}
2491
2492pub(super) fn build_wasm_canisters_cached_in_batch_with_progress<F>(
2493 spec: &WasmBuildSpec,
2494 index: usize,
2495 resolver: &mut WasmBuildBatchInputResolver<'_, '_>,
2496 config: WasmBuildProgressConfig,
2497 mut observer: F,
2498) -> WasmBuildBatchAttempt
2499where
2500 F: FnMut(WasmBuildProgressEvent),
2501{
2502 if config.heartbeat_interval == Some(Duration::ZERO) {
2503 return WasmBuildBatchAttempt {
2504 result: Err((
2505 WasmBuildError::InvalidSpec {
2506 message: "Wasm build progress heartbeat interval must be greater than zero"
2507 .to_owned(),
2508 },
2509 WasmBuildFailureDetails::specification(Duration::ZERO),
2510 )),
2511 };
2512 }
2513 let started = Instant::now();
2514 let mut progress = ProgressReporter::observed(config, &mut observer);
2515 let result = build_wasm_canisters_cached_internal(spec, &mut progress, Some((resolver, index)));
2516 if result
2517 .as_ref()
2518 .is_err_and(WasmBuildError::indicates_input_change)
2519 {
2520 resolver.invalidate_source_lease();
2521 }
2522 batch_result(result, &progress, started.elapsed())
2523}
2524
2525fn batch_result(
2526 result: Result<WasmBuildOutcome, WasmBuildError>,
2527 progress: &ProgressReporter<'_>,
2528 total: Duration,
2529) -> WasmBuildBatchAttempt {
2530 WasmBuildBatchAttempt {
2531 result: result.map_err(|error| {
2532 let details = progress.failure_details(&error, total);
2533 (error, details)
2534 }),
2535 }
2536}
2537
2538fn batch_source_assumptions(
2539 batch_resolution: Option<&(&mut WasmBuildBatchInputResolver<'_, '_>, usize)>,
2540) -> (bool, Option<Arc<RwLock<bool>>>) {
2541 batch_resolution.map_or((false, None), |(resolver, _)| {
2542 (
2543 resolver.assumes_sources_immutable(),
2544 resolver.prepared_invalidation(),
2545 )
2546 })
2547}
2548
2549fn build_wasm_canisters_cached_internal(
2550 spec: &WasmBuildSpec,
2551 progress: &mut ProgressReporter<'_>,
2552 mut batch_resolution: Option<(&mut WasmBuildBatchInputResolver<'_, '_>, usize)>,
2553) -> Result<WasmBuildOutcome, WasmBuildError> {
2554 let total_started = Instant::now();
2555 validate_spec(spec)?;
2556 let (assumes_sources_immutable, prepared_invalidation) =
2557 batch_source_assumptions(batch_resolution.as_ref());
2558 progress.emit(WasmBuildProgressEvent::Started);
2559 if spec.shared_incremental_maintenance_config.is_some() {
2560 let outcome = build_wasm_canisters_cached_with_scheduled_shared_maintenance(
2561 spec,
2562 total_started,
2563 progress,
2564 batch_resolution.take(),
2565 )?;
2566 emit_finished_progress(&outcome, progress);
2567 return Ok(outcome);
2568 }
2569 let (cache_lock, first_lock_wait) =
2570 lock_wasm_build_cache_with_progress(&spec.target_dir, progress)?;
2571 ensure_cache_directory_tag(&spec.target_dir)?;
2572
2573 let resolved = resolve_initial_inputs(spec, batch_resolution.take(), progress)?;
2574 let isolated_acquisition =
2575 WasmAcquisitionContext::isolated(prepared_invalidation.clone(), assumes_sources_immutable);
2576 if let Some(outcome) = try_reuse_wasm_artifacts(
2577 spec,
2578 &resolved,
2579 first_lock_wait,
2580 &isolated_acquisition,
2581 total_started,
2582 progress,
2583 )? {
2584 emit_finished_progress(&outcome, progress);
2585 return Ok(outcome);
2586 }
2587 progress.emit(WasmBuildProgressEvent::CacheMiss {
2588 fingerprint: resolved.fingerprint,
2589 });
2590
2591 let outcome = match &spec.cache_mode {
2592 WasmBuildCacheMode::Isolated => {
2593 let cache_entry = cache_entry_directory(spec, resolved.fingerprint);
2594 build_wasm_cache_miss(
2595 spec,
2596 resolved,
2597 first_lock_wait,
2598 isolated_acquisition,
2599 cache_entry,
2600 total_started,
2601 progress,
2602 )
2603 }
2604 WasmBuildCacheMode::SharedIncremental { .. } => {
2605 drop(cache_lock);
2606 build_wasm_with_shared_incremental(
2607 spec,
2608 resolved,
2609 first_lock_wait,
2610 assumes_sources_immutable,
2611 prepared_invalidation,
2612 total_started,
2613 progress,
2614 )
2615 }
2616 }?;
2617 emit_finished_progress(&outcome, progress);
2618 Ok(outcome)
2619}
2620
2621fn build_wasm_with_shared_incremental(
2622 spec: &WasmBuildSpec,
2623 resolved: ResolvedCargoBuildInputs,
2624 first_lock_wait: Duration,
2625 assumes_sources_immutable: bool,
2626 prepared_invalidation: Option<Arc<RwLock<bool>>>,
2627 total_started: Instant,
2628 progress: &mut ProgressReporter<'_>,
2629) -> Result<WasmBuildOutcome, WasmBuildError> {
2630 let (shared_lock, shared_lock_wait, shared_target) =
2631 lock_shared_incremental_target_with_progress(spec, progress)?;
2632 let (_cache_lock, second_lock_wait) =
2633 lock_wasm_build_cache_with_progress(&spec.target_dir, progress)?;
2634 ensure_cache_directory_tag(&spec.target_dir)?;
2635
2636 let current = if assumes_sources_immutable {
2637 resolved
2638 } else {
2639 let mut current = resolve_inputs_with_progress(spec, progress)?;
2640 current.timings.include(resolved.timings);
2641 current
2642 };
2643 let lock_wait = first_lock_wait.saturating_add(second_lock_wait);
2644 let shared_incremental = WasmAcquisitionContext::shared(
2645 shared_lock_wait,
2646 None,
2647 prepared_invalidation,
2648 assumes_sources_immutable,
2649 );
2650 if let Some(outcome) = try_reuse_wasm_artifacts(
2651 spec,
2652 ¤t,
2653 lock_wait,
2654 &shared_incremental,
2655 total_started,
2656 progress,
2657 )? {
2658 return Ok(outcome);
2659 }
2660
2661 let outcome = build_wasm_cache_miss(
2662 spec,
2663 current,
2664 lock_wait,
2665 shared_incremental,
2666 shared_target,
2667 total_started,
2668 progress,
2669 );
2670 drop(shared_lock);
2671 outcome
2672}
2673
2674fn build_wasm_canisters_cached_with_scheduled_shared_maintenance(
2675 spec: &WasmBuildSpec,
2676 total_started: Instant,
2677 progress: &mut ProgressReporter<'_>,
2678 batch_resolution: Option<(&mut WasmBuildBatchInputResolver<'_, '_>, usize)>,
2679) -> Result<WasmBuildOutcome, WasmBuildError> {
2680 let (assumes_sources_immutable, prepared_invalidation) =
2681 batch_source_assumptions(batch_resolution.as_ref());
2682 let (_shared_lock, shared_lock_wait, shared_target) =
2683 lock_shared_incremental_target_with_progress(spec, progress)?;
2684 let (_cache_lock, lock_wait) = lock_wasm_build_cache_with_progress(&spec.target_dir, progress)?;
2685 ensure_cache_directory_tag(&spec.target_dir)?;
2686
2687 let resolved = resolve_initial_inputs(spec, batch_resolution, progress)?;
2690 let shared_maintenance = perform_configured_shared_incremental_target_maintenance(
2691 spec,
2692 &shared_target,
2693 shared_lock_wait,
2694 progress,
2695 )?;
2696 let shared_incremental = WasmAcquisitionContext::shared(
2697 shared_lock_wait,
2698 Some(shared_maintenance),
2699 prepared_invalidation,
2700 assumes_sources_immutable,
2701 );
2702 if let Some(outcome) = try_reuse_wasm_artifacts(
2703 spec,
2704 &resolved,
2705 lock_wait,
2706 &shared_incremental,
2707 total_started,
2708 progress,
2709 )? {
2710 return Ok(outcome);
2711 }
2712 progress.emit(WasmBuildProgressEvent::CacheMiss {
2713 fingerprint: resolved.fingerprint,
2714 });
2715 build_wasm_cache_miss(
2716 spec,
2717 resolved,
2718 lock_wait,
2719 shared_incremental,
2720 shared_target,
2721 total_started,
2722 progress,
2723 )
2724}
2725
2726fn perform_configured_shared_incremental_target_maintenance(
2727 spec: &WasmBuildSpec,
2728 shared_target: &Path,
2729 lock_wait: Duration,
2730 progress: &mut ProgressReporter<'_>,
2731) -> Result<SharedIncrementalTargetMaintenanceOutcome, WasmBuildError> {
2732 let config = spec
2733 .shared_incremental_maintenance_config
2734 .expect("configured shared-target maintenance must have settings");
2735 progress.emit(WasmBuildProgressEvent::SharedTargetMaintenanceStarted {
2736 target_dir: shared_target.to_owned(),
2737 });
2738 let result = progress.run_phase(WasmBuildProgressPhase::SharedTargetMaintenance, || {
2739 let schedule = schedule_shared_incremental_target_maintenance(
2740 shared_target,
2741 config.policy,
2742 config.minimum_interval,
2743 lock_wait,
2744 )?;
2745 match schedule {
2746 SharedIncrementalTargetMaintenanceSchedule::Skipped(outcome) => Ok(outcome),
2747 SharedIncrementalTargetMaintenanceSchedule::Due(due) => {
2748 perform_due_shared_incremental_target_maintenance(
2749 shared_target,
2750 config.policy,
2751 lock_wait,
2752 due,
2753 )
2754 }
2755 }
2756 });
2757 let outcome = integrated_shared_maintenance_result(config, shared_target, lock_wait, result)?;
2758 progress.emit(WasmBuildProgressEvent::SharedTargetMaintenanceFinished {
2759 outcome: outcome.clone(),
2760 });
2761 Ok(outcome)
2762}
2763
2764fn integrated_shared_maintenance_result(
2765 config: SharedIncrementalTargetMaintenanceConfig,
2766 shared_target: &Path,
2767 lock_wait: Duration,
2768 result: Result<SharedIncrementalTargetMaintenanceOutcome, WasmBuildError>,
2769) -> Result<SharedIncrementalTargetMaintenanceOutcome, WasmBuildError> {
2770 match result {
2771 Ok(outcome) => Ok(outcome),
2772 Err(error)
2773 if config.failure_mode == SharedIncrementalTargetMaintenanceFailureMode::BestEffort =>
2774 {
2775 Ok(SharedIncrementalTargetMaintenanceOutcome::Failed {
2776 target_dir: shared_target.to_owned(),
2777 lock_wait,
2778 message: error.to_string(),
2779 })
2780 }
2781 Err(error) => Err(error),
2782 }
2783}
2784
2785fn resolve_inputs_with_progress(
2786 spec: &WasmBuildSpec,
2787 progress: &mut ProgressReporter<'_>,
2788) -> Result<ResolvedCargoBuildInputs, WasmBuildError> {
2789 let resolved = build_fingerprint_with_progress(spec, progress)?;
2790 progress.emit(WasmBuildProgressEvent::InputsResolved {
2791 fingerprint: resolved.fingerprint,
2792 input_digest: resolved.input_digest,
2793 elapsed: resolved.timings.total,
2794 });
2795 Ok(resolved)
2796}
2797
2798fn resolve_initial_inputs(
2799 spec: &WasmBuildSpec,
2800 batch_resolution: Option<(&mut WasmBuildBatchInputResolver<'_, '_>, usize)>,
2801 progress: &mut ProgressReporter<'_>,
2802) -> Result<ResolvedCargoBuildInputs, WasmBuildError> {
2803 let resolved = if let Some((resolver, index)) = batch_resolution {
2804 resolver.resolve(index, progress)?
2805 } else {
2806 build_fingerprint_with_progress(spec, progress)?
2807 };
2808 progress.emit(WasmBuildProgressEvent::InputsResolved {
2809 fingerprint: resolved.fingerprint,
2810 input_digest: resolved.input_digest,
2811 elapsed: resolved.timings.total,
2812 });
2813 Ok(resolved)
2814}
2815
2816fn emit_finished_progress(outcome: &WasmBuildOutcome, progress: &mut ProgressReporter<'_>) {
2817 let state = if outcome.is_reused() {
2818 progress.emit(WasmBuildProgressEvent::CacheHit {
2819 fingerprint: outcome.record().fingerprint,
2820 });
2821 WasmBuildProgressOutcome::Reused
2822 } else {
2823 WasmBuildProgressOutcome::Built
2824 };
2825 progress.emit(WasmBuildProgressEvent::Finished {
2826 outcome: state,
2827 fingerprint: outcome.record().fingerprint,
2828 elapsed: outcome.record().timings.total,
2829 });
2830}
2831
2832#[derive(Clone, Debug, Default)]
2833struct WasmAcquisitionContext {
2834 assumes_sources_immutable: bool,
2835 lock_wait: Option<Duration>,
2836 maintenance: Option<SharedIncrementalTargetMaintenanceOutcome>,
2837 prepared_invalidation: Option<Arc<RwLock<bool>>>,
2838}
2839
2840impl WasmAcquisitionContext {
2841 fn isolated(
2842 prepared_invalidation: Option<Arc<RwLock<bool>>>,
2843 assumes_sources_immutable: bool,
2844 ) -> Self {
2845 Self {
2846 assumes_sources_immutable,
2847 prepared_invalidation,
2848 ..Self::default()
2849 }
2850 }
2851
2852 const fn shared(
2853 lock_wait: Duration,
2854 maintenance: Option<SharedIncrementalTargetMaintenanceOutcome>,
2855 prepared_invalidation: Option<Arc<RwLock<bool>>>,
2856 assumes_sources_immutable: bool,
2857 ) -> Self {
2858 Self {
2859 assumes_sources_immutable,
2860 lock_wait: Some(lock_wait),
2861 maintenance,
2862 prepared_invalidation,
2863 }
2864 }
2865
2866 fn lock_prepared_publication(
2867 &self,
2868 ) -> Result<Option<std::sync::RwLockReadGuard<'_, bool>>, WasmBuildError> {
2869 let guard = self.prepared_invalidation.as_deref().map(|invalidation| {
2870 invalidation
2871 .read()
2872 .unwrap_or_else(std::sync::PoisonError::into_inner)
2873 });
2874 if guard.as_deref().is_some_and(|invalidated| *invalidated) {
2875 return Err(WasmBuildError::PreparedInputSnapshotInvalidated);
2876 }
2877 Ok(guard)
2878 }
2879}
2880
2881fn try_reuse_wasm_artifacts(
2882 spec: &WasmBuildSpec,
2883 resolved: &ResolvedCargoBuildInputs,
2884 lock_wait: Duration,
2885 shared_incremental: &WasmAcquisitionContext,
2886 total_started: Instant,
2887 progress: &mut ProgressReporter<'_>,
2888) -> Result<Option<WasmBuildOutcome>, WasmBuildError> {
2889 let _publication_guard = shared_incremental.lock_prepared_publication()?;
2890 let fingerprint = resolved.fingerprint;
2891 let artifacts = expected_artifacts(spec, &spec.target_dir);
2892 let cache_entry = cache_entry_directory(spec, fingerprint);
2893 let artifacts_match = progress.run_phase(WasmBuildProgressPhase::ArtifactPublication, || {
2894 artifact_set_matches(&artifacts, fingerprint)
2895 });
2896 if artifacts_match {
2897 ensure_exact_cache_entry(spec, &artifacts, &cache_entry, fingerprint, progress)?;
2898 } else {
2899 let cached_artifacts = expected_artifacts(spec, &cache_entry);
2900 let cached_artifacts_match = progress
2901 .run_phase(WasmBuildProgressPhase::ArtifactPublication, || {
2902 artifact_set_matches(&cached_artifacts, fingerprint)
2903 });
2904 if !cached_artifacts_match {
2905 return Ok(None);
2906 }
2907 progress.run_phase(WasmBuildProgressPhase::ArtifactPublication, || {
2908 materialize_artifacts(&cached_artifacts, &artifacts, fingerprint)?;
2909 record_cache_entry_use(&cache_entry)
2910 })?;
2911 }
2912 let input_resolution = validate_reused_inputs(spec, resolved, shared_incremental, progress)?;
2913 Ok(Some(WasmBuildOutcome::Reused(complete_build_record(
2914 spec,
2915 BuildRecordInput {
2916 fingerprint,
2917 input_digest: resolved.input_digest,
2918 lock_wait,
2919 shared_incremental: shared_incremental.clone(),
2920 input_resolution,
2921 cargo_build: None,
2922 active_entry: &cache_entry,
2923 },
2924 total_started,
2925 progress,
2926 )?)))
2927}
2928
2929fn validate_reused_inputs(
2930 spec: &WasmBuildSpec,
2931 resolved: &ResolvedCargoBuildInputs,
2932 context: &WasmAcquisitionContext,
2933 progress: &mut ProgressReporter<'_>,
2934) -> Result<WasmInputResolutionTimings, WasmBuildError> {
2935 let mut timings = resolved.timings;
2936 if !context.assumes_sources_immutable {
2937 let verified = verify_resolved_inputs(spec, resolved, progress)?;
2938 timings.include(verified.timings);
2939 }
2940 Ok(timings)
2941}
2942
2943fn verify_resolved_inputs(
2944 spec: &WasmBuildSpec,
2945 resolved: &ResolvedCargoBuildInputs,
2946 progress: &mut ProgressReporter<'_>,
2947) -> Result<ResolvedCargoBuildInputs, WasmBuildError> {
2948 let verified = resolve_inputs_with_progress(spec, progress)?;
2949 for (before, after) in [
2950 (resolved.validation_digest, verified.validation_digest),
2951 (resolved.fingerprint, verified.fingerprint),
2952 ] {
2953 if before != after {
2954 return Err(WasmBuildError::InputsChangedDuringAcquisition { before, after });
2955 }
2956 }
2957 Ok(verified)
2958}
2959
2960fn ensure_exact_cache_entry(
2961 spec: &WasmBuildSpec,
2962 artifacts: &[PathBuf],
2963 cache_entry: &Path,
2964 fingerprint: InputDigest,
2965 progress: &mut ProgressReporter<'_>,
2966) -> Result<(), WasmBuildError> {
2967 let cached_artifacts = expected_artifacts(spec, cache_entry);
2968 let entry_is_current =
2969 progress.run_phase(WasmBuildProgressPhase::ArtifactPublication, || {
2970 if artifact_set_matches(&cached_artifacts, fingerprint) {
2971 record_cache_entry_use(cache_entry)?;
2972 Ok::<_, WasmBuildError>(true)
2973 } else {
2974 Ok(false)
2975 }
2976 })?;
2977 if entry_is_current {
2978 return Ok(());
2979 }
2980 progress.run_phase(WasmBuildProgressPhase::ArtifactPublication, || {
2981 remove_unretained_entry(cache_entry).map_err(wasm_cache_fs_error)?;
2982 create_dir_all(
2983 cache_entry,
2984 "create content-addressed Cargo target directory",
2985 )
2986 })?;
2987 let incomplete = IncompleteBuildDirectory::new(cache_entry.to_owned());
2988 let result = progress.run_phase(WasmBuildProgressPhase::ArtifactPublication, || {
2989 copy_wasm_artifacts(artifacts, &cached_artifacts)?;
2990 publish_artifact_stamps(&cached_artifacts, fingerprint)?;
2991 record_cache_entry_use(cache_entry)
2992 });
2993 finish_fingerprint_build(result, incomplete, progress)
2994}
2995
2996fn build_wasm_cache_miss(
2997 spec: &WasmBuildSpec,
2998 resolved: ResolvedCargoBuildInputs,
2999 lock_wait: Duration,
3000 shared_incremental: WasmAcquisitionContext,
3001 cargo_target_dir: PathBuf,
3002 total_started: Instant,
3003 progress: &mut ProgressReporter<'_>,
3004) -> Result<WasmBuildOutcome, WasmBuildError> {
3005 let fingerprint = resolved.fingerprint;
3006 let mut input_resolution = resolved.timings;
3007 let artifacts = expected_artifacts(spec, &spec.target_dir);
3008 let cache_entry = cache_entry_directory(spec, fingerprint);
3009 let preparation_started = Instant::now();
3010 progress.begin_phase(WasmBuildFailurePhase::ArtifactPublication);
3011 let preparation_result = (|| {
3012 remove_unretained_entry(&cache_entry).map_err(wasm_cache_fs_error)?;
3013 create_dir_all(
3014 &cache_entry,
3015 "create content-addressed Cargo target directory",
3016 )
3017 })();
3018 progress.record_phase(
3019 WasmBuildFailurePhase::ArtifactPublication,
3020 preparation_started.elapsed(),
3021 );
3022 preparation_result?;
3023 let incomplete_directory = IncompleteBuildDirectory::new(cache_entry.clone());
3024 let build_result = (|| {
3025 if matches!(
3026 spec.cache_mode,
3027 WasmBuildCacheMode::SharedIncremental { .. }
3028 ) {
3029 record_cache_entry_use(&cargo_target_dir)?;
3030 }
3031 let build_started = Instant::now();
3032 progress.begin_phase(WasmBuildFailurePhase::CargoBuild);
3033 let cargo_result = run_cargo_build(spec, &cargo_target_dir, progress);
3034 let cargo_build = build_started.elapsed();
3035 progress.record_phase(WasmBuildFailurePhase::CargoBuild, cargo_build);
3036 cargo_result?;
3037 let built_artifacts = expected_artifacts(spec, &cargo_target_dir);
3038 let validation_started = Instant::now();
3039 progress.begin_phase(WasmBuildFailurePhase::ArtifactPublication);
3040 let missing = missing_artifacts(&built_artifacts);
3041 progress.record_phase(
3042 WasmBuildFailurePhase::ArtifactPublication,
3043 validation_started.elapsed(),
3044 );
3045 if !missing.is_empty() {
3046 return Err(WasmBuildError::MissingArtifacts { paths: missing });
3047 }
3048
3049 let verified = verify_resolved_inputs(spec, &resolved, progress)?;
3050 input_resolution.include(verified.timings);
3051
3052 let publication_guard = shared_incremental.lock_prepared_publication()?;
3057
3058 let cached_artifacts = expected_artifacts(spec, &cache_entry);
3059 progress.run_phase(WasmBuildProgressPhase::ArtifactPublication, || {
3060 if cargo_target_dir != cache_entry {
3061 copy_wasm_artifacts(&built_artifacts, &cached_artifacts)?;
3062 }
3063 publish_artifact_stamps(&cached_artifacts, fingerprint)?;
3064 materialize_artifacts(&cached_artifacts, &artifacts, fingerprint)?;
3065 record_cache_entry_use(&cache_entry)
3066 })?;
3067 drop(publication_guard);
3068
3069 Ok(WasmBuildOutcome::Built(complete_build_record(
3070 spec,
3071 BuildRecordInput {
3072 fingerprint,
3073 input_digest: resolved.input_digest,
3074 lock_wait,
3075 shared_incremental,
3076 input_resolution,
3077 cargo_build: Some(cargo_build),
3078 active_entry: &cache_entry,
3079 },
3080 total_started,
3081 progress,
3082 )?))
3083 })();
3084 finish_fingerprint_build(build_result, incomplete_directory, progress)
3085}
3086
3087pub fn prune_wasm_build_cache(
3096 target_dir: &Path,
3097 policy: ArtifactCachePrunePolicy,
3098) -> Result<ArtifactCachePruneReport, WasmBuildError> {
3099 let (_lock_file, _) = lock_wasm_build_cache(target_dir)?;
3100 ensure_cache_directory_tag(target_dir)?;
3101
3102 prune_wasm_build_cache_locked(target_dir, policy, None)
3103}
3104
3105struct BuildRecordInput<'a> {
3106 fingerprint: InputDigest,
3107 input_digest: InputDigest,
3108 lock_wait: Duration,
3109 shared_incremental: WasmAcquisitionContext,
3110 input_resolution: WasmInputResolutionTimings,
3111 cargo_build: Option<Duration>,
3112 active_entry: &'a Path,
3113}
3114
3115fn complete_build_record(
3116 spec: &WasmBuildSpec,
3117 input: BuildRecordInput<'_>,
3118 total_started: Instant,
3119 progress: &mut ProgressReporter<'_>,
3120) -> Result<WasmBuildRecord, WasmBuildError> {
3121 let retention = progress.run_phase(WasmBuildProgressPhase::ArtifactPublication, || {
3122 RetainedCacheEntry::acquire(input.active_entry).map_err(wasm_cache_fs_error)
3123 })?;
3124 let (maintenance, cache_maintenance) = spec.prune_policy.map_or((None, None), |policy| {
3125 progress.run_phase(WasmBuildProgressPhase::ExactCacheMaintenance, || {
3126 let cache_root = spec.target_dir.join(".ic-testkit/wasm-targets");
3127 let identity = policy.maintenance_identity();
3128 perform_scheduled_cache_maintenance(&cache_root, spec.prune_interval, &identity, || {
3129 prune_wasm_build_cache_locked(&spec.target_dir, policy, Some(input.active_entry))
3130 .map_err(|error| error.to_string())
3131 })
3132 })
3133 });
3134 Ok(WasmBuildRecord {
3135 fingerprint: input.fingerprint,
3136 input_digest: input.input_digest,
3137 artifacts: expected_artifacts(spec, input.active_entry),
3138 retention,
3139 timings: WasmBuildTimings {
3140 lock_wait: input.lock_wait,
3141 shared_incremental_lock_wait: input.shared_incremental.lock_wait,
3142 input_resolution: input.input_resolution,
3143 cargo_build: input.cargo_build,
3144 cache_maintenance,
3145 total: total_started.elapsed(),
3146 },
3147 maintenance,
3148 shared_incremental_maintenance: input.shared_incremental.maintenance,
3149 })
3150}
3151
3152fn prune_wasm_build_cache_locked(
3153 target_dir: &Path,
3154 policy: ArtifactCachePrunePolicy,
3155 protected_entry: Option<&Path>,
3156) -> Result<ArtifactCachePruneReport, WasmBuildError> {
3157 let cache_root = target_dir.join(".ic-testkit/wasm-targets");
3158 prune_direct_child_directories(&cache_root, policy, protected_entry, is_sha256_directory)
3159 .map_err(wasm_cache_fs_error)
3160}
3161
3162struct IncompleteBuildDirectory {
3163 path: PathBuf,
3164 armed: bool,
3165}
3166
3167impl IncompleteBuildDirectory {
3168 const fn new(path: PathBuf) -> Self {
3169 Self { path, armed: true }
3170 }
3171
3172 fn preserve(mut self) {
3173 self.armed = false;
3174 }
3175
3176 fn cleanup(mut self) -> io::Result<()> {
3177 let result = remove_path_if_present(&self.path);
3178 if result.is_ok() {
3179 self.armed = false;
3180 }
3181 result
3182 }
3183}
3184
3185impl Drop for IncompleteBuildDirectory {
3186 fn drop(&mut self) {
3187 if self.armed {
3188 let _ = remove_path_if_present(&self.path);
3189 }
3190 }
3191}
3192
3193fn finish_fingerprint_build<T>(
3194 result: Result<T, WasmBuildError>,
3195 incomplete_directory: IncompleteBuildDirectory,
3196 progress: &mut ProgressReporter<'_>,
3197) -> Result<T, WasmBuildError> {
3198 match result {
3199 Ok(outcome) => {
3200 incomplete_directory.preserve();
3201 Ok(outcome)
3202 }
3203 Err(build_error) => Err(cleanup_failed_fingerprint_build(
3204 build_error,
3205 incomplete_directory,
3206 progress,
3207 )),
3208 }
3209}
3210
3211fn cleanup_failed_fingerprint_build(
3212 build_error: WasmBuildError,
3213 incomplete_directory: IncompleteBuildDirectory,
3214 progress: &mut ProgressReporter<'_>,
3215) -> WasmBuildError {
3216 let path = incomplete_directory.path.clone();
3217 let primary_phase = progress.failure_phase;
3218 let cleanup_started = Instant::now();
3219 let cleanup = incomplete_directory.cleanup();
3220 progress.record_phase(WasmBuildFailurePhase::Cleanup, cleanup_started.elapsed());
3221 match cleanup {
3222 Ok(()) => {
3223 progress.failure_phase = primary_phase;
3224 build_error
3225 }
3226 Err(source) => WasmBuildError::FailedBuildCleanup {
3227 build_error: Box::new(build_error),
3228 path,
3229 source,
3230 },
3231 }
3232}
3233
3234fn lock_wasm_build_cache(target_dir: &Path) -> Result<(File, Duration), WasmBuildError> {
3235 create_dir_all(target_dir, "create Cargo target directory")?;
3236 let lock_path = target_dir.join(".ic-testkit/wasm-build.lock");
3237 lock_cache_file(&lock_path).map_err(wasm_cache_fs_error)
3238}
3239
3240fn lock_wasm_build_cache_with_progress(
3241 target_dir: &Path,
3242 progress: &mut ProgressReporter<'_>,
3243) -> Result<(File, Duration), WasmBuildError> {
3244 progress.begin_phase(WasmBuildFailurePhase::ExactCacheCoordination);
3245 create_dir_all(target_dir, "create Cargo target directory")?;
3246 let lock_path = target_dir.join(".ic-testkit/wasm-build.lock");
3247 lock_cache_file_with_progress(&lock_path, WasmBuildProgressPhase::ExactCacheLock, progress)
3248}
3249
3250fn lock_shared_incremental_target(
3251 spec: &WasmBuildSpec,
3252) -> Result<(File, Duration, PathBuf), WasmBuildError> {
3253 lock_shared_incremental_target_internal(spec, None)
3254}
3255
3256fn lock_shared_incremental_target_with_progress(
3257 spec: &WasmBuildSpec,
3258 progress: &mut ProgressReporter<'_>,
3259) -> Result<(File, Duration, PathBuf), WasmBuildError> {
3260 let target_dir = shared_incremental_target(spec)
3261 .expect("validated shared acquisition must have a shared Cargo target");
3262 progress.emit(WasmBuildProgressEvent::SharedTargetLockStarted { target_dir });
3263 let (lock, wait, canonical) = lock_shared_incremental_target_internal(spec, Some(progress))?;
3264 progress.emit(WasmBuildProgressEvent::SharedTargetLockAcquired {
3265 target_dir: canonical.clone(),
3266 wait,
3267 });
3268 Ok((lock, wait, canonical))
3269}
3270
3271fn lock_shared_incremental_target_internal(
3272 spec: &WasmBuildSpec,
3273 mut progress: Option<&mut ProgressReporter<'_>>,
3274) -> Result<(File, Duration, PathBuf), WasmBuildError> {
3275 if let Some(progress) = progress.as_deref_mut() {
3276 progress.begin_phase(WasmBuildFailurePhase::SharedTargetCoordination);
3277 }
3278 let target_dir =
3279 shared_incremental_target(spec).ok_or_else(|| WasmBuildError::InvalidSpec {
3280 message: "shared incremental target is not configured".to_owned(),
3281 })?;
3282 create_dir_all(
3283 &target_dir,
3284 "create shared incremental Cargo target directory",
3285 )?;
3286 ensure_cache_tag(&target_dir).map_err(wasm_cache_fs_error)?;
3287 let canonical = target_dir
3288 .canonicalize()
3289 .map_err(|source| WasmBuildError::Io {
3290 operation: "resolve shared incremental Cargo target directory",
3291 path: target_dir.clone(),
3292 source,
3293 })?;
3294 let lock_path = canonical.join(".ic-testkit/wasm-incremental.lock");
3295 let (lock, wait) = if let Some(progress) = progress {
3296 lock_cache_file_with_progress(
3297 &lock_path,
3298 WasmBuildProgressPhase::SharedTargetLock,
3299 progress,
3300 )?
3301 } else {
3302 lock_cache_file(&lock_path).map_err(wasm_cache_fs_error)?
3303 };
3304 Ok((lock, wait, canonical))
3305}
3306
3307fn lock_cache_file_with_progress(
3308 lock_path: &Path,
3309 phase: WasmBuildProgressPhase,
3310 progress: &mut ProgressReporter<'_>,
3311) -> Result<(File, Duration), WasmBuildError> {
3312 let failure_phase = progress_failure_phase(phase);
3313 let started = Instant::now();
3314 progress.begin_phase(failure_phase);
3315 let result = if !progress.is_observed() || progress.config.heartbeat_interval.is_none() {
3316 lock_cache_file(lock_path).map_err(wasm_cache_fs_error)
3317 } else {
3318 let heartbeat_interval = progress
3319 .config
3320 .heartbeat_interval
3321 .expect("observed cache lock must have a heartbeat interval");
3322 lock_cache_file_with_wait_observer(lock_path, heartbeat_interval, |elapsed| {
3323 progress.emit_heartbeat_if_due(phase, elapsed);
3324 })
3325 .map_err(wasm_cache_fs_error)
3326 };
3327 progress.record_phase(failure_phase, started.elapsed());
3328 result
3329}
3330
3331fn ensure_cache_directory_tag(target_dir: &Path) -> Result<(), WasmBuildError> {
3332 ensure_cache_tag(target_dir).map_err(wasm_cache_fs_error)
3333}
3334
3335fn record_cache_entry_use(path: &Path) -> Result<(), WasmBuildError> {
3336 record_entry_use(path).map_err(wasm_cache_fs_error)
3337}
3338
3339fn wasm_cache_fs_error(error: CacheFsError) -> WasmBuildError {
3340 WasmBuildError::Io {
3341 operation: error.operation,
3342 path: error.path,
3343 source: error.source,
3344 }
3345}
3346
3347fn validate_spec(spec: &WasmBuildSpec) -> Result<(), WasmBuildError> {
3348 if spec.packages.is_empty() {
3349 return Err(WasmBuildError::InvalidSpec {
3350 message: "at least one Cargo package is required".to_owned(),
3351 });
3352 }
3353 if spec.profile_target_dir.is_empty() {
3354 return Err(WasmBuildError::InvalidSpec {
3355 message: "Cargo profile target directory must not be empty".to_owned(),
3356 });
3357 }
3358 if spec.target.is_empty() {
3359 return Err(WasmBuildError::InvalidSpec {
3360 message: "Cargo compilation target must not be empty".to_owned(),
3361 });
3362 }
3363 if spec.extra_env.contains_key(OsStr::new("CARGO_TARGET_DIR"))
3364 || spec.cargo_profile_args.iter().any(|argument| {
3365 argument == OsStr::new("--target-dir")
3366 || argument.as_encoded_bytes().starts_with(b"--target-dir=")
3367 })
3368 {
3369 return Err(WasmBuildError::InvalidSpec {
3370 message: "Cargo target directories are owned by the build specification; use target_dir or with_shared_incremental_target instead of command overrides".to_owned(),
3371 });
3372 }
3373 if matches!(
3374 &spec.cache_mode,
3375 WasmBuildCacheMode::SharedIncremental { target_dir } if target_dir.as_os_str().is_empty()
3376 ) {
3377 return Err(WasmBuildError::InvalidSpec {
3378 message: "shared incremental Cargo target directory must not be empty".to_owned(),
3379 });
3380 }
3381 if spec.shared_incremental_maintenance_config.is_some()
3382 && !matches!(
3383 spec.cache_mode,
3384 WasmBuildCacheMode::SharedIncremental { .. }
3385 )
3386 {
3387 return Err(WasmBuildError::InvalidSpec {
3388 message:
3389 "scheduled shared-target maintenance requires a shared incremental Cargo target"
3390 .to_owned(),
3391 });
3392 }
3393 Ok(())
3394}
3395
3396fn build_fingerprint(spec: &WasmBuildSpec) -> Result<ResolvedCargoBuildInputs, WasmBuildError> {
3397 build_fingerprint_with_progress(spec, &mut ProgressReporter::silent())
3398}
3399
3400fn build_fingerprint_with_progress(
3401 spec: &WasmBuildSpec,
3402 progress: &mut ProgressReporter<'_>,
3403) -> Result<ResolvedCargoBuildInputs, WasmBuildError> {
3404 let total_started = Instant::now();
3405 let (cargo_identity, rustc_identity, tool_identity) = resolve_tool_identity(spec, progress)?;
3406
3407 let metadata_started = Instant::now();
3408 let metadata = progress.run_phase(WasmBuildProgressPhase::CargoMetadata, || {
3409 cargo_metadata(spec)
3410 })?;
3411 let cargo_metadata = metadata_started.elapsed();
3412
3413 let discovery_started = Instant::now();
3414 let (inputs, exclusions) =
3415 progress.run_phase(WasmBuildProgressPhase::InputDiscovery, || {
3416 let parsed = ParsedCargoMetadata::parse(&metadata)
3417 .map_err(|message| invalid_metadata(&message))?;
3418 resolve_local_inputs(spec, &parsed)
3419 })?;
3420 let input_discovery = discovery_started.elapsed();
3421
3422 let hashing_started = Instant::now();
3423 let (input_digest, validation_digest) =
3424 progress.run_phase(WasmBuildProgressPhase::ContentHashing, || {
3425 let mut cache = LabeledPathDigestCache::default();
3426 digest_resolved_local_inputs(
3427 &inputs,
3428 &exclusions,
3429 &mut cache,
3430 &spec.workspace_root,
3431 "hash Wasm build inputs",
3432 "hash semantic Wasm build inputs",
3433 )
3434 })?;
3435 let content_hashing = hashing_started.elapsed();
3436
3437 let fingerprint =
3438 finish_build_fingerprint(spec, &cargo_identity, &rustc_identity, input_digest);
3439 Ok(ResolvedCargoBuildInputs {
3440 fingerprint,
3441 input_digest,
3442 validation_digest,
3443 inputs: inputs
3444 .validation_inputs
3445 .into_iter()
3446 .map(|(label, path)| CargoBuildInput { label, path })
3447 .collect(),
3448 exclusions,
3449 timings: WasmInputResolutionTimings {
3450 tool_identity,
3451 cargo_metadata,
3452 input_discovery,
3453 content_hashing,
3454 total: total_started.elapsed(),
3455 },
3456 })
3457}
3458
3459fn finish_build_fingerprint(
3460 spec: &WasmBuildSpec,
3461 cargo_identity: &[u8],
3462 rustc_identity: &[u8],
3463 input_digest: InputDigest,
3464) -> InputDigest {
3465 let mut hasher = InputHasher::new(CACHE_FORMAT_VERSION);
3466 let mut packages = spec.packages.clone();
3467 packages.sort();
3468 packages.dedup();
3469 for package in packages {
3470 hasher.field("package", package.as_bytes());
3471 }
3472 hasher.field("target", spec.target.as_bytes());
3473 hasher.field("profile-target-dir", spec.profile_target_dir.as_bytes());
3474 for argument in &spec.cargo_profile_args {
3475 hasher.field("cargo-argument", &os_bytes(argument));
3476 }
3477 for (key, value) in effective_environment(spec) {
3478 hasher.field("environment-key", &os_bytes(&key));
3479 if let Some(value) = value {
3480 hasher.field("environment-value", &os_bytes(&value));
3481 } else {
3482 hasher.field("environment-unset", b"");
3483 }
3484 }
3485 hasher.field("cargo-identity", cargo_identity);
3486 hasher.field("rustc-identity", rustc_identity);
3487 hasher.field("source-input-digest", input_digest.as_bytes());
3488 hasher.finish()
3489}
3490
3491fn command_identity(
3492 spec: &WasmBuildSpec,
3493 phase: WasmBuildPhase,
3494 program: &OsStr,
3495 arguments: &[&str],
3496) -> Result<Vec<u8>, WasmBuildError> {
3497 let mut command = Command::new(program);
3498 command.current_dir(&spec.workspace_root).args(arguments);
3499 apply_command_environment(&mut command, spec);
3500 let output = command
3501 .output()
3502 .map_err(|source| WasmBuildError::CommandSpawn {
3503 phase,
3504 program: program.to_owned(),
3505 source,
3506 })?;
3507 ensure_command_success(phase, output).map(|output| {
3508 let mut identity = output.stdout;
3509 identity.extend_from_slice(&output.stderr);
3510 identity
3511 })
3512}
3513
3514fn cargo_metadata(spec: &WasmBuildSpec) -> Result<Value, WasmBuildError> {
3515 let mut command = Command::new(&spec.cargo_program);
3516 command
3517 .current_dir(&spec.workspace_root)
3518 .args(["metadata", "--format-version", "1"]);
3519 for argument in metadata_arguments(&spec.cargo_profile_args) {
3520 command.arg(argument);
3521 }
3522 apply_command_environment(&mut command, spec);
3523 let output = command
3524 .output()
3525 .map_err(|source| WasmBuildError::CommandSpawn {
3526 phase: WasmBuildPhase::CargoMetadata,
3527 program: spec.cargo_program.clone(),
3528 source,
3529 })?;
3530 let output = ensure_command_success(WasmBuildPhase::CargoMetadata, output)?;
3531 serde_json::from_slice(&output.stdout).map_err(|error| WasmBuildError::InvalidMetadata {
3532 message: format!("Cargo metadata was not valid JSON: {error}"),
3533 })
3534}
3535
3536fn metadata_arguments(arguments: &[OsString]) -> Vec<OsString> {
3537 let mut selected = Vec::new();
3538 let mut arguments = arguments.iter();
3539 while let Some(argument) = arguments.next() {
3540 let argument_text = argument.to_string_lossy();
3541 match argument_text.as_ref() {
3542 "--all-features" | "--no-default-features" | "--locked" | "--offline" | "--frozen" => {
3543 selected.push(argument.clone());
3544 }
3545 "--features" | "-F" | "--filter-platform" => {
3546 selected.push(argument.clone());
3547 if let Some(value) = arguments.next() {
3548 selected.push(value.clone());
3549 }
3550 }
3551 _ if argument_text.starts_with("--features=")
3552 || argument_text.starts_with("-F")
3553 || argument_text.starts_with("--filter-platform=") =>
3554 {
3555 selected.push(argument.clone());
3556 }
3557 _ => {}
3558 }
3559 }
3560 selected
3561}
3562
3563#[derive(Clone)]
3564struct MetadataPackage {
3565 id: String,
3566 name: String,
3567 version: String,
3568 manifest_path: PathBuf,
3569 is_local: bool,
3570 source: Option<String>,
3571 semantic_fields: Vec<(&'static str, Option<String>)>,
3572}
3573
3574struct ParsedCargoMetadata<'a> {
3577 packages: HashMap<String, MetadataPackage>,
3578 workspace_members: HashSet<&'a str>,
3579 nodes: HashMap<String, MetadataNode<'a>>,
3580 workspace_root: Option<&'a str>,
3581}
3582
3583struct MetadataNode<'a> {
3584 dependencies: Vec<String>,
3585 value: &'a Value,
3586}
3587
3588impl<'a> ParsedCargoMetadata<'a> {
3589 fn parse(metadata: &'a Value) -> Result<Self, String> {
3590 let packages = metadata_packages(metadata)?;
3591 let workspace_members = metadata
3592 .get("workspace_members")
3593 .and_then(Value::as_array)
3594 .ok_or_else(|| "Cargo metadata has no workspace member array".to_owned())?
3595 .iter()
3596 .filter_map(Value::as_str)
3597 .collect();
3598 let values = metadata
3599 .pointer("/resolve/nodes")
3600 .and_then(Value::as_array)
3601 .ok_or_else(|| "Cargo metadata has no resolved dependency nodes".to_owned())?;
3602 let mut nodes = HashMap::new();
3603 for value in values {
3604 let id = required_string(value, "id")?;
3605 let dependencies = value
3606 .get("deps")
3607 .and_then(Value::as_array)
3608 .ok_or_else(|| "Cargo metadata dependency node has no deps array".to_owned())?
3609 .iter()
3610 .map(|dependency| required_string(dependency, "pkg"))
3611 .collect::<Result<_, _>>()?;
3612 nodes.insert(
3613 id,
3614 MetadataNode {
3615 dependencies,
3616 value,
3617 },
3618 );
3619 }
3620 Ok(Self {
3621 packages,
3622 workspace_members,
3623 nodes,
3624 workspace_root: metadata.get("workspace_root").and_then(Value::as_str),
3625 })
3626 }
3627}
3628
3629const SEMANTIC_PACKAGE_FIELDS: &[&str] = &[
3630 "authors",
3631 "default_run",
3632 "description",
3633 "documentation",
3634 "edition",
3635 "homepage",
3636 "license",
3637 "license_file",
3638 "links",
3639 "metadata",
3640 "name",
3641 "readme",
3642 "repository",
3643 "rust_version",
3644 "version",
3645];
3646
3647struct LockedPackageIdentity {
3648 name: String,
3649 version: String,
3650 source: String,
3651 checksum: Option<String>,
3652}
3653
3654fn resolve_local_inputs(
3655 spec: &WasmBuildSpec,
3656 metadata: &ParsedCargoMetadata<'_>,
3657) -> Result<(ResolvedLocalInputs, Vec<PathBuf>), WasmBuildError> {
3658 let mut selected_ids = selected_package_ids(spec, metadata)?;
3659 let mut closure = BTreeSet::new();
3660 while let Some(id) = selected_ids.pop_front() {
3661 if !closure.insert(id.clone()) {
3662 continue;
3663 }
3664 if let Some(node) = metadata.nodes.get(&id) {
3665 selected_ids.extend(node.dependencies.iter().cloned());
3666 }
3667 }
3668
3669 let workspace_root = metadata
3670 .workspace_root
3671 .map_or_else(|| spec.workspace_root.clone(), PathBuf::from);
3672 let workspace_projection = semantic_workspace_projection(metadata, &closure, &workspace_root)?;
3673 let mut validation_inputs = workspace_configuration_inputs(spec, &workspace_root)?;
3674 append_package_inputs(
3675 &mut validation_inputs,
3676 &metadata.packages,
3677 closure,
3678 &workspace_root,
3679 )?;
3680 append_additional_inputs(&mut validation_inputs, spec, &workspace_root);
3681 validate_shared_incremental_target_boundary(spec, &validation_inputs)?;
3682 let exclusions = source_exclusions(spec, &validation_inputs);
3683 Ok((
3684 ResolvedLocalInputs {
3685 validation_inputs,
3686 workspace_projection,
3687 },
3688 exclusions,
3689 ))
3690}
3691
3692fn metadata_packages(metadata: &Value) -> Result<HashMap<String, MetadataPackage>, String> {
3693 let packages_value = metadata
3694 .get("packages")
3695 .and_then(Value::as_array)
3696 .ok_or_else(|| "Cargo metadata has no package array".to_owned())?;
3697 let mut packages = HashMap::new();
3698 for value in packages_value {
3699 let source = optional_string(value, "source")?;
3700 let package = MetadataPackage {
3701 id: required_string(value, "id")?,
3702 name: required_string(value, "name")?,
3703 version: required_string(value, "version")?,
3704 manifest_path: PathBuf::from(required_string(value, "manifest_path")?),
3705 is_local: value.get("source").is_some_and(Value::is_null),
3706 source,
3707 semantic_fields: SEMANTIC_PACKAGE_FIELDS
3708 .iter()
3709 .map(|field| (*field, value.get(*field).map(Value::to_string)))
3710 .collect(),
3711 };
3712 packages.insert(package.id.clone(), package);
3713 }
3714 Ok(packages)
3715}
3716
3717fn selected_package_ids(
3718 spec: &WasmBuildSpec,
3719 metadata: &ParsedCargoMetadata<'_>,
3720) -> Result<VecDeque<String>, WasmBuildError> {
3721 let mut selected_ids = VecDeque::new();
3722 for requested in &spec.packages {
3723 let matches = metadata
3724 .packages
3725 .values()
3726 .filter(|package| {
3727 package.name == *requested
3728 && metadata.workspace_members.contains(package.id.as_str())
3729 })
3730 .map(|package| package.id.clone())
3731 .collect::<Vec<_>>();
3732 match matches.as_slice() {
3733 [id] => selected_ids.push_back(id.clone()),
3734 [] => {
3735 return Err(WasmBuildError::InvalidSpec {
3736 message: format!("Cargo workspace contains no package named `{requested}`"),
3737 });
3738 }
3739 _ => {
3740 return Err(WasmBuildError::InvalidSpec {
3741 message: format!("Cargo workspace package name `{requested}` is ambiguous"),
3742 });
3743 }
3744 }
3745 }
3746 Ok(selected_ids)
3747}
3748
3749fn semantic_workspace_projection(
3750 metadata: &ParsedCargoMetadata<'_>,
3751 closure: &BTreeSet<String>,
3752 workspace_root: &Path,
3753) -> Result<Option<InputDigest>, WasmBuildError> {
3754 let locked_packages = locked_package_identities(workspace_root)?;
3757 let mut identities = HashMap::new();
3758 for id in closure {
3759 let package = metadata
3760 .packages
3761 .get(id)
3762 .ok_or_else(|| invalid_metadata(&format!("resolved package `{id}` is missing")))?;
3763 let Some(identity) = semantic_package_identity(package, workspace_root, &locked_packages)
3764 else {
3765 return Ok(None);
3766 };
3767 identities.insert(id.as_str(), identity);
3768 }
3769
3770 let mut projected_packages = closure
3771 .iter()
3772 .map(|id| {
3773 let package = metadata
3774 .packages
3775 .get(id)
3776 .expect("selected package closure was validated above");
3777 let identity = identities[id.as_str()];
3778 let node = metadata.nodes.get(id).ok_or_else(|| {
3779 invalid_metadata(&format!("resolved package `{id}` has no dependency node"))
3780 })?;
3781 let projection = semantic_package_projection(package, node.value, &identities)?;
3782 Ok::<_, WasmBuildError>((identity, projection))
3783 })
3784 .collect::<Result<Vec<_>, _>>()?;
3785 projected_packages.sort_by_key(|(identity, _)| *identity);
3786
3787 let root_manifest = workspace_root.join("Cargo.toml");
3788 let root_contents =
3789 fs::read_to_string(&root_manifest).map_err(|source| WasmBuildError::Io {
3790 operation: "read workspace manifest for semantic projection",
3791 path: root_manifest.clone(),
3792 source,
3793 })?;
3794 let root = toml::from_str::<TomlValue>(&root_contents).map_err(|error| {
3795 invalid_metadata(&format!(
3796 "workspace manifest could not be projected as TOML: {error}"
3797 ))
3798 })?;
3799
3800 let mut hasher = InputHasher::new("wasm-semantic-workspace-projection-v1");
3801 for (identity, projection) in projected_packages {
3802 hasher.field("package-identity", identity.as_bytes());
3803 hasher.field("package-projection", projection.as_bytes());
3804 }
3805 hash_toml_setting(&mut hasher, "cargo-features", root.get("cargo-features"));
3806 hash_toml_setting(&mut hasher, "profile", root.get("profile"));
3807 let workspace = root.get("workspace").and_then(TomlValue::as_table);
3808 hash_toml_setting(
3809 &mut hasher,
3810 "workspace-resolver",
3811 workspace.and_then(|table| table.get("resolver")),
3812 );
3813 hash_toml_setting(
3814 &mut hasher,
3815 "workspace-lints",
3816 workspace.and_then(|table| table.get("lints")),
3817 );
3818 Ok(Some(hasher.finish()))
3819}
3820
3821fn locked_package_identities(
3822 workspace_root: &Path,
3823) -> Result<Vec<LockedPackageIdentity>, WasmBuildError> {
3824 let lockfile = workspace_root.join("Cargo.lock");
3825 let contents = match fs::read_to_string(&lockfile) {
3826 Ok(contents) => contents,
3827 Err(error) if error.kind() == io::ErrorKind::NotFound => return Ok(Vec::new()),
3828 Err(source) => {
3829 return Err(WasmBuildError::Io {
3830 operation: "read Cargo lockfile for semantic projection",
3831 path: lockfile,
3832 source,
3833 });
3834 }
3835 };
3836 let lock = toml::from_str::<TomlValue>(&contents).map_err(|error| {
3837 invalid_metadata(&format!(
3838 "Cargo lockfile could not be projected as TOML: {error}"
3839 ))
3840 })?;
3841 let Some(packages) = lock.get("package").and_then(TomlValue::as_array) else {
3842 return Ok(Vec::new());
3843 };
3844 packages
3845 .iter()
3846 .filter_map(|package| {
3847 let Some(table) = package.as_table() else {
3848 return Some(Err(invalid_metadata(
3849 "Cargo lockfile package entry is not a table",
3850 )));
3851 };
3852 let source = table.get("source")?.as_str().map(str::to_owned);
3853 Some(
3854 source
3855 .ok_or_else(|| {
3856 invalid_metadata("Cargo lockfile package source is not a string")
3857 })
3858 .and_then(|source| {
3859 Ok(LockedPackageIdentity {
3860 name: required_toml_string(table, "name", "Cargo lockfile package")?,
3861 version: required_toml_string(
3862 table,
3863 "version",
3864 "Cargo lockfile package",
3865 )?,
3866 source,
3867 checksum: optional_toml_string(
3868 table,
3869 "checksum",
3870 "Cargo lockfile package",
3871 )?,
3872 })
3873 }),
3874 )
3875 })
3876 .collect()
3877}
3878
3879fn required_toml_string(
3880 table: &toml::Table,
3881 field: &str,
3882 context: &str,
3883) -> Result<String, WasmBuildError> {
3884 table
3885 .get(field)
3886 .and_then(TomlValue::as_str)
3887 .map(str::to_owned)
3888 .ok_or_else(|| invalid_metadata(&format!("{context} `{field}` is missing or not a string")))
3889}
3890
3891fn optional_toml_string(
3892 table: &toml::Table,
3893 field: &str,
3894 context: &str,
3895) -> Result<Option<String>, WasmBuildError> {
3896 match table.get(field) {
3897 None => Ok(None),
3898 Some(TomlValue::String(value)) => Ok(Some(value.clone())),
3899 Some(_) => Err(invalid_metadata(&format!(
3900 "{context} `{field}` is not a string"
3901 ))),
3902 }
3903}
3904
3905fn semantic_package_identity(
3906 package: &MetadataPackage,
3907 workspace_root: &Path,
3908 locked_packages: &[LockedPackageIdentity],
3909) -> Option<InputDigest> {
3910 let mut hasher = InputHasher::new("wasm-semantic-package-identity-v1");
3911 hasher.field("name", package.name.as_bytes());
3912 hasher.field("version", package.version.as_bytes());
3913 if package.is_local {
3914 let manifest = package.manifest_path.strip_prefix(workspace_root).ok()?;
3915 let package_root = package.manifest_path.parent()?;
3916 if package_root == workspace_root {
3917 return None;
3918 }
3919 hasher.field("local-manifest", &os_bytes(manifest.as_os_str()));
3920 } else {
3921 let metadata_source = package.source.as_deref()?;
3922 let locked = locked_packages.iter().find(|locked| {
3923 locked.name == package.name
3924 && locked.version == package.version
3925 && locked.source == metadata_source
3926 })?;
3927 match locked.source.as_str() {
3928 source if source.starts_with("registry+") && locked.checksum.is_some() => {}
3929 source if source.starts_with("git+") && source.contains('#') => {}
3930 _ => return None,
3931 }
3932 hasher.field("external-package-id", package.id.as_bytes());
3933 hasher.field("external-source", locked.source.as_bytes());
3934 hasher.field(
3935 "external-checksum",
3936 locked.checksum.as_deref().unwrap_or_default().as_bytes(),
3937 );
3938 }
3939 Some(hasher.finish())
3940}
3941
3942fn semantic_package_projection(
3943 package: &MetadataPackage,
3944 node: &Value,
3945 identities: &HashMap<&str, InputDigest>,
3946) -> Result<InputDigest, WasmBuildError> {
3947 let mut hasher = InputHasher::new("wasm-semantic-package-projection-v1");
3951 for (field, value) in &package.semantic_fields {
3952 hasher.field("package-field-name", field.as_bytes());
3953 match value {
3954 Some(value) => hasher.field("package-field-value", value.as_bytes()),
3955 None => hasher.field("package-field-missing", b""),
3956 }
3957 }
3958
3959 let mut features = node
3960 .get("features")
3961 .and_then(Value::as_array)
3962 .ok_or_else(|| invalid_metadata("Cargo metadata dependency node has no features array"))?
3963 .iter()
3964 .map(|feature| {
3965 feature.as_str().map(str::to_owned).ok_or_else(|| {
3966 invalid_metadata("Cargo metadata dependency feature is not a string")
3967 })
3968 })
3969 .collect::<Result<Vec<_>, _>>()?;
3970 features.sort();
3971 for feature in features {
3972 hasher.field("enabled-feature", feature.as_bytes());
3973 }
3974
3975 let mut dependencies = node
3976 .get("deps")
3977 .and_then(Value::as_array)
3978 .ok_or_else(|| invalid_metadata("Cargo metadata dependency node has no deps array"))?
3979 .iter()
3980 .map(|dependency| {
3981 let name = required_string(dependency, "name")
3982 .map_err(|message| invalid_metadata(&message))?;
3983 let package_id =
3984 required_string(dependency, "pkg").map_err(|message| invalid_metadata(&message))?;
3985 let identity = identities
3986 .get(package_id.as_str())
3987 .copied()
3988 .ok_or_else(|| {
3989 invalid_metadata(&format!(
3990 "dependency `{package_id}` is outside the selected package closure"
3991 ))
3992 })?;
3993 let kinds = dependency
3994 .get("dep_kinds")
3995 .ok_or_else(|| invalid_metadata("Cargo metadata dependency has no kind array"))?
3996 .to_string();
3997 Ok::<_, WasmBuildError>((name, identity, kinds))
3998 })
3999 .collect::<Result<Vec<_>, _>>()?;
4000 dependencies.sort();
4001 for (name, identity, kinds) in dependencies {
4002 hasher.field("dependency-name", name.as_bytes());
4003 hasher.field("dependency-identity", identity.as_bytes());
4004 hasher.field("dependency-kinds", kinds.as_bytes());
4005 }
4006 Ok(hasher.finish())
4007}
4008
4009fn hash_toml_setting(hasher: &mut InputHasher, label: &str, value: Option<&TomlValue>) {
4010 hasher.field("workspace-setting-name", label.as_bytes());
4011 match value {
4012 Some(value) => hasher.field("workspace-setting-value", value.to_string().as_bytes()),
4013 None => hasher.field("workspace-setting-missing", b""),
4014 }
4015}
4016
4017fn is_broad_workspace_input(label: &Path) -> bool {
4018 label == Path::new("workspace/Cargo.toml") || label == Path::new("workspace/Cargo.lock")
4019}
4020
4021fn digest_resolved_local_inputs(
4022 inputs: &ResolvedLocalInputs,
4023 exclusions: &[PathBuf],
4024 cache: &mut LabeledPathDigestCache,
4025 error_path: &Path,
4026 validation_operation: &'static str,
4027 semantic_operation: &'static str,
4028) -> Result<(InputDigest, InputDigest), WasmBuildError> {
4029 let validation_digest = digest_labeled_paths_composable(
4030 "wasm-source-inputs-v1",
4031 inputs
4032 .validation_inputs
4033 .iter()
4034 .map(|(label, path)| (label.as_path(), path.as_path())),
4035 exclusions,
4036 cache,
4037 )
4038 .map_err(|source| WasmBuildError::Io {
4039 operation: validation_operation,
4040 path: error_path.to_owned(),
4041 source,
4042 })?;
4043 let input_digest = semantic_input_digest(inputs, validation_digest, exclusions, cache)
4044 .map_err(|source| WasmBuildError::Io {
4045 operation: semantic_operation,
4046 path: error_path.to_owned(),
4047 source,
4048 })?;
4049 Ok((input_digest, validation_digest))
4050}
4051
4052fn semantic_input_digest(
4053 inputs: &ResolvedLocalInputs,
4054 validation_digest: InputDigest,
4055 exclusions: &[PathBuf],
4056 cache: &mut LabeledPathDigestCache,
4057) -> io::Result<InputDigest> {
4058 let Some(workspace) = inputs.workspace_projection else {
4059 return Ok(validation_digest);
4060 };
4061 let path_digest = digest_labeled_paths_composable(
4062 "wasm-source-inputs-v1",
4063 inputs
4064 .validation_inputs
4065 .iter()
4066 .filter(|(label, _)| !is_broad_workspace_input(label))
4067 .map(|(label, path)| (label.as_path(), path.as_path())),
4068 exclusions,
4069 cache,
4070 )?;
4071 let mut hasher = InputHasher::new("wasm-semantic-source-inputs-v1");
4072 hasher.field("path-input-digest", path_digest.as_bytes());
4073 hasher.field("workspace-projection", workspace.as_bytes());
4074 Ok(hasher.finish())
4075}
4076
4077fn workspace_configuration_inputs(
4078 spec: &WasmBuildSpec,
4079 workspace_root: &Path,
4080) -> Result<Vec<(PathBuf, PathBuf)>, WasmBuildError> {
4081 let mut inputs = Vec::new();
4082 add_if_present(
4083 &mut inputs,
4084 "workspace/Cargo.toml",
4085 workspace_root.join("Cargo.toml"),
4086 );
4087 add_if_present(
4088 &mut inputs,
4089 "workspace/Cargo.lock",
4090 workspace_root.join("Cargo.lock"),
4091 );
4092 add_if_present(
4093 &mut inputs,
4094 "workspace/rust-toolchain.toml",
4095 workspace_root.join("rust-toolchain.toml"),
4096 );
4097 add_if_present(
4098 &mut inputs,
4099 "workspace/rust-toolchain",
4100 workspace_root.join("rust-toolchain"),
4101 );
4102 append_cargo_configuration_inputs(&mut inputs, spec, workspace_root)?;
4103 Ok(inputs)
4104}
4105
4106fn append_cargo_configuration_inputs(
4107 inputs: &mut Vec<(PathBuf, PathBuf)>,
4108 spec: &WasmBuildSpec,
4109 workspace_root: &Path,
4110) -> Result<(), WasmBuildError> {
4111 let invocation_root =
4112 spec.workspace_root
4113 .canonicalize()
4114 .map_err(|source| WasmBuildError::Io {
4115 operation: "resolve Cargo invocation directory",
4116 path: spec.workspace_root.clone(),
4117 source,
4118 })?;
4119 let canonical_workspace =
4120 workspace_root
4121 .canonicalize()
4122 .map_err(|source| WasmBuildError::Io {
4123 operation: "resolve Cargo workspace directory",
4124 path: workspace_root.to_owned(),
4125 source,
4126 })?;
4127
4128 let mut roots = invocation_root
4129 .ancestors()
4130 .filter_map(|directory| effective_cargo_config(&directory.join(".cargo")))
4131 .collect::<Vec<_>>();
4132 if let Some(cargo_home) = effective_cargo_home(spec, &invocation_root)
4133 && let Some(config) = effective_cargo_config(&cargo_home)
4134 {
4135 roots.push(config);
4136 }
4137
4138 let mut visited = BTreeSet::new();
4139 for config in roots {
4140 append_cargo_configuration_tree(
4141 inputs,
4142 &config,
4143 &canonical_workspace,
4144 &mut visited,
4145 false,
4146 )?;
4147 }
4148 Ok(())
4149}
4150
4151fn effective_cargo_config(directory: &Path) -> Option<PathBuf> {
4152 let extensionless = directory.join("config");
4153 if extensionless.exists() {
4154 return Some(extensionless);
4155 }
4156 let toml = directory.join("config.toml");
4157 toml.exists().then_some(toml)
4158}
4159
4160fn effective_cargo_home(spec: &WasmBuildSpec, invocation_root: &Path) -> Option<PathBuf> {
4161 if let Some(cargo_home) = command_environment_value(spec, "CARGO_HOME") {
4162 let cargo_home = PathBuf::from(cargo_home);
4163 return Some(if cargo_home.is_absolute() {
4164 cargo_home
4165 } else {
4166 invocation_root.join(cargo_home)
4167 });
4168 }
4169
4170 default_home_directory(spec).map(|home| {
4171 let home = if home.is_absolute() {
4172 home
4173 } else {
4174 invocation_root.join(home)
4175 };
4176 home.join(".cargo")
4177 })
4178}
4179
4180#[cfg(windows)]
4181fn default_home_directory(spec: &WasmBuildSpec) -> Option<PathBuf> {
4182 command_environment_value(spec, "USERPROFILE")
4183 .or_else(|| command_environment_value(spec, "HOME"))
4184 .map(PathBuf::from)
4185}
4186
4187#[cfg(not(windows))]
4188fn default_home_directory(spec: &WasmBuildSpec) -> Option<PathBuf> {
4189 command_environment_value(spec, "HOME").map(PathBuf::from)
4190}
4191
4192fn command_environment_value(spec: &WasmBuildSpec, name: &str) -> Option<OsString> {
4193 spec.extra_env
4194 .get(OsStr::new(name))
4195 .cloned()
4196 .or_else(|| std::env::var_os(name))
4197}
4198
4199fn append_cargo_configuration_tree(
4200 inputs: &mut Vec<(PathBuf, PathBuf)>,
4201 config: &Path,
4202 workspace_root: &Path,
4203 visited: &mut BTreeSet<PathBuf>,
4204 optional: bool,
4205) -> Result<(), WasmBuildError> {
4206 let canonical = match config.canonicalize() {
4207 Ok(canonical) => canonical,
4208 Err(error) if optional && error.kind() == io::ErrorKind::NotFound => return Ok(()),
4209 Err(source) => {
4210 return Err(WasmBuildError::Io {
4211 operation: "resolve Cargo configuration",
4212 path: config.to_owned(),
4213 source,
4214 });
4215 }
4216 };
4217 if !visited.insert(canonical.clone()) {
4218 return Ok(());
4219 }
4220
4221 let contents = fs::read_to_string(&canonical).map_err(|source| WasmBuildError::Io {
4222 operation: "read Cargo configuration",
4223 path: canonical.clone(),
4224 source,
4225 })?;
4226 let configuration = toml::from_str::<TomlValue>(&contents).map_err(|error| {
4227 WasmBuildError::InvalidCargoConfiguration {
4228 path: canonical.clone(),
4229 message: error.to_string(),
4230 }
4231 })?;
4232 inputs.push((
4233 cargo_configuration_label(&canonical, workspace_root),
4234 canonical.clone(),
4235 ));
4236
4237 let Some(include) = configuration.get("include") else {
4238 return Ok(());
4239 };
4240 let parent = canonical
4241 .parent()
4242 .ok_or_else(|| WasmBuildError::InvalidCargoConfiguration {
4243 path: canonical.clone(),
4244 message: "configuration path has no parent directory".to_owned(),
4245 })?;
4246 for (included, optional) in cargo_configuration_includes(include, &canonical)? {
4247 let included = if included.is_absolute() {
4248 included
4249 } else {
4250 parent.join(included)
4251 };
4252 append_cargo_configuration_tree(inputs, &included, workspace_root, visited, optional)?;
4253 }
4254 Ok(())
4255}
4256
4257fn cargo_configuration_includes(
4258 include: &TomlValue,
4259 config: &Path,
4260) -> Result<Vec<(PathBuf, bool)>, WasmBuildError> {
4261 let values = match include {
4262 TomlValue::Array(values) => values.as_slice(),
4263 value => std::slice::from_ref(value),
4264 };
4265 values
4266 .iter()
4267 .map(|value| match value {
4268 TomlValue::String(path) => Ok((PathBuf::from(path), false)),
4269 TomlValue::Table(table) => {
4270 let path = table
4271 .get("path")
4272 .and_then(TomlValue::as_str)
4273 .ok_or_else(|| {
4274 invalid_cargo_configuration(
4275 config,
4276 "Cargo configuration include table requires a string `path`",
4277 )
4278 })?;
4279 let optional = table
4280 .get("optional")
4281 .map(|value| {
4282 value.as_bool().ok_or_else(|| {
4283 invalid_cargo_configuration(
4284 config,
4285 "Cargo configuration include `optional` must be a boolean",
4286 )
4287 })
4288 })
4289 .transpose()?
4290 .unwrap_or(false);
4291 Ok((PathBuf::from(path), optional))
4292 }
4293 _ => Err(invalid_cargo_configuration(
4294 config,
4295 "Cargo configuration `include` must contain paths or include tables",
4296 )),
4297 })
4298 .collect()
4299}
4300
4301fn cargo_configuration_label(config: &Path, workspace_root: &Path) -> PathBuf {
4302 if let Ok(relative) = config.strip_prefix(workspace_root) {
4303 return PathBuf::from("cargo-config/workspace").join(relative);
4304 }
4305 let location = digest_bytes("cargo-config-location-v1", &os_bytes(config.as_os_str()));
4306 PathBuf::from("cargo-config/external").join(location.to_hex())
4307}
4308
4309fn invalid_cargo_configuration(path: &Path, message: &str) -> WasmBuildError {
4310 WasmBuildError::InvalidCargoConfiguration {
4311 path: path.to_owned(),
4312 message: message.to_owned(),
4313 }
4314}
4315
4316fn append_package_inputs(
4317 inputs: &mut Vec<(PathBuf, PathBuf)>,
4318 packages: &HashMap<String, MetadataPackage>,
4319 closure: BTreeSet<String>,
4320 workspace_root: &Path,
4321) -> Result<(), WasmBuildError> {
4322 for id in closure {
4323 let Some(package) = packages.get(&id) else {
4324 return Err(invalid_metadata(&format!(
4325 "resolved package `{id}` is missing"
4326 )));
4327 };
4328 if !package.is_local {
4329 continue;
4330 }
4331 let root = package.manifest_path.parent().ok_or_else(|| {
4332 invalid_metadata(&format!(
4333 "package `{}` manifest has no parent",
4334 package.name
4335 ))
4336 })?;
4337 let relative_manifest = package
4338 .manifest_path
4339 .strip_prefix(workspace_root)
4340 .unwrap_or(&package.manifest_path);
4341 let label = PathBuf::from(format!("package/{}@{}", package.name, package.version))
4342 .join(relative_manifest.parent().unwrap_or_else(|| Path::new(".")));
4343 inputs.push((label, root.to_owned()));
4344 }
4345 Ok(())
4346}
4347
4348fn append_additional_inputs(
4349 inputs: &mut Vec<(PathBuf, PathBuf)>,
4350 spec: &WasmBuildSpec,
4351 workspace_root: &Path,
4352) {
4353 for additional in &spec.additional_inputs {
4354 let path = if additional.is_absolute() {
4355 additional.clone()
4356 } else {
4357 workspace_root.join(additional)
4358 };
4359 inputs.push((PathBuf::from("additional").join(additional), path));
4360 }
4361}
4362
4363fn source_exclusions(spec: &WasmBuildSpec, inputs: &[(PathBuf, PathBuf)]) -> Vec<PathBuf> {
4364 let mut exclusions = vec![
4365 spec.target_dir.clone(),
4366 spec.workspace_root.join("target"),
4367 spec.workspace_root.join(".git"),
4368 ];
4369 if let Some(shared_target) = shared_incremental_target(spec) {
4370 exclusions.push(shared_target);
4371 }
4372 for (_, path) in inputs {
4373 if path.is_dir() {
4374 exclusions.push(path.join("target"));
4375 exclusions.push(path.join(".git"));
4376 }
4377 }
4378 exclusions
4379}
4380
4381fn validate_shared_incremental_target_boundary(
4382 spec: &WasmBuildSpec,
4383 inputs: &[(PathBuf, PathBuf)],
4384) -> Result<(), WasmBuildError> {
4385 let Some(shared_target) = shared_incremental_target(spec) else {
4386 return Ok(());
4387 };
4388 let shared_target =
4389 canonicalize_allow_missing(&shared_target).map_err(|source| WasmBuildError::Io {
4390 operation: "resolve shared incremental Cargo target boundary",
4391 path: shared_target.clone(),
4392 source,
4393 })?;
4394 let exact_entries = spec.target_dir.join(".ic-testkit/wasm-targets");
4395 let exact_entries =
4396 canonicalize_allow_missing(&exact_entries).map_err(|source| WasmBuildError::Io {
4397 operation: "resolve exact Wasm cache boundary",
4398 path: exact_entries,
4399 source,
4400 })?;
4401 if shared_target.starts_with(&exact_entries)
4404 || (exact_entries.starts_with(&shared_target)
4405 && !exact_entries.starts_with(shared_target.join(".ic-testkit")))
4406 {
4407 return Err(WasmBuildError::InvalidSpec {
4408 message: "shared incremental target must not overlap removable exact Wasm cache state"
4409 .to_owned(),
4410 });
4411 }
4412 let resolved_inputs = inputs
4413 .iter()
4414 .map(|(_, input)| {
4415 let canonical = input.canonicalize().map_err(|source| WasmBuildError::Io {
4416 operation: "resolve Cargo input boundary",
4417 path: input.clone(),
4418 source,
4419 })?;
4420 let metadata = fs::metadata(&canonical).map_err(|source| WasmBuildError::Io {
4421 operation: "inspect Cargo input boundary",
4422 path: canonical.clone(),
4423 source,
4424 })?;
4425 Ok((canonical, metadata.is_dir()))
4426 })
4427 .collect::<Result<Vec<_>, WasmBuildError>>()?;
4428 let safe_generated_roots = std::iter::once(spec.target_dir.clone())
4429 .chain(std::iter::once(spec.workspace_root.join("target")))
4430 .chain(
4431 inputs
4432 .iter()
4433 .filter(|(_, path)| path.is_dir())
4434 .map(|(_, path)| path.join("target")),
4435 )
4436 .filter_map(|path| canonicalize_allow_missing(&path).ok())
4437 .filter(|root| {
4438 !resolved_inputs
4439 .iter()
4440 .any(|(input, _is_directory)| input.starts_with(root))
4441 })
4442 .collect::<Vec<_>>();
4443 if safe_generated_roots
4444 .iter()
4445 .any(|root| shared_target.starts_with(root))
4446 {
4447 return Ok(());
4448 }
4449
4450 for (input, is_directory) in resolved_inputs {
4451 if shared_target == input
4452 || (is_directory && shared_target.starts_with(&input))
4453 || input.starts_with(&shared_target)
4454 {
4455 return Err(WasmBuildError::InvalidSpec {
4456 message: format!(
4457 "shared incremental target {} must not overlap exact Cargo inputs unless it is inside a generated target directory",
4458 shared_target.display()
4459 ),
4460 });
4461 }
4462 }
4463 Ok(())
4464}
4465
4466pub(super) fn shared_incremental_target(spec: &WasmBuildSpec) -> Option<PathBuf> {
4467 let WasmBuildCacheMode::SharedIncremental { target_dir } = &spec.cache_mode else {
4468 return None;
4469 };
4470 Some(if target_dir.is_absolute() {
4471 target_dir.clone()
4472 } else {
4473 spec.workspace_root.join(target_dir)
4474 })
4475}
4476
4477fn shared_incremental_target_exists(
4478 spec: &WasmBuildSpec,
4479 operation: &'static str,
4480) -> Result<bool, WasmBuildError> {
4481 let target_dir =
4482 shared_incremental_target(spec).ok_or_else(|| WasmBuildError::InvalidSpec {
4483 message: "shared incremental target is not configured".to_owned(),
4484 })?;
4485 match fs::symlink_metadata(&target_dir) {
4486 Ok(metadata) if metadata.is_dir() => Ok(true),
4487 Ok(_) => Err(WasmBuildError::InvalidSpec {
4488 message: format!(
4489 "shared incremental Cargo target {} must be a directory",
4490 target_dir.display()
4491 ),
4492 }),
4493 Err(error) if error.kind() == io::ErrorKind::NotFound => Ok(false),
4494 Err(source) => Err(WasmBuildError::Io {
4495 operation,
4496 path: target_dir,
4497 source,
4498 }),
4499 }
4500}
4501
4502fn effective_environment(spec: &WasmBuildSpec) -> BTreeMap<OsString, Option<OsString>> {
4503 let mut names = spec.inherited_env.clone();
4504 names.extend(AUTOMATIC_ENVIRONMENT.iter().map(OsString::from));
4505 let mut environment = names
4506 .into_iter()
4507 .map(|name| {
4508 let value = std::env::var_os(&name);
4509 (name, value)
4510 })
4511 .collect::<BTreeMap<_, _>>();
4512 for (key, value) in &spec.extra_env {
4513 environment.insert(key.clone(), Some(value.clone()));
4514 }
4515 environment
4516}
4517
4518fn apply_command_environment(command: &mut Command, spec: &WasmBuildSpec) {
4519 for (key, value) in &spec.extra_env {
4520 command.env(key, value);
4521 }
4522}
4523
4524fn run_cargo_build(
4525 spec: &WasmBuildSpec,
4526 build_target_dir: &Path,
4527 progress: &mut ProgressReporter<'_>,
4528) -> Result<(), WasmBuildError> {
4529 let absolute_target_dir =
4530 canonicalize_allow_missing(build_target_dir).map_err(|source| WasmBuildError::Io {
4531 operation: "resolve Cargo build target directory",
4532 path: build_target_dir.to_owned(),
4533 source,
4534 })?;
4535 let mut command = Command::new(&spec.cargo_program);
4536 command
4537 .current_dir(&spec.workspace_root)
4538 .env("CARGO_TARGET_DIR", absolute_target_dir)
4539 .args(["build", "--target", &spec.target])
4540 .args(&spec.cargo_profile_args);
4541 apply_command_environment(&mut command, spec);
4542 for package in &spec.packages {
4543 command.args(["-p", package]);
4544 }
4545
4546 if !progress.is_observed() {
4547 let output = command
4548 .output()
4549 .map_err(|source| WasmBuildError::CommandSpawn {
4550 phase: WasmBuildPhase::CargoBuild,
4551 program: spec.cargo_program.clone(),
4552 source,
4553 })?;
4554 return ensure_command_success(WasmBuildPhase::CargoBuild, output).map(|_| ());
4555 }
4556
4557 run_observed_cargo_build(spec, build_target_dir, command, progress)
4558}
4559
4560fn run_observed_cargo_build(
4561 spec: &WasmBuildSpec,
4562 build_target_dir: &Path,
4563 mut command: Command,
4564 progress: &mut ProgressReporter<'_>,
4565) -> Result<(), WasmBuildError> {
4566 command.stdout(Stdio::piped()).stderr(Stdio::piped());
4567 let started = Instant::now();
4568 let child = command
4569 .spawn()
4570 .map_err(|source| WasmBuildError::CommandSpawn {
4571 phase: WasmBuildPhase::CargoBuild,
4572 program: spec.cargo_program.clone(),
4573 source,
4574 })?;
4575 let mut child = ObservedChild::new(child);
4576 progress.emit(WasmBuildProgressEvent::CargoStarted {
4577 target_dir: build_target_dir.to_owned(),
4578 });
4579
4580 let stdout = child
4581 .child_mut()
4582 .stdout
4583 .take()
4584 .expect("Cargo stdout must be piped");
4585 let stderr = child
4586 .child_mut()
4587 .stderr
4588 .take()
4589 .expect("Cargo stderr must be piped");
4590 let (sender, chunks) = mpsc::channel();
4591 let stdout_sender = sender.clone();
4592 let stdout_reader = thread::spawn(move || {
4593 read_process_output(stdout, WasmBuildOutputStream::Stdout, stdout_sender)
4594 });
4595 let stderr_reader =
4596 thread::spawn(move || read_process_output(stderr, WasmBuildOutputStream::Stderr, sender));
4597
4598 let captured = capture_observed_cargo_output(chunks, progress, started);
4599
4600 let status = child.wait().map_err(|source| WasmBuildError::Io {
4601 operation: "wait for observed cargo build",
4602 path: PathBuf::from(&spec.cargo_program),
4603 source,
4604 })?;
4605 join_output_reader(
4606 stdout_reader,
4607 "read observed cargo stdout",
4608 &spec.cargo_program,
4609 )?;
4610 join_output_reader(
4611 stderr_reader,
4612 "read observed cargo stderr",
4613 &spec.cargo_program,
4614 )?;
4615 let elapsed = started.elapsed();
4616 progress.emit(WasmBuildProgressEvent::CargoFinished {
4617 success: status.success(),
4618 code: status.code(),
4619 elapsed,
4620 });
4621
4622 ensure_command_success(
4623 WasmBuildPhase::CargoBuild,
4624 Output {
4625 status,
4626 stdout: captured.stdout,
4627 stderr: captured.stderr,
4628 },
4629 )
4630 .map(|_| ())
4631}
4632
4633struct CapturedProcessOutput {
4634 stdout: Vec<u8>,
4635 stderr: Vec<u8>,
4636}
4637
4638fn capture_observed_cargo_output(
4639 chunks: mpsc::Receiver<ProcessOutputChunk>,
4640 progress: &mut ProgressReporter<'_>,
4641 started: Instant,
4642) -> CapturedProcessOutput {
4643 let mut stdout = Vec::new();
4644 let mut stderr = Vec::new();
4645 loop {
4646 let message = match progress.heartbeat_due_in() {
4647 Some(wait) => match chunks.recv_timeout(wait) {
4648 Ok(chunk) => Some(chunk),
4649 Err(RecvTimeoutError::Timeout) => {
4650 progress.emit_heartbeat(WasmBuildProgressPhase::CargoBuild, started.elapsed());
4651 None
4652 }
4653 Err(RecvTimeoutError::Disconnected) => break,
4654 },
4655 None => match chunks.recv() {
4656 Ok(chunk) => Some(chunk),
4657 Err(_) => break,
4658 },
4659 };
4660 let Some(chunk) = message else {
4661 continue;
4662 };
4663 match chunk.stream {
4664 WasmBuildOutputStream::Stdout => stdout.extend_from_slice(&chunk.bytes),
4665 WasmBuildOutputStream::Stderr => stderr.extend_from_slice(&chunk.bytes),
4666 }
4667 if progress.config.emit_cargo_output {
4668 progress.emit(WasmBuildProgressEvent::CargoOutput {
4669 stream: chunk.stream,
4670 bytes: chunk.bytes,
4671 });
4672 }
4673 }
4674 CapturedProcessOutput { stdout, stderr }
4675}
4676
4677#[derive(Debug)]
4678struct ProcessOutputChunk {
4679 stream: WasmBuildOutputStream,
4680 bytes: Vec<u8>,
4681}
4682
4683fn read_process_output<R: io::Read>(
4684 mut reader: R,
4685 stream: WasmBuildOutputStream,
4686 sender: mpsc::Sender<ProcessOutputChunk>,
4687) -> io::Result<()> {
4688 let mut buffer = [0_u8; 8 * 1024];
4689 loop {
4690 let count = match reader.read(&mut buffer) {
4691 Ok(count) => count,
4692 Err(error) if error.kind() == io::ErrorKind::Interrupted => continue,
4693 Err(error) => return Err(error),
4694 };
4695 if count == 0 {
4696 return Ok(());
4697 }
4698 if sender
4699 .send(ProcessOutputChunk {
4700 stream,
4701 bytes: buffer[..count].to_vec(),
4702 })
4703 .is_err()
4704 {
4705 return Ok(());
4706 }
4707 }
4708}
4709
4710fn join_output_reader(
4711 reader: thread::JoinHandle<io::Result<()>>,
4712 operation: &'static str,
4713 cargo_program: &OsStr,
4714) -> Result<(), WasmBuildError> {
4715 let result = reader.join().map_err(|_| WasmBuildError::Io {
4716 operation,
4717 path: PathBuf::from(cargo_program),
4718 source: io::Error::other("Cargo output reader panicked"),
4719 })?;
4720 result.map_err(|source| WasmBuildError::Io {
4721 operation,
4722 path: PathBuf::from(cargo_program),
4723 source,
4724 })
4725}
4726
4727struct ObservedChild(Option<Child>);
4728
4729impl ObservedChild {
4730 const fn new(child: Child) -> Self {
4731 Self(Some(child))
4732 }
4733
4734 const fn child_mut(&mut self) -> &mut Child {
4735 self.0.as_mut().expect("observed child must be present")
4736 }
4737
4738 fn wait(&mut self) -> io::Result<ExitStatus> {
4739 let status = self.child_mut().wait()?;
4740 self.0.take();
4741 Ok(status)
4742 }
4743}
4744
4745impl Drop for ObservedChild {
4746 fn drop(&mut self) {
4747 if let Some(mut child) = self.0.take() {
4748 let _ = child.kill();
4749 let _ = child.wait();
4750 }
4751 }
4752}
4753
4754fn ensure_command_success(phase: WasmBuildPhase, output: Output) -> Result<Output, WasmBuildError> {
4755 if output.status.success() {
4756 return Ok(output);
4757 }
4758 Err(WasmBuildError::CommandFailed {
4759 phase,
4760 status: output.status,
4761 stdout: String::from_utf8_lossy(&output.stdout).into_owned(),
4762 stderr: String::from_utf8_lossy(&output.stderr).into_owned(),
4763 })
4764}
4765
4766fn expected_artifacts(spec: &WasmBuildSpec, target_dir: &Path) -> Vec<PathBuf> {
4767 let mut packages = spec.packages.iter().map(String::as_str).collect::<Vec<_>>();
4768 packages.sort_unstable();
4769 packages.dedup();
4770 packages
4771 .into_iter()
4772 .map(|package| {
4773 if spec.target == DEFAULT_TARGET {
4774 wasm_path(target_dir, package, &spec.profile_target_dir)
4775 } else {
4776 target_dir
4777 .join(&spec.target)
4778 .join(&spec.profile_target_dir)
4779 .join(format!("{package}.wasm"))
4780 }
4781 })
4782 .collect()
4783}
4784
4785fn cache_entry_directory(spec: &WasmBuildSpec, fingerprint: InputDigest) -> PathBuf {
4786 spec.target_dir
4787 .join(".ic-testkit/wasm-targets")
4788 .join(fingerprint.to_hex())
4789}
4790
4791fn artifact_set_matches(artifacts: &[PathBuf], fingerprint: InputDigest) -> bool {
4792 artifacts.iter().all(|path| {
4793 fs::metadata(path).is_ok_and(|metadata| metadata.is_file() && metadata.len() > 0)
4794 && cache_stamp_matches(path, fingerprint)
4795 })
4796}
4797
4798fn missing_artifacts(artifacts: &[PathBuf]) -> Vec<PathBuf> {
4799 artifacts
4800 .iter()
4801 .filter(|path| {
4802 fs::metadata(path).map_or(true, |metadata| !metadata.is_file() || metadata.len() == 0)
4803 })
4804 .cloned()
4805 .collect()
4806}
4807
4808fn cache_stamp_matches(artifact: &Path, fingerprint: InputDigest) -> bool {
4809 let stamp_path = artifact_stamp_path(artifact);
4810 let Ok(stamp) = fs::read_to_string(stamp_path) else {
4811 return false;
4812 };
4813 if !stamp.starts_with(&artifact_stamp_header(fingerprint)) {
4816 return false;
4817 }
4818 let Ok(expected) = artifact_stamp_contents(artifact, fingerprint) else {
4819 return false;
4820 };
4821 stamp == expected
4822}
4823
4824fn artifact_stamp_path(artifact: &Path) -> PathBuf {
4825 let mut name = artifact
4826 .file_name()
4827 .map_or_else(|| OsString::from("artifact"), OsString::from);
4828 name.push(".ic-testkit-build");
4829 artifact.with_file_name(name)
4830}
4831
4832fn artifact_stamp_header(fingerprint: InputDigest) -> String {
4833 format!("{CACHE_FORMAT_VERSION}\nbuild-sha256:{fingerprint}\n")
4834}
4835
4836fn artifact_stamp_contents(artifact: &Path, fingerprint: InputDigest) -> io::Result<String> {
4837 let (_, artifact_digest) = digest_file("wasm-artifact-v1", artifact)?;
4838 Ok(format!(
4839 "{}artifact-sha256:{artifact_digest}\n",
4840 artifact_stamp_header(fingerprint),
4841 ))
4842}
4843
4844fn publish_artifact_stamps(
4845 artifacts: &[PathBuf],
4846 fingerprint: InputDigest,
4847) -> Result<(), WasmBuildError> {
4848 for artifact in artifacts {
4849 let stamp_path = artifact_stamp_path(artifact);
4850 let stamp = artifact_stamp_contents(artifact, fingerprint).map_err(|source| {
4851 WasmBuildError::Io {
4852 operation: "hash built Wasm artifact",
4853 path: artifact.clone(),
4854 source,
4855 }
4856 })?;
4857 write_atomic(&stamp_path, stamp.as_bytes()).map_err(|source| WasmBuildError::Io {
4858 operation: "publish Wasm build stamp",
4859 path: stamp_path,
4860 source,
4861 })?;
4862 }
4863 Ok(())
4864}
4865
4866fn materialize_artifacts(
4867 cached_artifacts: &[PathBuf],
4868 artifacts: &[PathBuf],
4869 fingerprint: InputDigest,
4870) -> Result<(), WasmBuildError> {
4871 for (cached, artifact) in cached_artifacts.iter().zip(artifacts) {
4872 copy_file_atomic(cached, artifact).map_err(|source| WasmBuildError::Io {
4873 operation: "publish Wasm artifact",
4874 path: artifact.clone(),
4875 source,
4876 })?;
4877 }
4878 publish_artifact_stamps(artifacts, fingerprint)
4879}
4880
4881fn copy_wasm_artifacts(
4882 source_artifacts: &[PathBuf],
4883 cached_artifacts: &[PathBuf],
4884) -> Result<(), WasmBuildError> {
4885 for (source, cached) in source_artifacts.iter().zip(cached_artifacts) {
4886 copy_file_atomic(source, cached).map_err(|source_error| WasmBuildError::Io {
4887 operation: "cache shared-incremental Wasm artifact",
4888 path: cached.clone(),
4889 source: source_error,
4890 })?;
4891 }
4892 Ok(())
4893}
4894
4895fn create_dir_all(path: &Path, operation: &'static str) -> Result<(), WasmBuildError> {
4896 fs::create_dir_all(path).map_err(|source| WasmBuildError::Io {
4897 operation,
4898 path: path.to_owned(),
4899 source,
4900 })
4901}
4902
4903fn add_if_present(inputs: &mut Vec<(PathBuf, PathBuf)>, label: &str, path: PathBuf) {
4904 if path.exists() {
4905 inputs.push((PathBuf::from(label), path));
4906 }
4907}
4908
4909fn required_string(value: &Value, field: &str) -> Result<String, String> {
4910 value
4911 .get(field)
4912 .and_then(Value::as_str)
4913 .map(str::to_owned)
4914 .ok_or_else(|| format!("Cargo metadata field `{field}` is missing"))
4915}
4916
4917fn optional_string(value: &Value, field: &str) -> Result<Option<String>, String> {
4918 match value.get(field) {
4919 None | Some(Value::Null) => Ok(None),
4920 Some(Value::String(value)) => Ok(Some(value.clone())),
4921 Some(_) => Err(format!(
4922 "Cargo metadata field `{field}` is not a string or null"
4923 )),
4924 }
4925}
4926
4927fn invalid_metadata(message: &str) -> WasmBuildError {
4928 WasmBuildError::InvalidMetadata {
4929 message: message.to_owned(),
4930 }
4931}
4932
4933impl WasmBuildError {
4934 fn indicates_input_change(&self) -> bool {
4935 match self {
4936 Self::InputsChangedDuringAcquisition { .. } => true,
4937 Self::FailedBuildCleanup { build_error, .. } => build_error.indicates_input_change(),
4938 _ => false,
4939 }
4940 }
4941}
4942
4943impl std::fmt::Display for WasmBuildPhase {
4944 fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
4945 formatter.write_str(match self {
4946 Self::CargoMetadata => "cargo metadata",
4947 Self::CargoIdentity => "Cargo identity",
4948 Self::RustcIdentity => "Rust compiler identity",
4949 Self::CargoBuild => "cargo build",
4950 })
4951 }
4952}
4953
4954impl std::fmt::Display for WasmBuildProgressPhase {
4955 fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
4956 formatter.write_str(match self {
4957 Self::ExactCacheLock => "exact cache lock",
4958 Self::CargoIdentity => "Cargo identity",
4959 Self::RustcIdentity => "Rust compiler identity",
4960 Self::CargoMetadata => "Cargo metadata",
4961 Self::InputDiscovery => "input discovery",
4962 Self::ContentHashing => "content hashing",
4963 Self::SharedTargetLock => "shared target lock",
4964 Self::SharedTargetMaintenance => "shared target maintenance",
4965 Self::CargoBuild => "Cargo build",
4966 Self::ArtifactPublication => "artifact publication",
4967 Self::ExactCacheMaintenance => "exact cache maintenance",
4968 })
4969 }
4970}
4971
4972impl std::fmt::Display for WasmBuildError {
4973 fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
4974 match self {
4975 Self::InvalidSpec { message } => {
4976 write!(formatter, "invalid Wasm build spec: {message}")
4977 }
4978 Self::Io {
4979 operation,
4980 path,
4981 source,
4982 } => write!(
4983 formatter,
4984 "failed to {operation} at {}: {source}",
4985 path.display()
4986 ),
4987 Self::CommandSpawn {
4988 phase,
4989 program,
4990 source,
4991 } => write!(
4992 formatter,
4993 "failed to launch {phase} using `{}`: {source}",
4994 program.to_string_lossy(),
4995 ),
4996 Self::CommandFailed {
4997 phase,
4998 status,
4999 stdout,
5000 stderr,
5001 } => write!(
5002 formatter,
5003 "{phase} failed with {status}\nstdout:\n{stdout}\nstderr:\n{stderr}",
5004 ),
5005 Self::InvalidMetadata { message } => {
5006 write!(formatter, "invalid Cargo metadata: {message}")
5007 }
5008 Self::InvalidCargoConfiguration { path, message } => write!(
5009 formatter,
5010 "invalid Cargo configuration at {}: {message}",
5011 path.display(),
5012 ),
5013 Self::MissingArtifacts { paths } => write!(
5014 formatter,
5015 "cargo build succeeded without producing: {}",
5016 paths
5017 .iter()
5018 .map(|path| path.display().to_string())
5019 .collect::<Vec<_>>()
5020 .join(", "),
5021 ),
5022 Self::InputsChangedDuringAcquisition { before, after } => write!(
5023 formatter,
5024 "Wasm inputs changed during artifact acquisition: {before} -> {after}",
5025 ),
5026 Self::PreparedInputSnapshotInvalidated => formatter.write_str(
5027 "the prepared Wasm input snapshot was invalidated before artifact publication",
5028 ),
5029 Self::FailedBuildCleanup {
5030 build_error,
5031 path,
5032 source,
5033 } => write!(
5034 formatter,
5035 "Wasm build failed ({build_error}) and its incomplete target directory at {} could not be removed: {source}",
5036 path.display(),
5037 ),
5038 }
5039 }
5040}
5041
5042impl std::error::Error for WasmBuildError {
5043 fn source(&self) -> Option<&(dyn std::error::Error + 'static)> {
5044 match self {
5045 Self::Io { source, .. }
5046 | Self::CommandSpawn { source, .. }
5047 | Self::FailedBuildCleanup { source, .. } => Some(source),
5048 _ => None,
5049 }
5050 }
5051}
5052
5053#[cfg(test)]
5054mod tests;