1use std::collections::{BTreeMap, BTreeSet};
4use std::fs::{self, File, OpenOptions};
5use std::io::{Read, Write};
6#[cfg(unix)]
7use std::os::unix::fs::OpenOptionsExt;
8use std::path::{Path, PathBuf};
9use std::process::{Command, ExitStatus, Stdio};
10use std::time::{Instant, SystemTime, UNIX_EPOCH};
11
12use ferrum_native_ops::{
13 CudaNativeBuildUnit, NativeBuildArtifactCache, NativeBuildArtifactLookup,
14 NativeBuildArtifactSpec,
15};
16use ferrum_types::{is_sha256_digest, NativeOperatorBackend, NativeOperatorSourcePackage};
17use serde::{Deserialize, Serialize};
18use tempfile::NamedTempFile;
19
20use super::{
21 read_json, require_file, sha256_bytes, sha256_file, symbol_slug, validate_relative_path,
22 write_json, NativeOperatorBuilderError, NativeOperatorEvidenceFile, Result,
23};
24
25pub const NATIVE_OPERATOR_SOURCE_DEFINITION_SCHEMA_VERSION: u32 = 3;
26pub const NATIVE_OPERATOR_SOURCE_BUILD_PLAN_SCHEMA_VERSION: u32 = 3;
27pub const NATIVE_OPERATOR_SOURCE_BUILD_RECEIPT_SCHEMA_VERSION: u32 = 7;
28pub const NATIVE_OPERATOR_SOURCE_OBJECT_BUILD_CONTRACT_VERSION: u32 = 7;
29pub const NATIVE_OPERATOR_CUDA_TOOLKIT_MANIFEST_SCHEMA_VERSION: u32 = 1;
30pub const NATIVE_OPERATOR_HOST_TOOLCHAIN_MANIFEST_SCHEMA_VERSION: u32 = 2;
31pub const NATIVE_OPERATOR_OBJECT_DEPENDENCY_PROOF_SCHEMA_VERSION: u32 = 3;
32pub const MAX_NVCC_THREADS: u32 = 8;
33const REQUIRED_CUDA_TOOLKIT_FILES: [&str; 6] = [
34 "bin/bin2c",
35 "bin/cudafe++",
36 "bin/fatbinary",
37 "bin/nvcc",
38 "bin/nvlink",
39 "bin/ptxas",
40];
41const REQUIRED_CUDA_TOOLKIT_SCOPES: [&str; 4] =
42 ["bin/crt", "include", "nvvm/bin", "nvvm/libdevice"];
43const HOST_TOOLCHAIN_PROGRAMS: [&str; 5] = ["as", "cc1", "cc1plus", "collect2", "ld"];
44const MAX_HOST_TOOLCHAIN_FILES: usize = 250_000;
45const MAX_DEPFILE_BYTES: usize = 16 * 1024 * 1024;
46const MAX_DEPFILE_DEPENDENCIES: usize = 250_000;
47const MAX_DEPFILE_WORD_BYTES: usize = 16 * 1024;
48const MAX_DEPENDENCY_PROOF_BYTES: usize = 64 * 1024 * 1024;
49
50#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
51pub struct NativeOperatorSourceDefinition {
52 pub schema_version: u32,
53 pub operator: String,
54 pub source_package_kind: String,
55 pub source_package_revision: String,
56 pub upstream_sources: Vec<NativeOperatorUpstreamSource>,
57 pub translation_units: Vec<String>,
58 pub headers: Vec<String>,
59 pub dependency_closures: Vec<NativeOperatorTranslationUnitDependencies>,
60 pub include_dirs: Vec<String>,
61 pub defines: Vec<String>,
62 pub nvcc_policy: NativeOperatorNvccPolicy,
63 pub architecture: NativeOperatorCudaArchitecture,
64 pub archive_file: String,
65}
66
67#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
68pub struct NativeOperatorUpstreamSource {
69 pub repository: String,
70 pub revision: String,
71 pub license: String,
72}
73
74#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
75pub struct NativeOperatorTranslationUnitDependencies {
76 pub translation_unit: String,
77 pub headers: Vec<String>,
78}
79
80#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
81pub struct NativeOperatorSourceBuildPlan {
82 pub schema_version: u32,
83 pub operator: String,
84 pub source_package: NativeOperatorSourcePackage,
85 pub upstream_sources: Vec<NativeOperatorUpstreamSource>,
86 pub translation_units: Vec<NativeOperatorSourceFileLock>,
87 pub headers: Vec<NativeOperatorSourceFileLock>,
88 pub dependency_closures: Vec<NativeOperatorTranslationUnitDependencyLock>,
89 pub include_dirs: Vec<String>,
90 pub defines: Vec<String>,
91 pub nvcc_policy: NativeOperatorNvccPolicy,
92 pub architecture: NativeOperatorCudaArchitecture,
93 pub archive_file: String,
94}
95
96#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize)]
97pub struct NativeOperatorSourceFileLock {
98 pub path: String,
99 pub sha256: String,
100}
101
102#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
103pub struct NativeOperatorTranslationUnitDependencyLock {
104 pub translation_unit: String,
105 pub headers: Vec<NativeOperatorSourceFileLock>,
106 pub closure_sha256: String,
107}
108
109#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
110#[serde(rename_all = "snake_case")]
111pub enum NativeOperatorCudaArchitecture {
112 DeviceComputeCapability,
113 Compute80Ptx,
114}
115
116#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
117#[serde(rename_all = "snake_case")]
118pub enum NativeOperatorCppStandard {
119 Cpp17,
120}
121
122#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
123#[serde(rename_all = "snake_case")]
124pub enum NativeOperatorOptimization {
125 O3,
126}
127
128#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
129pub struct NativeOperatorNvccPolicy {
130 pub cpp_standard: NativeOperatorCppStandard,
131 pub optimization: NativeOperatorOptimization,
132 pub use_fast_math: bool,
133 pub relaxed_constexpr: bool,
134 pub extended_lambda: bool,
135 pub host_position_independent_code: bool,
136 pub host_default_visibility: bool,
137}
138
139#[derive(Debug, Clone)]
140pub struct NativeOperatorSourceBuildRequest {
141 pub plan_path: PathBuf,
142 pub source_root: PathBuf,
143 pub output_dir: PathBuf,
144 pub compute_capability: String,
145 pub builder_sha: String,
146 pub nvcc_path: PathBuf,
147 pub ccbin_path: PathBuf,
148 pub ar_path: PathBuf,
149 pub cuda_toolkit_root: PathBuf,
150 pub nvcc_threads: u32,
151 pub object_cache_dir: PathBuf,
152 pub plan_only: bool,
153}
154
155#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
156pub struct NativeOperatorSourceBuildReceipt {
157 pub schema_version: u32,
158 pub status: NativeOperatorSourceBuildStatus,
159 pub operator: String,
160 pub plan_only: bool,
161 pub plan_sha256: String,
162 pub source_package: NativeOperatorSourcePackage,
163 pub builder_sha: String,
164 pub compute_capability: String,
165 pub architecture_argument: String,
166 pub nvcc_threads: u32,
167 pub object_cache_root: String,
168 pub toolchain: Option<NativeOperatorSourceBuildToolchain>,
169 pub effective_environment: BTreeMap<String, String>,
170 pub inputs_sha256: String,
171 pub commands: Vec<NativeOperatorSourceBuildCommand>,
172 pub compiled_translation_units: Vec<String>,
173 pub cache_hit_translation_units: Vec<String>,
174 pub archive_file: Option<String>,
175 pub archive_sha256: Option<String>,
176 pub started_unix_ms: u64,
177 pub elapsed_ms: u64,
178 pub failure_class: Option<String>,
179}
180
181#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
182#[serde(rename_all = "snake_case")]
183pub enum NativeOperatorSourceBuildStatus {
184 Plan,
185 Pass,
186 Reject,
187}
188
189#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
190pub struct NativeOperatorSourceBuildToolchain {
191 pub static_identity: NativeOperatorSourceBuildStaticToolchain,
192 pub miss_probe: Option<NativeOperatorSourceBuildToolchainProbe>,
193}
194
195#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
196pub struct NativeOperatorSourceBuildStaticToolchain {
197 pub backend: NativeOperatorBackend,
198 pub compiler_driver: NativeOperatorSourceCompilerDriver,
199 pub cuda_toolkit: NativeOperatorCudaToolkitIdentity,
200 pub host_toolchain: NativeOperatorHostToolchainIdentity,
201 pub archiver: NativeOperatorToolFileIdentity,
202}
203
204#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
205#[serde(rename_all = "snake_case")]
206pub enum NativeOperatorSourceCompilerDriver {
207 CudaNvcc,
208}
209
210#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
211pub struct NativeOperatorCudaToolkitIdentity {
212 pub canonical_root: String,
213 pub invocation_root: String,
214 pub release_version: String,
215 pub nvcc: NativeOperatorToolFileIdentity,
216 pub manifest: NativeOperatorEvidenceFile,
217}
218
219#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
220pub struct NativeOperatorToolFileIdentity {
221 pub path: String,
222 pub sha256: String,
223 pub size_bytes: u64,
224}
225
226#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
227pub struct NativeOperatorHostToolchainIdentity {
228 pub compiler: NativeOperatorToolFileIdentity,
229 pub compiler_version: String,
230 pub target: String,
231 pub manifest: NativeOperatorEvidenceFile,
232}
233
234#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
235pub struct NativeOperatorHostToolchainManifest {
236 pub schema_version: u32,
237 pub compiler: NativeOperatorToolFileIdentity,
238 pub compiler_version: String,
239 pub target: String,
240 pub executable_inputs: Vec<NativeOperatorToolFileIdentity>,
241 pub include_roots: Vec<String>,
242 pub include_probe_sha256: String,
243 pub driver_probe_sha256: String,
244 pub discovery_roots: Vec<String>,
245 pub files: Vec<NativeOperatorHostToolchainFileIdentity>,
246}
247
248#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
249pub struct NativeOperatorHostToolchainFileIdentity {
250 pub logical_path: String,
251 pub resolved_path: String,
252 pub sha256: String,
253 pub size_bytes: u64,
254}
255
256#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
257pub struct NativeOperatorSourceBuildToolchainProbe {
258 pub nvcc_version: String,
259 pub host_compiler_version: String,
260 pub host_target: String,
261 pub archiver_version: String,
262 pub probed_for_misses: Vec<String>,
263}
264
265#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
266pub struct NativeOperatorCudaToolkitManifest {
267 pub schema_version: u32,
268 pub canonical_root: String,
269 pub entries: Vec<NativeOperatorCudaToolkitFileIdentity>,
270}
271
272#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
273pub struct NativeOperatorCudaToolkitFileIdentity {
274 pub logical_path: String,
275 pub resolved_path: String,
276 pub sha256: String,
277 pub size_bytes: u64,
278}
279
280#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
281pub struct NativeOperatorToolIdentity {
282 pub path: String,
283 pub sha256: String,
284 pub version: String,
285}
286
287#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
288pub struct NativeOperatorObjectIdentity {
289 pub format: NativeOperatorObjectFormat,
290 pub class_bits: u8,
291 pub endianness: NativeOperatorObjectEndianness,
292 pub machine: u32,
293}
294
295#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
296#[serde(rename_all = "snake_case")]
297pub enum NativeOperatorObjectFormat {
298 Elf,
299 MachO,
300 Coff,
301}
302
303#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
304#[serde(rename_all = "snake_case")]
305pub enum NativeOperatorObjectEndianness {
306 Little,
307 Big,
308}
309
310#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
311pub struct NativeOperatorSourceBuildCommand {
312 pub translation_unit: Option<String>,
313 pub working_directory: String,
314 pub argv: Vec<String>,
315 pub object_file: Option<String>,
316 pub stdout_log: String,
317 pub stderr_log: String,
318 pub object_cache_key: Option<String>,
319 pub object_cache_status: Option<NativeOperatorSourceObjectCacheStatus>,
320 pub object_cache_entry: Option<String>,
321 pub object_sha256: Option<String>,
322 pub object_size_bytes: Option<u64>,
323 pub object_identity: Option<NativeOperatorObjectIdentity>,
324 pub dependency_closure_sha256: Option<String>,
325 pub dependency_validation: Option<NativeOperatorDependencyValidation>,
326 pub compiler_depfile: Option<String>,
327 pub compiler_depfile_sha256: Option<String>,
328 pub depfile: Option<String>,
329 pub depfile_sha256: Option<String>,
330 pub depfile_producer_working_directory: Option<String>,
331 pub depfile_producer_object_file: Option<String>,
332 pub depfile_bindings: Vec<NativeOperatorDepfileDependencyBinding>,
333 pub observed_dependencies: Vec<NativeOperatorObservedDependency>,
334 pub compiler_executed: bool,
335 pub elapsed_ms: Option<u64>,
336 pub return_code: Option<i32>,
337}
338
339#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
340#[serde(rename_all = "snake_case")]
341pub enum NativeOperatorDependencyValidation {
342 Plan,
343 Pending,
344 CacheProof,
345 Depfile,
346}
347
348#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize)]
349#[serde(rename_all = "snake_case")]
350pub enum NativeOperatorDependencyDomain {
351 Source,
352 BackendToolchain,
353 HostToolchain,
354}
355
356#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize)]
357pub struct NativeOperatorObservedDependency {
358 pub domain: NativeOperatorDependencyDomain,
359 pub path: String,
360 pub sha256: String,
361}
362
363#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
364pub struct NativeOperatorDepfileDependencyBinding {
365 pub producer_path: String,
366 pub portable_path: String,
367 pub dependency: NativeOperatorObservedDependency,
368}
369
370#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
371pub struct NativeOperatorObjectDependencyProof {
372 pub schema_version: u32,
373 pub object_cache_key: String,
374 pub object_sha256: String,
375 pub dependency_closure_sha256: String,
376 pub dependency_set_sha256: String,
377 pub compiler_depfile_sha256: String,
378 pub depfile_sha256: String,
379 pub producer_working_directory: String,
380 pub producer_object_file: String,
381 pub depfile_bindings: Vec<NativeOperatorDepfileDependencyBinding>,
382 pub observed_dependencies: Vec<NativeOperatorObservedDependency>,
383}
384
385#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
386#[serde(rename_all = "snake_case")]
387pub enum NativeOperatorSourceObjectCacheStatus {
388 Plan,
389 Pending,
390 Hit,
391 Miss,
392 Published,
393 Rejected,
394}
395
396#[derive(Debug, Serialize)]
397struct NativeOperatorSourceInventoryIdentity<'a> {
398 operator: &'a str,
399 upstream_sources: &'a [NativeOperatorUpstreamSource],
400 translation_units: &'a [NativeOperatorSourceFileLock],
401 headers: &'a [NativeOperatorSourceFileLock],
402}
403
404#[derive(Debug, Serialize)]
405struct NativeOperatorDependencyClosureIdentity<'a> {
406 translation_unit: &'a NativeOperatorSourceFileLock,
407 headers: &'a [NativeOperatorSourceFileLock],
408}
409
410#[derive(Debug, Serialize)]
411struct NativeOperatorBuildInputIdentity<'a> {
412 plan_sha256: &'a str,
413 source_package_sha256: &'a str,
414 builder_contract_version: u32,
415 architecture_argument: &'a str,
416 effective_environment: &'a BTreeMap<String, String>,
417 toolchain: Option<&'a NativeOperatorSourceBuildStaticToolchain>,
418}
419
420#[derive(Debug, Serialize)]
421struct NativeOperatorObjectInputIdentity<'a> {
422 schema_version: u32,
423 operator: &'a str,
424 translation_unit: &'a NativeOperatorSourceFileLock,
425 dependency_closure_sha256: &'a str,
426 headers: &'a [NativeOperatorSourceFileLock],
427 include_dirs: &'a [String],
428 defines: &'a [String],
429 nvcc_policy: &'a NativeOperatorNvccPolicy,
430 architecture_argument: &'a str,
431 builder_contract_version: u32,
432 effective_environment: &'a BTreeMap<String, String>,
433 toolchain: &'a NativeOperatorSourceBuildStaticToolchain,
434}
435
436pub fn lock_native_operator_source_definition(
437 definition_path: &Path,
438 source_root: &Path,
439 output_plan_path: &Path,
440) -> Result<NativeOperatorSourceBuildPlan> {
441 require_file(definition_path)?;
442 if output_plan_path.exists() {
443 return Err(NativeOperatorBuilderError::OutputExists(
444 output_plan_path.to_path_buf(),
445 ));
446 }
447 let definition: NativeOperatorSourceDefinition = read_json(definition_path)?;
448 validate_definition(&definition)?;
449 let canonical_root =
450 source_root
451 .canonicalize()
452 .map_err(|source| NativeOperatorBuilderError::Io {
453 path: source_root.to_path_buf(),
454 source,
455 })?;
456 if !canonical_root.is_dir() {
457 return Err(NativeOperatorBuilderError::Invalid(format!(
458 "source_root is not a directory: {}",
459 canonical_root.display()
460 )));
461 }
462 for include_dir in &definition.include_dirs {
463 let path = canonical_root.join(include_dir);
464 let canonical = path
465 .canonicalize()
466 .map_err(|source| NativeOperatorBuilderError::Io {
467 path: path.clone(),
468 source,
469 })?;
470 if !canonical.starts_with(&canonical_root) || !canonical.is_dir() {
471 return Err(NativeOperatorBuilderError::Invalid(format!(
472 "include directory escapes source root or does not exist: {}",
473 path.display()
474 )));
475 }
476 }
477 let translation_units = lock_source_files(&canonical_root, &definition.translation_units)?;
478 let headers = lock_source_files(&canonical_root, &definition.headers)?;
479 let translation_unit_by_path = translation_units
480 .iter()
481 .map(|locked| (locked.path.as_str(), locked))
482 .collect::<BTreeMap<_, _>>();
483 let header_by_path = headers
484 .iter()
485 .map(|locked| (locked.path.as_str(), locked))
486 .collect::<BTreeMap<_, _>>();
487 let dependency_closures = definition
488 .dependency_closures
489 .iter()
490 .map(|closure| {
491 let translation_unit = translation_unit_by_path
492 .get(closure.translation_unit.as_str())
493 .copied()
494 .ok_or_else(|| {
495 NativeOperatorBuilderError::Invalid(format!(
496 "dependency closure references unknown translation unit: {}",
497 closure.translation_unit
498 ))
499 })?;
500 let locked_headers = closure
501 .headers
502 .iter()
503 .map(|path| {
504 header_by_path
505 .get(path.as_str())
506 .copied()
507 .cloned()
508 .ok_or_else(|| {
509 NativeOperatorBuilderError::Invalid(format!(
510 "dependency closure for {} references unknown header: {path}",
511 closure.translation_unit
512 ))
513 })
514 })
515 .collect::<Result<Vec<_>>>()?;
516 let closure_sha256 =
517 dependency_closure_sha256(translation_unit, &locked_headers, output_plan_path)?;
518 Ok(NativeOperatorTranslationUnitDependencyLock {
519 translation_unit: closure.translation_unit.clone(),
520 headers: locked_headers,
521 closure_sha256,
522 })
523 })
524 .collect::<Result<Vec<_>>>()?;
525 let inventory_identity = NativeOperatorSourceInventoryIdentity {
526 operator: &definition.operator,
527 upstream_sources: &definition.upstream_sources,
528 translation_units: &translation_units,
529 headers: &headers,
530 };
531 let source_package_sha256 =
532 sha256_bytes(&serde_json::to_vec(&inventory_identity).map_err(|source| {
533 NativeOperatorBuilderError::Json {
534 path: output_plan_path.to_path_buf(),
535 source,
536 }
537 })?);
538 let plan = NativeOperatorSourceBuildPlan {
539 schema_version: NATIVE_OPERATOR_SOURCE_BUILD_PLAN_SCHEMA_VERSION,
540 operator: definition.operator,
541 source_package: NativeOperatorSourcePackage {
542 kind: definition.source_package_kind,
543 revision: definition.source_package_revision,
544 sha256: source_package_sha256,
545 },
546 upstream_sources: definition.upstream_sources,
547 translation_units,
548 headers,
549 dependency_closures,
550 include_dirs: definition.include_dirs,
551 defines: definition.defines,
552 nvcc_policy: definition.nvcc_policy,
553 architecture: definition.architecture,
554 archive_file: definition.archive_file,
555 };
556 validate_plan(&plan)?;
557 if let Some(parent) = output_plan_path.parent() {
558 fs::create_dir_all(parent).map_err(|source| NativeOperatorBuilderError::Io {
559 path: parent.to_path_buf(),
560 source,
561 })?;
562 }
563 write_json(output_plan_path, &plan)?;
564 Ok(plan)
565}
566
567pub fn run_native_operator_source_build(
568 request: &NativeOperatorSourceBuildRequest,
569) -> Result<NativeOperatorSourceBuildReceipt> {
570 require_file(&request.plan_path)?;
571 if request.output_dir.exists() {
572 return Err(NativeOperatorBuilderError::OutputExists(
573 request.output_dir.clone(),
574 ));
575 }
576 if !request.output_dir.is_absolute() {
577 return Err(NativeOperatorBuilderError::Invalid(
578 "source build output_dir must be absolute".to_string(),
579 ));
580 }
581 if !request.object_cache_dir.is_absolute() {
582 return Err(NativeOperatorBuilderError::Invalid(
583 "source build object_cache_dir must be absolute".to_string(),
584 ));
585 }
586 if [
587 &request.nvcc_path,
588 &request.ccbin_path,
589 &request.ar_path,
590 &request.cuda_toolkit_root,
591 ]
592 .iter()
593 .any(|path| !path.is_absolute())
594 {
595 return Err(NativeOperatorBuilderError::Invalid(
596 "source build tool paths and cuda_toolkit_root must be absolute".to_string(),
597 ));
598 }
599 validate_compute_capability(&request.compute_capability)?;
600 if request.nvcc_threads == 0 || request.nvcc_threads > MAX_NVCC_THREADS {
601 return Err(NativeOperatorBuilderError::Invalid(format!(
602 "nvcc_threads must be in [1,{MAX_NVCC_THREADS}]"
603 )));
604 }
605 if !is_git_oid(&request.builder_sha) {
606 return Err(NativeOperatorBuilderError::Invalid(
607 "builder_sha must be a lowercase 40- or 64-hex git object id".to_string(),
608 ));
609 }
610 let plan: NativeOperatorSourceBuildPlan = read_json(&request.plan_path)?;
611 validate_plan(&plan)?;
612 let plan_sha256 = sha256_file(&request.plan_path)?;
613 let canonical_root =
614 request
615 .source_root
616 .canonicalize()
617 .map_err(|source| NativeOperatorBuilderError::Io {
618 path: request.source_root.clone(),
619 source,
620 })?;
621 validate_locked_source_tree(&canonical_root, &plan)?;
622 let architecture_argument =
623 architecture_argument(plan.architecture, &request.compute_capability);
624 let started_unix_ms = unix_ms();
625 let started = Instant::now();
626 fs::create_dir_all(&request.output_dir).map_err(|source| NativeOperatorBuilderError::Io {
627 path: request.output_dir.clone(),
628 source,
629 })?;
630 let receipt_path = request.output_dir.join("source-build.receipt.json");
631 let logs_dir = request.output_dir.join("logs");
632 let objects_dir = request.output_dir.join("objects");
633 let depfiles_dir = request.output_dir.join("depfiles");
634 fs::create_dir_all(&logs_dir).map_err(|source| NativeOperatorBuilderError::Io {
635 path: logs_dir.clone(),
636 source,
637 })?;
638 if !request.plan_only {
639 fs::create_dir_all(&objects_dir).map_err(|source| NativeOperatorBuilderError::Io {
640 path: objects_dir.clone(),
641 source,
642 })?;
643 fs::create_dir_all(&depfiles_dir).map_err(|source| NativeOperatorBuilderError::Io {
644 path: depfiles_dir.clone(),
645 source,
646 })?;
647 }
648
649 let (toolchain, toolchain_failure) = if request.plan_only {
650 (None, None)
651 } else {
652 match resolve_static_toolchain(request) {
653 Ok(toolchain) => (Some(toolchain), None),
654 Err(error) => (None, Some(error.to_string())),
655 }
656 };
657 let effective_environment = effective_build_environment(request, toolchain.as_ref())?;
658 let inputs_sha256 = build_inputs_sha256(
659 &plan_sha256,
660 &plan.source_package.sha256,
661 &architecture_argument,
662 &effective_environment,
663 toolchain
664 .as_ref()
665 .map(|toolchain| &toolchain.static_identity),
666 &receipt_path,
667 )?;
668 let mut commands = build_commands(
669 request,
670 &plan,
671 &canonical_root,
672 &architecture_argument,
673 &objects_dir,
674 &logs_dir,
675 toolchain.as_ref(),
676 &effective_environment,
677 );
678 let initial_log = if request.plan_only {
679 b"plan-only: command was not executed\n".as_slice()
680 } else {
681 b"pending: command has not executed\n".as_slice()
682 };
683 for command in &commands {
684 write_command_stream(
685 &request.output_dir.join(&command.stdout_log),
686 "stdout",
687 &command.argv,
688 initial_log,
689 )?;
690 write_command_stream(
691 &request.output_dir.join(&command.stderr_log),
692 "stderr",
693 &command.argv,
694 initial_log,
695 )?;
696 }
697 let mut receipt = NativeOperatorSourceBuildReceipt {
698 schema_version: NATIVE_OPERATOR_SOURCE_BUILD_RECEIPT_SCHEMA_VERSION,
699 status: if request.plan_only {
700 NativeOperatorSourceBuildStatus::Plan
701 } else {
702 NativeOperatorSourceBuildStatus::Reject
703 },
704 operator: plan.operator.clone(),
705 plan_only: request.plan_only,
706 plan_sha256,
707 source_package: plan.source_package.clone(),
708 builder_sha: request.builder_sha.clone(),
709 compute_capability: request.compute_capability.clone(),
710 architecture_argument,
711 nvcc_threads: request.nvcc_threads,
712 object_cache_root: request.object_cache_dir.display().to_string(),
713 toolchain,
714 effective_environment,
715 inputs_sha256,
716 commands: commands.clone(),
717 compiled_translation_units: Vec::new(),
718 cache_hit_translation_units: Vec::new(),
719 archive_file: None,
720 archive_sha256: None,
721 started_unix_ms,
722 elapsed_ms: 0,
723 failure_class: None,
724 };
725 if request.plan_only {
726 receipt.elapsed_ms = millis(started.elapsed());
727 write_json(&receipt_path, &receipt)?;
728 return Ok(receipt);
729 }
730 if let Some(error) = toolchain_failure {
731 receipt.elapsed_ms = millis(started.elapsed());
732 return reject_source_build(
733 &receipt_path,
734 &mut receipt,
735 format!("toolchain_preflight_failed:{error}"),
736 );
737 }
738 let toolchain_dependency_scope = match load_toolchain_dependency_scope(
739 &request.output_dir,
740 &receipt
741 .toolchain
742 .as_ref()
743 .expect("actual source build has a resolved toolchain")
744 .static_identity,
745 ) {
746 Ok(scope) => scope,
747 Err(error) => {
748 receipt.elapsed_ms = millis(started.elapsed());
749 return reject_source_build(
750 &receipt_path,
751 &mut receipt,
752 format!("toolchain_dependency_scope_failed:{error}"),
753 );
754 }
755 };
756 let object_cache = match NativeBuildArtifactCache::new(&request.object_cache_dir) {
757 Ok(cache) => cache,
758 Err(error) => {
759 receipt.elapsed_ms = millis(started.elapsed());
760 return reject_source_build(
761 &receipt_path,
762 &mut receipt,
763 format!("object_cache_init_failed:{error}"),
764 );
765 }
766 };
767 let object_specs = match build_object_cache_specs(
768 &plan,
769 &receipt.architecture_argument,
770 receipt
771 .toolchain
772 .as_ref()
773 .map(|toolchain| &toolchain.static_identity)
774 .expect("actual source build has a resolved toolchain"),
775 &receipt.effective_environment,
776 ) {
777 Ok(specs) => specs,
778 Err(error) => {
779 receipt.elapsed_ms = millis(started.elapsed());
780 return reject_source_build(
781 &receipt_path,
782 &mut receipt,
783 format!("object_cache_spec_failed:{error}"),
784 );
785 }
786 };
787 for (command, spec) in commands.iter_mut().zip(object_specs.iter()) {
788 command.object_cache_key = Some(spec.input_signature_sha256().to_string());
789 }
790 receipt.commands = commands.clone();
791 receipt.failure_class = Some("build_incomplete".to_string());
792 write_json(&receipt_path, &receipt)?;
793
794 let mut expected_object_identity: Option<NativeOperatorObjectIdentity> = None;
795 let mut miss_indices = Vec::new();
796 for index in 0..plan.translation_units.len() {
797 let translation_unit = &plan.translation_units[index];
798 let object_path = PathBuf::from(
799 commands[index]
800 .object_file
801 .as_deref()
802 .expect("translation-unit command has an object file"),
803 );
804 let lookup_started = Instant::now();
805 match object_cache.restore(&object_specs[index], &object_path) {
806 Ok(NativeBuildArtifactLookup::Hit(cache_receipt)) => {
807 let compiler_depfile_relative = commands[index]
808 .compiler_depfile
809 .as_deref()
810 .expect("translation-unit command has compiler depfile output")
811 .to_string();
812 let compiler_depfile_path = request.output_dir.join(&compiler_depfile_relative);
813 let depfile_relative = commands[index]
814 .depfile
815 .as_deref()
816 .expect("translation-unit command has depfile output")
817 .to_string();
818 let depfile_path = request.output_dir.join(&depfile_relative);
819 let dependency_proof = match restore_object_dependency_proof(
820 &cache_receipt.cache_entry,
821 object_specs[index].input_signature_sha256(),
822 &cache_receipt.artifact_sha256,
823 &plan.dependency_closures[index],
824 translation_unit,
825 &object_path,
826 &compiler_depfile_path,
827 &depfile_path,
828 &toolchain_dependency_scope,
829 ) {
830 Ok(Some(proof)) => proof,
831 Ok(None) => {
832 fs::remove_file(&object_path).map_err(|source| {
833 NativeOperatorBuilderError::Io {
834 path: object_path.clone(),
835 source,
836 }
837 })?;
838 commands[index].object_cache_status =
839 Some(NativeOperatorSourceObjectCacheStatus::Miss);
840 append_command_stream(
841 &request.output_dir.join(&commands[index].stdout_log),
842 b"object-cache-miss: dependency-proof-absent\n",
843 )?;
844 miss_indices.push(index);
845 continue;
846 }
847 Err(error) => {
848 commands[index].object_cache_status =
849 Some(NativeOperatorSourceObjectCacheStatus::Rejected);
850 receipt.commands = commands.clone();
851 receipt.elapsed_ms = millis(started.elapsed());
852 return reject_source_build(
853 &receipt_path,
854 &mut receipt,
855 format!("cached_dependency_proof_failed:{index}:{error}"),
856 );
857 }
858 };
859 commands[index].object_cache_status =
860 Some(NativeOperatorSourceObjectCacheStatus::Hit);
861 commands[index].object_cache_entry =
862 Some(cache_receipt.cache_entry.display().to_string());
863 commands[index].object_sha256 = Some(cache_receipt.artifact_sha256);
864 let object_size_bytes = match native_object_size(&object_path) {
865 Ok(size_bytes) => size_bytes,
866 Err(error) => {
867 commands[index].object_cache_status =
868 Some(NativeOperatorSourceObjectCacheStatus::Rejected);
869 receipt.commands = commands.clone();
870 receipt.elapsed_ms = millis(started.elapsed());
871 return reject_source_build(
872 &receipt_path,
873 &mut receipt,
874 format!("cached_object_size_failed:{index}:{error}"),
875 );
876 }
877 };
878 commands[index].object_size_bytes = Some(object_size_bytes);
879 let object_identity = match native_object_identity_file(&object_path) {
880 Ok(identity) => identity,
881 Err(error) => {
882 commands[index].object_cache_status =
883 Some(NativeOperatorSourceObjectCacheStatus::Rejected);
884 receipt.commands = commands.clone();
885 receipt.elapsed_ms = millis(started.elapsed());
886 return reject_source_build(
887 &receipt_path,
888 &mut receipt,
889 format!("cached_object_identity_failed:{index}:{error}"),
890 );
891 }
892 };
893 if expected_object_identity
894 .as_ref()
895 .is_some_and(|expected| expected != &object_identity)
896 {
897 commands[index].object_cache_status =
898 Some(NativeOperatorSourceObjectCacheStatus::Rejected);
899 receipt.commands = commands.clone();
900 receipt.elapsed_ms = millis(started.elapsed());
901 return reject_source_build(
902 &receipt_path,
903 &mut receipt,
904 format!("cached_object_target_mismatch:{index}"),
905 );
906 }
907 expected_object_identity.get_or_insert_with(|| object_identity.clone());
908 commands[index].object_identity = Some(object_identity);
909 commands[index].dependency_validation =
910 Some(NativeOperatorDependencyValidation::CacheProof);
911 commands[index].compiler_depfile_sha256 =
912 Some(dependency_proof.compiler_depfile_sha256);
913 commands[index].depfile_sha256 = Some(dependency_proof.depfile_sha256);
914 commands[index].depfile_producer_working_directory =
915 Some(dependency_proof.producer_working_directory);
916 commands[index].depfile_producer_object_file =
917 Some(dependency_proof.producer_object_file);
918 commands[index].depfile_bindings = dependency_proof.depfile_bindings;
919 commands[index].observed_dependencies = dependency_proof.observed_dependencies;
920 commands[index].elapsed_ms = Some(millis(lookup_started.elapsed()));
921 append_command_stream(
922 &request.output_dir.join(&commands[index].stdout_log),
923 b"object-cache-hit: compiler and toolchain probes were not executed\n",
924 )?;
925 receipt
926 .cache_hit_translation_units
927 .push(translation_unit.path.clone());
928 }
929 Ok(NativeBuildArtifactLookup::Miss { reason }) => {
930 commands[index].object_cache_status =
931 Some(NativeOperatorSourceObjectCacheStatus::Miss);
932 append_command_stream(
933 &request.output_dir.join(&commands[index].stdout_log),
934 format!("object-cache-miss: {reason}\n").as_bytes(),
935 )?;
936 miss_indices.push(index);
937 }
938 Err(error) => {
939 commands[index].object_cache_status =
940 Some(NativeOperatorSourceObjectCacheStatus::Rejected);
941 receipt.commands = commands.clone();
942 receipt.elapsed_ms = millis(started.elapsed());
943 return reject_source_build(
944 &receipt_path,
945 &mut receipt,
946 format!("object_cache_restore_failed:{index}:{error}"),
947 );
948 }
949 }
950 }
951 receipt.commands = commands.clone();
952 receipt.elapsed_ms = millis(started.elapsed());
953 write_json(&receipt_path, &receipt)?;
954
955 if !miss_indices.is_empty() {
956 let missed_translation_units = miss_indices
957 .iter()
958 .map(|index| plan.translation_units[*index].path.clone())
959 .collect::<Vec<_>>();
960 let static_identity = receipt
961 .toolchain
962 .as_ref()
963 .expect("actual source build has a resolved toolchain")
964 .static_identity
965 .clone();
966 let probe = match probe_source_toolchain(&static_identity, missed_translation_units) {
967 Ok(probe) => probe,
968 Err(error) => {
969 receipt.commands = commands.clone();
970 receipt.elapsed_ms = millis(started.elapsed());
971 return reject_source_build(
972 &receipt_path,
973 &mut receipt,
974 format!("toolchain_miss_probe_failed:{error}"),
975 );
976 }
977 };
978 receipt
979 .toolchain
980 .as_mut()
981 .expect("actual source build has a resolved toolchain")
982 .miss_probe = Some(probe);
983 write_json(&receipt_path, &receipt)?;
984 }
985
986 for index in miss_indices {
987 let translation_unit = &plan.translation_units[index];
988 let object_path = PathBuf::from(
989 commands[index]
990 .object_file
991 .as_deref()
992 .expect("translation-unit command has an object file"),
993 );
994 let command_started = Instant::now();
995 let stdout_path = request.output_dir.join(&commands[index].stdout_log);
996 let stderr_path = request.output_dir.join(&commands[index].stderr_log);
997 commands[index].compiler_executed = true;
998 let status = match run_logged_command(
999 &commands[index].argv,
1000 &stdout_path,
1001 &stderr_path,
1002 &commands[index].working_directory,
1003 &receipt.effective_environment,
1004 ) {
1005 Ok(status) => status,
1006 Err(error) => {
1007 append_command_stream(&stderr_path, format!("spawn failed: {error}\n").as_bytes())?;
1008 receipt.commands = commands.clone();
1009 receipt.elapsed_ms = millis(started.elapsed());
1010 return reject_source_build(
1011 &receipt_path,
1012 &mut receipt,
1013 format!("nvcc_translation_unit_{index}_spawn_failed"),
1014 );
1015 }
1016 };
1017 commands[index].elapsed_ms = Some(millis(command_started.elapsed()));
1018 commands[index].return_code = status.code();
1019 receipt
1020 .compiled_translation_units
1021 .push(translation_unit.path.clone());
1022 receipt.commands = commands.clone();
1023 receipt.elapsed_ms = millis(started.elapsed());
1024 if !status.success() {
1025 return reject_source_build(
1026 &receipt_path,
1027 &mut receipt,
1028 format!("nvcc_translation_unit_{index}_failed"),
1029 );
1030 }
1031 if let Err(error) = require_file(&object_path) {
1032 commands[index].object_cache_status =
1033 Some(NativeOperatorSourceObjectCacheStatus::Rejected);
1034 receipt.commands = commands.clone();
1035 receipt.elapsed_ms = millis(started.elapsed());
1036 return reject_source_build(
1037 &receipt_path,
1038 &mut receipt,
1039 format!("compiled_object_missing:{index}:{error}"),
1040 );
1041 }
1042 let compiler_depfile_relative = commands[index]
1043 .compiler_depfile
1044 .as_deref()
1045 .expect("cache-miss command has a compiler depfile")
1046 .to_string();
1047 let compiler_depfile_path = request.output_dir.join(&compiler_depfile_relative);
1048 let depfile_relative = commands[index]
1049 .depfile
1050 .as_deref()
1051 .expect("cache-miss command has a portable depfile")
1052 .to_string();
1053 let depfile_path = request.output_dir.join(&depfile_relative);
1054 let validated_depfile = match validate_translation_unit_depfile(
1055 &compiler_depfile_path,
1056 &depfile_path,
1057 &object_path,
1058 &canonical_root,
1059 translation_unit,
1060 &plan.dependency_closures[index],
1061 &toolchain_dependency_scope,
1062 ) {
1063 Ok(dependencies) => dependencies,
1064 Err(error) => {
1065 commands[index].object_cache_status =
1066 Some(NativeOperatorSourceObjectCacheStatus::Rejected);
1067 receipt.commands = commands.clone();
1068 receipt.elapsed_ms = millis(started.elapsed());
1069 return reject_source_build(
1070 &receipt_path,
1071 &mut receipt,
1072 format!("dependency_validation_failed:{index}:{error}"),
1073 );
1074 }
1075 };
1076 commands[index].compiler_depfile_sha256 = Some(validated_depfile.compiler_sha256.clone());
1077 commands[index].depfile_sha256 = Some(validated_depfile.portable_sha256.clone());
1078 commands[index].depfile_producer_working_directory =
1079 Some(commands[index].working_directory.clone());
1080 commands[index].depfile_producer_object_file = commands[index].object_file.clone();
1081 commands[index].depfile_bindings = validated_depfile.bindings.clone();
1082 commands[index].observed_dependencies = validated_depfile.observed_dependencies.clone();
1083 commands[index].dependency_validation = Some(NativeOperatorDependencyValidation::Depfile);
1084 let object_identity = match native_object_identity_file(&object_path) {
1085 Ok(identity) => identity,
1086 Err(error) => {
1087 commands[index].object_cache_status =
1088 Some(NativeOperatorSourceObjectCacheStatus::Rejected);
1089 receipt.commands = commands.clone();
1090 receipt.elapsed_ms = millis(started.elapsed());
1091 return reject_source_build(
1092 &receipt_path,
1093 &mut receipt,
1094 format!("compiled_object_identity_failed:{index}:{error}"),
1095 );
1096 }
1097 };
1098 if expected_object_identity
1099 .as_ref()
1100 .is_some_and(|expected| expected != &object_identity)
1101 {
1102 commands[index].object_cache_status =
1103 Some(NativeOperatorSourceObjectCacheStatus::Rejected);
1104 receipt.commands = commands.clone();
1105 receipt.elapsed_ms = millis(started.elapsed());
1106 return reject_source_build(
1107 &receipt_path,
1108 &mut receipt,
1109 format!("compiled_object_target_mismatch:{index}"),
1110 );
1111 }
1112 expected_object_identity.get_or_insert_with(|| object_identity.clone());
1113 commands[index].object_identity = Some(object_identity);
1114 let object_size_bytes = match native_object_size(&object_path) {
1115 Ok(size_bytes) => size_bytes,
1116 Err(error) => {
1117 commands[index].object_cache_status =
1118 Some(NativeOperatorSourceObjectCacheStatus::Rejected);
1119 receipt.commands = commands.clone();
1120 receipt.elapsed_ms = millis(started.elapsed());
1121 return reject_source_build(
1122 &receipt_path,
1123 &mut receipt,
1124 format!("compiled_object_size_failed:{index}:{error}"),
1125 );
1126 }
1127 };
1128 commands[index].object_size_bytes = Some(object_size_bytes);
1129 let cache_receipt = match object_cache.publish(&object_specs[index], &object_path) {
1130 Ok(cache_receipt) => cache_receipt,
1131 Err(error) => {
1132 commands[index].object_cache_status =
1133 Some(NativeOperatorSourceObjectCacheStatus::Rejected);
1134 receipt.commands = commands.clone();
1135 receipt.elapsed_ms = millis(started.elapsed());
1136 return reject_source_build(
1137 &receipt_path,
1138 &mut receipt,
1139 format!("object_cache_publish_failed:{index}:{error}"),
1140 );
1141 }
1142 };
1143 if let Err(error) = publish_object_dependency_proof(
1144 &cache_receipt.cache_entry,
1145 object_specs[index].input_signature_sha256(),
1146 &cache_receipt.artifact_sha256,
1147 translation_unit,
1148 &plan.dependency_closures[index],
1149 &validated_depfile.compiler_raw,
1150 &validated_depfile.compiler_sha256,
1151 &validated_depfile.portable_raw,
1152 &validated_depfile.portable_sha256,
1153 commands[index]
1154 .depfile_producer_working_directory
1155 .as_deref()
1156 .expect("compiled depfile records its producer working directory"),
1157 commands[index]
1158 .depfile_producer_object_file
1159 .as_deref()
1160 .expect("compiled depfile records its producer object"),
1161 &commands[index].depfile_bindings,
1162 &commands[index].observed_dependencies,
1163 &toolchain_dependency_scope,
1164 ) {
1165 commands[index].object_cache_status =
1166 Some(NativeOperatorSourceObjectCacheStatus::Rejected);
1167 receipt.commands = commands.clone();
1168 receipt.elapsed_ms = millis(started.elapsed());
1169 return reject_source_build(
1170 &receipt_path,
1171 &mut receipt,
1172 format!("object_dependency_proof_publish_failed:{index}:{error}"),
1173 );
1174 }
1175 commands[index].object_cache_status =
1176 Some(NativeOperatorSourceObjectCacheStatus::Published);
1177 commands[index].object_cache_entry = Some(cache_receipt.cache_entry.display().to_string());
1178 commands[index].object_sha256 = Some(cache_receipt.artifact_sha256);
1179 receipt.commands = commands.clone();
1180 write_json(&receipt_path, &receipt)?;
1181 }
1182
1183 let archive_index = plan.translation_units.len();
1184 let archive_command = commands
1185 .get_mut(archive_index)
1186 .expect("archive command follows translation units");
1187 let archive_started = Instant::now();
1188 let archive_stdout_path = request.output_dir.join(&archive_command.stdout_log);
1189 let archive_stderr_path = request.output_dir.join(&archive_command.stderr_log);
1190 let archive_status = match run_logged_command(
1191 &archive_command.argv,
1192 &archive_stdout_path,
1193 &archive_stderr_path,
1194 &archive_command.working_directory,
1195 &receipt.effective_environment,
1196 ) {
1197 Ok(status) => status,
1198 Err(error) => {
1199 append_command_stream(
1200 &archive_stderr_path,
1201 format!("spawn failed: {error}\n").as_bytes(),
1202 )?;
1203 receipt.commands = commands.clone();
1204 receipt.elapsed_ms = millis(started.elapsed());
1205 return reject_source_build(
1206 &receipt_path,
1207 &mut receipt,
1208 "archive_spawn_failed".to_string(),
1209 );
1210 }
1211 };
1212 archive_command.elapsed_ms = Some(millis(archive_started.elapsed()));
1213 archive_command.return_code = archive_status.code();
1214 receipt.commands = commands;
1215 receipt.elapsed_ms = millis(started.elapsed());
1216 if !archive_status.success() {
1217 return reject_source_build(&receipt_path, &mut receipt, "archive_failed".to_string());
1218 }
1219
1220 let archive_path = request.output_dir.join(&plan.archive_file);
1221 if let Err(error) = require_file(&archive_path) {
1222 return reject_source_build(
1223 &receipt_path,
1224 &mut receipt,
1225 format!("archive_output_missing:{error}"),
1226 );
1227 }
1228 let archive_sha256 = match sha256_file(&archive_path) {
1229 Ok(sha256) => sha256,
1230 Err(error) => {
1231 return reject_source_build(
1232 &receipt_path,
1233 &mut receipt,
1234 format!("archive_output_hash_failed:{error}"),
1235 );
1236 }
1237 };
1238 let static_identity = receipt
1239 .toolchain
1240 .as_ref()
1241 .expect("actual source build has a resolved toolchain")
1242 .static_identity
1243 .clone();
1244 if let Err(error) = validate_tool_file_unchanged(&static_identity.archiver) {
1245 return reject_source_build(
1246 &receipt_path,
1247 &mut receipt,
1248 format!("archiver_changed_during_build:{error}"),
1249 );
1250 }
1251 if receipt
1252 .toolchain
1253 .as_ref()
1254 .is_some_and(|toolchain| toolchain.miss_probe.is_some())
1255 {
1256 if let Err(error) = validate_host_toolchain_unchanged(
1257 &static_identity.host_toolchain,
1258 &request.output_dir,
1259 &receipt.effective_environment,
1260 )
1261 .and_then(|()| validate_tool_file_unchanged(&static_identity.cuda_toolkit.nvcc))
1262 .and_then(|()| validate_cuda_toolkit_unchanged(&static_identity.cuda_toolkit))
1263 {
1264 return reject_source_build(
1265 &receipt_path,
1266 &mut receipt,
1267 format!("compiler_toolchain_changed_during_build:{error}"),
1268 );
1269 }
1270 }
1271 receipt.archive_sha256 = Some(archive_sha256);
1272 receipt.archive_file = Some(plan.archive_file);
1273 receipt.status = NativeOperatorSourceBuildStatus::Pass;
1274 receipt.failure_class = None;
1275 receipt.elapsed_ms = millis(started.elapsed());
1276 write_json(&receipt_path, &receipt)?;
1277 Ok(receipt)
1278}
1279
1280fn validate_definition(definition: &NativeOperatorSourceDefinition) -> Result<()> {
1281 if definition.schema_version != NATIVE_OPERATOR_SOURCE_DEFINITION_SCHEMA_VERSION {
1282 return Err(NativeOperatorBuilderError::Invalid(format!(
1283 "source definition schema_version must be {NATIVE_OPERATOR_SOURCE_DEFINITION_SCHEMA_VERSION}"
1284 )));
1285 }
1286 validate_common(
1287 &definition.operator,
1288 &definition.upstream_sources,
1289 &definition.translation_units,
1290 &definition.headers,
1291 &definition.include_dirs,
1292 &definition.defines,
1293 &definition.archive_file,
1294 )?;
1295 validate_definition_dependency_closures(
1296 &definition.translation_units,
1297 &definition.headers,
1298 &definition.dependency_closures,
1299 )?;
1300 if definition.source_package_kind.trim().is_empty()
1301 || definition.source_package_revision.trim().is_empty()
1302 {
1303 return Err(NativeOperatorBuilderError::Invalid(
1304 "source package kind and revision must be non-empty".to_string(),
1305 ));
1306 }
1307 Ok(())
1308}
1309
1310fn validate_plan(plan: &NativeOperatorSourceBuildPlan) -> Result<()> {
1311 if plan.schema_version != NATIVE_OPERATOR_SOURCE_BUILD_PLAN_SCHEMA_VERSION {
1312 return Err(NativeOperatorBuilderError::Invalid(format!(
1313 "source build plan schema_version must be {NATIVE_OPERATOR_SOURCE_BUILD_PLAN_SCHEMA_VERSION}"
1314 )));
1315 }
1316 let translation_units = plan
1317 .translation_units
1318 .iter()
1319 .map(|file| file.path.clone())
1320 .collect::<Vec<_>>();
1321 let headers = plan
1322 .headers
1323 .iter()
1324 .map(|file| file.path.clone())
1325 .collect::<Vec<_>>();
1326 validate_common(
1327 &plan.operator,
1328 &plan.upstream_sources,
1329 &translation_units,
1330 &headers,
1331 &plan.include_dirs,
1332 &plan.defines,
1333 &plan.archive_file,
1334 )?;
1335 if plan.source_package.kind.trim().is_empty()
1336 || plan.source_package.revision.trim().is_empty()
1337 || !is_sha256_digest(&plan.source_package.sha256)
1338 {
1339 return Err(NativeOperatorBuilderError::Invalid(
1340 "source_package kind/revision must be non-empty and sha256 must be lowercase"
1341 .to_string(),
1342 ));
1343 }
1344 for file in plan.translation_units.iter().chain(plan.headers.iter()) {
1345 if !is_sha256_digest(&file.sha256) {
1346 return Err(NativeOperatorBuilderError::Invalid(format!(
1347 "{} has an invalid sha256",
1348 file.path
1349 )));
1350 }
1351 }
1352 validate_plan_dependency_closures(plan)?;
1353 let identity = NativeOperatorSourceInventoryIdentity {
1354 operator: &plan.operator,
1355 upstream_sources: &plan.upstream_sources,
1356 translation_units: &plan.translation_units,
1357 headers: &plan.headers,
1358 };
1359 let actual = sha256_bytes(&serde_json::to_vec(&identity).map_err(|source| {
1360 NativeOperatorBuilderError::Json {
1361 path: PathBuf::from("<source-build-plan>"),
1362 source,
1363 }
1364 })?);
1365 if actual != plan.source_package.sha256 {
1366 return Err(NativeOperatorBuilderError::Invalid(format!(
1367 "source_package.sha256 differs from locked file inventory: expected={} actual={actual}",
1368 plan.source_package.sha256
1369 )));
1370 }
1371 Ok(())
1372}
1373
1374fn validate_definition_dependency_closures(
1375 translation_units: &[String],
1376 headers: &[String],
1377 closures: &[NativeOperatorTranslationUnitDependencies],
1378) -> Result<()> {
1379 let closure_translation_units = closures
1380 .iter()
1381 .map(|closure| closure.translation_unit.as_str())
1382 .collect::<Vec<_>>();
1383 let expected_translation_units = translation_units
1384 .iter()
1385 .map(String::as_str)
1386 .collect::<Vec<_>>();
1387 if closure_translation_units != expected_translation_units {
1388 return Err(NativeOperatorBuilderError::Invalid(
1389 "dependency_closures must contain exactly one row per translation unit in translation_units order"
1390 .to_string(),
1391 ));
1392 }
1393 let declared_headers = headers.iter().map(String::as_str).collect::<BTreeSet<_>>();
1394 let mut attached_headers = BTreeSet::new();
1395 for closure in closures {
1396 require_sorted_optional_paths(
1397 &format!("dependency_closures[{}].headers", closure.translation_unit),
1398 &closure.headers,
1399 )?;
1400 for header in &closure.headers {
1401 if !declared_headers.contains(header.as_str()) {
1402 return Err(NativeOperatorBuilderError::Invalid(format!(
1403 "dependency closure for {} references undeclared header: {header}",
1404 closure.translation_unit
1405 )));
1406 }
1407 attached_headers.insert(header.as_str());
1408 }
1409 }
1410 if attached_headers != declared_headers {
1411 let missing = declared_headers
1412 .difference(&attached_headers)
1413 .copied()
1414 .collect::<Vec<_>>()
1415 .join(",");
1416 return Err(NativeOperatorBuilderError::Invalid(format!(
1417 "every declared header must belong to at least one dependency closure; unattached={missing}"
1418 )));
1419 }
1420 Ok(())
1421}
1422
1423fn validate_plan_dependency_closures(plan: &NativeOperatorSourceBuildPlan) -> Result<()> {
1424 let translation_units = plan
1425 .translation_units
1426 .iter()
1427 .map(|file| file.path.clone())
1428 .collect::<Vec<_>>();
1429 let headers = plan
1430 .headers
1431 .iter()
1432 .map(|file| file.path.clone())
1433 .collect::<Vec<_>>();
1434 let closures = plan
1435 .dependency_closures
1436 .iter()
1437 .map(|closure| NativeOperatorTranslationUnitDependencies {
1438 translation_unit: closure.translation_unit.clone(),
1439 headers: closure
1440 .headers
1441 .iter()
1442 .map(|header| header.path.clone())
1443 .collect(),
1444 })
1445 .collect::<Vec<_>>();
1446 validate_definition_dependency_closures(&translation_units, &headers, &closures)?;
1447
1448 let translation_unit_by_path = plan
1449 .translation_units
1450 .iter()
1451 .map(|locked| (locked.path.as_str(), locked))
1452 .collect::<BTreeMap<_, _>>();
1453 let header_by_path = plan
1454 .headers
1455 .iter()
1456 .map(|locked| (locked.path.as_str(), locked))
1457 .collect::<BTreeMap<_, _>>();
1458 for closure in &plan.dependency_closures {
1459 let translation_unit = translation_unit_by_path
1460 .get(closure.translation_unit.as_str())
1461 .copied()
1462 .expect("definition-shaped closure already validated");
1463 for header in &closure.headers {
1464 let global = header_by_path
1465 .get(header.path.as_str())
1466 .copied()
1467 .expect("definition-shaped closure already validated");
1468 if global != header {
1469 return Err(NativeOperatorBuilderError::Invalid(format!(
1470 "dependency closure header lock differs from the global lock: {}",
1471 header.path
1472 )));
1473 }
1474 }
1475 if !is_sha256_digest(&closure.closure_sha256) {
1476 return Err(NativeOperatorBuilderError::Invalid(format!(
1477 "dependency closure for {} has an invalid sha256",
1478 closure.translation_unit
1479 )));
1480 }
1481 let actual = dependency_closure_sha256(
1482 translation_unit,
1483 &closure.headers,
1484 Path::new("<source-build-plan>"),
1485 )?;
1486 if actual != closure.closure_sha256 {
1487 return Err(NativeOperatorBuilderError::Invalid(format!(
1488 "dependency closure hash mismatch for {}: expected={} actual={actual}",
1489 closure.translation_unit, closure.closure_sha256
1490 )));
1491 }
1492 }
1493 Ok(())
1494}
1495
1496fn dependency_closure_sha256(
1497 translation_unit: &NativeOperatorSourceFileLock,
1498 headers: &[NativeOperatorSourceFileLock],
1499 context: &Path,
1500) -> Result<String> {
1501 let identity = NativeOperatorDependencyClosureIdentity {
1502 translation_unit,
1503 headers,
1504 };
1505 let bytes =
1506 serde_json::to_vec(&identity).map_err(|source| NativeOperatorBuilderError::Json {
1507 path: context.to_path_buf(),
1508 source,
1509 })?;
1510 Ok(sha256_bytes(&bytes))
1511}
1512
1513pub(crate) fn verify_source_build_receipt_against_plan(
1514 receipt: &NativeOperatorSourceBuildReceipt,
1515 receipt_root: &Path,
1516 plan_path: &Path,
1517 source_root: &Path,
1518) -> Result<NativeOperatorSourceBuildPlan> {
1519 let plan = verify_source_build_receipt_against_plan_portable(receipt, plan_path)?;
1520 verify_source_build_evidence(receipt, receipt_root, &plan)?;
1521 let canonical_source_root =
1522 source_root
1523 .canonicalize()
1524 .map_err(|source| NativeOperatorBuilderError::Io {
1525 path: source_root.to_path_buf(),
1526 source,
1527 })?;
1528 validate_locked_source_tree(&canonical_source_root, &plan)?;
1529 Ok(plan)
1530}
1531
1532pub(crate) fn verify_source_build_receipt_against_plan_portable(
1533 receipt: &NativeOperatorSourceBuildReceipt,
1534 plan_path: &Path,
1535) -> Result<NativeOperatorSourceBuildPlan> {
1536 require_file(plan_path)?;
1537 let plan: NativeOperatorSourceBuildPlan = read_json(plan_path)?;
1538 validate_plan(&plan)?;
1539 let plan_sha256 = sha256_file(plan_path)?;
1540 if receipt.plan_sha256 != plan_sha256 {
1541 return Err(NativeOperatorBuilderError::Invalid(format!(
1542 "{} source-build receipt plan_sha256 mismatch: expected={plan_sha256} actual={}",
1543 receipt.operator, receipt.plan_sha256
1544 )));
1545 }
1546 if receipt.operator != plan.operator || receipt.source_package != plan.source_package {
1547 return Err(NativeOperatorBuilderError::Invalid(format!(
1548 "{} source-build receipt does not match its locked plan identity",
1549 receipt.operator
1550 )));
1551 }
1552
1553 let expected_architecture =
1554 architecture_argument(plan.architecture, &receipt.compute_capability);
1555 if receipt.architecture_argument != expected_architecture {
1556 return Err(NativeOperatorBuilderError::Invalid(format!(
1557 "{} source-build architecture argument differs from its plan: expected={expected_architecture} actual={}",
1558 receipt.operator, receipt.architecture_argument
1559 )));
1560 }
1561 let toolchain = receipt.toolchain.as_ref().ok_or_else(|| {
1562 NativeOperatorBuilderError::Invalid(format!(
1563 "{} source-build receipt is missing toolchain provenance",
1564 receipt.operator
1565 ))
1566 })?;
1567 validate_static_toolchain_identity(&receipt.operator, &toolchain.static_identity)?;
1568 let static_identity = &toolchain.static_identity;
1569 let expected_environment = effective_environment_for_tool_paths([
1570 static_identity.cuda_toolkit.nvcc.path.as_str(),
1571 static_identity.host_toolchain.compiler.path.as_str(),
1572 static_identity.archiver.path.as_str(),
1573 ])?;
1574 if receipt.effective_environment != expected_environment {
1575 return Err(NativeOperatorBuilderError::Invalid(format!(
1576 "{} source-build effective environment differs from the deterministic policy",
1577 receipt.operator
1578 )));
1579 }
1580 let expected_inputs_sha256 = build_inputs_sha256(
1581 &plan_sha256,
1582 &plan.source_package.sha256,
1583 &expected_architecture,
1584 &expected_environment,
1585 Some(static_identity),
1586 plan_path,
1587 )?;
1588 if receipt.inputs_sha256 != expected_inputs_sha256 {
1589 return Err(NativeOperatorBuilderError::Invalid(format!(
1590 "{} source-build inputs_sha256 mismatch: expected={expected_inputs_sha256} actual={}",
1591 receipt.operator, receipt.inputs_sha256
1592 )));
1593 }
1594 let object_specs = build_object_cache_specs(
1595 &plan,
1596 &expected_architecture,
1597 static_identity,
1598 &expected_environment,
1599 )?;
1600 if receipt.commands.len() != plan.translation_units.len() + 1 {
1601 return Err(NativeOperatorBuilderError::Invalid(format!(
1602 "{} source-build command count differs from its plan",
1603 receipt.operator
1604 )));
1605 }
1606 for (index, ((translation_unit, object_spec), command)) in plan
1607 .translation_units
1608 .iter()
1609 .zip(object_specs.iter())
1610 .zip(receipt.commands.iter())
1611 .enumerate()
1612 {
1613 let expected_object_file = object_file_name(index, translation_unit);
1614 let closure = &plan.dependency_closures[index];
1615 if command.translation_unit.as_deref() != Some(translation_unit.path.as_str())
1616 || command.object_cache_key.as_deref() != Some(object_spec.input_signature_sha256())
1617 || command.dependency_closure_sha256.as_deref() != Some(closure.closure_sha256.as_str())
1618 || command
1619 .object_file
1620 .as_deref()
1621 .and_then(|path| Path::new(path).file_name())
1622 != Some(std::ffi::OsStr::new(&expected_object_file))
1623 {
1624 return Err(NativeOperatorBuilderError::Invalid(format!(
1625 "{} source-build object identity for {} differs from its plan",
1626 receipt.operator, translation_unit.path
1627 )));
1628 }
1629 let stem = Path::new(&translation_unit.path)
1630 .file_stem()
1631 .and_then(|value| value.to_str())
1632 .unwrap_or("translation_unit");
1633 let expected_depfile = format!("{index:08}-{stem}.d");
1634 let expected_compiler_depfile = format!("{index:08}-{stem}.compiler.raw.d");
1635 let mut expected_argv = vec![
1636 static_identity.cuda_toolkit.nvcc.path.clone(),
1637 "-c".to_string(),
1638 translation_unit.path.clone(),
1639 "-o".to_string(),
1640 expected_object_file.clone(),
1641 expected_architecture.clone(),
1642 "-ccbin".to_string(),
1643 static_identity.host_toolchain.compiler.path.clone(),
1644 "-MMD".to_string(),
1645 "-MF".to_string(),
1646 expected_compiler_depfile.clone(),
1647 "-MT".to_string(),
1648 expected_object_file.clone(),
1649 ];
1650 expected_argv.extend(plan.include_dirs.iter().map(|path| format!("-I{path}")));
1651 expected_argv.extend(plan.defines.iter().map(|define| format!("-D{define}")));
1652 expected_argv.extend(nvcc_policy_flags(&plan.nvcc_policy));
1653 expected_argv.push("--threads".to_string());
1654 expected_argv.push(receipt.nvcc_threads.to_string());
1655 let mut actual_argv = command.argv.clone();
1656 if let Some(output) = actual_argv.get_mut(4) {
1657 *output = Path::new(output)
1658 .file_name()
1659 .unwrap_or_default()
1660 .to_string_lossy()
1661 .into_owned();
1662 }
1663 for argument in [10_usize, 12] {
1664 if let Some(path) = actual_argv.get_mut(argument) {
1665 *path = Path::new(path)
1666 .file_name()
1667 .unwrap_or_default()
1668 .to_string_lossy()
1669 .into_owned();
1670 }
1671 }
1672 if actual_argv != expected_argv {
1673 return Err(NativeOperatorBuilderError::Invalid(format!(
1674 "{} source-build argv for {} differs from its locked plan",
1675 receipt.operator, translation_unit.path
1676 )));
1677 }
1678 validate_observed_dependencies(
1679 &format!("{}:{}", receipt.operator, translation_unit.path),
1680 &command.observed_dependencies,
1681 )?;
1682 let expected_source = expected_source_dependencies(translation_unit, closure);
1683 let observed_source = command
1684 .observed_dependencies
1685 .iter()
1686 .filter(|dependency| dependency.domain == NativeOperatorDependencyDomain::Source)
1687 .cloned()
1688 .collect::<BTreeSet<_>>();
1689 if observed_source != expected_source {
1690 return Err(NativeOperatorBuilderError::Invalid(format!(
1691 "{} source-build source dependency evidence for {} differs from its locked closure",
1692 receipt.operator, translation_unit.path
1693 )));
1694 }
1695 let expected_depfile_relative = format!("depfiles/{expected_depfile}");
1696 let expected_compiler_depfile_relative = format!("depfiles/{expected_compiler_depfile}");
1697 let binding_dependencies = validate_depfile_bindings_basic(
1698 &format!("{}:{}", receipt.operator, translation_unit.path),
1699 &command.depfile_bindings,
1700 )?;
1701 match command.object_cache_status {
1702 Some(NativeOperatorSourceObjectCacheStatus::Published)
1703 if command.dependency_validation
1704 == Some(NativeOperatorDependencyValidation::Depfile)
1705 && command.compiler_depfile.as_deref()
1706 == Some(expected_compiler_depfile_relative.as_str())
1707 && command
1708 .compiler_depfile_sha256
1709 .as_deref()
1710 .is_some_and(is_sha256_digest)
1711 && command.depfile.as_deref() == Some(expected_depfile_relative.as_str())
1712 && command
1713 .depfile_sha256
1714 .as_deref()
1715 .is_some_and(is_sha256_digest)
1716 && command.depfile_producer_working_directory.as_deref()
1717 == Some(command.working_directory.as_str())
1718 && command.depfile_producer_object_file.as_deref()
1719 == command.object_file.as_deref()
1720 && binding_dependencies == command.observed_dependencies => {}
1721 Some(NativeOperatorSourceObjectCacheStatus::Hit)
1722 if command.dependency_validation
1723 == Some(NativeOperatorDependencyValidation::CacheProof)
1724 && command.compiler_depfile.as_deref()
1725 == Some(expected_compiler_depfile_relative.as_str())
1726 && command
1727 .compiler_depfile_sha256
1728 .as_deref()
1729 .is_some_and(is_sha256_digest)
1730 && command.depfile.as_deref() == Some(expected_depfile_relative.as_str())
1731 && command
1732 .depfile_sha256
1733 .as_deref()
1734 .is_some_and(is_sha256_digest)
1735 && command
1736 .depfile_producer_working_directory
1737 .as_deref()
1738 .is_some_and(|path| Path::new(path).is_absolute())
1739 && command
1740 .depfile_producer_object_file
1741 .as_deref()
1742 .is_some_and(|path| {
1743 Path::new(path).is_absolute()
1744 && Path::new(path).file_name()
1745 == Some(std::ffi::OsStr::new(&expected_object_file))
1746 })
1747 && binding_dependencies == command.observed_dependencies => {}
1748 _ => {
1749 return Err(NativeOperatorBuilderError::Invalid(format!(
1750 "{} source-build dependency evidence for {} differs from its locked plan",
1751 receipt.operator, translation_unit.path
1752 )))
1753 }
1754 }
1755 }
1756 match &toolchain.miss_probe {
1757 Some(probe)
1758 if probe.probed_for_misses == receipt.compiled_translation_units
1759 && !receipt.compiled_translation_units.is_empty()
1760 && !probe.nvcc_version.trim().is_empty()
1761 && probe.host_compiler_version
1762 == static_identity.host_toolchain.compiler_version
1763 && !probe.archiver_version.trim().is_empty()
1764 && probe.host_target == static_identity.host_toolchain.target => {}
1765 None if receipt.compiled_translation_units.is_empty() => {}
1766 _ => {
1767 return Err(NativeOperatorBuilderError::Invalid(format!(
1768 "{} source-build miss-only toolchain probe differs from compiled translation units",
1769 receipt.operator
1770 )))
1771 }
1772 }
1773 let archive_command = receipt.commands.last().expect("command count checked");
1774 let actual_archive_argv = archive_command
1775 .argv
1776 .iter()
1777 .enumerate()
1778 .map(|(index, value)| {
1779 if index >= 2 {
1780 Path::new(value)
1781 .file_name()
1782 .unwrap_or_default()
1783 .to_string_lossy()
1784 .into_owned()
1785 } else {
1786 value.clone()
1787 }
1788 })
1789 .collect::<Vec<_>>();
1790 let mut expected_archive_argv = vec![
1791 static_identity.archiver.path.clone(),
1792 "rcs".to_string(),
1793 plan.archive_file.clone(),
1794 ];
1795 expected_archive_argv.extend(
1796 plan.translation_units
1797 .iter()
1798 .enumerate()
1799 .map(|(index, translation_unit)| object_file_name(index, translation_unit)),
1800 );
1801 if actual_archive_argv != expected_archive_argv
1802 || receipt.archive_file.as_deref() != Some(plan.archive_file.as_str())
1803 {
1804 return Err(NativeOperatorBuilderError::Invalid(format!(
1805 "{} source-build archive command differs from its locked plan",
1806 receipt.operator
1807 )));
1808 }
1809 Ok(plan)
1810}
1811
1812pub(crate) fn verify_source_build_evidence(
1813 receipt: &NativeOperatorSourceBuildReceipt,
1814 receipt_root: &Path,
1815 plan: &NativeOperatorSourceBuildPlan,
1816) -> Result<()> {
1817 let toolchain = receipt.toolchain.as_ref().ok_or_else(|| {
1818 NativeOperatorBuilderError::Invalid(format!(
1819 "{} source-build receipt is missing toolchain provenance",
1820 receipt.operator
1821 ))
1822 })?;
1823 let manifest_evidence = &toolchain.static_identity.cuda_toolkit.manifest;
1824 let manifest_path =
1825 resolve_source_build_evidence_file(receipt_root, &receipt.operator, manifest_evidence)?;
1826 let manifest: NativeOperatorCudaToolkitManifest = read_json(&manifest_path)?;
1827 validate_cuda_toolkit_manifest(
1828 &receipt.operator,
1829 &toolchain.static_identity.cuda_toolkit,
1830 &manifest,
1831 )?;
1832 let host_identity = &toolchain.static_identity.host_toolchain;
1833 let host_manifest_path = resolve_source_build_evidence_file(
1834 receipt_root,
1835 &receipt.operator,
1836 &host_identity.manifest,
1837 )?;
1838 let host_manifest: NativeOperatorHostToolchainManifest = read_json(&host_manifest_path)?;
1839 validate_host_toolchain_manifest(&receipt.operator, &host_manifest)?;
1840 if host_manifest.compiler != host_identity.compiler
1841 || host_manifest.compiler_version != host_identity.compiler_version
1842 || host_manifest.target != host_identity.target
1843 {
1844 return Err(NativeOperatorBuilderError::Invalid(format!(
1845 "{} host toolchain manifest differs from its receipt identity",
1846 receipt.operator
1847 )));
1848 }
1849 let toolchain_dependency_scope =
1850 toolchain_dependency_scope(&toolchain.static_identity, &manifest, &host_manifest)?;
1851 for (index, command) in receipt
1852 .commands
1853 .iter()
1854 .take(plan.translation_units.len())
1855 .enumerate()
1856 {
1857 if !matches!(
1858 command.dependency_validation,
1859 Some(
1860 NativeOperatorDependencyValidation::Depfile
1861 | NativeOperatorDependencyValidation::CacheProof
1862 )
1863 ) {
1864 continue;
1865 }
1866 let compiler_relative = command.compiler_depfile.as_deref().ok_or_else(|| {
1867 NativeOperatorBuilderError::Invalid(format!(
1868 "{} compiled source-build command is missing compiler depfile evidence",
1869 receipt.operator
1870 ))
1871 })?;
1872 let compiler_sha256 = command.compiler_depfile_sha256.as_deref().ok_or_else(|| {
1873 NativeOperatorBuilderError::Invalid(format!(
1874 "{} compiled source-build command is missing compiler depfile sha256",
1875 receipt.operator
1876 ))
1877 })?;
1878 let compiler_path = resolve_source_build_relative_file(receipt_root, compiler_relative)?;
1879 let compiler_bytes = read_bounded_regular_file(
1880 &compiler_path,
1881 MAX_DEPFILE_BYTES,
1882 "source-build compiler depfile",
1883 )?;
1884 let compiler_actual = sha256_bytes(&compiler_bytes);
1885 if compiler_actual != compiler_sha256 {
1886 return Err(NativeOperatorBuilderError::Invalid(format!(
1887 "{} source-build compiler depfile sha256 mismatch: path={compiler_relative} expected={compiler_sha256} actual={compiler_actual}",
1888 receipt.operator
1889 )));
1890 }
1891 let compiler_raw = std::str::from_utf8(&compiler_bytes).map_err(|_| {
1892 NativeOperatorBuilderError::Invalid(format!(
1893 "{} source-build compiler depfile is not UTF-8: {compiler_relative}",
1894 receipt.operator
1895 ))
1896 })?;
1897 let relative = command.depfile.as_deref().ok_or_else(|| {
1898 NativeOperatorBuilderError::Invalid(format!(
1899 "{} compiled source-build command is missing portable depfile evidence",
1900 receipt.operator
1901 ))
1902 })?;
1903 let sha256 = command.depfile_sha256.as_deref().ok_or_else(|| {
1904 NativeOperatorBuilderError::Invalid(format!(
1905 "{} compiled source-build command is missing depfile sha256",
1906 receipt.operator
1907 ))
1908 })?;
1909 let path = resolve_source_build_relative_file(receipt_root, relative)?;
1910 let bytes =
1911 read_bounded_regular_file(&path, MAX_DEPFILE_BYTES, "source-build portable depfile")?;
1912 let actual = sha256_bytes(&bytes);
1913 if actual != sha256 {
1914 return Err(NativeOperatorBuilderError::Invalid(format!(
1915 "{} source-build depfile sha256 mismatch: path={relative} expected={sha256} actual={actual}",
1916 receipt.operator
1917 )));
1918 }
1919 let raw = std::str::from_utf8(&bytes).map_err(|_| {
1920 NativeOperatorBuilderError::Invalid(format!(
1921 "{} source-build portable depfile is not UTF-8: {relative}",
1922 receipt.operator
1923 ))
1924 })?;
1925 let producer_object_file =
1926 command
1927 .depfile_producer_object_file
1928 .as_deref()
1929 .ok_or_else(|| {
1930 NativeOperatorBuilderError::Invalid(format!(
1931 "{} compiled source-build command is missing depfile producer object",
1932 receipt.operator
1933 ))
1934 })?;
1935 let producer_working_directory = command
1936 .depfile_producer_working_directory
1937 .as_deref()
1938 .ok_or_else(|| {
1939 NativeOperatorBuilderError::Invalid(format!(
1940 "{} compiled source-build command is missing depfile producer working directory",
1941 receipt.operator
1942 ))
1943 })?;
1944 command.object_file.as_deref().ok_or_else(|| {
1945 NativeOperatorBuilderError::Invalid(format!(
1946 "{} source-build command is missing object path for depfile validation",
1947 receipt.operator
1948 ))
1949 })?;
1950 let observed = validate_portable_depfile_pair(
1951 compiler_raw,
1952 &compiler_path,
1953 raw,
1954 &path,
1955 producer_object_file,
1956 producer_working_directory,
1957 &plan.translation_units[index],
1958 &plan.dependency_closures[index],
1959 &command.depfile_bindings,
1960 &toolchain_dependency_scope,
1961 )?;
1962 if observed != command.observed_dependencies {
1963 return Err(NativeOperatorBuilderError::Invalid(format!(
1964 "{} source-build depfile semantics differ from receipt evidence: path={relative}",
1965 receipt.operator
1966 )));
1967 }
1968 }
1969 Ok(())
1970}
1971
1972fn resolve_source_build_evidence_file(
1973 root: &Path,
1974 operator: &str,
1975 evidence: &NativeOperatorEvidenceFile,
1976) -> Result<PathBuf> {
1977 if !is_sha256_digest(&evidence.sha256) || evidence.size_bytes == 0 {
1978 return Err(NativeOperatorBuilderError::Invalid(format!(
1979 "{operator} source-build evidence identity is incomplete: {}",
1980 evidence.path
1981 )));
1982 }
1983 let path = resolve_source_build_relative_file(root, &evidence.path)?;
1984 let size_bytes = fs::metadata(&path)
1985 .map_err(|source| NativeOperatorBuilderError::Io {
1986 path: path.clone(),
1987 source,
1988 })?
1989 .len();
1990 let sha256 = sha256_file(&path)?;
1991 if size_bytes != evidence.size_bytes || sha256 != evidence.sha256 {
1992 return Err(NativeOperatorBuilderError::Invalid(format!(
1993 "{operator} source-build evidence mismatch: path={} expected_size={} actual_size={size_bytes} expected_sha256={} actual_sha256={sha256}",
1994 evidence.path, evidence.size_bytes, evidence.sha256
1995 )));
1996 }
1997 Ok(path)
1998}
1999
2000fn resolve_source_build_relative_file(root: &Path, relative: &str) -> Result<PathBuf> {
2001 validate_relative_path(relative)?;
2002 let canonical_root = root
2003 .canonicalize()
2004 .map_err(|source| NativeOperatorBuilderError::Io {
2005 path: root.to_path_buf(),
2006 source,
2007 })?;
2008 let path = canonical_root.join(relative);
2009 let canonical = path
2010 .canonicalize()
2011 .map_err(|source| NativeOperatorBuilderError::Io {
2012 path: path.clone(),
2013 source,
2014 })?;
2015 if !canonical.starts_with(&canonical_root) || !canonical.is_file() {
2016 return Err(NativeOperatorBuilderError::Invalid(format!(
2017 "source-build evidence escapes its root or is not a file: {relative}"
2018 )));
2019 }
2020 Ok(canonical)
2021}
2022
2023fn validate_cuda_toolkit_manifest(
2024 operator: &str,
2025 identity: &NativeOperatorCudaToolkitIdentity,
2026 manifest: &NativeOperatorCudaToolkitManifest,
2027) -> Result<()> {
2028 if manifest.schema_version != NATIVE_OPERATOR_CUDA_TOOLKIT_MANIFEST_SCHEMA_VERSION
2029 || manifest.canonical_root != identity.canonical_root
2030 || manifest.entries.is_empty()
2031 || manifest
2032 .entries
2033 .windows(2)
2034 .any(|pair| pair[0].logical_path >= pair[1].logical_path)
2035 {
2036 return Err(NativeOperatorBuilderError::Invalid(format!(
2037 "{operator} cuda toolkit manifest header/order is invalid"
2038 )));
2039 }
2040 let mut scopes = BTreeSet::new();
2041 let mut required_files = REQUIRED_CUDA_TOOLKIT_FILES
2042 .iter()
2043 .copied()
2044 .collect::<BTreeSet<_>>();
2045 let mut selected_nvcc = None;
2046 for entry in &manifest.entries {
2047 validate_relative_path(&entry.logical_path)?;
2048 validate_relative_path(&entry.resolved_path)?;
2049 if !is_sha256_digest(&entry.sha256) {
2050 return Err(NativeOperatorBuilderError::Invalid(format!(
2051 "{operator} cuda toolkit manifest entry is incomplete: {}",
2052 entry.logical_path
2053 )));
2054 }
2055 for scope in ["bin/crt/", "include/", "nvvm/bin/", "nvvm/libdevice/"] {
2056 if entry.logical_path.starts_with(scope) {
2057 scopes.insert(scope);
2058 }
2059 }
2060 required_files.remove(entry.logical_path.as_str());
2061 if Path::new(&identity.canonical_root).join(&entry.resolved_path)
2062 == Path::new(&identity.nvcc.path)
2063 {
2064 selected_nvcc = Some(entry);
2065 }
2066 }
2067 if scopes.len() != REQUIRED_CUDA_TOOLKIT_SCOPES.len() || !required_files.is_empty() {
2068 return Err(NativeOperatorBuilderError::Invalid(format!(
2069 "{operator} cuda toolkit manifest does not cover every required compiler input; missing={}",
2070 required_files.into_iter().collect::<Vec<_>>().join(",")
2071 )));
2072 }
2073 if selected_nvcc.is_none_or(|entry| {
2074 entry.sha256 != identity.nvcc.sha256 || entry.size_bytes != identity.nvcc.size_bytes
2075 }) {
2076 return Err(NativeOperatorBuilderError::Invalid(format!(
2077 "{operator} cuda toolkit manifest does not bind the selected nvcc"
2078 )));
2079 }
2080 Ok(())
2081}
2082
2083#[allow(clippy::too_many_arguments)]
2084fn validate_common(
2085 operator: &str,
2086 upstream_sources: &[NativeOperatorUpstreamSource],
2087 translation_units: &[String],
2088 headers: &[String],
2089 include_dirs: &[String],
2090 defines: &[String],
2091 archive_file: &str,
2092) -> Result<()> {
2093 symbol_slug(operator)?;
2094 if CudaNativeBuildUnit::from_artifact_operator(operator).is_none() {
2095 return Err(NativeOperatorBuilderError::Invalid(format!(
2096 "source build operator is not a registered CUDA build unit: {operator}"
2097 )));
2098 }
2099 if upstream_sources.is_empty() {
2100 return Err(NativeOperatorBuilderError::Invalid(
2101 "upstream_sources must be non-empty".to_string(),
2102 ));
2103 }
2104 if upstream_sources.windows(2).any(|pair| {
2105 (&pair[0].repository, &pair[0].revision) >= (&pair[1].repository, &pair[1].revision)
2106 }) {
2107 return Err(NativeOperatorBuilderError::Invalid(
2108 "upstream_sources must be sorted and unique by repository/revision".to_string(),
2109 ));
2110 }
2111 for upstream in upstream_sources {
2112 if upstream.repository.trim().is_empty()
2113 || upstream.revision.trim().is_empty()
2114 || upstream.license.trim().is_empty()
2115 {
2116 return Err(NativeOperatorBuilderError::Invalid(
2117 "upstream source repository, revision, and license must be non-empty".to_string(),
2118 ));
2119 }
2120 }
2121 require_sorted_paths("translation_units", translation_units)?;
2122 require_sorted_optional_paths("headers", headers)?;
2123 if translation_units.iter().any(|path| !path.ends_with(".cu")) {
2124 return Err(NativeOperatorBuilderError::Invalid(
2125 "translation_units must use .cu paths".to_string(),
2126 ));
2127 }
2128 let mut all_files = translation_units
2129 .iter()
2130 .chain(headers.iter())
2131 .collect::<Vec<_>>();
2132 all_files.sort();
2133 if all_files.windows(2).any(|pair| pair[0] == pair[1]) {
2134 return Err(NativeOperatorBuilderError::Invalid(
2135 "translation_units and headers overlap".to_string(),
2136 ));
2137 }
2138 require_sorted_optional_paths("include_dirs", include_dirs)?;
2139 if defines.windows(2).any(|pair| pair[0] >= pair[1])
2140 || defines
2141 .iter()
2142 .any(|define| define.is_empty() || define.chars().any(char::is_whitespace))
2143 {
2144 return Err(NativeOperatorBuilderError::Invalid(
2145 "defines must be sorted, unique, non-empty single arguments".to_string(),
2146 ));
2147 }
2148 validate_relative_path(archive_file)?;
2149 if Path::new(archive_file).parent() != Some(Path::new(""))
2150 || !archive_file.starts_with("lib")
2151 || !archive_file.ends_with(".a")
2152 {
2153 return Err(NativeOperatorBuilderError::Invalid(
2154 "archive_file must be a lib*.a filename without directories".to_string(),
2155 ));
2156 }
2157 Ok(())
2158}
2159
2160fn require_sorted_paths(field: &str, paths: &[String]) -> Result<()> {
2161 if paths.is_empty() {
2162 return Err(NativeOperatorBuilderError::Invalid(format!(
2163 "{field} must be sorted, unique, non-empty normalized relative paths"
2164 )));
2165 }
2166 require_sorted_optional_paths(field, paths)
2167}
2168
2169fn require_sorted_optional_paths(field: &str, paths: &[String]) -> Result<()> {
2170 if paths.windows(2).any(|pair| pair[0] >= pair[1])
2171 || paths
2172 .iter()
2173 .any(|path| validate_relative_path(path).is_err())
2174 {
2175 return Err(NativeOperatorBuilderError::Invalid(format!(
2176 "{field} must be sorted, unique normalized relative paths"
2177 )));
2178 }
2179 Ok(())
2180}
2181
2182fn lock_source_files(root: &Path, paths: &[String]) -> Result<Vec<NativeOperatorSourceFileLock>> {
2183 paths
2184 .iter()
2185 .map(|relative| {
2186 let path = locked_source_file(root, relative)?;
2187 Ok(NativeOperatorSourceFileLock {
2188 path: relative.clone(),
2189 sha256: sha256_file(&path)?,
2190 })
2191 })
2192 .collect()
2193}
2194
2195fn validate_locked_source_tree(root: &Path, plan: &NativeOperatorSourceBuildPlan) -> Result<()> {
2196 if !root.is_dir() {
2197 return Err(NativeOperatorBuilderError::Invalid(format!(
2198 "source_root is not a directory: {}",
2199 root.display()
2200 )));
2201 }
2202 for include_dir in &plan.include_dirs {
2203 let path = root.join(include_dir);
2204 let canonical = path
2205 .canonicalize()
2206 .map_err(|source| NativeOperatorBuilderError::Io {
2207 path: path.clone(),
2208 source,
2209 })?;
2210 if !canonical.starts_with(root) || !canonical.is_dir() {
2211 return Err(NativeOperatorBuilderError::Invalid(format!(
2212 "locked include directory escapes source root or is missing: {include_dir}"
2213 )));
2214 }
2215 }
2216 for locked in plan.translation_units.iter().chain(plan.headers.iter()) {
2217 let path = locked_source_file(root, &locked.path)?;
2218 let actual = sha256_file(&path)?;
2219 if actual != locked.sha256 {
2220 return Err(NativeOperatorBuilderError::Invalid(format!(
2221 "locked source drift: path={} expected={} actual={actual}",
2222 locked.path, locked.sha256
2223 )));
2224 }
2225 }
2226 Ok(())
2227}
2228
2229struct NativeOperatorToolchainDependencyScope {
2230 by_absolute_path: BTreeMap<String, NativeOperatorObservedDependency>,
2231}
2232
2233fn load_toolchain_dependency_scope(
2234 receipt_root: &Path,
2235 toolchain: &NativeOperatorSourceBuildStaticToolchain,
2236) -> Result<NativeOperatorToolchainDependencyScope> {
2237 let cuda_manifest_path = resolve_source_build_evidence_file(
2238 receipt_root,
2239 "<dependency-scope>",
2240 &toolchain.cuda_toolkit.manifest,
2241 )?;
2242 let cuda_manifest: NativeOperatorCudaToolkitManifest = read_json(&cuda_manifest_path)?;
2243 validate_cuda_toolkit_manifest(
2244 "<dependency-scope>",
2245 &toolchain.cuda_toolkit,
2246 &cuda_manifest,
2247 )?;
2248 let host_manifest_path = resolve_source_build_evidence_file(
2249 receipt_root,
2250 "<dependency-scope>",
2251 &toolchain.host_toolchain.manifest,
2252 )?;
2253 let host_manifest: NativeOperatorHostToolchainManifest = read_json(&host_manifest_path)?;
2254 validate_host_toolchain_manifest("<dependency-scope>", &host_manifest)?;
2255 toolchain_dependency_scope(toolchain, &cuda_manifest, &host_manifest)
2256}
2257
2258fn toolchain_dependency_scope(
2259 toolchain: &NativeOperatorSourceBuildStaticToolchain,
2260 cuda_manifest: &NativeOperatorCudaToolkitManifest,
2261 host_manifest: &NativeOperatorHostToolchainManifest,
2262) -> Result<NativeOperatorToolchainDependencyScope> {
2263 let mut by_absolute_path = BTreeMap::new();
2264 let cuda_roots = [
2265 Path::new(&toolchain.cuda_toolkit.canonical_root),
2266 Path::new(&toolchain.cuda_toolkit.invocation_root),
2267 ];
2268 for entry in &cuda_manifest.entries {
2269 let dependency = NativeOperatorObservedDependency {
2270 domain: NativeOperatorDependencyDomain::BackendToolchain,
2271 path: entry.logical_path.clone(),
2272 sha256: entry.sha256.clone(),
2273 };
2274 for root in cuda_roots {
2275 for relative in [&entry.logical_path, &entry.resolved_path] {
2276 insert_toolchain_dependency(
2277 &mut by_absolute_path,
2278 root.join(relative).display().to_string(),
2279 dependency.clone(),
2280 )?;
2281 }
2282 }
2283 }
2284 for entry in &host_manifest.files {
2285 let dependency = NativeOperatorObservedDependency {
2286 domain: NativeOperatorDependencyDomain::HostToolchain,
2287 path: entry.resolved_path.clone(),
2288 sha256: entry.sha256.clone(),
2289 };
2290 for absolute in [&entry.logical_path, &entry.resolved_path] {
2291 insert_toolchain_dependency(
2292 &mut by_absolute_path,
2293 absolute.clone(),
2294 dependency.clone(),
2295 )?;
2296 }
2297 }
2298 Ok(NativeOperatorToolchainDependencyScope { by_absolute_path })
2299}
2300
2301fn insert_toolchain_dependency(
2302 dependencies: &mut BTreeMap<String, NativeOperatorObservedDependency>,
2303 absolute_path: String,
2304 dependency: NativeOperatorObservedDependency,
2305) -> Result<()> {
2306 validate_normalized_absolute_path(&absolute_path, "toolchain dependency path")?;
2307 if let Some(existing) = dependencies.get(&absolute_path) {
2308 if existing != &dependency {
2309 return Err(NativeOperatorBuilderError::Invalid(format!(
2310 "toolchain manifests ambiguously own dependency path: {absolute_path}"
2311 )));
2312 }
2313 return Ok(());
2314 }
2315 dependencies.insert(absolute_path, dependency);
2316 Ok(())
2317}
2318
2319fn expected_source_dependencies(
2320 translation_unit: &NativeOperatorSourceFileLock,
2321 closure: &NativeOperatorTranslationUnitDependencyLock,
2322) -> BTreeSet<NativeOperatorObservedDependency> {
2323 std::iter::once(translation_unit)
2324 .chain(closure.headers.iter())
2325 .map(|locked| NativeOperatorObservedDependency {
2326 domain: NativeOperatorDependencyDomain::Source,
2327 path: locked.path.clone(),
2328 sha256: locked.sha256.clone(),
2329 })
2330 .collect()
2331}
2332
2333fn validate_observed_dependency(
2334 context: &str,
2335 dependency: &NativeOperatorObservedDependency,
2336) -> Result<()> {
2337 let path_is_valid = match dependency.domain {
2338 NativeOperatorDependencyDomain::Source
2339 | NativeOperatorDependencyDomain::BackendToolchain => {
2340 validate_relative_path(&dependency.path).is_ok()
2341 }
2342 NativeOperatorDependencyDomain::HostToolchain => Path::new(&dependency.path).is_absolute(),
2343 };
2344 if !path_is_valid || !is_sha256_digest(&dependency.sha256) {
2345 return Err(NativeOperatorBuilderError::Invalid(format!(
2346 "{context} observed dependency identity is invalid: {:?}:{}",
2347 dependency.domain, dependency.path
2348 )));
2349 }
2350 Ok(())
2351}
2352
2353fn validate_observed_dependencies(
2354 context: &str,
2355 dependencies: &[NativeOperatorObservedDependency],
2356) -> Result<()> {
2357 if dependencies.is_empty()
2358 || dependencies.windows(2).any(|pair| pair[0] >= pair[1])
2359 || dependencies
2360 .iter()
2361 .any(|dependency| validate_observed_dependency(context, dependency).is_err())
2362 {
2363 return Err(NativeOperatorBuilderError::Invalid(format!(
2364 "{context} observed dependencies are empty, unsorted, duplicated, or invalid"
2365 )));
2366 }
2367 Ok(())
2368}
2369
2370fn observed_dependency_set_sha256(
2371 dependencies: &[NativeOperatorObservedDependency],
2372) -> Result<String> {
2373 validate_observed_dependencies("<dependency-set>", dependencies)?;
2374 let bytes =
2375 serde_json::to_vec(dependencies).map_err(|source| NativeOperatorBuilderError::Json {
2376 path: PathBuf::from("<dependency-set>"),
2377 source,
2378 })?;
2379 Ok(sha256_bytes(&bytes))
2380}
2381
2382fn validate_translation_unit_depfile(
2383 compiler_depfile_path: &Path,
2384 portable_depfile_path: &Path,
2385 object_path: &Path,
2386 source_root: &Path,
2387 translation_unit: &NativeOperatorSourceFileLock,
2388 closure: &NativeOperatorTranslationUnitDependencyLock,
2389 toolchain_scope: &NativeOperatorToolchainDependencyScope,
2390) -> Result<ValidatedTranslationUnitDepfile> {
2391 if closure.translation_unit != translation_unit.path {
2392 return Err(NativeOperatorBuilderError::Invalid(format!(
2393 "dependency closure translation unit mismatch: expected={} actual={}",
2394 translation_unit.path, closure.translation_unit
2395 )));
2396 }
2397 require_file(compiler_depfile_path)?;
2398 let raw =
2399 read_bounded_regular_file(compiler_depfile_path, MAX_DEPFILE_BYTES, "compiler depfile")?;
2400 let raw_text = std::str::from_utf8(&raw).map_err(|_| {
2401 NativeOperatorBuilderError::Invalid(format!(
2402 "compiler depfile is not valid UTF-8: {}",
2403 compiler_depfile_path.display()
2404 ))
2405 })?;
2406 let (target, dependencies) = parse_make_depfile(raw_text, compiler_depfile_path)?;
2407 let expected_target = object_path.display().to_string();
2408 validate_normalized_absolute_path(&expected_target, "compiler depfile object target")?;
2409 if target != expected_target {
2410 return Err(NativeOperatorBuilderError::Invalid(format!(
2411 "compiler depfile target differs from its exact -MT object: expected={expected_target} actual={target}"
2412 )));
2413 }
2414
2415 let mut observed = BTreeSet::new();
2416 let mut portable_paths = BTreeMap::new();
2417 let mut bindings = Vec::with_capacity(dependencies.len());
2418 for dependency in dependencies {
2419 let candidate = if Path::new(&dependency).is_absolute() {
2420 PathBuf::from(&dependency)
2421 } else {
2422 source_root.join(&dependency)
2423 };
2424 let canonical =
2425 candidate
2426 .canonicalize()
2427 .map_err(|source| NativeOperatorBuilderError::Io {
2428 path: candidate.clone(),
2429 source,
2430 })?;
2431 if !canonical.is_file() {
2432 return Err(NativeOperatorBuilderError::Invalid(format!(
2433 "compiler depfile dependency is not a file: {}",
2434 canonical.display()
2435 )));
2436 }
2437 let (identity, portable_path) = if canonical.starts_with(source_root) {
2438 let relative = canonical.strip_prefix(source_root).map_err(|_| {
2439 NativeOperatorBuilderError::Invalid(format!(
2440 "compiler depfile dependency escapes the source root: {}",
2441 canonical.display()
2442 ))
2443 })?;
2444 let identity = NativeOperatorObservedDependency {
2445 domain: NativeOperatorDependencyDomain::Source,
2446 path: path_with_forward_slashes(relative)?,
2447 sha256: sha256_file(&canonical)?,
2448 };
2449 let portable_path = identity.path.clone();
2450 (identity, portable_path)
2451 } else {
2452 let candidate_path = candidate.display().to_string();
2453 let canonical_path = canonical.display().to_string();
2454 let identity = toolchain_scope
2455 .by_absolute_path
2456 .get(&candidate_path)
2457 .or_else(|| toolchain_scope.by_absolute_path.get(&canonical_path))
2458 .ok_or_else(|| {
2459 NativeOperatorBuilderError::Invalid(format!(
2460 "compiler depfile dependency is outside the locked source and toolchain manifests: {}",
2461 canonical.display()
2462 ))
2463 })?
2464 .clone();
2465 let actual = sha256_file(&canonical)?;
2466 if actual != identity.sha256 {
2467 return Err(NativeOperatorBuilderError::Invalid(format!(
2468 "toolchain dependency changed while compiling: path={} expected={} actual={actual}",
2469 canonical.display(),
2470 identity.sha256
2471 )));
2472 }
2473 validate_normalized_absolute_path(
2474 &canonical_path,
2475 "compiled depfile canonical toolchain dependency",
2476 )?;
2477 (identity, canonical_path)
2478 };
2479 validate_observed_dependency("<compiled-depfile>", &identity)?;
2480 if !observed.insert(identity.clone()) {
2481 return Err(NativeOperatorBuilderError::Invalid(format!(
2482 "compiler depfile contains a duplicate dependency identity: {:?}:{}",
2483 identity.domain, identity.path
2484 )));
2485 }
2486 portable_paths.insert(identity.clone(), portable_path.clone());
2487 bindings.push(NativeOperatorDepfileDependencyBinding {
2488 producer_path: dependency,
2489 portable_path,
2490 dependency: identity,
2491 });
2492 }
2493
2494 let expected = expected_source_dependencies(translation_unit, closure);
2495 let observed_source = observed
2496 .iter()
2497 .filter(|dependency| dependency.domain == NativeOperatorDependencyDomain::Source)
2498 .cloned()
2499 .collect::<BTreeSet<_>>();
2500 if observed_source != expected {
2501 let missing = expected
2502 .difference(&observed_source)
2503 .cloned()
2504 .collect::<Vec<_>>();
2505 let undeclared = observed_source
2506 .difference(&expected)
2507 .cloned()
2508 .collect::<Vec<_>>();
2509 return Err(NativeOperatorBuilderError::Invalid(format!(
2510 "depfile differs from the declared source dependency closure: missing={missing:?} undeclared={undeclared:?}"
2511 )));
2512 }
2513
2514 for locked in std::iter::once(translation_unit).chain(closure.headers.iter()) {
2515 let path = locked_source_file(source_root, &locked.path)?;
2516 let actual = sha256_file(&path)?;
2517 if actual != locked.sha256 {
2518 return Err(NativeOperatorBuilderError::Invalid(format!(
2519 "dependency changed while compiling: path={} expected={} actual={actual}",
2520 locked.path, locked.sha256
2521 )));
2522 }
2523 }
2524 let observed_dependencies = observed.into_iter().collect::<Vec<_>>();
2525 let portable_dependencies = observed_dependencies
2526 .iter()
2527 .map(|identity| {
2528 portable_paths.get(identity).cloned().ok_or_else(|| {
2529 NativeOperatorBuilderError::Invalid(format!(
2530 "validated dependency has no portable depfile path: {:?}:{}",
2531 identity.domain, identity.path
2532 ))
2533 })
2534 })
2535 .collect::<Result<Vec<_>>>()?;
2536 let portable_raw =
2537 serialize_portable_depfile(&object_path.display().to_string(), &portable_dependencies)?;
2538 let portable_text = std::str::from_utf8(&portable_raw).expect("portable depfile is UTF-8");
2539 let verified_dependencies = validate_portable_depfile_pair(
2540 raw_text,
2541 compiler_depfile_path,
2542 portable_text,
2543 portable_depfile_path,
2544 &object_path.display().to_string(),
2545 &source_root.display().to_string(),
2546 translation_unit,
2547 closure,
2548 &bindings,
2549 toolchain_scope,
2550 )?;
2551 if verified_dependencies != observed_dependencies {
2552 return Err(NativeOperatorBuilderError::Invalid(
2553 "producer depfile verification changed the typed dependency set".to_string(),
2554 ));
2555 }
2556 atomic_write_bytes(portable_depfile_path, &portable_raw)?;
2557 Ok(ValidatedTranslationUnitDepfile {
2558 compiler_sha256: sha256_bytes(&raw),
2559 compiler_raw: raw,
2560 portable_sha256: sha256_bytes(&portable_raw),
2561 portable_raw,
2562 bindings,
2563 observed_dependencies,
2564 })
2565}
2566
2567struct ValidatedTranslationUnitDepfile {
2568 compiler_raw: Vec<u8>,
2569 compiler_sha256: String,
2570 portable_raw: Vec<u8>,
2571 portable_sha256: String,
2572 bindings: Vec<NativeOperatorDepfileDependencyBinding>,
2573 observed_dependencies: Vec<NativeOperatorObservedDependency>,
2574}
2575
2576fn serialize_portable_depfile(target: &str, dependencies: &[String]) -> Result<Vec<u8>> {
2577 validate_normalized_absolute_path(target, "portable depfile target")?;
2578 if dependencies.is_empty() || dependencies.len() > MAX_DEPFILE_DEPENDENCIES {
2579 return Err(NativeOperatorBuilderError::Invalid(
2580 "portable depfile dependency count is invalid".to_string(),
2581 ));
2582 }
2583 let mut result = escape_make_word(target)?;
2584 result.push(':');
2585 for dependency in dependencies {
2586 if Path::new(dependency).is_absolute() {
2587 validate_normalized_absolute_path(dependency, "portable depfile dependency")?;
2588 } else {
2589 validate_relative_path(dependency)?;
2590 }
2591 result.push(' ');
2592 result.push_str(&escape_make_word(dependency)?);
2593 }
2594 result.push('\n');
2595 if result.len() > MAX_DEPFILE_BYTES {
2596 return Err(NativeOperatorBuilderError::Invalid(format!(
2597 "portable depfile exceeds {MAX_DEPFILE_BYTES} bytes"
2598 )));
2599 }
2600 let (parsed_target, parsed_dependencies) =
2601 parse_make_depfile(&result, Path::new("<portable-depfile>"))?;
2602 if parsed_target != target || parsed_dependencies != dependencies {
2603 return Err(NativeOperatorBuilderError::Invalid(
2604 "portable depfile serialization did not round-trip".to_string(),
2605 ));
2606 }
2607 Ok(result.into_bytes())
2608}
2609
2610fn escape_make_word(value: &str) -> Result<String> {
2611 if value.is_empty()
2612 || value.len() > MAX_DEPFILE_WORD_BYTES
2613 || value
2614 .chars()
2615 .any(|character| matches!(character, '\0' | '\n' | '\r'))
2616 {
2617 return Err(NativeOperatorBuilderError::Invalid(
2618 "make depfile word must be non-empty and single-line".to_string(),
2619 ));
2620 }
2621 let mut result = String::with_capacity(value.len());
2622 for character in value.chars() {
2623 if matches!(character, '\\' | ' ' | '\t' | ':' | '#' | '$') {
2624 result.push('\\');
2625 }
2626 result.push(character);
2627 }
2628 Ok(result)
2629}
2630
2631fn publish_object_dependency_proof(
2632 cache_entry: &Path,
2633 object_cache_key: &str,
2634 object_sha256: &str,
2635 translation_unit: &NativeOperatorSourceFileLock,
2636 closure: &NativeOperatorTranslationUnitDependencyLock,
2637 compiler_depfile_raw: &[u8],
2638 expected_compiler_depfile_sha256: &str,
2639 depfile_raw: &[u8],
2640 expected_depfile_sha256: &str,
2641 producer_working_directory: &str,
2642 producer_object_file: &str,
2643 depfile_bindings: &[NativeOperatorDepfileDependencyBinding],
2644 observed_dependencies: &[NativeOperatorObservedDependency],
2645 toolchain_scope: &NativeOperatorToolchainDependencyScope,
2646) -> Result<()> {
2647 let compiler_depfile_sha256 = sha256_bytes(compiler_depfile_raw);
2648 if compiler_depfile_sha256 != expected_compiler_depfile_sha256 {
2649 return Err(NativeOperatorBuilderError::Invalid(format!(
2650 "compiler depfile bytes differ from their recorded sha256: expected={expected_compiler_depfile_sha256} actual={compiler_depfile_sha256}"
2651 )));
2652 }
2653 let depfile_sha256 = sha256_bytes(depfile_raw);
2654 if depfile_sha256 != expected_depfile_sha256 {
2655 return Err(NativeOperatorBuilderError::Invalid(format!(
2656 "validated depfile bytes differ from their recorded sha256: expected={expected_depfile_sha256} actual={depfile_sha256}"
2657 )));
2658 }
2659 let proof = NativeOperatorObjectDependencyProof {
2660 schema_version: NATIVE_OPERATOR_OBJECT_DEPENDENCY_PROOF_SCHEMA_VERSION,
2661 object_cache_key: object_cache_key.to_string(),
2662 object_sha256: object_sha256.to_string(),
2663 dependency_closure_sha256: closure.closure_sha256.clone(),
2664 dependency_set_sha256: observed_dependency_set_sha256(observed_dependencies)?,
2665 compiler_depfile_sha256,
2666 depfile_sha256: depfile_sha256.clone(),
2667 producer_working_directory: producer_working_directory.to_string(),
2668 producer_object_file: producer_object_file.to_string(),
2669 depfile_bindings: depfile_bindings.to_vec(),
2670 observed_dependencies: observed_dependencies.to_vec(),
2671 };
2672 validate_object_dependency_proof(
2673 "<object-cache-publish>",
2674 &proof,
2675 object_cache_key,
2676 object_sha256,
2677 translation_unit,
2678 closure,
2679 )?;
2680 let proof_dir = cache_entry.join("dependency-proof");
2681 if proof_dir.exists() {
2682 return validate_existing_dependency_proof(
2683 &proof_dir,
2684 object_cache_key,
2685 object_sha256,
2686 translation_unit,
2687 closure,
2688 producer_object_file,
2689 toolchain_scope,
2690 );
2691 }
2692
2693 let staging = tempfile::Builder::new()
2694 .prefix(".dependency-proof-")
2695 .tempdir_in(cache_entry)
2696 .map_err(|source| NativeOperatorBuilderError::Io {
2697 path: cache_entry.to_path_buf(),
2698 source,
2699 })?;
2700 let staging_path = staging.path().to_path_buf();
2701 atomic_write_bytes(
2702 &staging_path.join("compiler-dependency.raw.d"),
2703 compiler_depfile_raw,
2704 )?;
2705 atomic_write_bytes(&staging_path.join("dependency.d"), depfile_raw)?;
2706 write_json(&staging_path.join("proof.json"), &proof)?;
2707 sync_directory(&staging_path)?;
2708 let staging_path = staging.keep();
2709 match fs::rename(&staging_path, &proof_dir) {
2710 Ok(()) => {
2711 sync_directory(cache_entry)?;
2712 validate_existing_dependency_proof(
2713 &proof_dir,
2714 object_cache_key,
2715 object_sha256,
2716 translation_unit,
2717 closure,
2718 producer_object_file,
2719 toolchain_scope,
2720 )
2721 }
2722 Err(_source) if proof_dir.exists() => {
2723 fs::remove_dir_all(&staging_path).map_err(|cleanup_source| {
2724 NativeOperatorBuilderError::Io {
2725 path: staging_path.clone(),
2726 source: cleanup_source,
2727 }
2728 })?;
2729 validate_existing_dependency_proof(
2730 &proof_dir,
2731 object_cache_key,
2732 object_sha256,
2733 translation_unit,
2734 closure,
2735 producer_object_file,
2736 toolchain_scope,
2737 )
2738 }
2739 Err(source) => {
2740 let _ = fs::remove_dir_all(&staging_path);
2741 Err(NativeOperatorBuilderError::Io {
2742 path: proof_dir,
2743 source,
2744 })
2745 }
2746 }
2747}
2748
2749fn validate_dependency_proof_directory(proof_dir: &Path) -> Result<()> {
2750 let metadata =
2751 fs::symlink_metadata(proof_dir).map_err(|source| NativeOperatorBuilderError::Io {
2752 path: proof_dir.to_path_buf(),
2753 source,
2754 })?;
2755 if !metadata.file_type().is_dir() || metadata.file_type().is_symlink() {
2756 return Err(NativeOperatorBuilderError::Invalid(format!(
2757 "object dependency proof is not a real directory: {}",
2758 proof_dir.display()
2759 )));
2760 }
2761 let mut entries = fs::read_dir(proof_dir)
2762 .map_err(|source| NativeOperatorBuilderError::Io {
2763 path: proof_dir.to_path_buf(),
2764 source,
2765 })?
2766 .map(|entry| {
2767 entry
2768 .map(|entry| entry.file_name())
2769 .map_err(|source| NativeOperatorBuilderError::Io {
2770 path: proof_dir.to_path_buf(),
2771 source,
2772 })
2773 })
2774 .collect::<Result<Vec<_>>>()?;
2775 entries.sort();
2776 let expected = [
2777 std::ffi::OsString::from("compiler-dependency.raw.d"),
2778 std::ffi::OsString::from("dependency.d"),
2779 std::ffi::OsString::from("proof.json"),
2780 ];
2781 if entries != expected {
2782 return Err(NativeOperatorBuilderError::Invalid(format!(
2783 "object dependency proof directory is incomplete or contains extra files: {}",
2784 proof_dir.display()
2785 )));
2786 }
2787 for name in expected {
2788 let path = proof_dir.join(name);
2789 let metadata =
2790 fs::symlink_metadata(&path).map_err(|source| NativeOperatorBuilderError::Io {
2791 path: path.clone(),
2792 source,
2793 })?;
2794 if !metadata.file_type().is_file() || metadata.file_type().is_symlink() {
2795 return Err(NativeOperatorBuilderError::Invalid(format!(
2796 "object dependency proof member is not a regular file: {}",
2797 path.display()
2798 )));
2799 }
2800 }
2801 Ok(())
2802}
2803
2804fn sync_directory(path: &Path) -> Result<()> {
2805 let directory = File::open(path).map_err(|source| NativeOperatorBuilderError::Io {
2806 path: path.to_path_buf(),
2807 source,
2808 })?;
2809 directory
2810 .sync_all()
2811 .map_err(|source| NativeOperatorBuilderError::Io {
2812 path: path.to_path_buf(),
2813 source,
2814 })
2815}
2816
2817struct ValidatedCachedDependencyProof {
2818 proof: NativeOperatorObjectDependencyProof,
2819 compiler_raw: Vec<u8>,
2820 portable_raw: Vec<u8>,
2821}
2822
2823#[allow(clippy::too_many_arguments)]
2824fn load_validated_dependency_proof(
2825 proof_dir: &Path,
2826 object_cache_key: &str,
2827 object_sha256: &str,
2828 translation_unit: &NativeOperatorSourceFileLock,
2829 closure: &NativeOperatorTranslationUnitDependencyLock,
2830 toolchain_scope: &NativeOperatorToolchainDependencyScope,
2831) -> Result<ValidatedCachedDependencyProof> {
2832 validate_dependency_proof_directory(proof_dir)?;
2833 let proof_path = proof_dir.join("proof.json");
2834 let proof_bytes = read_bounded_regular_file(
2835 &proof_path,
2836 MAX_DEPENDENCY_PROOF_BYTES,
2837 "object dependency proof",
2838 )?;
2839 let proof: NativeOperatorObjectDependencyProof =
2840 serde_json::from_slice(&proof_bytes).map_err(|source| {
2841 NativeOperatorBuilderError::Json {
2842 path: proof_path.clone(),
2843 source,
2844 }
2845 })?;
2846 validate_object_dependency_proof(
2847 "<object-cache-proof>",
2848 &proof,
2849 object_cache_key,
2850 object_sha256,
2851 translation_unit,
2852 closure,
2853 )?;
2854
2855 let compiler_depfile_path = proof_dir.join("compiler-dependency.raw.d");
2856 let compiler_raw = read_bounded_regular_file(
2857 &compiler_depfile_path,
2858 MAX_DEPFILE_BYTES,
2859 "cached compiler depfile",
2860 )?;
2861 if sha256_bytes(&compiler_raw) != proof.compiler_depfile_sha256 {
2862 return Err(NativeOperatorBuilderError::Invalid(format!(
2863 "cached object compiler depfile hash mismatch: {}",
2864 compiler_depfile_path.display()
2865 )));
2866 }
2867 let depfile_path = proof_dir.join("dependency.d");
2868 let portable_raw =
2869 read_bounded_regular_file(&depfile_path, MAX_DEPFILE_BYTES, "cached portable depfile")?;
2870 if sha256_bytes(&portable_raw) != proof.depfile_sha256 {
2871 return Err(NativeOperatorBuilderError::Invalid(format!(
2872 "cached object dependency depfile hash mismatch: {}",
2873 depfile_path.display()
2874 )));
2875 }
2876 let compiler_text = std::str::from_utf8(&compiler_raw).map_err(|_| {
2877 NativeOperatorBuilderError::Invalid(format!(
2878 "cached object compiler depfile is not UTF-8: {}",
2879 compiler_depfile_path.display()
2880 ))
2881 })?;
2882 let portable_text = std::str::from_utf8(&portable_raw).map_err(|_| {
2883 NativeOperatorBuilderError::Invalid(format!(
2884 "cached object dependency depfile is not UTF-8: {}",
2885 depfile_path.display()
2886 ))
2887 })?;
2888 let observed = validate_portable_depfile_pair(
2889 compiler_text,
2890 &compiler_depfile_path,
2891 portable_text,
2892 &depfile_path,
2893 &proof.producer_object_file,
2894 &proof.producer_working_directory,
2895 translation_unit,
2896 closure,
2897 &proof.depfile_bindings,
2898 toolchain_scope,
2899 )?;
2900 if observed != proof.observed_dependencies {
2901 return Err(NativeOperatorBuilderError::Invalid(format!(
2902 "cached object dependency proof observed list differs from depfile: {}",
2903 proof_path.display()
2904 )));
2905 }
2906 Ok(ValidatedCachedDependencyProof {
2907 proof,
2908 compiler_raw,
2909 portable_raw,
2910 })
2911}
2912
2913#[allow(clippy::too_many_arguments)]
2914fn validate_existing_dependency_proof(
2915 proof_dir: &Path,
2916 object_cache_key: &str,
2917 object_sha256: &str,
2918 translation_unit: &NativeOperatorSourceFileLock,
2919 closure: &NativeOperatorTranslationUnitDependencyLock,
2920 producer_object_file: &str,
2921 toolchain_scope: &NativeOperatorToolchainDependencyScope,
2922) -> Result<()> {
2923 let validated = load_validated_dependency_proof(
2924 proof_dir,
2925 object_cache_key,
2926 object_sha256,
2927 translation_unit,
2928 closure,
2929 toolchain_scope,
2930 )?;
2931 if Path::new(&validated.proof.producer_object_file).file_name()
2932 != Path::new(producer_object_file).file_name()
2933 {
2934 return Err(NativeOperatorBuilderError::Invalid(format!(
2935 "published dependency proof object name differs from the current object: {}",
2936 proof_dir.display()
2937 )));
2938 }
2939 Ok(())
2940}
2941
2942#[allow(clippy::too_many_arguments)]
2943fn restore_object_dependency_proof(
2944 cache_entry: &Path,
2945 object_cache_key: &str,
2946 object_sha256: &str,
2947 closure: &NativeOperatorTranslationUnitDependencyLock,
2948 translation_unit: &NativeOperatorSourceFileLock,
2949 object_path: &Path,
2950 output_compiler_depfile: &Path,
2951 output_depfile: &Path,
2952 toolchain_scope: &NativeOperatorToolchainDependencyScope,
2953) -> Result<Option<NativeOperatorObjectDependencyProof>> {
2954 let proof_dir = cache_entry.join("dependency-proof");
2955 if !proof_dir.exists() {
2956 return Ok(None);
2957 }
2958 let validated = load_validated_dependency_proof(
2959 &proof_dir,
2960 object_cache_key,
2961 object_sha256,
2962 translation_unit,
2963 closure,
2964 toolchain_scope,
2965 )?;
2966 if Path::new(&validated.proof.producer_object_file).file_name() != object_path.file_name() {
2967 return Err(NativeOperatorBuilderError::Invalid(format!(
2968 "cached dependency proof object name differs from restored object: {}",
2969 proof_dir.display()
2970 )));
2971 }
2972 atomic_write_bytes(output_compiler_depfile, &validated.compiler_raw)?;
2973 atomic_write_bytes(output_depfile, &validated.portable_raw)?;
2974 Ok(Some(validated.proof))
2975}
2976
2977fn validate_object_dependency_proof(
2978 context: &str,
2979 proof: &NativeOperatorObjectDependencyProof,
2980 object_cache_key: &str,
2981 object_sha256: &str,
2982 translation_unit: &NativeOperatorSourceFileLock,
2983 closure: &NativeOperatorTranslationUnitDependencyLock,
2984) -> Result<()> {
2985 let dependency_set_sha256 = observed_dependency_set_sha256(&proof.observed_dependencies)?;
2986 let binding_dependencies = validate_depfile_bindings_basic(context, &proof.depfile_bindings)?;
2987 let expected_source = expected_source_dependencies(translation_unit, closure);
2988 let observed_source = proof
2989 .observed_dependencies
2990 .iter()
2991 .filter(|dependency| dependency.domain == NativeOperatorDependencyDomain::Source)
2992 .cloned()
2993 .collect::<BTreeSet<_>>();
2994 validate_normalized_absolute_path(
2995 &proof.producer_working_directory,
2996 &format!("{context} producer_working_directory"),
2997 )?;
2998 validate_normalized_absolute_path(
2999 &proof.producer_object_file,
3000 &format!("{context} producer_object_file"),
3001 )?;
3002 if proof.schema_version != NATIVE_OPERATOR_OBJECT_DEPENDENCY_PROOF_SCHEMA_VERSION
3003 || proof.object_cache_key != object_cache_key
3004 || proof.object_sha256 != object_sha256
3005 || proof.dependency_closure_sha256 != closure.closure_sha256
3006 || proof.dependency_set_sha256 != dependency_set_sha256
3007 || !is_sha256_digest(&proof.object_cache_key)
3008 || !is_sha256_digest(&proof.object_sha256)
3009 || !is_sha256_digest(&proof.dependency_closure_sha256)
3010 || !is_sha256_digest(&proof.dependency_set_sha256)
3011 || !is_sha256_digest(&proof.compiler_depfile_sha256)
3012 || !is_sha256_digest(&proof.depfile_sha256)
3013 || binding_dependencies != proof.observed_dependencies
3014 || observed_source != expected_source
3015 {
3016 return Err(NativeOperatorBuilderError::Invalid(format!(
3017 "{context} object dependency proof is invalid"
3018 )));
3019 }
3020 Ok(())
3021}
3022
3023fn validate_depfile_bindings_basic(
3024 context: &str,
3025 bindings: &[NativeOperatorDepfileDependencyBinding],
3026) -> Result<Vec<NativeOperatorObservedDependency>> {
3027 if bindings.is_empty() || bindings.len() > MAX_DEPFILE_DEPENDENCIES {
3028 return Err(NativeOperatorBuilderError::Invalid(format!(
3029 "{context} depfile bindings are empty or exceed {MAX_DEPFILE_DEPENDENCIES}"
3030 )));
3031 }
3032 let mut dependencies = BTreeSet::new();
3033 for binding in bindings {
3034 if binding.producer_path.is_empty()
3035 || binding.producer_path.len() > MAX_DEPFILE_WORD_BYTES
3036 || binding
3037 .producer_path
3038 .chars()
3039 .any(|character| matches!(character, '\0' | '\n' | '\r'))
3040 || binding.portable_path.is_empty()
3041 || binding.portable_path.len() > MAX_DEPFILE_WORD_BYTES
3042 {
3043 return Err(NativeOperatorBuilderError::Invalid(format!(
3044 "{context} depfile binding path is invalid"
3045 )));
3046 }
3047 validate_observed_dependency(context, &binding.dependency)?;
3048 match binding.dependency.domain {
3049 NativeOperatorDependencyDomain::Source => {
3050 validate_relative_path(&binding.portable_path)?;
3051 if binding.portable_path != binding.dependency.path {
3052 return Err(NativeOperatorBuilderError::Invalid(format!(
3053 "{context} source depfile binding does not use its locked path"
3054 )));
3055 }
3056 }
3057 NativeOperatorDependencyDomain::BackendToolchain
3058 | NativeOperatorDependencyDomain::HostToolchain => {
3059 validate_normalized_absolute_path(
3060 &binding.portable_path,
3061 &format!("{context} toolchain depfile binding"),
3062 )?;
3063 }
3064 }
3065 if !dependencies.insert(binding.dependency.clone()) {
3066 return Err(NativeOperatorBuilderError::Invalid(format!(
3067 "{context} depfile bindings duplicate a typed dependency"
3068 )));
3069 }
3070 }
3071 Ok(dependencies.into_iter().collect())
3072}
3073
3074#[allow(clippy::too_many_arguments)]
3075fn validate_portable_depfile_pair(
3076 compiler_raw: &str,
3077 compiler_depfile_path: &Path,
3078 portable_raw: &str,
3079 depfile_path: &Path,
3080 object_path: &str,
3081 working_directory: &str,
3082 translation_unit: &NativeOperatorSourceFileLock,
3083 closure: &NativeOperatorTranslationUnitDependencyLock,
3084 bindings: &[NativeOperatorDepfileDependencyBinding],
3085 toolchain_scope: &NativeOperatorToolchainDependencyScope,
3086) -> Result<Vec<NativeOperatorObservedDependency>> {
3087 let (compiler_target, compiler_dependencies) =
3088 parse_make_depfile(compiler_raw, compiler_depfile_path)?;
3089 let (target, dependencies) = parse_make_depfile(portable_raw, depfile_path)?;
3090 validate_normalized_absolute_path(object_path, "portable depfile producer object")?;
3091 if compiler_target != object_path || target != object_path {
3092 return Err(NativeOperatorBuilderError::Invalid(format!(
3093 "compiler or portable depfile target differs from object file: depfile={} compiler_target={compiler_target} portable_target={target}",
3094 depfile_path.display()
3095 )));
3096 }
3097 validate_normalized_absolute_path(working_directory, "portable depfile working directory")?;
3098 let working_directory = Path::new(working_directory);
3099 let expected = expected_source_dependencies(translation_unit, closure);
3100 let observed = validate_depfile_bindings_basic("<portable-depfile>", bindings)?;
3101 let expected_compiler_dependencies = bindings
3102 .iter()
3103 .map(|binding| binding.producer_path.as_str())
3104 .collect::<Vec<_>>();
3105 if compiler_dependencies
3106 != expected_compiler_dependencies
3107 .iter()
3108 .map(|value| (*value).to_string())
3109 .collect::<Vec<_>>()
3110 {
3111 return Err(NativeOperatorBuilderError::Invalid(
3112 "compiler depfile bytes differ from their ordered typed bindings".to_string(),
3113 ));
3114 }
3115 let portable_by_dependency = bindings
3116 .iter()
3117 .map(|binding| (binding.dependency.clone(), binding.portable_path.clone()))
3118 .collect::<BTreeMap<_, _>>();
3119 let expected_portable_dependencies = observed
3120 .iter()
3121 .map(|dependency| {
3122 portable_by_dependency
3123 .get(dependency)
3124 .cloned()
3125 .expect("binding dependencies were collected from the same rows")
3126 })
3127 .collect::<Vec<_>>();
3128 let canonical_portable =
3129 serialize_portable_depfile(object_path, &expected_portable_dependencies)?;
3130 if dependencies != expected_portable_dependencies
3131 || portable_raw.as_bytes() != canonical_portable.as_slice()
3132 {
3133 return Err(NativeOperatorBuilderError::Invalid(
3134 "portable depfile bytes differ from their canonical typed bindings".to_string(),
3135 ));
3136 }
3137 for binding in bindings {
3138 match binding.dependency.domain {
3139 NativeOperatorDependencyDomain::Source => {
3140 let producer = Path::new(&binding.producer_path);
3141 let relative = if producer.is_absolute() {
3142 producer.strip_prefix(working_directory).map_err(|_| {
3143 NativeOperatorBuilderError::Invalid(format!(
3144 "source compiler depfile path escapes its recorded working directory: {}",
3145 binding.producer_path
3146 ))
3147 })?
3148 } else {
3149 producer
3150 };
3151 if normalize_portable_relative_path(relative)? != binding.dependency.path {
3152 return Err(NativeOperatorBuilderError::Invalid(format!(
3153 "source compiler depfile path differs from its locked identity: {}",
3154 binding.producer_path
3155 )));
3156 }
3157 }
3158 NativeOperatorDependencyDomain::BackendToolchain
3159 | NativeOperatorDependencyDomain::HostToolchain => {
3160 let normalized_producer = normalize_absolute_posix_path_lexically(
3161 &binding.producer_path,
3162 "toolchain compiler depfile binding",
3163 )?;
3164 if toolchain_scope.by_absolute_path.get(&normalized_producer)
3165 != Some(&binding.dependency)
3166 || toolchain_scope.by_absolute_path.get(&binding.portable_path)
3167 != Some(&binding.dependency)
3168 {
3169 return Err(NativeOperatorBuilderError::Invalid(format!(
3170 "toolchain depfile binding is outside its typed manifest: producer={} portable={}",
3171 binding.producer_path, binding.portable_path
3172 )));
3173 }
3174 }
3175 }
3176 }
3177 let observed_source = observed
3178 .iter()
3179 .filter(|dependency| dependency.domain == NativeOperatorDependencyDomain::Source)
3180 .cloned()
3181 .collect::<BTreeSet<_>>();
3182 if observed_source != expected {
3183 return Err(NativeOperatorBuilderError::Invalid(format!(
3184 "portable depfile differs from locked source dependency closure: expected={expected:?} observed={observed_source:?}"
3185 )));
3186 }
3187 validate_observed_dependencies("<portable-depfile>", &observed)?;
3188 for dependency in &observed {
3189 if !bindings
3190 .iter()
3191 .any(|binding| &binding.dependency == dependency)
3192 {
3193 return Err(NativeOperatorBuilderError::Invalid(format!(
3194 "portable depfile is missing a binding for {:?}:{}",
3195 dependency.domain, dependency.path
3196 )));
3197 }
3198 }
3199 Ok(observed)
3200}
3201
3202fn normalize_absolute_posix_path_lexically(value: &str, label: &str) -> Result<String> {
3203 if !value.starts_with('/')
3204 || value.contains('\\')
3205 || value
3206 .chars()
3207 .any(|character| matches!(character, '\0' | '\n' | '\r'))
3208 {
3209 return Err(NativeOperatorBuilderError::Invalid(format!(
3210 "{label} must be an absolute POSIX path: {value}"
3211 )));
3212 }
3213 let mut components = Vec::new();
3214 for component in value.split('/') {
3215 match component {
3216 "" | "." => {}
3217 ".." => {
3218 if components.pop().is_none() {
3219 return Err(NativeOperatorBuilderError::Invalid(format!(
3220 "{label} escapes the filesystem root: {value}"
3221 )));
3222 }
3223 }
3224 _ => components.push(component),
3225 }
3226 }
3227 if components.is_empty() {
3228 Ok("/".to_string())
3229 } else {
3230 Ok(format!("/{}", components.join("/")))
3231 }
3232}
3233
3234fn validate_normalized_absolute_path(value: &str, label: &str) -> Result<()> {
3235 if value == "/" {
3236 return Ok(());
3237 }
3238 if !value.starts_with('/')
3239 || value.contains('\\')
3240 || value.ends_with('/')
3241 || value[1..]
3242 .split('/')
3243 .any(|component| component.is_empty() || component == "." || component == "..")
3244 {
3245 return Err(NativeOperatorBuilderError::Invalid(format!(
3246 "{label} must be a normalized absolute POSIX path: {value}"
3247 )));
3248 }
3249 Ok(())
3250}
3251
3252fn normalize_portable_relative_path(path: &Path) -> Result<String> {
3253 let raw = path.to_str().ok_or_else(|| {
3254 NativeOperatorBuilderError::Invalid(format!(
3255 "depfile path is not valid UTF-8: {}",
3256 path.display()
3257 ))
3258 })?;
3259 if raw.contains('\\')
3260 || raw
3261 .chars()
3262 .any(|character| matches!(character, '\0' | '\n' | '\r'))
3263 {
3264 return Err(NativeOperatorBuilderError::Invalid(format!(
3265 "depfile path is not a relative POSIX path: {}",
3266 path.display()
3267 )));
3268 }
3269 let mut components = Vec::new();
3270 for component in path.components() {
3271 match component {
3272 std::path::Component::Normal(value) => {
3273 components.push(value.to_str().ok_or_else(|| {
3274 NativeOperatorBuilderError::Invalid(format!(
3275 "depfile path is not valid UTF-8: {}",
3276 path.display()
3277 ))
3278 })?)
3279 }
3280 std::path::Component::CurDir => {}
3281 std::path::Component::ParentDir => {
3282 if components.pop().is_none() {
3283 return Err(NativeOperatorBuilderError::Invalid(format!(
3284 "depfile path escapes its working directory: {}",
3285 path.display()
3286 )));
3287 }
3288 }
3289 _ => {
3290 return Err(NativeOperatorBuilderError::Invalid(format!(
3291 "depfile path is not normalized beneath its working directory: {}",
3292 path.display()
3293 )))
3294 }
3295 }
3296 }
3297 let normalized = components.join("/");
3298 validate_relative_path(&normalized)?;
3299 Ok(normalized)
3300}
3301
3302fn read_bounded_regular_file(path: &Path, max_bytes: usize, label: &str) -> Result<Vec<u8>> {
3303 let mut options = OpenOptions::new();
3304 options.read(true);
3305 #[cfg(unix)]
3306 options.custom_flags(libc::O_NOFOLLOW);
3307 let file = options
3308 .open(path)
3309 .map_err(|source| NativeOperatorBuilderError::Io {
3310 path: path.to_path_buf(),
3311 source,
3312 })?;
3313 let metadata = file
3314 .metadata()
3315 .map_err(|source| NativeOperatorBuilderError::Io {
3316 path: path.to_path_buf(),
3317 source,
3318 })?;
3319 if !metadata.is_file() || metadata.len() > max_bytes as u64 {
3320 return Err(NativeOperatorBuilderError::Invalid(format!(
3321 "{label} is not a regular file or exceeds {max_bytes} bytes: {}",
3322 path.display()
3323 )));
3324 }
3325 let mut bytes = Vec::with_capacity(metadata.len() as usize);
3326 file.take(max_bytes as u64 + 1)
3327 .read_to_end(&mut bytes)
3328 .map_err(|source| NativeOperatorBuilderError::Io {
3329 path: path.to_path_buf(),
3330 source,
3331 })?;
3332 if bytes.len() > max_bytes {
3333 return Err(NativeOperatorBuilderError::Invalid(format!(
3334 "{label} grew beyond {max_bytes} bytes while reading: {}",
3335 path.display()
3336 )));
3337 }
3338 Ok(bytes)
3339}
3340
3341fn atomic_write_bytes(destination: &Path, bytes: &[u8]) -> Result<()> {
3342 let parent = destination.parent().ok_or_else(|| {
3343 NativeOperatorBuilderError::Invalid(format!(
3344 "atomic write destination has no parent: {}",
3345 destination.display()
3346 ))
3347 })?;
3348 fs::create_dir_all(parent).map_err(|source| NativeOperatorBuilderError::Io {
3349 path: parent.to_path_buf(),
3350 source,
3351 })?;
3352 let mut temporary =
3353 NamedTempFile::new_in(parent).map_err(|source| NativeOperatorBuilderError::Io {
3354 path: parent.to_path_buf(),
3355 source,
3356 })?;
3357 temporary
3358 .write_all(bytes)
3359 .map_err(|source| NativeOperatorBuilderError::Io {
3360 path: temporary.path().to_path_buf(),
3361 source,
3362 })?;
3363 temporary
3364 .as_file_mut()
3365 .flush()
3366 .and_then(|()| temporary.as_file().sync_all())
3367 .map_err(|source| NativeOperatorBuilderError::Io {
3368 path: temporary.path().to_path_buf(),
3369 source,
3370 })?;
3371 temporary
3372 .persist(destination)
3373 .map_err(|error| NativeOperatorBuilderError::Io {
3374 path: destination.to_path_buf(),
3375 source: error.error,
3376 })?;
3377 Ok(())
3378}
3379
3380fn parse_make_depfile(raw: &str, path: &Path) -> Result<(String, Vec<String>)> {
3381 if raw.len() > MAX_DEPFILE_BYTES || raw.trim().is_empty() || raw.as_bytes().contains(&0) {
3382 return Err(NativeOperatorBuilderError::Invalid(format!(
3383 "depfile is too large, empty, or contains NUL: {}",
3384 path.display()
3385 )));
3386 }
3387 let normalized = raw.replace("\\\r\n", "").replace("\\\n", "");
3388 let normalized = normalized.trim_end_matches(&['\r', '\n'][..]);
3389 if normalized.contains('\n') || normalized.contains('\r') {
3390 return Err(NativeOperatorBuilderError::Invalid(format!(
3391 "depfile contains more than one make rule: {}",
3392 path.display()
3393 )));
3394 }
3395 let mut escaped = false;
3396 let mut delimiter = None;
3397 for (index, character) in normalized.char_indices() {
3398 if escaped {
3399 escaped = false;
3400 continue;
3401 }
3402 match character {
3403 '\\' => escaped = true,
3404 ':' => {
3405 delimiter = Some(index);
3406 break;
3407 }
3408 '\n' | '\r' => {
3409 return Err(NativeOperatorBuilderError::Invalid(format!(
3410 "depfile contains multiple or unterminated rules: {}",
3411 path.display()
3412 )))
3413 }
3414 _ => {}
3415 }
3416 }
3417 let delimiter = delimiter.ok_or_else(|| {
3418 NativeOperatorBuilderError::Invalid(format!(
3419 "depfile has no make target delimiter: {}",
3420 path.display()
3421 ))
3422 })?;
3423 let targets = parse_make_words(&normalized[..delimiter], path, 1)?;
3424 let dependencies =
3425 parse_make_words(&normalized[delimiter + 1..], path, MAX_DEPFILE_DEPENDENCIES)?;
3426 if targets.len() != 1 || dependencies.is_empty() {
3427 return Err(NativeOperatorBuilderError::Invalid(format!(
3428 "depfile target/dependency count or word size is invalid: {}",
3429 path.display()
3430 )));
3431 }
3432 Ok((targets[0].clone(), dependencies))
3433}
3434
3435fn parse_make_words(value: &str, path: &Path, max_words: usize) -> Result<Vec<String>> {
3436 let mut words = Vec::new();
3437 let mut word = String::new();
3438 let mut escaped = false;
3439 for character in value.chars() {
3440 if escaped {
3441 word.push(character);
3442 if word.len() > MAX_DEPFILE_WORD_BYTES {
3443 return Err(NativeOperatorBuilderError::Invalid(format!(
3444 "depfile word exceeds {MAX_DEPFILE_WORD_BYTES} bytes: {}",
3445 path.display()
3446 )));
3447 }
3448 escaped = false;
3449 continue;
3450 }
3451 match character {
3452 '\\' => escaped = true,
3453 '\n' | '\r' => unreachable!("newlines rejected before make-word parsing"),
3454 character if character.is_whitespace() => {
3455 if !word.is_empty() {
3456 if words.len() >= max_words {
3457 return Err(NativeOperatorBuilderError::Invalid(format!(
3458 "depfile exceeds its word-count limit: {}",
3459 path.display()
3460 )));
3461 }
3462 words.push(std::mem::take(&mut word));
3463 }
3464 }
3465 _ => {
3466 word.push(character);
3467 if word.len() > MAX_DEPFILE_WORD_BYTES {
3468 return Err(NativeOperatorBuilderError::Invalid(format!(
3469 "depfile word exceeds {MAX_DEPFILE_WORD_BYTES} bytes: {}",
3470 path.display()
3471 )));
3472 }
3473 }
3474 }
3475 }
3476 if escaped {
3477 return Err(NativeOperatorBuilderError::Invalid(format!(
3478 "depfile ends with an incomplete escape: {}",
3479 path.display()
3480 )));
3481 }
3482 if !word.is_empty() {
3483 if words.len() >= max_words {
3484 return Err(NativeOperatorBuilderError::Invalid(format!(
3485 "depfile exceeds its word-count limit: {}",
3486 path.display()
3487 )));
3488 }
3489 words.push(word);
3490 }
3491 Ok(words)
3492}
3493
3494fn locked_source_file(root: &Path, relative: &str) -> Result<PathBuf> {
3495 validate_relative_path(relative)?;
3496 let path = root.join(relative);
3497 let metadata =
3498 fs::symlink_metadata(&path).map_err(|source| NativeOperatorBuilderError::Io {
3499 path: path.clone(),
3500 source,
3501 })?;
3502 if metadata.file_type().is_symlink() {
3503 return Err(NativeOperatorBuilderError::Invalid(format!(
3504 "locked source files must not be symlinks: {relative}"
3505 )));
3506 }
3507 let canonical = path
3508 .canonicalize()
3509 .map_err(|source| NativeOperatorBuilderError::Io {
3510 path: path.clone(),
3511 source,
3512 })?;
3513 if !canonical.starts_with(root) || !canonical.is_file() {
3514 return Err(NativeOperatorBuilderError::Invalid(format!(
3515 "locked source file escapes source root or is not a file: {relative}"
3516 )));
3517 }
3518 Ok(canonical)
3519}
3520
3521fn resolve_static_toolchain(
3522 request: &NativeOperatorSourceBuildRequest,
3523) -> Result<NativeOperatorSourceBuildToolchain> {
3524 let invocation_root_path = if request.cuda_toolkit_root.is_absolute() {
3525 request.cuda_toolkit_root.clone()
3526 } else {
3527 std::env::current_dir()
3528 .map_err(|source| NativeOperatorBuilderError::Io {
3529 path: PathBuf::from("."),
3530 source,
3531 })?
3532 .join(&request.cuda_toolkit_root)
3533 };
3534 let invocation_root = invocation_root_path.to_str().ok_or_else(|| {
3535 NativeOperatorBuilderError::Invalid(format!(
3536 "cuda_toolkit_root is not valid UTF-8: {}",
3537 invocation_root_path.display()
3538 ))
3539 })?;
3540 validate_normalized_absolute_path(invocation_root, "cuda_toolkit_root")?;
3541 let canonical_root = request.cuda_toolkit_root.canonicalize().map_err(|source| {
3542 NativeOperatorBuilderError::Io {
3543 path: request.cuda_toolkit_root.clone(),
3544 source,
3545 }
3546 })?;
3547 if !canonical_root.is_dir() {
3548 return Err(NativeOperatorBuilderError::Invalid(format!(
3549 "cuda_toolkit_root is not a directory: {}",
3550 canonical_root.display()
3551 )));
3552 }
3553 let canonical_root_string = canonical_root.to_str().ok_or_else(|| {
3554 NativeOperatorBuilderError::Invalid(format!(
3555 "canonical cuda_toolkit_root is not valid UTF-8: {}",
3556 canonical_root.display()
3557 ))
3558 })?;
3559 validate_normalized_absolute_path(canonical_root_string, "canonical cuda_toolkit_root")?;
3560 let nvcc = tool_file_identity(&request.nvcc_path)?;
3561 let canonical_nvcc = Path::new(&nvcc.path);
3562 if !canonical_nvcc.starts_with(&canonical_root) {
3563 return Err(NativeOperatorBuilderError::Invalid(format!(
3564 "nvcc must resolve inside cuda_toolkit_root: nvcc={} root={}",
3565 canonical_nvcc.display(),
3566 canonical_root.display()
3567 )));
3568 }
3569 let manifest = build_cuda_toolkit_manifest(&canonical_root)?;
3570 if !manifest.entries.iter().any(|entry| {
3571 canonical_root.join(&entry.resolved_path) == canonical_nvcc
3572 && entry.sha256 == nvcc.sha256
3573 && entry.size_bytes == nvcc.size_bytes
3574 }) {
3575 return Err(NativeOperatorBuilderError::Invalid(format!(
3576 "cuda toolkit manifest does not contain the selected nvcc: {}",
3577 canonical_nvcc.display()
3578 )));
3579 }
3580 let manifest_relative = "toolchain/cuda-static-manifest.json";
3581 let manifest_path = request.output_dir.join(manifest_relative);
3582 let manifest_parent = manifest_path
3583 .parent()
3584 .expect("cuda toolkit manifest has a parent directory");
3585 fs::create_dir_all(manifest_parent).map_err(|source| NativeOperatorBuilderError::Io {
3586 path: manifest_parent.to_path_buf(),
3587 source,
3588 })?;
3589 write_json(&manifest_path, &manifest)?;
3590 let manifest_size = fs::metadata(&manifest_path)
3591 .map_err(|source| NativeOperatorBuilderError::Io {
3592 path: manifest_path.clone(),
3593 source,
3594 })?
3595 .len();
3596 let manifest_evidence = NativeOperatorEvidenceFile {
3597 path: manifest_relative.to_string(),
3598 sha256: sha256_file(&manifest_path)?,
3599 size_bytes: manifest_size,
3600 };
3601 let host_toolchain = resolve_host_toolchain(request)?;
3602 Ok(NativeOperatorSourceBuildToolchain {
3603 static_identity: NativeOperatorSourceBuildStaticToolchain {
3604 backend: NativeOperatorBackend::Cuda,
3605 compiler_driver: NativeOperatorSourceCompilerDriver::CudaNvcc,
3606 cuda_toolkit: NativeOperatorCudaToolkitIdentity {
3607 canonical_root: canonical_root_string.to_string(),
3608 invocation_root: invocation_root.to_string(),
3609 release_version: cuda_toolkit_release_version(&canonical_root)?,
3610 nvcc,
3611 manifest: manifest_evidence,
3612 },
3613 host_toolchain,
3614 archiver: tool_file_identity(&request.ar_path)?,
3615 },
3616 miss_probe: None,
3617 })
3618}
3619
3620fn resolve_host_toolchain(
3621 request: &NativeOperatorSourceBuildRequest,
3622) -> Result<NativeOperatorHostToolchainIdentity> {
3623 let compiler = tool_file_identity(&request.ccbin_path)?;
3624 let cache_key = sha256_bytes(format!("{}\n{}\n", compiler.path, compiler.sha256).as_bytes());
3625 let cache_dir = request
3626 .object_cache_dir
3627 .join(".host-toolchains")
3628 .join(cache_key);
3629 fs::create_dir_all(&cache_dir).map_err(|source| NativeOperatorBuilderError::Io {
3630 path: cache_dir.clone(),
3631 source,
3632 })?;
3633 let cached_path = cache_dir.join("manifest.json");
3634 let environment = effective_environment_for_tool_paths([
3635 request.nvcc_path.to_str().unwrap_or(""),
3636 compiler.path.as_str(),
3637 request.ar_path.to_str().unwrap_or(""),
3638 ])?;
3639 let cached = cached_path
3640 .is_file()
3641 .then(|| read_json::<NativeOperatorHostToolchainManifest>(&cached_path))
3642 .transpose()
3643 .ok()
3644 .flatten()
3645 .filter(|manifest| {
3646 validate_host_toolchain_manifest("<host-toolchain-cache>", manifest).is_ok()
3647 });
3648 let manifest = match cached {
3649 Some(cached)
3650 if host_toolchain_manifest_matches_current(&cached, &compiler, &environment)? =>
3651 {
3652 cached
3653 }
3654 _ => {
3655 let probed = probe_host_toolchain_manifest(&compiler, &environment)?;
3656 write_json(&cached_path, &probed)?;
3657 probed
3658 }
3659 };
3660
3661 let relative = "toolchain/host-static-manifest.json";
3662 let output_path = request.output_dir.join(relative);
3663 write_json(&output_path, &manifest)?;
3664 let size_bytes = fs::metadata(&output_path)
3665 .map_err(|source| NativeOperatorBuilderError::Io {
3666 path: output_path.clone(),
3667 source,
3668 })?
3669 .len();
3670 Ok(NativeOperatorHostToolchainIdentity {
3671 compiler: manifest.compiler.clone(),
3672 compiler_version: manifest.compiler_version.clone(),
3673 target: manifest.target.clone(),
3674 manifest: NativeOperatorEvidenceFile {
3675 path: relative.to_string(),
3676 sha256: sha256_file(&output_path)?,
3677 size_bytes,
3678 },
3679 })
3680}
3681
3682fn probe_host_toolchain_manifest(
3683 compiler: &NativeOperatorToolFileIdentity,
3684 environment: &BTreeMap<String, String>,
3685) -> Result<NativeOperatorHostToolchainManifest> {
3686 let compiler_path = Path::new(&compiler.path);
3687 let compiler_version = host_compiler_output(compiler_path, &["--version"], b"", environment)?;
3688 let target = host_compiler_output(compiler_path, &["-dumpmachine"], b"", environment)?;
3689 if target.is_empty() || target.len() > 256 || target.chars().any(char::is_whitespace) {
3690 return Err(NativeOperatorBuilderError::Invalid(format!(
3691 "host compiler produced an invalid target: {}",
3692 compiler.path
3693 )));
3694 }
3695
3696 let discovery_probe = probe_host_compiler_discovery(compiler_path, environment)?;
3697
3698 let mut executable_inputs = BTreeMap::new();
3699 executable_inputs.insert(compiler.path.clone(), compiler.clone());
3700 for program in HOST_TOOLCHAIN_PROGRAMS {
3701 let value = host_compiler_output(
3702 compiler_path,
3703 &[&format!("-print-prog-name={program}")],
3704 b"",
3705 environment,
3706 )?;
3707 if let Some(path) = resolve_host_program(&value, environment)? {
3708 let identity = tool_file_identity(&path)?;
3709 executable_inputs.insert(identity.path.clone(), identity);
3710 }
3711 }
3712
3713 let mut discovery_roots = executable_inputs
3714 .values()
3715 .filter_map(|identity| Path::new(&identity.path).parent())
3716 .map(|path| path.display().to_string())
3717 .collect::<Vec<_>>();
3718 discovery_roots.sort();
3719 discovery_roots.dedup();
3720 let scope_roots = discovery_probe
3721 .include_roots
3722 .iter()
3723 .chain(discovery_roots.iter())
3724 .cloned()
3725 .collect::<Vec<_>>();
3726 let files = collect_host_toolchain_scope_files(&scope_roots)?;
3727 let manifest = NativeOperatorHostToolchainManifest {
3728 schema_version: NATIVE_OPERATOR_HOST_TOOLCHAIN_MANIFEST_SCHEMA_VERSION,
3729 compiler: compiler.clone(),
3730 compiler_version,
3731 target,
3732 executable_inputs: executable_inputs.into_values().collect(),
3733 include_roots: discovery_probe.include_roots,
3734 include_probe_sha256: discovery_probe.include_probe_sha256,
3735 driver_probe_sha256: discovery_probe.driver_probe_sha256,
3736 discovery_roots,
3737 files,
3738 };
3739 validate_host_toolchain_manifest("<host-toolchain-probe>", &manifest)?;
3740 Ok(manifest)
3741}
3742
3743fn rebuild_host_toolchain_manifest(
3744 recorded: &NativeOperatorHostToolchainManifest,
3745) -> Result<NativeOperatorHostToolchainManifest> {
3746 validate_host_toolchain_manifest("<host-toolchain-recorded>", recorded)?;
3747 let compiler = tool_file_identity(Path::new(&recorded.compiler.path))?;
3748 let executable_inputs = recorded
3749 .executable_inputs
3750 .iter()
3751 .map(|identity| tool_file_identity(Path::new(&identity.path)))
3752 .collect::<Result<Vec<_>>>()?;
3753 let scope_roots = recorded
3754 .include_roots
3755 .iter()
3756 .chain(recorded.discovery_roots.iter())
3757 .cloned()
3758 .collect::<Vec<_>>();
3759 let files = collect_host_toolchain_scope_files(&scope_roots)?;
3760 let current = NativeOperatorHostToolchainManifest {
3761 schema_version: NATIVE_OPERATOR_HOST_TOOLCHAIN_MANIFEST_SCHEMA_VERSION,
3762 compiler,
3763 compiler_version: recorded.compiler_version.clone(),
3764 target: recorded.target.clone(),
3765 executable_inputs,
3766 include_roots: recorded.include_roots.clone(),
3767 include_probe_sha256: recorded.include_probe_sha256.clone(),
3768 driver_probe_sha256: recorded.driver_probe_sha256.clone(),
3769 discovery_roots: recorded.discovery_roots.clone(),
3770 files,
3771 };
3772 validate_host_toolchain_manifest("<host-toolchain-current>", ¤t)?;
3773 Ok(current)
3774}
3775
3776struct HostCompilerDiscoveryProbe {
3777 include_roots: Vec<String>,
3778 include_probe_sha256: String,
3779 driver_probe_sha256: String,
3780}
3781
3782fn probe_host_compiler_discovery(
3783 compiler: &Path,
3784 environment: &BTreeMap<String, String>,
3785) -> Result<HostCompilerDiscoveryProbe> {
3786 let include_probe = host_compiler_raw_output(
3787 compiler,
3788 &["-E", "-x", "c++", "-", "-v"],
3789 b"\n",
3790 environment,
3791 )?;
3792 let include_roots = parse_host_compiler_include_roots(
3793 &String::from_utf8_lossy(&include_probe.stderr),
3794 compiler,
3795 )?;
3796 let driver_probe = host_compiler_raw_output(
3797 compiler,
3798 &["-###", "-pipe", "-x", "c++", "-c", "-", "-o", "/dev/null"],
3799 b"\n",
3800 environment,
3801 )?;
3802 Ok(HostCompilerDiscoveryProbe {
3803 include_roots,
3804 include_probe_sha256: compiler_probe_sha256(&include_probe),
3805 driver_probe_sha256: compiler_probe_sha256(&driver_probe),
3806 })
3807}
3808
3809fn compiler_probe_sha256(output: &std::process::Output) -> String {
3810 let mut identity = Vec::with_capacity(output.stdout.len() + output.stderr.len() + 16);
3811 identity.extend_from_slice(&(output.stdout.len() as u64).to_le_bytes());
3812 identity.extend_from_slice(&output.stdout);
3813 identity.extend_from_slice(&(output.stderr.len() as u64).to_le_bytes());
3814 identity.extend_from_slice(&output.stderr);
3815 sha256_bytes(&identity)
3816}
3817
3818fn host_toolchain_manifest_matches_current(
3819 recorded: &NativeOperatorHostToolchainManifest,
3820 compiler: &NativeOperatorToolFileIdentity,
3821 environment: &BTreeMap<String, String>,
3822) -> Result<bool> {
3823 if &recorded.compiler != compiler || rebuild_host_toolchain_manifest(recorded)? != *recorded {
3824 return Ok(false);
3825 }
3826 let discovery = probe_host_compiler_discovery(Path::new(&compiler.path), environment)?;
3827 Ok(discovery.include_roots == recorded.include_roots
3828 && discovery.include_probe_sha256 == recorded.include_probe_sha256
3829 && discovery.driver_probe_sha256 == recorded.driver_probe_sha256)
3830}
3831
3832fn host_compiler_output(
3833 compiler: &Path,
3834 args: &[&str],
3835 stdin: &[u8],
3836 environment: &BTreeMap<String, String>,
3837) -> Result<String> {
3838 let output = host_compiler_raw_output(compiler, args, stdin, environment)?;
3839 let value = format!(
3840 "{}{}",
3841 String::from_utf8_lossy(&output.stdout),
3842 String::from_utf8_lossy(&output.stderr)
3843 )
3844 .trim()
3845 .chars()
3846 .take(16_384)
3847 .collect::<String>();
3848 if value.is_empty() {
3849 return Err(NativeOperatorBuilderError::Invalid(format!(
3850 "host compiler produced no output for {:?}: {}",
3851 args,
3852 compiler.display()
3853 )));
3854 }
3855 Ok(value)
3856}
3857
3858fn host_compiler_raw_output(
3859 compiler: &Path,
3860 args: &[&str],
3861 stdin: &[u8],
3862 environment: &BTreeMap<String, String>,
3863) -> Result<std::process::Output> {
3864 let mut child = Command::new(compiler)
3865 .args(args)
3866 .env_clear()
3867 .envs(environment)
3868 .stdin(Stdio::piped())
3869 .stdout(Stdio::piped())
3870 .stderr(Stdio::piped())
3871 .spawn()
3872 .map_err(|source| NativeOperatorBuilderError::Io {
3873 path: compiler.to_path_buf(),
3874 source,
3875 })?;
3876 let stdin_result = {
3877 let mut child_stdin = child.stdin.take().expect("host compiler stdin is piped");
3878 let result = child_stdin.write_all(stdin);
3879 drop(child_stdin);
3880 result
3881 };
3882 let output = child
3883 .wait_with_output()
3884 .map_err(|source| NativeOperatorBuilderError::Io {
3885 path: compiler.to_path_buf(),
3886 source,
3887 })?;
3888 if let Err(source) = stdin_result {
3889 if source.kind() != std::io::ErrorKind::BrokenPipe {
3894 return Err(NativeOperatorBuilderError::Io {
3895 path: compiler.to_path_buf(),
3896 source,
3897 });
3898 }
3899 }
3900 if !output.status.success() {
3901 return Err(NativeOperatorBuilderError::Invalid(format!(
3902 "host compiler probe failed for {:?}: path={} status={}",
3903 args,
3904 compiler.display(),
3905 output.status
3906 )));
3907 }
3908 Ok(output)
3909}
3910
3911fn parse_host_compiler_include_roots(stderr: &str, compiler: &Path) -> Result<Vec<String>> {
3912 let mut in_search_list = false;
3913 let mut roots = Vec::new();
3914 let mut seen = BTreeSet::new();
3915 for line in stderr.lines() {
3916 let trimmed = line.trim();
3917 if trimmed == "#include <...> search starts here:" {
3918 in_search_list = true;
3919 continue;
3920 }
3921 if in_search_list && trimmed == "End of search list." {
3922 break;
3923 }
3924 if !in_search_list || trimmed.is_empty() {
3925 continue;
3926 }
3927 let path = trimmed
3928 .strip_suffix(" (framework directory)")
3929 .unwrap_or(trimmed);
3930 validate_normalized_absolute_path(path, "host compiler include search entry")?;
3931 let canonical =
3932 Path::new(path)
3933 .canonicalize()
3934 .map_err(|source| NativeOperatorBuilderError::Io {
3935 path: PathBuf::from(path),
3936 source,
3937 })?;
3938 if !canonical.is_dir() {
3939 return Err(NativeOperatorBuilderError::Invalid(format!(
3940 "host compiler include search entry is not a directory: {}",
3941 canonical.display()
3942 )));
3943 }
3944 if seen.insert(path.to_string()) {
3945 roots.push(path.to_string());
3946 }
3947 }
3948 if roots.is_empty() {
3949 return Err(NativeOperatorBuilderError::Invalid(format!(
3950 "host compiler include search probe produced no roots: {}",
3951 compiler.display()
3952 )));
3953 }
3954 Ok(roots)
3955}
3956
3957fn resolve_host_program(
3958 value: &str,
3959 environment: &BTreeMap<String, String>,
3960) -> Result<Option<PathBuf>> {
3961 if value.is_empty()
3962 || value
3963 .chars()
3964 .any(|character| matches!(character, '\n' | '\r'))
3965 {
3966 return Err(NativeOperatorBuilderError::Invalid(format!(
3967 "host compiler returned an invalid program path: {value:?}"
3968 )));
3969 }
3970 let candidate = Path::new(value);
3971 if candidate.is_absolute() {
3972 return Ok(candidate.is_file().then(|| candidate.to_path_buf()));
3973 }
3974 let path = environment.get("PATH").ok_or_else(|| {
3975 NativeOperatorBuilderError::Invalid(
3976 "host compiler probe environment has no PATH".to_string(),
3977 )
3978 })?;
3979 Ok(std::env::split_paths(std::ffi::OsStr::new(path))
3980 .map(|directory| directory.join(candidate))
3981 .find(|candidate| candidate.is_file()))
3982}
3983
3984fn collect_host_toolchain_scope_files(
3985 roots: &[String],
3986) -> Result<Vec<NativeOperatorHostToolchainFileIdentity>> {
3987 let mut files = BTreeMap::new();
3988 for root in roots {
3989 collect_host_toolchain_files(Path::new(root), &mut BTreeSet::new(), &mut files)?;
3990 }
3991 if files.len() > MAX_HOST_TOOLCHAIN_FILES {
3992 return Err(NativeOperatorBuilderError::Invalid(format!(
3993 "host toolchain manifest must contain at most {MAX_HOST_TOOLCHAIN_FILES} sorted unique files"
3994 )));
3995 }
3996 Ok(files.into_values().collect())
3997}
3998
3999fn collect_host_toolchain_files(
4000 directory: &Path,
4001 active_directories: &mut BTreeSet<PathBuf>,
4002 files: &mut BTreeMap<String, NativeOperatorHostToolchainFileIdentity>,
4003) -> Result<()> {
4004 let resolved_directory =
4005 directory
4006 .canonicalize()
4007 .map_err(|source| NativeOperatorBuilderError::Io {
4008 path: directory.to_path_buf(),
4009 source,
4010 })?;
4011 if !resolved_directory.is_dir() {
4012 return Err(NativeOperatorBuilderError::Invalid(format!(
4013 "host toolchain scope is not a directory: {}",
4014 directory.display()
4015 )));
4016 }
4017 if !active_directories.insert(resolved_directory.clone()) {
4018 return Ok(());
4019 }
4020 let mut children = fs::read_dir(directory)
4021 .map_err(|source| NativeOperatorBuilderError::Io {
4022 path: directory.to_path_buf(),
4023 source,
4024 })?
4025 .collect::<std::result::Result<Vec<_>, _>>()
4026 .map_err(|source| NativeOperatorBuilderError::Io {
4027 path: directory.to_path_buf(),
4028 source,
4029 })?;
4030 children.sort_by_key(|entry| entry.file_name());
4031 for child in children {
4032 let logical = child.path();
4033 let resolved = logical
4034 .canonicalize()
4035 .map_err(|source| NativeOperatorBuilderError::Io {
4036 path: logical.clone(),
4037 source,
4038 })?;
4039 if resolved.is_dir() {
4040 collect_host_toolchain_files(&logical, active_directories, files)?;
4041 } else if resolved.is_file() {
4042 let logical_path = logical.display().to_string();
4043 if !files.contains_key(&logical_path) && files.len() >= MAX_HOST_TOOLCHAIN_FILES {
4044 return Err(NativeOperatorBuilderError::Invalid(format!(
4045 "host toolchain manifest exceeds {MAX_HOST_TOOLCHAIN_FILES} files"
4046 )));
4047 }
4048 let size_bytes = fs::metadata(&resolved)
4049 .map_err(|source| NativeOperatorBuilderError::Io {
4050 path: resolved.clone(),
4051 source,
4052 })?
4053 .len();
4054 let identity = NativeOperatorHostToolchainFileIdentity {
4055 logical_path,
4056 resolved_path: resolved.display().to_string(),
4057 sha256: sha256_file(&resolved)?,
4058 size_bytes,
4059 };
4060 if let Some(existing) = files.insert(identity.logical_path.clone(), identity.clone()) {
4061 if existing != identity {
4062 return Err(NativeOperatorBuilderError::Invalid(format!(
4063 "host toolchain scope resolved inconsistently: {}",
4064 identity.logical_path
4065 )));
4066 }
4067 }
4068 } else {
4069 return Err(NativeOperatorBuilderError::Invalid(format!(
4070 "host toolchain scope contains a non-file entry: {}",
4071 logical.display()
4072 )));
4073 }
4074 }
4075 active_directories.remove(&resolved_directory);
4076 Ok(())
4077}
4078
4079fn validate_host_toolchain_manifest(
4080 context: &str,
4081 manifest: &NativeOperatorHostToolchainManifest,
4082) -> Result<()> {
4083 if manifest.schema_version != NATIVE_OPERATOR_HOST_TOOLCHAIN_MANIFEST_SCHEMA_VERSION
4084 || manifest.compiler_version.trim().is_empty()
4085 || manifest.target.trim().is_empty()
4086 || manifest.target.len() > 256
4087 || manifest.target.chars().any(char::is_whitespace)
4088 || manifest.executable_inputs.is_empty()
4089 || manifest.include_roots.is_empty()
4090 || !is_sha256_digest(&manifest.include_probe_sha256)
4091 || !is_sha256_digest(&manifest.driver_probe_sha256)
4092 || manifest.discovery_roots.is_empty()
4093 || manifest.files.is_empty()
4094 || manifest
4095 .executable_inputs
4096 .windows(2)
4097 .any(|pair| pair[0].path >= pair[1].path)
4098 || manifest.include_roots.iter().collect::<BTreeSet<_>>().len()
4099 != manifest.include_roots.len()
4100 || manifest
4101 .discovery_roots
4102 .windows(2)
4103 .any(|pair| pair[0] >= pair[1])
4104 || manifest
4105 .files
4106 .windows(2)
4107 .any(|pair| pair[0].logical_path >= pair[1].logical_path)
4108 {
4109 return Err(NativeOperatorBuilderError::Invalid(format!(
4110 "{context} host toolchain manifest header/order is invalid"
4111 )));
4112 }
4113 for tool in std::iter::once(&manifest.compiler).chain(manifest.executable_inputs.iter()) {
4114 validate_normalized_absolute_path(
4115 &tool.path,
4116 &format!("{context} host toolchain executable path"),
4117 )?;
4118 if !is_sha256_digest(&tool.sha256) || tool.size_bytes == 0 {
4119 return Err(NativeOperatorBuilderError::Invalid(format!(
4120 "{context} host toolchain executable identity is invalid: {}",
4121 tool.path
4122 )));
4123 }
4124 }
4125 for root in manifest
4126 .include_roots
4127 .iter()
4128 .chain(manifest.discovery_roots.iter())
4129 {
4130 validate_normalized_absolute_path(root, &format!("{context} host toolchain scope root"))?;
4131 }
4132 if !manifest
4133 .executable_inputs
4134 .iter()
4135 .any(|tool| tool == &manifest.compiler)
4136 || manifest.executable_inputs.iter().any(|tool| {
4137 Path::new(&tool.path).parent().map_or(true, |parent| {
4138 !manifest
4139 .discovery_roots
4140 .iter()
4141 .any(|root| Path::new(root) == parent)
4142 })
4143 })
4144 || manifest.discovery_roots.iter().any(|root| {
4145 !manifest
4146 .executable_inputs
4147 .iter()
4148 .any(|tool| Path::new(&tool.path).parent() == Some(Path::new(root)))
4149 })
4150 {
4151 return Err(NativeOperatorBuilderError::Invalid(format!(
4152 "{context} host toolchain manifest does not bind its compiler/search roots"
4153 )));
4154 }
4155 for file in &manifest.files {
4156 validate_normalized_absolute_path(
4157 &file.logical_path,
4158 &format!("{context} host toolchain logical path"),
4159 )?;
4160 validate_normalized_absolute_path(
4161 &file.resolved_path,
4162 &format!("{context} host toolchain resolved path"),
4163 )?;
4164 if !is_sha256_digest(&file.sha256)
4165 || !manifest
4166 .include_roots
4167 .iter()
4168 .chain(manifest.discovery_roots.iter())
4169 .any(|root| Path::new(&file.logical_path).starts_with(root))
4170 {
4171 return Err(NativeOperatorBuilderError::Invalid(format!(
4172 "{context} host toolchain file identity is invalid: {}",
4173 file.logical_path
4174 )));
4175 }
4176 }
4177 Ok(())
4178}
4179
4180fn build_cuda_toolkit_manifest(root: &Path) -> Result<NativeOperatorCudaToolkitManifest> {
4181 let canonical_root = root
4182 .canonicalize()
4183 .map_err(|source| NativeOperatorBuilderError::Io {
4184 path: root.to_path_buf(),
4185 source,
4186 })?;
4187 let root = canonical_root.as_path();
4188 let mut entries = Vec::new();
4189 for relative in REQUIRED_CUDA_TOOLKIT_FILES {
4190 collect_cuda_toolkit_single_file(root, relative, &mut entries)?;
4191 }
4192 for optional in ["bin/cudafe", "bin/nvcc.profile"] {
4193 if root.join(optional).exists() {
4194 collect_cuda_toolkit_single_file(root, optional, &mut entries)?;
4195 }
4196 }
4197 for scope in REQUIRED_CUDA_TOOLKIT_SCOPES {
4198 let scope_path = root.join(scope);
4199 if !scope_path.is_dir() {
4200 return Err(NativeOperatorBuilderError::Invalid(format!(
4201 "cuda toolkit compiler scope is missing: {scope}"
4202 )));
4203 }
4204 collect_cuda_toolkit_files(root, &scope_path, &mut BTreeSet::new(), &mut entries)?;
4205 }
4206 entries.sort_by(|left, right| left.logical_path.cmp(&right.logical_path));
4207 if entries.is_empty()
4208 || entries
4209 .windows(2)
4210 .any(|pair| pair[0].logical_path >= pair[1].logical_path)
4211 {
4212 return Err(NativeOperatorBuilderError::Invalid(
4213 "cuda toolkit manifest entries must be non-empty, sorted, and unique".to_string(),
4214 ));
4215 }
4216 Ok(NativeOperatorCudaToolkitManifest {
4217 schema_version: NATIVE_OPERATOR_CUDA_TOOLKIT_MANIFEST_SCHEMA_VERSION,
4218 canonical_root: root.display().to_string(),
4219 entries,
4220 })
4221}
4222
4223fn collect_cuda_toolkit_single_file(
4224 root: &Path,
4225 relative: &str,
4226 entries: &mut Vec<NativeOperatorCudaToolkitFileIdentity>,
4227) -> Result<()> {
4228 validate_relative_path(relative)?;
4229 let logical = root.join(relative);
4230 let resolved = logical
4231 .canonicalize()
4232 .map_err(|source| NativeOperatorBuilderError::Io {
4233 path: logical.clone(),
4234 source,
4235 })?;
4236 if !resolved.starts_with(root) || !resolved.is_file() {
4237 return Err(NativeOperatorBuilderError::Invalid(format!(
4238 "cuda toolkit compiler input escapes root or is not a file: {relative}"
4239 )));
4240 }
4241 entries.push(cuda_toolkit_file_identity(root, &logical, &resolved)?);
4242 Ok(())
4243}
4244
4245fn cuda_toolkit_release_version(root: &Path) -> Result<String> {
4246 let cuda_header = root.join("include/cuda.h");
4247 require_file(&cuda_header)?;
4248 let contents =
4249 fs::read_to_string(&cuda_header).map_err(|source| NativeOperatorBuilderError::Io {
4250 path: cuda_header.clone(),
4251 source,
4252 })?;
4253 let encoded = contents
4254 .lines()
4255 .find_map(|line| {
4256 let mut fields = line.split_whitespace();
4257 match (fields.next(), fields.next(), fields.next(), fields.next()) {
4258 (Some("#define"), Some("CUDA_VERSION"), Some(value), None) => {
4259 value.parse::<u32>().ok()
4260 }
4261 _ => None,
4262 }
4263 })
4264 .filter(|value| *value >= 1000)
4265 .ok_or_else(|| {
4266 NativeOperatorBuilderError::Invalid(format!(
4267 "cuda toolkit include/cuda.h has no valid CUDA_VERSION: {}",
4268 cuda_header.display()
4269 ))
4270 })?;
4271 Ok(format!(
4272 "{}.{}.{}",
4273 encoded / 1000,
4274 (encoded % 1000) / 10,
4275 encoded % 10
4276 ))
4277}
4278
4279fn collect_cuda_toolkit_files(
4280 root: &Path,
4281 directory: &Path,
4282 active_directories: &mut BTreeSet<PathBuf>,
4283 entries: &mut Vec<NativeOperatorCudaToolkitFileIdentity>,
4284) -> Result<()> {
4285 let resolved_directory =
4286 directory
4287 .canonicalize()
4288 .map_err(|source| NativeOperatorBuilderError::Io {
4289 path: directory.to_path_buf(),
4290 source,
4291 })?;
4292 if !resolved_directory.starts_with(root) || !resolved_directory.is_dir() {
4293 return Err(NativeOperatorBuilderError::Invalid(format!(
4294 "cuda toolkit compiler directory escapes its canonical root: {}",
4295 directory.display()
4296 )));
4297 }
4298 if !active_directories.insert(resolved_directory.clone()) {
4299 return Err(NativeOperatorBuilderError::Invalid(format!(
4300 "cuda toolkit compiler directory contains a symlink cycle: {}",
4301 directory.display()
4302 )));
4303 }
4304 let mut children = fs::read_dir(directory)
4305 .map_err(|source| NativeOperatorBuilderError::Io {
4306 path: directory.to_path_buf(),
4307 source,
4308 })?
4309 .collect::<std::result::Result<Vec<_>, _>>()
4310 .map_err(|source| NativeOperatorBuilderError::Io {
4311 path: directory.to_path_buf(),
4312 source,
4313 })?;
4314 children.sort_by_key(|entry| entry.file_name());
4315 for child in children {
4316 let logical_path = child.path();
4317 let resolved =
4318 logical_path
4319 .canonicalize()
4320 .map_err(|source| NativeOperatorBuilderError::Io {
4321 path: logical_path.clone(),
4322 source,
4323 })?;
4324 if !resolved.starts_with(root) {
4325 return Err(NativeOperatorBuilderError::Invalid(format!(
4326 "cuda toolkit symlink escapes its canonical root: {}",
4327 logical_path.display()
4328 )));
4329 }
4330 if resolved.is_dir() {
4331 collect_cuda_toolkit_files(root, &logical_path, active_directories, entries)?;
4332 } else if resolved.is_file() {
4333 entries.push(cuda_toolkit_file_identity(root, &logical_path, &resolved)?);
4334 } else {
4335 return Err(NativeOperatorBuilderError::Invalid(format!(
4336 "cuda toolkit compiler scope contains a non-file entry: {}",
4337 logical_path.display()
4338 )));
4339 }
4340 }
4341 active_directories.remove(&resolved_directory);
4342 Ok(())
4343}
4344
4345fn cuda_toolkit_file_identity(
4346 root: &Path,
4347 logical: &Path,
4348 resolved: &Path,
4349) -> Result<NativeOperatorCudaToolkitFileIdentity> {
4350 let logical_path = logical.strip_prefix(root).map_err(|_| {
4351 NativeOperatorBuilderError::Invalid(format!(
4352 "cuda toolkit logical path escapes root: {}",
4353 logical.display()
4354 ))
4355 })?;
4356 let resolved_path = resolved.strip_prefix(root).map_err(|_| {
4357 NativeOperatorBuilderError::Invalid(format!(
4358 "cuda toolkit resolved path escapes root: {}",
4359 resolved.display()
4360 ))
4361 })?;
4362 let size_bytes = fs::metadata(resolved)
4363 .map_err(|source| NativeOperatorBuilderError::Io {
4364 path: resolved.to_path_buf(),
4365 source,
4366 })?
4367 .len();
4368 Ok(NativeOperatorCudaToolkitFileIdentity {
4369 logical_path: path_with_forward_slashes(logical_path)?,
4370 resolved_path: path_with_forward_slashes(resolved_path)?,
4371 sha256: sha256_file(resolved)?,
4372 size_bytes,
4373 })
4374}
4375
4376fn path_with_forward_slashes(path: &Path) -> Result<String> {
4377 let components = path
4378 .components()
4379 .map(|component| match component {
4380 std::path::Component::Normal(value) => {
4381 value.to_str().map(str::to_string).ok_or_else(|| {
4382 NativeOperatorBuilderError::Invalid(format!(
4383 "native build path is not valid UTF-8: {}",
4384 path.display()
4385 ))
4386 })
4387 }
4388 _ => Err(NativeOperatorBuilderError::Invalid(format!(
4389 "native build path is not normalized and relative: {}",
4390 path.display()
4391 ))),
4392 })
4393 .collect::<Result<Vec<_>>>()?;
4394 Ok(components.join("/"))
4395}
4396
4397fn tool_file_identity(path: &Path) -> Result<NativeOperatorToolFileIdentity> {
4398 require_file(path)?;
4399 let canonical = path
4400 .canonicalize()
4401 .map_err(|source| NativeOperatorBuilderError::Io {
4402 path: path.to_path_buf(),
4403 source,
4404 })?;
4405 let size_bytes = fs::metadata(&canonical)
4406 .map_err(|source| NativeOperatorBuilderError::Io {
4407 path: canonical.clone(),
4408 source,
4409 })?
4410 .len();
4411 if size_bytes == 0 {
4412 return Err(NativeOperatorBuilderError::Invalid(format!(
4413 "source build tool is empty: {}",
4414 canonical.display()
4415 )));
4416 }
4417 Ok(NativeOperatorToolFileIdentity {
4418 path: canonical.display().to_string(),
4419 sha256: sha256_file(&canonical)?,
4420 size_bytes,
4421 })
4422}
4423
4424fn probe_source_toolchain(
4425 static_identity: &NativeOperatorSourceBuildStaticToolchain,
4426 missed_translation_units: Vec<String>,
4427) -> Result<NativeOperatorSourceBuildToolchainProbe> {
4428 Ok(NativeOperatorSourceBuildToolchainProbe {
4429 nvcc_version: tool_version(Path::new(&static_identity.cuda_toolkit.nvcc.path))?,
4430 host_compiler_version: static_identity.host_toolchain.compiler_version.clone(),
4431 host_target: static_identity.host_toolchain.target.clone(),
4432 archiver_version: tool_version(Path::new(&static_identity.archiver.path))?,
4433 probed_for_misses: missed_translation_units,
4434 })
4435}
4436
4437fn validate_static_toolchain_identity(
4438 operator: &str,
4439 toolchain: &NativeOperatorSourceBuildStaticToolchain,
4440) -> Result<()> {
4441 if toolchain.backend != NativeOperatorBackend::Cuda
4442 || toolchain.compiler_driver != NativeOperatorSourceCompilerDriver::CudaNvcc
4443 {
4444 return Err(NativeOperatorBuilderError::Invalid(format!(
4445 "{operator} source-build toolchain must use the CUDA nvcc driver"
4446 )));
4447 }
4448 validate_normalized_absolute_path(
4449 &toolchain.cuda_toolkit.canonical_root,
4450 &format!("{operator} cuda toolkit canonical_root"),
4451 )?;
4452 validate_normalized_absolute_path(
4453 &toolchain.cuda_toolkit.invocation_root,
4454 &format!("{operator} cuda toolkit invocation_root"),
4455 )?;
4456 if toolchain.cuda_toolkit.release_version.trim().is_empty()
4457 || toolchain
4458 .cuda_toolkit
4459 .release_version
4460 .chars()
4461 .any(|character| !(character.is_ascii_digit() || character == '.'))
4462 {
4463 return Err(NativeOperatorBuilderError::Invalid(format!(
4464 "{operator} cuda toolkit release_version is invalid"
4465 )));
4466 }
4467 for (name, tool) in [
4468 ("nvcc", &toolchain.cuda_toolkit.nvcc),
4469 ("host_compiler", &toolchain.host_toolchain.compiler),
4470 ("archiver", &toolchain.archiver),
4471 ] {
4472 if validate_normalized_absolute_path(
4473 &tool.path,
4474 &format!("{operator} source-build static {name} path"),
4475 )
4476 .is_err()
4477 || !is_sha256_digest(&tool.sha256)
4478 || tool.size_bytes == 0
4479 {
4480 return Err(NativeOperatorBuilderError::Invalid(format!(
4481 "{operator} source-build static {name} identity is incomplete"
4482 )));
4483 }
4484 }
4485 if !Path::new(&toolchain.cuda_toolkit.nvcc.path)
4486 .starts_with(&toolchain.cuda_toolkit.canonical_root)
4487 {
4488 return Err(NativeOperatorBuilderError::Invalid(format!(
4489 "{operator} nvcc identity escapes cuda toolkit root"
4490 )));
4491 }
4492 let manifest = &toolchain.cuda_toolkit.manifest;
4493 if manifest.path != "toolchain/cuda-static-manifest.json"
4494 || !is_sha256_digest(&manifest.sha256)
4495 || manifest.size_bytes == 0
4496 {
4497 return Err(NativeOperatorBuilderError::Invalid(format!(
4498 "{operator} cuda toolkit manifest evidence is incomplete"
4499 )));
4500 }
4501 let host = &toolchain.host_toolchain;
4502 if host.compiler_version.trim().is_empty()
4503 || host.target.trim().is_empty()
4504 || host.target.len() > 256
4505 || host.target.chars().any(char::is_whitespace)
4506 || host.manifest.path != "toolchain/host-static-manifest.json"
4507 || !is_sha256_digest(&host.manifest.sha256)
4508 || host.manifest.size_bytes == 0
4509 {
4510 return Err(NativeOperatorBuilderError::Invalid(format!(
4511 "{operator} host toolchain manifest evidence is incomplete"
4512 )));
4513 }
4514 Ok(())
4515}
4516
4517fn validate_tool_file_unchanged(identity: &NativeOperatorToolFileIdentity) -> Result<()> {
4518 let current = tool_file_identity(Path::new(&identity.path))?;
4519 if ¤t != identity {
4520 return Err(NativeOperatorBuilderError::Invalid(format!(
4521 "tool file changed after static identity was recorded: {}",
4522 identity.path
4523 )));
4524 }
4525 Ok(())
4526}
4527
4528fn validate_cuda_toolkit_unchanged(identity: &NativeOperatorCudaToolkitIdentity) -> Result<()> {
4529 let manifest_path = Path::new(&identity.canonical_root);
4530 let current = build_cuda_toolkit_manifest(manifest_path)?;
4531 validate_cuda_toolkit_manifest("<source-build-finalize>", identity, ¤t)?;
4532 let recorded_path = Path::new(&identity.canonical_root);
4533 if current.canonical_root != recorded_path.display().to_string() {
4534 return Err(NativeOperatorBuilderError::Invalid(
4535 "cuda toolkit canonical root changed during source build".to_string(),
4536 ));
4537 }
4538 let recorded_manifest = identity.manifest.sha256.as_str();
4539 let serialized_with_newline = {
4540 let mut bytes = serde_json::to_vec_pretty(¤t).map_err(|source| {
4541 NativeOperatorBuilderError::Json {
4542 path: PathBuf::from("<cuda-static-manifest>"),
4543 source,
4544 }
4545 })?;
4546 bytes.push(b'\n');
4547 sha256_bytes(&bytes)
4548 };
4549 if serialized_with_newline != recorded_manifest {
4550 return Err(NativeOperatorBuilderError::Invalid(format!(
4551 "cuda toolkit manifest changed during source build: expected={recorded_manifest} actual={serialized_with_newline}"
4552 )));
4553 }
4554 Ok(())
4555}
4556
4557fn validate_host_toolchain_unchanged(
4558 identity: &NativeOperatorHostToolchainIdentity,
4559 receipt_root: &Path,
4560 environment: &BTreeMap<String, String>,
4561) -> Result<()> {
4562 let manifest_path = resolve_source_build_evidence_file(
4563 receipt_root,
4564 "<source-build-finalize>",
4565 &identity.manifest,
4566 )?;
4567 let recorded: NativeOperatorHostToolchainManifest = read_json(&manifest_path)?;
4568 validate_host_toolchain_manifest("<source-build-finalize>", &recorded)?;
4569 if recorded.compiler != identity.compiler
4570 || recorded.compiler_version != identity.compiler_version
4571 || recorded.target != identity.target
4572 {
4573 return Err(NativeOperatorBuilderError::Invalid(
4574 "host toolchain identity differs from its manifest".to_string(),
4575 ));
4576 }
4577 if !host_toolchain_manifest_matches_current(&recorded, &identity.compiler, environment)? {
4578 return Err(NativeOperatorBuilderError::Invalid(
4579 "host toolchain files or driver configuration changed during source build".to_string(),
4580 ));
4581 }
4582 Ok(())
4583}
4584
4585pub(crate) fn compiler_target(path: &Path) -> Result<String> {
4586 require_file(path)?;
4587 let canonical = path
4588 .canonicalize()
4589 .map_err(|source| NativeOperatorBuilderError::Io {
4590 path: path.to_path_buf(),
4591 source,
4592 })?;
4593 let output = Command::new(&canonical)
4594 .arg("-dumpmachine")
4595 .env_clear()
4596 .env("LANG", "C")
4597 .env("LC_ALL", "C")
4598 .env("TZ", "UTC")
4599 .output()
4600 .map_err(|source| NativeOperatorBuilderError::Io {
4601 path: canonical.clone(),
4602 source,
4603 })?;
4604 let target = String::from_utf8_lossy(&output.stdout).trim().to_string();
4605 if !output.status.success()
4606 || target.is_empty()
4607 || target.len() > 256
4608 || target.chars().any(char::is_whitespace)
4609 {
4610 return Err(NativeOperatorBuilderError::Invalid(format!(
4611 "compiler produced no valid target identity: {}",
4612 canonical.display()
4613 )));
4614 }
4615 Ok(target)
4616}
4617
4618pub(crate) fn native_object_identity_file(path: &Path) -> Result<NativeOperatorObjectIdentity> {
4619 let bytes = fs::read(path).map_err(|source| NativeOperatorBuilderError::Io {
4620 path: path.to_path_buf(),
4621 source,
4622 })?;
4623 native_object_identity_bytes(&bytes, &path.display().to_string())
4624}
4625
4626fn native_object_size(path: &Path) -> Result<u64> {
4627 let size_bytes = fs::metadata(path)
4628 .map_err(|source| NativeOperatorBuilderError::Io {
4629 path: path.to_path_buf(),
4630 source,
4631 })?
4632 .len();
4633 if size_bytes == 0 {
4634 return Err(NativeOperatorBuilderError::Invalid(format!(
4635 "native object is empty: {}",
4636 path.display()
4637 )));
4638 }
4639 Ok(size_bytes)
4640}
4641
4642pub(crate) fn native_object_identity_bytes(
4643 bytes: &[u8],
4644 context: &str,
4645) -> Result<NativeOperatorObjectIdentity> {
4646 if bytes.len() >= 20 && bytes.starts_with(b"\x7fELF") {
4647 let class_bits = match bytes[4] {
4648 1 => 32,
4649 2 => 64,
4650 value => {
4651 return Err(NativeOperatorBuilderError::Invalid(format!(
4652 "unsupported ELF class in {context}: {value}"
4653 )))
4654 }
4655 };
4656 let endianness = match bytes[5] {
4657 1 => NativeOperatorObjectEndianness::Little,
4658 2 => NativeOperatorObjectEndianness::Big,
4659 value => {
4660 return Err(NativeOperatorBuilderError::Invalid(format!(
4661 "unsupported ELF endianness in {context}: {value}"
4662 )))
4663 }
4664 };
4665 let machine = u32::from(read_u16(&bytes[18..20], endianness));
4666 let identity = NativeOperatorObjectIdentity {
4667 format: NativeOperatorObjectFormat::Elf,
4668 class_bits,
4669 endianness,
4670 machine,
4671 };
4672 validate_native_object_identity(&identity, context)?;
4673 return Ok(identity);
4674 }
4675
4676 if bytes.len() >= 8 {
4677 let (class_bits, endianness) = match &bytes[..4] {
4678 [0xce, 0xfa, 0xed, 0xfe] => (32, NativeOperatorObjectEndianness::Little),
4679 [0xfe, 0xed, 0xfa, 0xce] => (32, NativeOperatorObjectEndianness::Big),
4680 [0xcf, 0xfa, 0xed, 0xfe] => (64, NativeOperatorObjectEndianness::Little),
4681 [0xfe, 0xed, 0xfa, 0xcf] => (64, NativeOperatorObjectEndianness::Big),
4682 _ => (0, NativeOperatorObjectEndianness::Little),
4683 };
4684 if class_bits != 0 {
4685 let machine = read_u32(&bytes[4..8], endianness);
4686 let identity = NativeOperatorObjectIdentity {
4687 format: NativeOperatorObjectFormat::MachO,
4688 class_bits,
4689 endianness,
4690 machine,
4691 };
4692 validate_native_object_identity(&identity, context)?;
4693 return Ok(identity);
4694 }
4695 }
4696
4697 if bytes.len() >= 20 {
4698 let machine = u16::from_le_bytes([bytes[0], bytes[1]]);
4699 if matches!(machine, 0x014c | 0x01c0 | 0x01c4 | 0x8664 | 0xaa64) {
4700 let identity = NativeOperatorObjectIdentity {
4701 format: NativeOperatorObjectFormat::Coff,
4702 class_bits: if matches!(machine, 0x8664 | 0xaa64) {
4703 64
4704 } else {
4705 32
4706 },
4707 endianness: NativeOperatorObjectEndianness::Little,
4708 machine: u32::from(machine),
4709 };
4710 validate_native_object_identity(&identity, context)?;
4711 return Ok(identity);
4712 }
4713 }
4714
4715 Err(NativeOperatorBuilderError::Invalid(format!(
4716 "native object has no supported ELF, Mach-O, or COFF header: {context}"
4717 )))
4718}
4719
4720pub(crate) fn validate_native_object_identity(
4721 identity: &NativeOperatorObjectIdentity,
4722 context: &str,
4723) -> Result<()> {
4724 if !matches!(identity.class_bits, 32 | 64) || identity.machine == 0 {
4725 return Err(NativeOperatorBuilderError::Invalid(format!(
4726 "native object identity is incomplete for {context}: {identity:?}"
4727 )));
4728 }
4729 if identity.format == NativeOperatorObjectFormat::Coff
4730 && identity.endianness != NativeOperatorObjectEndianness::Little
4731 {
4732 return Err(NativeOperatorBuilderError::Invalid(format!(
4733 "COFF object must be little-endian for {context}"
4734 )));
4735 }
4736 Ok(())
4737}
4738
4739fn read_u16(bytes: &[u8], endianness: NativeOperatorObjectEndianness) -> u16 {
4740 let bytes = [bytes[0], bytes[1]];
4741 match endianness {
4742 NativeOperatorObjectEndianness::Little => u16::from_le_bytes(bytes),
4743 NativeOperatorObjectEndianness::Big => u16::from_be_bytes(bytes),
4744 }
4745}
4746
4747fn read_u32(bytes: &[u8], endianness: NativeOperatorObjectEndianness) -> u32 {
4748 let bytes = [bytes[0], bytes[1], bytes[2], bytes[3]];
4749 match endianness {
4750 NativeOperatorObjectEndianness::Little => u32::from_le_bytes(bytes),
4751 NativeOperatorObjectEndianness::Big => u32::from_be_bytes(bytes),
4752 }
4753}
4754
4755pub(crate) fn tool_identity(path: &Path) -> Result<NativeOperatorToolIdentity> {
4756 require_file(path)?;
4757 let canonical = path
4758 .canonicalize()
4759 .map_err(|source| NativeOperatorBuilderError::Io {
4760 path: path.to_path_buf(),
4761 source,
4762 })?;
4763 Ok(NativeOperatorToolIdentity {
4764 path: canonical.display().to_string(),
4765 sha256: sha256_file(&canonical)?,
4766 version: tool_version(&canonical)?,
4767 })
4768}
4769
4770fn tool_version(path: &Path) -> Result<String> {
4771 require_file(path)?;
4772 let canonical = path
4773 .canonicalize()
4774 .map_err(|source| NativeOperatorBuilderError::Io {
4775 path: path.to_path_buf(),
4776 source,
4777 })?;
4778 let output = Command::new(&canonical)
4779 .arg("--version")
4780 .env_clear()
4781 .env("LANG", "C")
4782 .env("LC_ALL", "C")
4783 .env("TZ", "UTC")
4784 .output()
4785 .map_err(|source| NativeOperatorBuilderError::Io {
4786 path: canonical.clone(),
4787 source,
4788 })?;
4789 let version = format!(
4790 "{}{}",
4791 String::from_utf8_lossy(&output.stdout),
4792 String::from_utf8_lossy(&output.stderr)
4793 )
4794 .trim()
4795 .chars()
4796 .take(4000)
4797 .collect::<String>();
4798 if version.is_empty() {
4799 return Err(NativeOperatorBuilderError::Invalid(format!(
4800 "tool produced no version identity: {}",
4801 canonical.display()
4802 )));
4803 }
4804 Ok(version)
4805}
4806
4807#[allow(clippy::too_many_arguments)]
4808fn build_inputs_sha256(
4809 plan_sha256: &str,
4810 source_package_sha256: &str,
4811 architecture_argument: &str,
4812 effective_environment: &BTreeMap<String, String>,
4813 toolchain: Option<&NativeOperatorSourceBuildStaticToolchain>,
4814 receipt_path: &Path,
4815) -> Result<String> {
4816 let identity = NativeOperatorBuildInputIdentity {
4817 plan_sha256,
4818 source_package_sha256,
4819 builder_contract_version: NATIVE_OPERATOR_SOURCE_OBJECT_BUILD_CONTRACT_VERSION,
4820 architecture_argument,
4821 effective_environment,
4822 toolchain,
4823 };
4824 let bytes =
4825 serde_json::to_vec(&identity).map_err(|source| NativeOperatorBuilderError::Json {
4826 path: receipt_path.to_path_buf(),
4827 source,
4828 })?;
4829 Ok(sha256_bytes(&bytes))
4830}
4831
4832fn effective_build_environment(
4833 request: &NativeOperatorSourceBuildRequest,
4834 toolchain: Option<&NativeOperatorSourceBuildToolchain>,
4835) -> Result<BTreeMap<String, String>> {
4836 let tool_paths = if let Some(toolchain) = toolchain {
4837 [
4838 toolchain.static_identity.cuda_toolkit.nvcc.path.as_str(),
4839 toolchain
4840 .static_identity
4841 .host_toolchain
4842 .compiler
4843 .path
4844 .as_str(),
4845 toolchain.static_identity.archiver.path.as_str(),
4846 ]
4847 } else {
4848 [
4849 request.nvcc_path.to_str().unwrap_or(""),
4850 request.ccbin_path.to_str().unwrap_or(""),
4851 request.ar_path.to_str().unwrap_or(""),
4852 ]
4853 };
4854 effective_environment_for_tool_paths(tool_paths)
4855}
4856
4857fn effective_environment_for_tool_paths(tool_paths: [&str; 3]) -> Result<BTreeMap<String, String>> {
4858 let mut path_entries = tool_paths
4859 .iter()
4860 .filter_map(|path| Path::new(path).parent())
4861 .map(Path::to_path_buf)
4862 .collect::<Vec<_>>();
4863 path_entries.extend([PathBuf::from("/bin"), PathBuf::from("/usr/bin")]);
4864 path_entries.sort();
4865 path_entries.dedup();
4866 if path_entries.iter().any(|path| path.as_os_str().is_empty()) {
4867 return Err(NativeOperatorBuilderError::Invalid(
4868 "source build tool paths must have parent directories".to_string(),
4869 ));
4870 }
4871 let path = std::env::join_paths(&path_entries)
4872 .map_err(|error| {
4873 NativeOperatorBuilderError::Invalid(format!(
4874 "source build tool PATH cannot be represented: {error}"
4875 ))
4876 })?
4877 .into_string()
4878 .map_err(|_| {
4879 NativeOperatorBuilderError::Invalid(
4880 "source build tool PATH is not valid UTF-8".to_string(),
4881 )
4882 })?;
4883 let mut environment = BTreeMap::new();
4884 environment.insert("LANG".to_string(), "C".to_string());
4885 environment.insert("LC_ALL".to_string(), "C".to_string());
4886 environment.insert("PATH".to_string(), path);
4887 environment.insert("SOURCE_DATE_EPOCH".to_string(), "0".to_string());
4888 environment.insert("TMPDIR".to_string(), "/tmp".to_string());
4889 environment.insert("TZ".to_string(), "UTC".to_string());
4890 environment.insert("ZERO_AR_DATE".to_string(), "1".to_string());
4891 Ok(environment)
4892}
4893
4894fn build_object_cache_specs(
4895 plan: &NativeOperatorSourceBuildPlan,
4896 architecture_argument: &str,
4897 toolchain: &NativeOperatorSourceBuildStaticToolchain,
4898 effective_environment: &BTreeMap<String, String>,
4899) -> Result<Vec<NativeBuildArtifactSpec>> {
4900 plan.translation_units
4901 .iter()
4902 .enumerate()
4903 .map(|(index, translation_unit)| {
4904 let closure = plan.dependency_closures.get(index).ok_or_else(|| {
4905 NativeOperatorBuilderError::Invalid(format!(
4906 "missing dependency closure for {}",
4907 translation_unit.path
4908 ))
4909 })?;
4910 if closure.translation_unit != translation_unit.path {
4911 return Err(NativeOperatorBuilderError::Invalid(format!(
4912 "dependency closure order differs from translation units: expected={} actual={}",
4913 translation_unit.path, closure.translation_unit
4914 )));
4915 }
4916 let identity = NativeOperatorObjectInputIdentity {
4917 schema_version: NATIVE_OPERATOR_SOURCE_OBJECT_BUILD_CONTRACT_VERSION,
4918 operator: &plan.operator,
4919 translation_unit,
4920 dependency_closure_sha256: &closure.closure_sha256,
4921 headers: &closure.headers,
4922 include_dirs: &plan.include_dirs,
4923 defines: &plan.defines,
4924 nvcc_policy: &plan.nvcc_policy,
4925 architecture_argument,
4926 builder_contract_version: NATIVE_OPERATOR_SOURCE_OBJECT_BUILD_CONTRACT_VERSION,
4927 effective_environment,
4928 toolchain,
4929 };
4930 let input_signature = serde_json::to_string(&identity).map_err(|source| {
4931 NativeOperatorBuilderError::Json {
4932 path: PathBuf::from("<object-cache-input>"),
4933 source,
4934 }
4935 })?;
4936 NativeBuildArtifactSpec::new(
4937 format!("{}.object.{index:02}", plan.operator),
4938 object_file_name(index, translation_unit),
4939 input_signature,
4940 )
4941 .map_err(NativeOperatorBuilderError::from)
4942 })
4943 .collect()
4944}
4945
4946fn object_file_name(index: usize, translation_unit: &NativeOperatorSourceFileLock) -> String {
4947 let stem = Path::new(&translation_unit.path)
4948 .file_stem()
4949 .and_then(|value| value.to_str())
4950 .unwrap_or("translation_unit");
4951 format!(
4952 "{index:08}_{}_{}.o",
4953 safe_component(stem),
4954 &translation_unit.sha256[..8]
4955 )
4956}
4957
4958fn nvcc_policy_flags(policy: &NativeOperatorNvccPolicy) -> Vec<String> {
4959 let mut flags = vec![
4960 match policy.cpp_standard {
4961 NativeOperatorCppStandard::Cpp17 => "-std=c++17",
4962 }
4963 .to_string(),
4964 match policy.optimization {
4965 NativeOperatorOptimization::O3 => "-O3",
4966 }
4967 .to_string(),
4968 ];
4969 if policy.use_fast_math {
4970 flags.push("--use_fast_math".to_string());
4971 }
4972 if policy.relaxed_constexpr {
4973 flags.push("--expt-relaxed-constexpr".to_string());
4974 }
4975 if policy.extended_lambda {
4976 flags.push("--expt-extended-lambda".to_string());
4977 }
4978 if policy.host_position_independent_code {
4979 flags.extend(["-Xcompiler".to_string(), "-fPIC".to_string()]);
4980 }
4981 if policy.host_default_visibility {
4982 flags.extend(["-Xcompiler".to_string(), "-fvisibility=default".to_string()]);
4983 }
4984 flags
4985}
4986
4987fn build_commands(
4988 request: &NativeOperatorSourceBuildRequest,
4989 plan: &NativeOperatorSourceBuildPlan,
4990 source_root: &Path,
4991 architecture_argument: &str,
4992 objects_dir: &Path,
4993 logs_dir: &Path,
4994 toolchain: Option<&NativeOperatorSourceBuildToolchain>,
4995 _effective_environment: &BTreeMap<String, String>,
4996) -> Vec<NativeOperatorSourceBuildCommand> {
4997 let nvcc_path = toolchain
4998 .map(|toolchain| toolchain.static_identity.cuda_toolkit.nvcc.path.as_str())
4999 .unwrap_or_else(|| request.nvcc_path.to_str().unwrap_or("<non-utf8-nvcc>"));
5000 let ccbin_path = toolchain
5001 .map(|toolchain| {
5002 toolchain
5003 .static_identity
5004 .host_toolchain
5005 .compiler
5006 .path
5007 .as_str()
5008 })
5009 .unwrap_or_else(|| request.ccbin_path.to_str().unwrap_or("<non-utf8-ccbin>"));
5010 let ar_path = toolchain
5011 .map(|toolchain| toolchain.static_identity.archiver.path.as_str())
5012 .unwrap_or_else(|| request.ar_path.to_str().unwrap_or("<non-utf8-ar>"));
5013 let mut commands = Vec::with_capacity(plan.translation_units.len() + 1);
5014 let mut object_paths = Vec::with_capacity(plan.translation_units.len());
5015 for (index, translation_unit) in plan.translation_units.iter().enumerate() {
5016 let stem = Path::new(&translation_unit.path)
5017 .file_stem()
5018 .and_then(|value| value.to_str())
5019 .unwrap_or("translation_unit");
5020 let object_name = object_file_name(index, translation_unit);
5021 let object_path = objects_dir.join(object_name);
5022 let depfile_name = format!("{index:08}-{stem}.d");
5023 let depfile_relative = format!("depfiles/{depfile_name}");
5024 let compiler_depfile_relative = format!("depfiles/{index:08}-{stem}.compiler.raw.d");
5025 let compiler_depfile_path = request.output_dir.join(&compiler_depfile_relative);
5026 object_paths.push(object_path.clone());
5027 let mut argv = vec![
5028 nvcc_path.to_string(),
5029 "-c".to_string(),
5030 translation_unit.path.clone(),
5031 "-o".to_string(),
5032 object_path.display().to_string(),
5033 architecture_argument.to_string(),
5034 "-ccbin".to_string(),
5035 ccbin_path.to_string(),
5036 "-MMD".to_string(),
5037 "-MF".to_string(),
5038 compiler_depfile_path.display().to_string(),
5039 "-MT".to_string(),
5040 object_path.display().to_string(),
5041 ];
5042 argv.extend(plan.include_dirs.iter().map(|path| format!("-I{path}")));
5043 argv.extend(plan.defines.iter().map(|define| format!("-D{define}")));
5044 argv.extend(nvcc_policy_flags(&plan.nvcc_policy));
5045 argv.push("--threads".to_string());
5046 argv.push(request.nvcc_threads.to_string());
5047 commands.push(NativeOperatorSourceBuildCommand {
5048 translation_unit: Some(translation_unit.path.clone()),
5049 working_directory: source_root.display().to_string(),
5050 argv,
5051 object_file: Some(object_path.display().to_string()),
5052 stdout_log: relative_log(logs_dir, &format!("{index:02}-{stem}.stdout.log")),
5053 stderr_log: relative_log(logs_dir, &format!("{index:02}-{stem}.stderr.log")),
5054 object_cache_key: None,
5055 object_cache_status: Some(if request.plan_only {
5056 NativeOperatorSourceObjectCacheStatus::Plan
5057 } else {
5058 NativeOperatorSourceObjectCacheStatus::Pending
5059 }),
5060 object_cache_entry: None,
5061 object_sha256: None,
5062 object_size_bytes: None,
5063 object_identity: None,
5064 dependency_closure_sha256: plan
5065 .dependency_closures
5066 .get(index)
5067 .map(|closure| closure.closure_sha256.clone()),
5068 dependency_validation: Some(if request.plan_only {
5069 NativeOperatorDependencyValidation::Plan
5070 } else {
5071 NativeOperatorDependencyValidation::Pending
5072 }),
5073 compiler_depfile: Some(compiler_depfile_relative),
5074 compiler_depfile_sha256: None,
5075 depfile: Some(depfile_relative),
5076 depfile_sha256: None,
5077 depfile_producer_working_directory: None,
5078 depfile_producer_object_file: None,
5079 depfile_bindings: Vec::new(),
5080 observed_dependencies: Vec::new(),
5081 compiler_executed: false,
5082 elapsed_ms: None,
5083 return_code: None,
5084 });
5085 }
5086 let archive_path = request.output_dir.join(&plan.archive_file);
5087 let mut archive_argv = vec![
5088 ar_path.to_string(),
5089 "rcs".to_string(),
5090 archive_path.display().to_string(),
5091 ];
5092 archive_argv.extend(object_paths.iter().map(|path| path.display().to_string()));
5093 commands.push(NativeOperatorSourceBuildCommand {
5094 translation_unit: None,
5095 working_directory: source_root.display().to_string(),
5096 argv: archive_argv,
5097 object_file: None,
5098 stdout_log: relative_log(logs_dir, "archive.stdout.log"),
5099 stderr_log: relative_log(logs_dir, "archive.stderr.log"),
5100 object_cache_key: None,
5101 object_cache_status: None,
5102 object_cache_entry: None,
5103 object_sha256: None,
5104 object_size_bytes: None,
5105 object_identity: None,
5106 dependency_closure_sha256: None,
5107 dependency_validation: None,
5108 compiler_depfile: None,
5109 compiler_depfile_sha256: None,
5110 depfile: None,
5111 depfile_sha256: None,
5112 depfile_producer_working_directory: None,
5113 depfile_producer_object_file: None,
5114 depfile_bindings: Vec::new(),
5115 observed_dependencies: Vec::new(),
5116 compiler_executed: false,
5117 elapsed_ms: None,
5118 return_code: None,
5119 });
5120 commands
5121}
5122
5123fn run_logged_command(
5124 argv: &[String],
5125 stdout_path: &Path,
5126 stderr_path: &Path,
5127 working_directory: &str,
5128 effective_environment: &BTreeMap<String, String>,
5129) -> Result<ExitStatus> {
5130 let (program, args) = argv.split_first().ok_or_else(|| {
5131 NativeOperatorBuilderError::Invalid("source build command is empty".to_string())
5132 })?;
5133 let stdout = append_command_file(stdout_path)?;
5134 let stderr = append_command_file(stderr_path)?;
5135 Command::new(program)
5136 .args(args)
5137 .current_dir(working_directory)
5138 .env_clear()
5139 .envs(effective_environment)
5140 .stdout(Stdio::from(stdout))
5141 .stderr(Stdio::from(stderr))
5142 .status()
5143 .map_err(|source| NativeOperatorBuilderError::Io {
5144 path: PathBuf::from(program),
5145 source,
5146 })
5147}
5148
5149fn append_command_file(path: &Path) -> Result<fs::File> {
5150 let mut file = OpenOptions::new()
5151 .append(true)
5152 .open(path)
5153 .map_err(|source| NativeOperatorBuilderError::Io {
5154 path: path.to_path_buf(),
5155 source,
5156 })?;
5157 file.write_all(b"execution-start\n")
5158 .and_then(|()| file.flush())
5159 .map_err(|source| NativeOperatorBuilderError::Io {
5160 path: path.to_path_buf(),
5161 source,
5162 })?;
5163 Ok(file)
5164}
5165
5166fn write_command_stream(path: &Path, stream: &str, argv: &[String], payload: &[u8]) -> Result<()> {
5167 let command =
5168 serde_json::to_string(argv).map_err(|source| NativeOperatorBuilderError::Json {
5169 path: path.to_path_buf(),
5170 source,
5171 })?;
5172 let mut bytes = format!("stream={stream}\nargv={command}\n").into_bytes();
5173 bytes.extend_from_slice(payload);
5174 if !payload.ends_with(b"\n") {
5175 bytes.push(b'\n');
5176 }
5177 fs::write(path, bytes).map_err(|source| NativeOperatorBuilderError::Io {
5178 path: path.to_path_buf(),
5179 source,
5180 })
5181}
5182
5183fn append_command_stream(path: &Path, payload: &[u8]) -> Result<()> {
5184 let mut file = OpenOptions::new()
5185 .append(true)
5186 .open(path)
5187 .map_err(|source| NativeOperatorBuilderError::Io {
5188 path: path.to_path_buf(),
5189 source,
5190 })?;
5191 file.write_all(payload)
5192 .and_then(|()| {
5193 if payload.ends_with(b"\n") {
5194 Ok(())
5195 } else {
5196 file.write_all(b"\n")
5197 }
5198 })
5199 .and_then(|()| file.flush())
5200 .map_err(|source| NativeOperatorBuilderError::Io {
5201 path: path.to_path_buf(),
5202 source,
5203 })
5204}
5205
5206fn reject_source_build<T>(
5207 receipt_path: &Path,
5208 receipt: &mut NativeOperatorSourceBuildReceipt,
5209 reason: String,
5210) -> Result<T> {
5211 receipt.status = NativeOperatorSourceBuildStatus::Reject;
5212 receipt.failure_class = Some(reason.clone());
5213 write_json(receipt_path, receipt)?;
5214 Err(NativeOperatorBuilderError::SourceBuildRejected {
5215 receipt_path: receipt_path.to_path_buf(),
5216 reason,
5217 })
5218}
5219
5220fn architecture_argument(
5221 architecture: NativeOperatorCudaArchitecture,
5222 compute_capability: &str,
5223) -> String {
5224 match architecture {
5225 NativeOperatorCudaArchitecture::DeviceComputeCapability => {
5226 format!("-arch={compute_capability}")
5227 }
5228 NativeOperatorCudaArchitecture::Compute80Ptx => "-arch=compute_80".to_string(),
5229 }
5230}
5231
5232fn validate_compute_capability(value: &str) -> Result<()> {
5233 if value.len() >= 5
5234 && value.starts_with("sm_")
5235 && value[3..].bytes().all(|byte| byte.is_ascii_digit())
5236 {
5237 Ok(())
5238 } else {
5239 Err(NativeOperatorBuilderError::Invalid(
5240 "compute_capability must use sm_<digits> form".to_string(),
5241 ))
5242 }
5243}
5244
5245fn is_git_oid(value: &str) -> bool {
5246 matches!(value.len(), 40 | 64)
5247 && value
5248 .bytes()
5249 .all(|byte| byte.is_ascii_digit() || (b'a'..=b'f').contains(&byte))
5250}
5251
5252fn safe_component(value: &str) -> String {
5253 value
5254 .chars()
5255 .map(|character| {
5256 if character.is_ascii_alphanumeric() {
5257 character
5258 } else {
5259 '_'
5260 }
5261 })
5262 .collect()
5263}
5264
5265fn relative_log(logs_dir: &Path, file: &str) -> String {
5266 Path::new("logs")
5267 .join(logs_dir.join(file).file_name().expect("log file name"))
5268 .to_string_lossy()
5269 .replace('\\', "/")
5270}
5271
5272fn unix_ms() -> u64 {
5273 SystemTime::now()
5274 .duration_since(UNIX_EPOCH)
5275 .unwrap_or_default()
5276 .as_millis()
5277 .try_into()
5278 .unwrap_or(u64::MAX)
5279}
5280
5281fn millis(duration: std::time::Duration) -> u64 {
5282 duration.as_millis().try_into().unwrap_or(u64::MAX)
5283}
5284
5285#[cfg(test)]
5286mod tests {
5287 use std::os::unix::fs::{symlink, PermissionsExt};
5288
5289 use super::*;
5290
5291 fn definition() -> NativeOperatorSourceDefinition {
5292 NativeOperatorSourceDefinition {
5293 schema_version: NATIVE_OPERATOR_SOURCE_DEFINITION_SCHEMA_VERSION,
5294 operator: CudaNativeBuildUnit::Marlin.artifact_operator().to_string(),
5295 source_package_kind: "ferrum-native-source-bundle".to_string(),
5296 source_package_revision: "fixture".to_string(),
5297 upstream_sources: vec![NativeOperatorUpstreamSource {
5298 repository: "https://example.invalid/marlin.git".to_string(),
5299 revision: "abc123".to_string(),
5300 license: "Apache-2.0".to_string(),
5301 }],
5302 translation_units: vec!["kernels/marlin.cu".to_string()],
5303 headers: vec!["kernels/marlin.h".to_string()],
5304 dependency_closures: vec![NativeOperatorTranslationUnitDependencies {
5305 translation_unit: "kernels/marlin.cu".to_string(),
5306 headers: vec!["kernels/marlin.h".to_string()],
5307 }],
5308 include_dirs: vec!["kernels".to_string()],
5309 defines: vec!["FERRUM_FIXTURE=1".to_string()],
5310 nvcc_policy: NativeOperatorNvccPolicy {
5311 cpp_standard: NativeOperatorCppStandard::Cpp17,
5312 optimization: NativeOperatorOptimization::O3,
5313 use_fast_math: false,
5314 relaxed_constexpr: false,
5315 extended_lambda: false,
5316 host_position_independent_code: true,
5317 host_default_visibility: false,
5318 },
5319 architecture: NativeOperatorCudaArchitecture::Compute80Ptx,
5320 archive_file: "libmarlin.a".to_string(),
5321 }
5322 }
5323
5324 fn write_fixture(root: &Path) -> (PathBuf, PathBuf) {
5325 let source_root = root.join("source");
5326 fs::create_dir_all(source_root.join("kernels")).unwrap();
5327 fs::create_dir_all(source_root.join("kernels/core")).unwrap();
5328 fs::write(
5329 source_root.join("kernels/marlin.cu"),
5330 "#include \"marlin.h\"\n\
5331 int marlin_cuda(void) { return MARLIN - 1; }\n\
5332 int marlin_cuda_moe(void) { return 0; }\n",
5333 )
5334 .unwrap();
5335 fs::write(source_root.join("kernels/marlin.h"), "#define MARLIN 1\n").unwrap();
5336 let definition_path = root.join("source-definition.json");
5337 write_json(&definition_path, &definition()).unwrap();
5338 (source_root, definition_path)
5339 }
5340
5341 struct FakeCudaToolkit {
5342 root: PathBuf,
5343 nvcc: PathBuf,
5344 ccbin: PathBuf,
5345 empty_host_include_root: PathBuf,
5346 compile_counter: PathBuf,
5347 invocation_counter: PathBuf,
5348 host_compiler_invocation_counter: PathBuf,
5349 host_driver_config: PathBuf,
5350 }
5351
5352 #[derive(Clone, Copy)]
5353 enum FakeDepfileMode {
5354 Valid,
5355 MissingDeclaredHeader,
5356 UndeclaredExternal,
5357 }
5358
5359 fn write_fake_nvcc(root: &Path) -> FakeCudaToolkit {
5360 write_fake_nvcc_with_mode(root, FakeDepfileMode::Valid)
5361 }
5362
5363 fn write_fake_nvcc_with_mode(root: &Path, mode: FakeDepfileMode) -> FakeCudaToolkit {
5364 let toolkit_root = root.join("fake-cuda");
5365 for directory in ["bin/crt", "include", "nvvm/bin", "nvvm/libdevice"] {
5366 fs::create_dir_all(toolkit_root.join(directory)).unwrap();
5367 }
5368 for (relative, contents) in [
5369 ("bin/bin2c", "fake bin2c\n"),
5370 ("bin/crt/link.stub", "fake link stub\n"),
5371 ("bin/cudafe++", "fake cudafe\n"),
5372 ("bin/ptxas", "fake ptxas\n"),
5373 ("bin/fatbinary", "fake fatbinary\n"),
5374 ("bin/nvlink", "fake nvlink\n"),
5375 ("include/cuda.h", "#define CUDA_VERSION 12040\n"),
5376 ("nvvm/bin/cicc", "fake cicc\n"),
5377 ("nvvm/libdevice/libdevice.10.bc", "fake libdevice\n"),
5378 ] {
5379 fs::write(toolkit_root.join(relative), contents).unwrap();
5380 }
5381 let path = toolkit_root.join("bin/nvcc");
5382 let counter = root.join("fake-nvcc-compile-count");
5383 let invocation_counter = root.join("fake-nvcc-invocation-count");
5384 let host_root = root.join("fake-host-toolchain");
5385 let compile_tail = match mode {
5386 FakeDepfileMode::Valid => format!(
5387 "/usr/bin/cc -x c -c \"$src\" -o \"$out\" || exit $?\n\
5388 declared_header=''\n\
5389 case \"$src\" in */marlin.cu) declared_header=' kernels/core/../marlin.h' ;; esac\n\
5390 printf '%s: %s%s %s %s\\n' \"$dep_target\" \"$src\" \"$declared_header\" '{}' '{}' > \"$depfile\"\n",
5391 toolkit_root.join("bin/../include/cuda.h").display(),
5392 host_root.join("include/stddef.h").display(),
5393 ),
5394 FakeDepfileMode::MissingDeclaredHeader => {
5395 "/usr/bin/cc -x c -MMD -MF \"$depfile\" -MT \"$dep_target\" -c \"$src\" -o \"$out\" || exit $?\n\
5396 printf '%s: %s\\n' \"$dep_target\" \"$src\" > \"$depfile\"\n"
5397 .to_string()
5398 }
5399 FakeDepfileMode::UndeclaredExternal => {
5400 "/usr/bin/cc -x c -MMD -MF \"$depfile\" -MT \"$dep_target\" -c \"$src\" -o \"$out\" || exit $?\n\
5401 printf '%s: %s kernels/marlin.h /etc/hosts\\n' \"$dep_target\" \"$src\" > \"$depfile\"\n"
5402 .to_string()
5403 }
5404 };
5405 fs::write(
5406 &path,
5407 format!(
5408 "#!/bin/sh\n\
5409 printf 'invoke:%s\\n' \"$*\" >> '{}'\n\
5410 if [ \"$1\" = \"--version\" ]; then echo 'fake nvcc 12.4'; exit 0; fi\n\
5411 src=''\n\
5412 out=''\n\
5413 depfile=''\n\
5414 dep_target=''\n\
5415 while [ \"$#\" -gt 0 ]; do\n\
5416 case \"$1\" in\n\
5417 -c) src=\"$2\"; shift 2 ;;\n\
5418 -o) out=\"$2\"; shift 2 ;;\n\
5419 -MF) depfile=\"$2\"; shift 2 ;;\n\
5420 -MT) dep_target=\"$2\"; shift 2 ;;\n\
5421 *) shift ;;\n\
5422 esac\n\
5423 done\n\
5424 build_dir=$(dirname \"$(dirname \"$out\")\")\n\
5425 receipt=\"$build_dir/source-build.receipt.json\"\n\
5426 test -s \"$receipt\"\n\
5427 grep -q '\"status\": \"reject\"' \"$receipt\"\n\
5428 grep -q '\"failure_class\": \"build_incomplete\"' \"$receipt\"\n\
5429 printf 'compile\\n' >> '{}'\n\
5430 {}",
5431 invocation_counter.display(),
5432 counter.display(),
5433 compile_tail
5434 ),
5435 )
5436 .unwrap();
5437 let mut permissions = fs::metadata(&path).unwrap().permissions();
5438 permissions.set_mode(0o755);
5439 fs::set_permissions(&path, permissions).unwrap();
5440
5441 fs::create_dir_all(host_root.join("bin")).unwrap();
5442 fs::create_dir_all(host_root.join("include")).unwrap();
5443 let empty_host_include_root = host_root.join("empty-include");
5444 fs::create_dir_all(&empty_host_include_root).unwrap();
5445 fs::write(
5446 host_root.join("include/stddef.h"),
5447 "#define FAKE_SIZE_T 1\n",
5448 )
5449 .unwrap();
5450 for program in HOST_TOOLCHAIN_PROGRAMS {
5451 fs::write(
5452 host_root.join("bin").join(program),
5453 format!("fake host tool {program}\n"),
5454 )
5455 .unwrap();
5456 }
5457 fs::write(
5458 host_root.join("bin/driver.specs"),
5459 "fake host driver configuration\n",
5460 )
5461 .unwrap();
5462 let ccbin = host_root.join("bin/c++");
5463 let host_compiler_invocation_counter = root.join("fake-host-compiler-invocation-count");
5464 let host_driver_config = root.join("fake-host-driver.conf");
5465 fs::write(&host_driver_config, "external driver option v1\n").unwrap();
5466 fs::write(
5467 &ccbin,
5468 format!(
5469 "#!/bin/sh\n\
5470 printf 'invoke:%s\\n' \"$*\" >> '{}'\n\
5471 case \"$1\" in\n\
5472 --version) echo 'fake host compiler 1.0'; exit 0 ;;\n\
5473 -dumpmachine) echo 'x86_64-ferrum-linux-gnu'; exit 0 ;;\n\
5474 -E) echo '#include <...> search starts here:' >&2; echo ' {}' >&2; echo ' {}' >&2; echo 'End of search list.' >&2; exit 0 ;;\n\
5475 -###) test \"$2\" = '-pipe' || exit 3; echo 'fake cc1plus -O2 -x c++' >&2; cat '{}' >&2; exit 0 ;;\n\
5476 -print-prog-name=*) name=${{1#*=}}; echo '{}/bin/'\"$name\"; exit 0 ;;\n\
5477 esac\n\
5478 exit 2\n",
5479 host_compiler_invocation_counter.display(),
5480 host_root.join("include").display(),
5481 empty_host_include_root.display(),
5482 host_driver_config.display(),
5483 host_root.display(),
5484 ),
5485 )
5486 .unwrap();
5487 let mut permissions = fs::metadata(&ccbin).unwrap().permissions();
5488 permissions.set_mode(0o755);
5489 fs::set_permissions(&ccbin, permissions).unwrap();
5490 FakeCudaToolkit {
5491 root: toolkit_root,
5492 nvcc: path,
5493 ccbin,
5494 empty_host_include_root,
5495 compile_counter: counter,
5496 invocation_counter,
5497 host_compiler_invocation_counter,
5498 host_driver_config,
5499 }
5500 }
5501
5502 #[test]
5503 fn object_file_names_preserve_lexical_order_past_one_hundred_units() {
5504 let translation_unit = NativeOperatorSourceFileLock {
5505 path: "kernels/unit.cu".to_string(),
5506 sha256: "a".repeat(64),
5507 };
5508
5509 assert!(object_file_name(99, &translation_unit) < object_file_name(100, &translation_unit));
5510 }
5511
5512 #[test]
5513 fn portable_depfile_serialization_round_trips_restricted_make_words() {
5514 let target = "/tmp/build path/object:name#$value.o";
5515 let dependencies = vec![
5516 "kernels/header name.h".to_string(),
5517 "/tmp/tool chain/header:name#$value.h".to_string(),
5518 ];
5519
5520 let raw = serialize_portable_depfile(target, &dependencies).unwrap();
5521 let text = std::str::from_utf8(&raw).unwrap();
5522 let (parsed_target, parsed_dependencies) =
5523 parse_make_depfile(text, Path::new("<round-trip>")).unwrap();
5524
5525 assert_eq!(parsed_target, target);
5526 assert_eq!(parsed_dependencies, dependencies);
5527 assert!(text.contains("\\ "));
5528 assert!(text.contains("\\:"));
5529 assert!(text.contains("\\#"));
5530 assert!(text.contains("\\$"));
5531 }
5532
5533 #[test]
5534 fn source_depfile_paths_lexically_normalize_without_escaping_working_directory() {
5535 assert_eq!(
5536 normalize_portable_relative_path(Path::new(
5537 "kernels/vllm_marlin_moe/core/../vllm_torch_shim.h"
5538 ))
5539 .unwrap(),
5540 "kernels/vllm_marlin_moe/vllm_torch_shim.h"
5541 );
5542 for path in ["../outside.h", "kernels/../../outside.h"] {
5543 let error = normalize_portable_relative_path(Path::new(path)).unwrap_err();
5544 assert!(error.to_string().contains("escapes its working directory"));
5545 }
5546 let error = normalize_portable_relative_path(Path::new("kernels\\outside.h")).unwrap_err();
5547 assert!(error.to_string().contains("relative POSIX path"));
5548 }
5549
5550 #[test]
5551 fn host_include_probe_preserves_search_order_and_deduplicates_first_occurrence() {
5552 let root = tempfile::tempdir().unwrap();
5553 let first = root.path().join("first include");
5554 let second = root.path().join("second include");
5555 fs::create_dir_all(&first).unwrap();
5556 fs::create_dir_all(&second).unwrap();
5557 let raw = format!(
5558 "#include <...> search starts here:\n {}\n {}\n {}\nEnd of search list.\n",
5559 second.display(),
5560 first.display(),
5561 second.display()
5562 );
5563
5564 let roots = parse_host_compiler_include_roots(&raw, Path::new("/fake/compiler")).unwrap();
5565
5566 assert_eq!(
5567 roots,
5568 [second.display().to_string(), first.display().to_string(),]
5569 );
5570 }
5571
5572 #[test]
5573 fn successful_host_probe_may_close_stdin_before_parent_finishes_writing() {
5574 let root = tempfile::tempdir().unwrap();
5575 let compiler = root.path().join("successful-probe");
5576 fs::write(
5577 &compiler,
5578 "#!/bin/sh\nexec 0<&-\nprintf 'probe complete\\n'\nexit 0\n",
5579 )
5580 .unwrap();
5581 let mut permissions = fs::metadata(&compiler).unwrap().permissions();
5582 permissions.set_mode(0o755);
5583 fs::set_permissions(&compiler, permissions).unwrap();
5584 let environment = BTreeMap::from([
5585 ("LANG".to_string(), "C".to_string()),
5586 ("LC_ALL".to_string(), "C".to_string()),
5587 ("TZ".to_string(), "UTC".to_string()),
5588 ]);
5589
5590 let output =
5591 host_compiler_raw_output(&compiler, &[], &vec![b'x'; 1024 * 1024], &environment)
5592 .unwrap();
5593
5594 assert!(output.status.success());
5595 assert_eq!(output.stdout, b"probe complete\n");
5596 }
5597
5598 #[test]
5599 fn host_program_resolution_and_effective_path_support_spaces() {
5600 let root = tempfile::tempdir().unwrap();
5601 let tool_dir = root.path().join("tool chain");
5602 fs::create_dir_all(&tool_dir).unwrap();
5603 let helper = tool_dir.join("cc helper");
5604 fs::write(&helper, "fixture\n").unwrap();
5605 let path = std::env::join_paths([tool_dir.clone()])
5606 .unwrap()
5607 .into_string()
5608 .unwrap();
5609 let environment = BTreeMap::from([("PATH".to_string(), path)]);
5610
5611 assert_eq!(
5612 resolve_host_program("cc helper", &environment).unwrap(),
5613 Some(helper)
5614 );
5615
5616 let nvcc = tool_dir.join("nvcc");
5617 let ccbin = tool_dir.join("c++");
5618 let ar = tool_dir.join("ar");
5619 let effective = effective_environment_for_tool_paths([
5620 nvcc.to_str().unwrap(),
5621 ccbin.to_str().unwrap(),
5622 ar.to_str().unwrap(),
5623 ])
5624 .unwrap();
5625 assert!(
5626 std::env::split_paths(std::ffi::OsStr::new(&effective["PATH"]))
5627 .any(|entry| entry == tool_dir)
5628 );
5629 }
5630
5631 #[test]
5632 fn toolchain_dependency_scope_rejects_cross_domain_aliases() {
5633 let absolute = "/toolchain/include/shared.h".to_string();
5634 let mut dependencies = BTreeMap::new();
5635 insert_toolchain_dependency(
5636 &mut dependencies,
5637 absolute.clone(),
5638 NativeOperatorObservedDependency {
5639 domain: NativeOperatorDependencyDomain::BackendToolchain,
5640 path: "include/shared.h".to_string(),
5641 sha256: "a".repeat(64),
5642 },
5643 )
5644 .unwrap();
5645
5646 let error = insert_toolchain_dependency(
5647 &mut dependencies,
5648 absolute,
5649 NativeOperatorObservedDependency {
5650 domain: NativeOperatorDependencyDomain::HostToolchain,
5651 path: "/toolchain/include/shared.h".to_string(),
5652 sha256: "a".repeat(64),
5653 },
5654 )
5655 .unwrap_err();
5656
5657 assert!(error
5658 .to_string()
5659 .contains("toolchain manifests ambiguously own dependency path"));
5660 }
5661
5662 #[test]
5663 fn locks_self_contained_translation_unit_without_auxiliary_inputs() {
5664 let root = tempfile::tempdir().unwrap();
5665 let (source_root, _) = write_fixture(root.path());
5666 let mut definition = definition();
5667 definition.headers.clear();
5668 definition.dependency_closures[0].headers.clear();
5669 definition.include_dirs.clear();
5670 let definition_path = root.path().join("self-contained-definition.json");
5671 write_json(&definition_path, &definition).unwrap();
5672 let plan_path = root.path().join("source-build.plan.json");
5673
5674 let plan =
5675 lock_native_operator_source_definition(&definition_path, &source_root, &plan_path)
5676 .unwrap();
5677
5678 assert!(plan.headers.is_empty());
5679 assert!(plan.include_dirs.is_empty());
5680 assert_eq!(plan.translation_units.len(), 1);
5681 }
5682
5683 #[test]
5684 fn locks_files_and_rejects_source_drift_before_rendering_commands() {
5685 let root = tempfile::tempdir().unwrap();
5686 let (source_root, definition_path) = write_fixture(root.path());
5687 let plan_path = root.path().join("source-build.plan.json");
5688 lock_native_operator_source_definition(&definition_path, &source_root, &plan_path).unwrap();
5689 fs::write(source_root.join("kernels/marlin.h"), "#define MARLIN 2\n").unwrap();
5690
5691 let error = run_native_operator_source_build(&NativeOperatorSourceBuildRequest {
5692 plan_path,
5693 source_root,
5694 output_dir: root.path().join("build"),
5695 compute_capability: "sm_89".to_string(),
5696 builder_sha: "7".repeat(40),
5697 nvcc_path: PathBuf::from("/missing/nvcc"),
5698 cuda_toolkit_root: PathBuf::from("/missing/cuda"),
5699 ccbin_path: PathBuf::from("/missing/c++"),
5700 ar_path: PathBuf::from("/missing/ar"),
5701 nvcc_threads: 4,
5702 object_cache_dir: root.path().join("object-cache"),
5703 plan_only: true,
5704 })
5705 .unwrap_err();
5706
5707 assert!(error.to_string().contains("locked source drift"));
5708 assert!(!root.path().join("build").exists());
5709 }
5710
5711 #[test]
5712 fn plan_only_records_exact_commands_without_requiring_cuda_tools() {
5713 let root = tempfile::tempdir().unwrap();
5714 let (source_root, definition_path) = write_fixture(root.path());
5715 let plan_path = root.path().join("source-build.plan.json");
5716 lock_native_operator_source_definition(&definition_path, &source_root, &plan_path).unwrap();
5717
5718 let receipt = run_native_operator_source_build(&NativeOperatorSourceBuildRequest {
5719 plan_path,
5720 source_root,
5721 output_dir: root.path().join("plan"),
5722 compute_capability: "sm_89".to_string(),
5723 builder_sha: "7".repeat(40),
5724 nvcc_path: PathBuf::from("/missing/nvcc"),
5725 cuda_toolkit_root: PathBuf::from("/missing/cuda"),
5726 ccbin_path: PathBuf::from("/missing/c++"),
5727 ar_path: PathBuf::from("/missing/ar"),
5728 nvcc_threads: 4,
5729 object_cache_dir: root.path().join("object-cache"),
5730 plan_only: true,
5731 })
5732 .unwrap();
5733
5734 assert_eq!(receipt.status, NativeOperatorSourceBuildStatus::Plan);
5735 assert_eq!(receipt.architecture_argument, "-arch=compute_80");
5736 assert_eq!(receipt.commands.len(), 2);
5737 assert!(receipt.commands[0]
5738 .argv
5739 .windows(2)
5740 .any(|pair| pair == ["--threads", "4"]));
5741 assert!(receipt.toolchain.is_none());
5742 assert!(receipt.commands.iter().all(|command| {
5743 [
5744 root.path().join("plan").join(&command.stdout_log),
5745 root.path().join("plan").join(&command.stderr_log),
5746 ]
5747 .iter()
5748 .all(|path| fs::metadata(path).is_ok_and(|metadata| metadata.len() > 0))
5749 }));
5750 }
5751
5752 #[test]
5753 fn missing_toolchain_writes_reject_receipt_before_any_compiler_spawn() {
5754 let root = tempfile::tempdir().unwrap();
5755 let (source_root, definition_path) = write_fixture(root.path());
5756 let plan_path = root.path().join("source-build.plan.json");
5757 lock_native_operator_source_definition(&definition_path, &source_root, &plan_path).unwrap();
5758 let output_dir = root.path().join("toolchain-reject");
5759
5760 let error = run_native_operator_source_build(&NativeOperatorSourceBuildRequest {
5761 plan_path,
5762 source_root,
5763 output_dir: output_dir.clone(),
5764 compute_capability: "sm_89".to_string(),
5765 builder_sha: "7".repeat(40),
5766 nvcc_path: PathBuf::from("/missing/nvcc"),
5767 cuda_toolkit_root: PathBuf::from("/missing/cuda"),
5768 ccbin_path: PathBuf::from("/missing/c++"),
5769 ar_path: PathBuf::from("/missing/ar"),
5770 nvcc_threads: 4,
5771 object_cache_dir: root.path().join("object-cache"),
5772 plan_only: false,
5773 })
5774 .unwrap_err();
5775
5776 assert!(matches!(
5777 error,
5778 NativeOperatorBuilderError::SourceBuildRejected { .. }
5779 ));
5780 let receipt: NativeOperatorSourceBuildReceipt =
5781 read_json(&output_dir.join("source-build.receipt.json")).unwrap();
5782 assert_eq!(receipt.status, NativeOperatorSourceBuildStatus::Reject);
5783 assert!(receipt
5784 .failure_class
5785 .as_deref()
5786 .is_some_and(|failure| failure.starts_with("toolchain_preflight_failed:")));
5787 assert!(receipt.commands.iter().all(|command| {
5788 [
5789 output_dir.join(&command.stdout_log),
5790 output_dir.join(&command.stderr_log),
5791 ]
5792 .iter()
5793 .all(|path| fs::metadata(path).is_ok_and(|metadata| metadata.len() > 0))
5794 }));
5795 }
5796
5797 #[test]
5798 fn rejects_incomplete_or_undeclared_depfiles_before_cache_publish() {
5799 for (name, mode) in [
5800 ("missing-header", FakeDepfileMode::MissingDeclaredHeader),
5801 ("undeclared-external", FakeDepfileMode::UndeclaredExternal),
5802 ] {
5803 let root = tempfile::tempdir().unwrap();
5804 let (source_root, definition_path) = write_fixture(root.path());
5805 let plan_path = root.path().join("source-build.plan.json");
5806 lock_native_operator_source_definition(&definition_path, &source_root, &plan_path)
5807 .unwrap();
5808 let fake_cuda = write_fake_nvcc_with_mode(root.path(), mode);
5809 let output_dir = root.path().join(name);
5810 let object_cache_dir = root.path().join("object-cache");
5811
5812 let error = run_native_operator_source_build(&NativeOperatorSourceBuildRequest {
5813 plan_path,
5814 source_root,
5815 output_dir: output_dir.clone(),
5816 compute_capability: "sm_89".to_string(),
5817 builder_sha: "7".repeat(40),
5818 nvcc_path: fake_cuda.nvcc,
5819 cuda_toolkit_root: fake_cuda.root,
5820 ccbin_path: fake_cuda.ccbin.clone(),
5821 ar_path: PathBuf::from("/usr/bin/ar"),
5822 nvcc_threads: 2,
5823 object_cache_dir: object_cache_dir.clone(),
5824 plan_only: false,
5825 })
5826 .unwrap_err();
5827
5828 assert!(matches!(
5829 error,
5830 NativeOperatorBuilderError::SourceBuildRejected { .. }
5831 ));
5832 let receipt: NativeOperatorSourceBuildReceipt =
5833 read_json(&output_dir.join("source-build.receipt.json")).unwrap();
5834 assert!(receipt
5835 .failure_class
5836 .as_deref()
5837 .is_some_and(|failure| failure.starts_with("dependency_validation_failed:")));
5838 assert_eq!(
5839 receipt.commands[0].object_cache_status,
5840 Some(NativeOperatorSourceObjectCacheStatus::Rejected)
5841 );
5842 assert!(receipt.commands[0].object_cache_entry.is_none());
5843 assert!(
5844 fs::read_dir(object_cache_dir).unwrap().all(|entry| {
5845 entry
5846 .is_ok_and(|entry| entry.file_name() == ".host-toolchains")
5847 }),
5848 "invalid dependency evidence may cache toolchain inventory but must not publish an object"
5849 );
5850 }
5851 }
5852
5853 #[test]
5854 fn rejects_tampered_cached_depfile_proof_before_compiler_start() {
5855 let root = tempfile::tempdir().unwrap();
5856 let (source_root, definition_path) = write_fixture(root.path());
5857 let plan_path = root.path().join("source-build.plan.json");
5858 lock_native_operator_source_definition(&definition_path, &source_root, &plan_path).unwrap();
5859 let fake_cuda = write_fake_nvcc(root.path());
5860 let object_cache_dir = root.path().join("object-cache");
5861 let request = |name: &str| NativeOperatorSourceBuildRequest {
5862 plan_path: plan_path.clone(),
5863 source_root: source_root.clone(),
5864 output_dir: root.path().join(name),
5865 compute_capability: "sm_89".to_string(),
5866 builder_sha: "7".repeat(40),
5867 nvcc_path: fake_cuda.nvcc.clone(),
5868 cuda_toolkit_root: fake_cuda.root.clone(),
5869 ccbin_path: fake_cuda.ccbin.clone(),
5870 ar_path: PathBuf::from("/usr/bin/ar"),
5871 nvcc_threads: 2,
5872 object_cache_dir: object_cache_dir.clone(),
5873 plan_only: false,
5874 };
5875
5876 let cold = run_native_operator_source_build(&request("cold")).unwrap();
5877 let cache_entry = PathBuf::from(
5878 cold.commands[0]
5879 .object_cache_entry
5880 .as_deref()
5881 .expect("published object records its cache entry"),
5882 );
5883 let proof_dir = cache_entry.join("dependency-proof");
5884 fs::write(
5885 proof_dir.join("dependency.d"),
5886 "forged.o: kernels/marlin.cu\n",
5887 )
5888 .unwrap();
5889
5890 let error = run_native_operator_source_build(&request("tampered")).unwrap_err();
5891 assert!(matches!(
5892 error,
5893 NativeOperatorBuilderError::SourceBuildRejected { .. }
5894 ));
5895 let receipt: NativeOperatorSourceBuildReceipt =
5896 read_json(&root.path().join("tampered/source-build.receipt.json")).unwrap();
5897 assert!(receipt
5898 .failure_class
5899 .as_deref()
5900 .is_some_and(|failure| failure.starts_with("cached_dependency_proof_failed:")));
5901 assert_eq!(
5902 fs::read_to_string(&fake_cuda.compile_counter)
5903 .unwrap()
5904 .lines()
5905 .count(),
5906 1,
5907 "tampered cache proof must reject before another compiler starts"
5908 );
5909 }
5910
5911 #[test]
5912 fn rejects_tampered_cached_compiler_depfile_before_compiler_start() {
5913 let root = tempfile::tempdir().unwrap();
5914 let (source_root, definition_path) = write_fixture(root.path());
5915 let plan_path = root.path().join("source-build.plan.json");
5916 lock_native_operator_source_definition(&definition_path, &source_root, &plan_path).unwrap();
5917 let fake_cuda = write_fake_nvcc(root.path());
5918 let object_cache_dir = root.path().join("object-cache");
5919 let request = |name: &str| NativeOperatorSourceBuildRequest {
5920 plan_path: plan_path.clone(),
5921 source_root: source_root.clone(),
5922 output_dir: root.path().join(name),
5923 compute_capability: "sm_89".to_string(),
5924 builder_sha: "7".repeat(40),
5925 nvcc_path: fake_cuda.nvcc.clone(),
5926 cuda_toolkit_root: fake_cuda.root.clone(),
5927 ccbin_path: fake_cuda.ccbin.clone(),
5928 ar_path: PathBuf::from("/usr/bin/ar"),
5929 nvcc_threads: 2,
5930 object_cache_dir: object_cache_dir.clone(),
5931 plan_only: false,
5932 };
5933
5934 let cold = run_native_operator_source_build(&request("cold")).unwrap();
5935 let cache_entry = PathBuf::from(
5936 cold.commands[0]
5937 .object_cache_entry
5938 .as_deref()
5939 .expect("published object records its cache entry"),
5940 );
5941 let proof_path = cache_entry.join("dependency-proof/proof.json");
5942 let compiler_depfile_path = cache_entry.join("dependency-proof/compiler-dependency.raw.d");
5943 let mut proof: NativeOperatorObjectDependencyProof = read_json(&proof_path).unwrap();
5944 let backend_binding = proof
5945 .depfile_bindings
5946 .iter_mut()
5947 .find(|binding| {
5948 binding.dependency.domain == NativeOperatorDependencyDomain::BackendToolchain
5949 })
5950 .expect("fixture depfile contains a backend toolchain dependency");
5951 let original_producer = backend_binding.producer_path.clone();
5952 let forged_producer =
5953 original_producer.replace("/bin/../include/cuda.h", "/forged/include/cuda.h");
5954 assert_ne!(forged_producer, original_producer);
5955 backend_binding.producer_path = forged_producer.clone();
5956 let compiler_raw = fs::read_to_string(&compiler_depfile_path)
5957 .unwrap()
5958 .replace(&original_producer, &forged_producer);
5959 proof.compiler_depfile_sha256 = sha256_bytes(compiler_raw.as_bytes());
5960 fs::write(&compiler_depfile_path, compiler_raw).unwrap();
5961 write_json(&proof_path, &proof).unwrap();
5962
5963 let error = run_native_operator_source_build(&request("tampered")).unwrap_err();
5964 assert!(matches!(
5965 error,
5966 NativeOperatorBuilderError::SourceBuildRejected { .. }
5967 ));
5968 let receipt: NativeOperatorSourceBuildReceipt =
5969 read_json(&root.path().join("tampered/source-build.receipt.json")).unwrap();
5970 assert!(receipt
5971 .failure_class
5972 .as_deref()
5973 .is_some_and(|failure| failure.starts_with("cached_dependency_proof_failed:")));
5974 assert_eq!(
5975 fs::read_to_string(&fake_cuda.compile_counter)
5976 .unwrap()
5977 .lines()
5978 .count(),
5979 1,
5980 "tampered compiler depfile must reject before another compiler starts"
5981 );
5982 }
5983
5984 #[test]
5985 fn rejects_partial_cached_dependency_proof_before_compiler_start() {
5986 let root = tempfile::tempdir().unwrap();
5987 let (source_root, definition_path) = write_fixture(root.path());
5988 let plan_path = root.path().join("source-build.plan.json");
5989 lock_native_operator_source_definition(&definition_path, &source_root, &plan_path).unwrap();
5990 let fake_cuda = write_fake_nvcc(root.path());
5991 let object_cache_dir = root.path().join("object-cache");
5992 let request = |name: &str| NativeOperatorSourceBuildRequest {
5993 plan_path: plan_path.clone(),
5994 source_root: source_root.clone(),
5995 output_dir: root.path().join(name),
5996 compute_capability: "sm_89".to_string(),
5997 builder_sha: "7".repeat(40),
5998 nvcc_path: fake_cuda.nvcc.clone(),
5999 cuda_toolkit_root: fake_cuda.root.clone(),
6000 ccbin_path: fake_cuda.ccbin.clone(),
6001 ar_path: PathBuf::from("/usr/bin/ar"),
6002 nvcc_threads: 2,
6003 object_cache_dir: object_cache_dir.clone(),
6004 plan_only: false,
6005 };
6006
6007 let cold = run_native_operator_source_build(&request("cold")).unwrap();
6008 let cache_entry = PathBuf::from(
6009 cold.commands[0]
6010 .object_cache_entry
6011 .as_deref()
6012 .expect("published object records its cache entry"),
6013 );
6014 fs::remove_file(cache_entry.join("dependency-proof/proof.json")).unwrap();
6015
6016 let error = run_native_operator_source_build(&request("partial")).unwrap_err();
6017 assert!(matches!(
6018 error,
6019 NativeOperatorBuilderError::SourceBuildRejected { .. }
6020 ));
6021 let receipt: NativeOperatorSourceBuildReceipt =
6022 read_json(&root.path().join("partial/source-build.receipt.json")).unwrap();
6023 assert!(receipt
6024 .failure_class
6025 .as_deref()
6026 .is_some_and(|failure| failure.starts_with("cached_dependency_proof_failed:")));
6027 assert_eq!(
6028 fs::read_to_string(&fake_cuda.compile_counter)
6029 .unwrap()
6030 .lines()
6031 .count(),
6032 1,
6033 "partial proof publication must reject before another compiler starts"
6034 );
6035 }
6036
6037 #[test]
6038 fn rejects_tampered_cached_dependency_identity_before_compiler_start() {
6039 let root = tempfile::tempdir().unwrap();
6040 let (source_root, definition_path) = write_fixture(root.path());
6041 let plan_path = root.path().join("source-build.plan.json");
6042 lock_native_operator_source_definition(&definition_path, &source_root, &plan_path).unwrap();
6043 let fake_cuda = write_fake_nvcc(root.path());
6044 let object_cache_dir = root.path().join("object-cache");
6045 let request = |name: &str| NativeOperatorSourceBuildRequest {
6046 plan_path: plan_path.clone(),
6047 source_root: source_root.clone(),
6048 output_dir: root.path().join(name),
6049 compute_capability: "sm_89".to_string(),
6050 builder_sha: "7".repeat(40),
6051 nvcc_path: fake_cuda.nvcc.clone(),
6052 cuda_toolkit_root: fake_cuda.root.clone(),
6053 ccbin_path: fake_cuda.ccbin.clone(),
6054 ar_path: PathBuf::from("/usr/bin/ar"),
6055 nvcc_threads: 2,
6056 object_cache_dir: object_cache_dir.clone(),
6057 plan_only: false,
6058 };
6059
6060 let cold = run_native_operator_source_build(&request("cold")).unwrap();
6061 let cache_entry = PathBuf::from(
6062 cold.commands[0]
6063 .object_cache_entry
6064 .as_deref()
6065 .expect("published object records its cache entry"),
6066 );
6067 let proof_path = cache_entry.join("dependency-proof/proof.json");
6068 let mut proof: NativeOperatorObjectDependencyProof = read_json(&proof_path).unwrap();
6069 let backend_dependency = proof
6070 .observed_dependencies
6071 .iter_mut()
6072 .find(|dependency| {
6073 dependency.domain == NativeOperatorDependencyDomain::BackendToolchain
6074 })
6075 .expect("fixture depfile contains a backend toolchain dependency");
6076 backend_dependency.sha256 = "b".repeat(64);
6077 proof.dependency_set_sha256 =
6078 observed_dependency_set_sha256(&proof.observed_dependencies).unwrap();
6079 write_json(&proof_path, &proof).unwrap();
6080
6081 let error = run_native_operator_source_build(&request("tampered")).unwrap_err();
6082 assert!(matches!(
6083 error,
6084 NativeOperatorBuilderError::SourceBuildRejected { .. }
6085 ));
6086 let receipt: NativeOperatorSourceBuildReceipt =
6087 read_json(&root.path().join("tampered/source-build.receipt.json")).unwrap();
6088 assert!(receipt
6089 .failure_class
6090 .as_deref()
6091 .is_some_and(|failure| failure.starts_with("cached_dependency_proof_failed:")));
6092 assert_eq!(
6093 fs::read_to_string(&fake_cuda.compile_counter)
6094 .unwrap()
6095 .lines()
6096 .count(),
6097 1,
6098 "tampered typed dependency identity must reject before another compiler starts"
6099 );
6100 }
6101
6102 #[test]
6103 fn compiler_inputs_invalidate_object_cache_without_hidden_probe_hits() {
6104 let root = tempfile::tempdir().unwrap();
6105 let (source_root, definition_path) = write_fixture(root.path());
6106 let plan_path = root.path().join("source-build.plan.json");
6107 lock_native_operator_source_definition(&definition_path, &source_root, &plan_path).unwrap();
6108 let fake_cuda = write_fake_nvcc(root.path());
6109 let object_cache_dir = root.path().join("object-cache");
6110 let run_build = |name: &str| {
6111 run_native_operator_source_build(&NativeOperatorSourceBuildRequest {
6112 plan_path: plan_path.clone(),
6113 source_root: source_root.clone(),
6114 output_dir: root.path().join(name),
6115 compute_capability: "sm_89".to_string(),
6116 builder_sha: "7".repeat(40),
6117 nvcc_path: fake_cuda.nvcc.clone(),
6118 cuda_toolkit_root: fake_cuda.root.clone(),
6119 ccbin_path: fake_cuda.ccbin.clone(),
6120 ar_path: PathBuf::from("/usr/bin/ar"),
6121 nvcc_threads: 2,
6122 object_cache_dir: object_cache_dir.clone(),
6123 plan_only: false,
6124 })
6125 .unwrap()
6126 };
6127
6128 assert_eq!(run_build("cold").compiled_translation_units.len(), 1);
6129 assert_eq!(run_build("hit").cache_hit_translation_units.len(), 1);
6130 for (index, relative) in [
6131 "bin/ptxas",
6132 "include/cuda.h",
6133 "nvvm/libdevice/libdevice.10.bc",
6134 ]
6135 .iter()
6136 .enumerate()
6137 {
6138 let path = fake_cuda.root.join(relative);
6139 let mut contents = fs::read_to_string(&path).unwrap();
6140 contents.push_str(&format!("mutation-{index}\n"));
6141 fs::write(path, contents).unwrap();
6142 let receipt = run_build(&format!("mutation-{index}"));
6143 assert_eq!(receipt.compiled_translation_units, ["kernels/marlin.cu"]);
6144 assert!(receipt.cache_hit_translation_units.is_empty());
6145 }
6146 let host_root = fake_cuda
6147 .ccbin
6148 .parent()
6149 .and_then(Path::parent)
6150 .unwrap()
6151 .to_path_buf();
6152 for (index, relative) in ["include/stddef.h", "bin/cc1plus", "bin/driver.specs"]
6153 .iter()
6154 .enumerate()
6155 {
6156 let path = host_root.join(relative);
6157 let mut contents = fs::read_to_string(&path).unwrap();
6158 contents.push_str(&format!("host-mutation-{index}\n"));
6159 fs::write(path, contents).unwrap();
6160 let receipt = run_build(&format!("host-mutation-{index}"));
6161 assert_eq!(receipt.compiled_translation_units, ["kernels/marlin.cu"]);
6162 assert!(receipt.cache_hit_translation_units.is_empty());
6163 }
6164 fs::write(&fake_cuda.host_driver_config, "external driver option v2\n").unwrap();
6165 let external_config_receipt = run_build("external-driver-config-mutation");
6166 assert_eq!(
6167 external_config_receipt.compiled_translation_units,
6168 ["kernels/marlin.cu"]
6169 );
6170 assert!(external_config_receipt
6171 .cache_hit_translation_units
6172 .is_empty());
6173 assert_eq!(
6174 fs::read_to_string(&fake_cuda.compile_counter)
6175 .unwrap()
6176 .lines()
6177 .count(),
6178 8
6179 );
6180 assert_eq!(
6181 fs::read_to_string(&fake_cuda.invocation_counter)
6182 .unwrap()
6183 .lines()
6184 .count(),
6185 16,
6186 "each cache miss invokes one nvcc version probe and one compile; the full hit invokes zero"
6187 );
6188 }
6189
6190 #[test]
6191 fn empty_host_include_root_is_locked_and_new_header_invalidates_object_cache() {
6192 let root = tempfile::tempdir().unwrap();
6193 let (source_root, definition_path) = write_fixture(root.path());
6194 let plan_path = root.path().join("source-build.plan.json");
6195 lock_native_operator_source_definition(&definition_path, &source_root, &plan_path).unwrap();
6196 let fake_cuda = write_fake_nvcc(root.path());
6197 let object_cache_dir = root.path().join("object-cache");
6198 let run_build = |name: &str| {
6199 run_native_operator_source_build(&NativeOperatorSourceBuildRequest {
6200 plan_path: plan_path.clone(),
6201 source_root: source_root.clone(),
6202 output_dir: root.path().join(name),
6203 compute_capability: "sm_89".to_string(),
6204 builder_sha: "7".repeat(40),
6205 nvcc_path: fake_cuda.nvcc.clone(),
6206 cuda_toolkit_root: fake_cuda.root.clone(),
6207 ccbin_path: fake_cuda.ccbin.clone(),
6208 ar_path: PathBuf::from("/usr/bin/ar"),
6209 nvcc_threads: 2,
6210 object_cache_dir: object_cache_dir.clone(),
6211 plan_only: false,
6212 })
6213 .unwrap()
6214 };
6215
6216 assert_eq!(
6217 fs::read_dir(&fake_cuda.empty_host_include_root)
6218 .unwrap()
6219 .count(),
6220 0
6221 );
6222 let cold = run_build("cold-empty-root");
6223 assert_eq!(cold.compiled_translation_units, ["kernels/marlin.cu"]);
6224 let cold_manifest: NativeOperatorHostToolchainManifest = read_json(
6225 &root
6226 .path()
6227 .join("cold-empty-root/toolchain/host-static-manifest.json"),
6228 )
6229 .unwrap();
6230 let empty_root = fake_cuda.empty_host_include_root.display().to_string();
6231 assert!(cold_manifest.include_roots.contains(&empty_root));
6232 assert!(!cold_manifest
6233 .files
6234 .iter()
6235 .any(|file| Path::new(&file.logical_path).starts_with(&empty_root)));
6236
6237 let hit = run_build("unchanged-empty-root");
6238 assert!(hit.compiled_translation_units.is_empty());
6239 assert_eq!(hit.cache_hit_translation_units, ["kernels/marlin.cu"]);
6240 assert!(!hit.commands[0].compiler_executed);
6241
6242 let late_header = fake_cuda.empty_host_include_root.join("late-header.h");
6243 fs::write(&late_header, "#define LATE_HEADER 1\n").unwrap();
6244 let changed = run_build("populated-root");
6245 assert_eq!(changed.compiled_translation_units, ["kernels/marlin.cu"]);
6246 assert!(changed.cache_hit_translation_units.is_empty());
6247 let changed_manifest: NativeOperatorHostToolchainManifest = read_json(
6248 &root
6249 .path()
6250 .join("populated-root/toolchain/host-static-manifest.json"),
6251 )
6252 .unwrap();
6253 assert!(changed_manifest.files.iter().any(|file| {
6254 file.logical_path == late_header.display().to_string()
6255 && file.resolved_path == late_header.canonicalize().unwrap().display().to_string()
6256 }));
6257 assert_eq!(
6258 fs::read_to_string(&fake_cuda.compile_counter)
6259 .unwrap()
6260 .lines()
6261 .count(),
6262 2
6263 );
6264 assert_eq!(
6265 fs::read_to_string(&fake_cuda.invocation_counter)
6266 .unwrap()
6267 .lines()
6268 .count(),
6269 4,
6270 "the unchanged empty root is an nvcc-free hit; adding a header forces one probe and compile"
6271 );
6272 }
6273
6274 #[test]
6275 fn cuda_toolkit_manifest_accepts_internal_symlink_directories_and_rejects_escapes() {
6276 let root = tempfile::tempdir().unwrap();
6277 let fake_cuda = write_fake_nvcc(root.path());
6278 let internal_target = fake_cuda.root.join("targets/headers");
6279 fs::create_dir_all(&internal_target).unwrap();
6280 fs::write(internal_target.join("linked.h"), "#define LINKED 1\n").unwrap();
6281 let internal_link = fake_cuda.root.join("include/linked");
6282 symlink(&internal_target, &internal_link).unwrap();
6283
6284 let manifest = build_cuda_toolkit_manifest(&fake_cuda.root).unwrap();
6285 assert!(manifest.entries.iter().any(|entry| {
6286 entry.logical_path == "include/linked/linked.h"
6287 && entry.resolved_path == "targets/headers/linked.h"
6288 }));
6289
6290 fs::remove_file(&internal_link).unwrap();
6291 symlink("/etc", &internal_link).unwrap();
6292 let error = build_cuda_toolkit_manifest(&fake_cuda.root).unwrap_err();
6293 assert!(error.to_string().contains("escapes its canonical root"));
6294 }
6295
6296 #[test]
6297 fn bounded_fixture_build_writes_pass_receipt_and_archive_hash() {
6298 let root = tempfile::tempdir().unwrap();
6299 let (source_root, definition_path) = write_fixture(root.path());
6300 let plan_path = root.path().join("source-build.plan.json");
6301 lock_native_operator_source_definition(&definition_path, &source_root, &plan_path).unwrap();
6302 let output_dir = root.path().join("build");
6303 let fake_cuda = write_fake_nvcc(root.path());
6304 let object_cache_dir = root.path().join("object-cache");
6305
6306 let receipt = run_native_operator_source_build(&NativeOperatorSourceBuildRequest {
6307 plan_path: plan_path.clone(),
6308 source_root: source_root.clone(),
6309 output_dir: output_dir.clone(),
6310 compute_capability: "sm_89".to_string(),
6311 builder_sha: "7".repeat(40),
6312 nvcc_path: fake_cuda.nvcc.clone(),
6313 cuda_toolkit_root: fake_cuda.root.clone(),
6314 ccbin_path: fake_cuda.ccbin.clone(),
6315 ar_path: PathBuf::from("/usr/bin/ar"),
6316 nvcc_threads: 2,
6317 object_cache_dir: object_cache_dir.clone(),
6318 plan_only: false,
6319 })
6320 .unwrap();
6321
6322 assert_eq!(receipt.status, NativeOperatorSourceBuildStatus::Pass);
6323 let static_toolchain = &receipt
6324 .toolchain
6325 .as_ref()
6326 .expect("completed build records toolchain identity")
6327 .static_identity;
6328 assert_eq!(static_toolchain.backend, NativeOperatorBackend::Cuda);
6329 assert_eq!(
6330 static_toolchain.compiler_driver,
6331 NativeOperatorSourceCompilerDriver::CudaNvcc
6332 );
6333 assert!(is_sha256_digest(receipt.archive_sha256.as_deref().unwrap()));
6334 assert!(output_dir.join("libmarlin.a").is_file());
6335 assert!(output_dir.join("source-build.receipt.json").is_file());
6336 assert_eq!(receipt.compiled_translation_units, ["kernels/marlin.cu"]);
6337 assert!(receipt.cache_hit_translation_units.is_empty());
6338 assert_eq!(
6339 receipt.commands[0]
6340 .observed_dependencies
6341 .iter()
6342 .map(|dependency| dependency.domain)
6343 .collect::<Vec<_>>(),
6344 [
6345 NativeOperatorDependencyDomain::Source,
6346 NativeOperatorDependencyDomain::Source,
6347 NativeOperatorDependencyDomain::BackendToolchain,
6348 NativeOperatorDependencyDomain::HostToolchain,
6349 ]
6350 );
6351 assert!(receipt.commands[0]
6352 .observed_dependencies
6353 .iter()
6354 .all(|dependency| is_sha256_digest(&dependency.sha256)));
6355 let compiler_depfile = fs::read_to_string(
6356 output_dir.join(
6357 receipt.commands[0]
6358 .compiler_depfile
6359 .as_deref()
6360 .expect("cold build records its compiler depfile"),
6361 ),
6362 )
6363 .unwrap();
6364 let portable_depfile = fs::read_to_string(
6365 output_dir.join(
6366 receipt.commands[0]
6367 .depfile
6368 .as_deref()
6369 .expect("cold build records its portable depfile"),
6370 ),
6371 )
6372 .unwrap();
6373 assert!(compiler_depfile.contains("/bin/../include/cuda.h"));
6374 assert!(!portable_depfile.contains("/../"));
6375 let plan: NativeOperatorSourceBuildPlan = read_json(&plan_path).unwrap();
6376 let toolchain_scope =
6377 load_toolchain_dependency_scope(&output_dir, static_toolchain).unwrap();
6378 let cache_entry = PathBuf::from(
6379 receipt.commands[0]
6380 .object_cache_entry
6381 .as_deref()
6382 .expect("cold build records its cache entry"),
6383 );
6384 let object_name = Path::new(
6385 receipt.commands[0]
6386 .object_file
6387 .as_deref()
6388 .expect("cold build records its object"),
6389 )
6390 .file_name()
6391 .unwrap()
6392 .to_str()
6393 .unwrap();
6394 validate_existing_dependency_proof(
6395 &cache_entry.join("dependency-proof"),
6396 receipt.commands[0].object_cache_key.as_deref().unwrap(),
6397 receipt.commands[0].object_sha256.as_deref().unwrap(),
6398 &plan.translation_units[0],
6399 &plan.dependency_closures[0],
6400 &format!("/another-worktree/objects/{object_name}"),
6401 &toolchain_scope,
6402 )
6403 .expect("a concurrent cache winner from another worktree remains valid");
6404 assert!(receipt.commands.iter().all(|command| {
6405 [
6406 output_dir.join(&command.stdout_log),
6407 output_dir.join(&command.stderr_log),
6408 ]
6409 .iter()
6410 .all(|path| {
6411 fs::read_to_string(path).is_ok_and(|content| content.contains("execution-start"))
6412 })
6413 }));
6414 let host_probe_count = fs::read_to_string(&fake_cuda.host_compiler_invocation_counter)
6415 .unwrap()
6416 .lines()
6417 .count();
6418
6419 let cached_output_dir = root.path().join("cached-build");
6420 let cached_receipt = run_native_operator_source_build(&NativeOperatorSourceBuildRequest {
6421 plan_path: plan_path.clone(),
6422 source_root: source_root.clone(),
6423 output_dir: cached_output_dir.clone(),
6424 compute_capability: "sm_89".to_string(),
6425 builder_sha: "8".repeat(40),
6426 nvcc_path: fake_cuda.nvcc.clone(),
6427 cuda_toolkit_root: fake_cuda.root.clone(),
6428 ccbin_path: fake_cuda.ccbin.clone(),
6429 ar_path: PathBuf::from("/usr/bin/ar"),
6430 nvcc_threads: 8,
6431 object_cache_dir,
6432 plan_only: false,
6433 })
6434 .unwrap();
6435
6436 assert!(cached_receipt.compiled_translation_units.is_empty());
6437 assert_eq!(
6438 cached_receipt.cache_hit_translation_units,
6439 ["kernels/marlin.cu"]
6440 );
6441 assert!(!cached_receipt.commands[0].compiler_executed);
6442 assert_eq!(
6443 cached_receipt.commands[0].object_cache_status,
6444 Some(NativeOperatorSourceObjectCacheStatus::Hit)
6445 );
6446 assert_eq!(
6447 cached_receipt.commands[0].dependency_validation,
6448 Some(NativeOperatorDependencyValidation::CacheProof)
6449 );
6450 assert_eq!(
6451 cached_receipt.commands[0].observed_dependencies,
6452 receipt.commands[0].observed_dependencies
6453 );
6454 assert_ne!(
6455 cached_receipt.commands[0].object_file,
6456 cached_receipt.commands[0].depfile_producer_object_file,
6457 "portable cache proof must preserve the producer object while restoring to a new output root"
6458 );
6459 assert!(cached_receipt.commands[0]
6460 .depfile
6461 .as_deref()
6462 .is_some_and(|relative| cached_output_dir.join(relative).is_file()));
6463 assert_eq!(
6464 fs::read_to_string(&fake_cuda.compile_counter)
6465 .unwrap()
6466 .lines()
6467 .count(),
6468 1
6469 );
6470 assert_eq!(
6471 fs::read_to_string(&fake_cuda.invocation_counter)
6472 .unwrap()
6473 .lines()
6474 .count(),
6475 2,
6476 "cold build invokes one miss-only version probe plus one compile; full cache hit invokes neither"
6477 );
6478 assert_eq!(
6479 fs::read_to_string(&fake_cuda.host_compiler_invocation_counter)
6480 .unwrap()
6481 .lines()
6482 .count(),
6483 host_probe_count + 2,
6484 "full cache hit runs only the bounded include/driver configuration probes"
6485 );
6486 assert_eq!(
6487 receipt.archive_sha256, cached_receipt.archive_sha256,
6488 "worker-count changes must not change the object or archive"
6489 );
6490 assert_eq!(
6491 receipt.inputs_sha256, cached_receipt.inputs_sha256,
6492 "worker-count and provenance commit changes are not output-content inputs"
6493 );
6494
6495 fs::write(source_root.join("LICENSE"), "fixture license\n").unwrap();
6496 let package_spec = crate::NativeOperatorPackageSpec {
6497 schema_version: crate::NATIVE_OPERATOR_PACKAGE_SPEC_SCHEMA_VERSION,
6498 operator: CudaNativeBuildUnit::Marlin.artifact_operator().to_string(),
6499 operator_abi_version: "1".to_string(),
6500 backend: ferrum_types::NativeOperatorBackend::Cuda,
6501 compute_capabilities: vec!["sm_89".to_string()],
6502 operation_bindings: vec![ferrum_types::NativeOperatorBinding {
6503 operation_id: "operation.dense_linear".to_string(),
6504 operation_contract_version: ferrum_types::NativeOperatorContractVersion::new(1, 0),
6505 provider_id: "provider.cuda.dense_linear.f16.marlin".to_string(),
6506 provider_version: ferrum_types::NativeOperatorContractVersion::new(1, 0),
6507 provider_implementation_fingerprint: "a".repeat(64),
6508 entrypoints: CudaNativeBuildUnit::Marlin
6509 .required_exports()
6510 .iter()
6511 .map(|value| (*value).to_string())
6512 .collect(),
6513 }],
6514 required_exports: CudaNativeBuildUnit::Marlin
6515 .required_exports()
6516 .iter()
6517 .map(|value| (*value).to_string())
6518 .collect(),
6519 license_files: vec![crate::NativeOperatorLicenseInput {
6520 source_path: "LICENSE".to_string(),
6521 output_path: "licenses/LICENSE".to_string(),
6522 }],
6523 cuda_toolkit: Some("12.4".to_string()),
6524 cuda_runtime_min: Some("12.4".to_string()),
6525 system_libraries: vec![
6526 ferrum_native_ops::NativeOperatorSystemLibrary::CudaRuntime,
6527 ferrum_native_ops::NativeOperatorSystemLibrary::StdCxx,
6528 ],
6529 };
6530 let package_spec_path = root.path().join("package-spec.json");
6531 write_json(&package_spec_path, &package_spec).unwrap();
6532 let catalog_path = root.path().join("operation-catalog.json");
6533 let abi_path = root.path().join("native-abi.json");
6534 write_json(
6535 &catalog_path,
6536 &ferrum_types::NativeOperatorProviderCatalog {
6537 schema_version: ferrum_types::NATIVE_OPERATOR_PROVIDER_CATALOG_SCHEMA_VERSION,
6538 backend: ferrum_types::NativeOperatorBackend::Cuda,
6539 providers: vec![ferrum_types::NativeOperatorProviderCatalogRow {
6540 operation_id: "operation.dense_linear".to_string(),
6541 operation_contract_version: ferrum_types::NativeOperatorContractVersion::new(
6542 1, 0,
6543 ),
6544 operation_fingerprint: "b".repeat(64),
6545 provider_id: "provider.cuda.dense_linear.f16.marlin".to_string(),
6546 provider_version: ferrum_types::NativeOperatorContractVersion::new(1, 0),
6547 provider_implementation_fingerprint: "a".repeat(64),
6548 }],
6549 },
6550 )
6551 .unwrap();
6552 write_json(
6553 &abi_path,
6554 &ferrum_types::NativeOperatorAbiContract {
6555 schema_version: ferrum_types::NATIVE_OPERATOR_ABI_CONTRACT_SCHEMA_VERSION,
6556 ferrum_native_abi_version: ferrum_types::FERRUM_NATIVE_OPERATOR_ABI_VERSION
6557 .to_string(),
6558 descriptor_struct: "FerrumNativeOperatorDescriptorV2".to_string(),
6559 descriptor_symbol_policy: "operator_namespaced".to_string(),
6560 descriptor_fields: [
6561 ("struct_size", "uint32_t"),
6562 ("ferrum_native_abi_version", "uint32_t"),
6563 ("operator_name", "const char *"),
6564 ("operator_abi_version", "const char *"),
6565 ("g03_catalog_sha256", "const char *"),
6566 ("abi_contract_sha256", "const char *"),
6567 ]
6568 .into_iter()
6569 .map(|(name, c_type)| ferrum_types::NativeOperatorAbiField {
6570 name: name.to_string(),
6571 c_type: c_type.to_string(),
6572 })
6573 .collect(),
6574 },
6575 )
6576 .unwrap();
6577 let package_output = root.path().join("package");
6578
6579 let package_receipt =
6580 crate::package_native_operator(&crate::NativeOperatorPackageRequest {
6581 spec_path: package_spec_path,
6582 source_root: source_root.clone(),
6583 license_root: source_root.clone(),
6584 source_build_receipt_path: output_dir.join("source-build.receipt.json"),
6585 source_build_plan_path: plan_path.clone(),
6586 g03_catalog_path: catalog_path,
6587 abi_contract_path: abi_path,
6588 output_dir: package_output.clone(),
6589 cc: PathBuf::from("/usr/bin/cc"),
6590 ar: PathBuf::from("/usr/bin/ar"),
6591 })
6592 .unwrap();
6593
6594 assert_eq!(
6595 package_receipt.source_build_plan.sha256,
6596 receipt.plan_sha256
6597 );
6598 assert_eq!(
6599 package_receipt.source_archive_sha256,
6600 receipt.archive_sha256.unwrap()
6601 );
6602 assert!(package_output.join("package.receipt.json").is_file());
6603
6604 let cached_package_spec_path = root.path().join("cached-package-spec.json");
6605 write_json(&cached_package_spec_path, &package_spec).unwrap();
6606 let cached_package_output = root.path().join("cached-package");
6607 crate::package_native_operator(&crate::NativeOperatorPackageRequest {
6608 spec_path: cached_package_spec_path,
6609 source_root: source_root.clone(),
6610 license_root: source_root,
6611 source_build_receipt_path: cached_output_dir.join("source-build.receipt.json"),
6612 source_build_plan_path: plan_path,
6613 g03_catalog_path: root.path().join("operation-catalog.json"),
6614 abi_contract_path: root.path().join("native-abi.json"),
6615 output_dir: cached_package_output.clone(),
6616 cc: PathBuf::from("/usr/bin/cc"),
6617 ar: PathBuf::from("/usr/bin/ar"),
6618 })
6619 .unwrap();
6620 let cached_manifest: ferrum_types::NativeOperatorManifest =
6621 read_json(&cached_package_output.join("native_operator_manifest.json")).unwrap();
6622 assert_eq!(
6623 cached_manifest.build_summary.nvcc_version.as_deref(),
6624 Some("cuda-toolkit-static 12.4.0")
6625 );
6626 }
6627
6628 #[test]
6629 fn changing_one_translation_unit_recompiles_only_that_unit() {
6630 let root = tempfile::tempdir().unwrap();
6631 let (source_root, _) = write_fixture(root.path());
6632 fs::write(
6633 source_root.join("kernels/other.cu"),
6634 "int other_cuda(void) { return 1; }\n",
6635 )
6636 .unwrap();
6637 let mut source_definition = definition();
6638 source_definition.translation_units = vec![
6639 "kernels/marlin.cu".to_string(),
6640 "kernels/other.cu".to_string(),
6641 ];
6642 source_definition.dependency_closures = vec![
6643 NativeOperatorTranslationUnitDependencies {
6644 translation_unit: "kernels/marlin.cu".to_string(),
6645 headers: vec!["kernels/marlin.h".to_string()],
6646 },
6647 NativeOperatorTranslationUnitDependencies {
6648 translation_unit: "kernels/other.cu".to_string(),
6649 headers: Vec::new(),
6650 },
6651 ];
6652 let definition_path = root.path().join("two-tu-definition.json");
6653 write_json(&definition_path, &source_definition).unwrap();
6654 let first_plan_path = root.path().join("first.plan.json");
6655 lock_native_operator_source_definition(&definition_path, &source_root, &first_plan_path)
6656 .unwrap();
6657 let fake_cuda = write_fake_nvcc(root.path());
6658 let object_cache_dir = root.path().join("object-cache");
6659
6660 let first_receipt = run_native_operator_source_build(&NativeOperatorSourceBuildRequest {
6661 plan_path: first_plan_path,
6662 source_root: source_root.clone(),
6663 output_dir: root.path().join("first-build"),
6664 compute_capability: "sm_89".to_string(),
6665 builder_sha: "7".repeat(40),
6666 nvcc_path: fake_cuda.nvcc.clone(),
6667 cuda_toolkit_root: fake_cuda.root.clone(),
6668 ccbin_path: fake_cuda.ccbin.clone(),
6669 ar_path: PathBuf::from("/usr/bin/ar"),
6670 nvcc_threads: 2,
6671 object_cache_dir: object_cache_dir.clone(),
6672 plan_only: false,
6673 })
6674 .unwrap();
6675 assert_eq!(first_receipt.compiled_translation_units.len(), 2);
6676 assert!(first_receipt.cache_hit_translation_units.is_empty());
6677
6678 fs::write(
6679 source_root.join("kernels/other.cu"),
6680 "int other_cuda(void) { return 2; }\n",
6681 )
6682 .unwrap();
6683 let second_plan_path = root.path().join("second.plan.json");
6684 lock_native_operator_source_definition(&definition_path, &source_root, &second_plan_path)
6685 .unwrap();
6686 let second_receipt = run_native_operator_source_build(&NativeOperatorSourceBuildRequest {
6687 plan_path: second_plan_path,
6688 source_root: source_root.clone(),
6689 output_dir: root.path().join("second-build"),
6690 compute_capability: "sm_89".to_string(),
6691 builder_sha: "8".repeat(40),
6692 nvcc_path: fake_cuda.nvcc.clone(),
6693 cuda_toolkit_root: fake_cuda.root.clone(),
6694 ccbin_path: fake_cuda.ccbin.clone(),
6695 ar_path: PathBuf::from("/usr/bin/ar"),
6696 nvcc_threads: 4,
6697 object_cache_dir: object_cache_dir.clone(),
6698 plan_only: false,
6699 })
6700 .unwrap();
6701
6702 assert_eq!(
6703 second_receipt.compiled_translation_units,
6704 ["kernels/other.cu"]
6705 );
6706 assert_eq!(
6707 second_receipt.cache_hit_translation_units,
6708 ["kernels/marlin.cu"]
6709 );
6710 assert_eq!(
6711 fs::read_to_string(&fake_cuda.compile_counter)
6712 .unwrap()
6713 .lines()
6714 .count(),
6715 3
6716 );
6717
6718 fs::write(source_root.join("kernels/marlin.h"), "#define MARLIN 2\n").unwrap();
6719 let third_plan_path = root.path().join("third.plan.json");
6720 lock_native_operator_source_definition(&definition_path, &source_root, &third_plan_path)
6721 .unwrap();
6722 let third_receipt = run_native_operator_source_build(&NativeOperatorSourceBuildRequest {
6723 plan_path: third_plan_path,
6724 source_root,
6725 output_dir: root.path().join("third-build"),
6726 compute_capability: "sm_89".to_string(),
6727 builder_sha: "9".repeat(40),
6728 nvcc_path: fake_cuda.nvcc,
6729 cuda_toolkit_root: fake_cuda.root,
6730 ccbin_path: fake_cuda.ccbin.clone(),
6731 ar_path: PathBuf::from("/usr/bin/ar"),
6732 nvcc_threads: 4,
6733 object_cache_dir,
6734 plan_only: false,
6735 })
6736 .unwrap();
6737 assert_eq!(
6738 third_receipt.compiled_translation_units,
6739 ["kernels/marlin.cu"]
6740 );
6741 assert_eq!(
6742 third_receipt.cache_hit_translation_units,
6743 ["kernels/other.cu"]
6744 );
6745 assert_eq!(
6746 fs::read_to_string(fake_cuda.compile_counter)
6747 .unwrap()
6748 .lines()
6749 .count(),
6750 4,
6751 "private header drift must recompile only its owning translation unit"
6752 );
6753 }
6754}