1use crate::{
4 CcActionContext, CcActionInput, CcBypassReason, CcCompilerFamily, MAX_INPUT_BYTES,
5 MAX_MANIFEST_ENTRIES, MAX_PREDICTED_INPUTS, normalize_components,
6};
7use mbx_cache_core::{
8 CacheDigest, FileDigestCache, FileDigestResolution, FileDigestScope, FileIdentity,
9 FileSnapshot, RecordedFileDigest, digest_file,
10};
11use std::collections::{BTreeMap, BTreeSet};
12use std::io::Read;
13use std::path::{Path, PathBuf};
14use std::time::SystemTime;
15
16pub const INCLUDE_MANIFEST_PREFIX: &str = "@include-manifest:";
18
19const ASSEMBLER_INPUT_DIRECTIVES: &[&[u8]] = &[b".include", b".incbin", b".sinclude"];
22
23const SCAN_CHUNK_BYTES: usize = 64 * 1024;
24
25#[derive(Debug, Clone, PartialEq, Eq, Default)]
27pub struct CcDepfile {
28 pub files: Vec<PathBuf>,
30}
31
32impl CcDepfile {
33 pub fn read(path: &Path) -> Result<Self, CcBypassReason> {
35 let contents =
36 std::fs::read_to_string(path).map_err(|error| CcBypassReason::DepfileRead {
37 path: path.to_path_buf(),
38 message: error.to_string(),
39 })?;
40 Self::parse(&contents)
41 }
42
43 pub fn read_for(path: &Path, family: CcCompilerFamily) -> Result<Self, CcBypassReason> {
45 if family.is_msvc() {
46 Self::read_msvc(path)
47 } else {
48 Self::read(path)
49 }
50 }
51
52 pub fn read_msvc(path: &Path) -> Result<Self, CcBypassReason> {
54 let contents = std::fs::read(path).map_err(|error| CcBypassReason::DepfileRead {
55 path: path.to_path_buf(),
56 message: error.to_string(),
57 })?;
58 let value: serde_json::Value = serde_json::from_slice(&contents)
59 .map_err(|error| CcBypassReason::MalformedDepfile(error.to_string()))?;
60 let data = value
61 .get("Data")
62 .and_then(serde_json::Value::as_object)
63 .ok_or_else(|| CcBypassReason::MalformedDepfile("missing Data object".into()))?;
64 if data
65 .get("ImportedModules")
66 .and_then(serde_json::Value::as_array)
67 .is_some_and(|modules| !modules.is_empty())
68 || data.get("ProvidedModule").is_some_and(|module| {
69 !module.is_null() && module.as_str().is_none_or(|s| !s.is_empty())
70 })
71 {
72 return Err(CcBypassReason::MalformedDepfile(
73 "C++ module dependencies are not modeled".into(),
74 ));
75 }
76 let includes = data
77 .get("Includes")
78 .and_then(serde_json::Value::as_array)
79 .ok_or_else(|| CcBypassReason::MalformedDepfile("missing Includes array".into()))?;
80 let files = includes
81 .iter()
82 .map(|entry| {
83 entry.as_str().map(PathBuf::from).ok_or_else(|| {
84 CcBypassReason::MalformedDepfile("non-string include path".into())
85 })
86 })
87 .collect::<Result<Vec<_>, _>>()?;
88 Ok(Self { files })
89 }
90
91 pub fn parse(contents: &str) -> Result<Self, CcBypassReason> {
97 let joined = join_continuations(contents)?;
98 let (_, prerequisites) = joined
99 .lines()
100 .find_map(|line| line.split_once(RULE_SEPARATOR))
101 .ok_or_else(|| CcBypassReason::MalformedDepfile("no dependency rule".into()))?;
102 let files = split_prerequisites(prerequisites)?;
103 Ok(Self { files })
104 }
105}
106
107const RULE_SEPARATOR: &str = ": ";
108
109impl CcDepfile {
110 pub fn render(
120 targets: &[crate::DepfileTarget],
121 files: &[PathBuf],
122 source: &Path,
123 phony_targets: bool,
124 ) -> String {
125 let mut rendered = String::new();
126 for (index, target) in targets.iter().enumerate() {
127 if index > 0 {
128 rendered.push(' ');
129 }
130 if target.quoted {
131 rendered.push_str(&escape_make_word(&target.name));
132 } else {
133 rendered.push_str(&target.name);
134 }
135 }
136 rendered.push(':');
137 for file in files {
138 rendered.push_str(" \\\n ");
139 rendered.push_str(&escape_make_word(&file.to_string_lossy()));
140 }
141 rendered.push('\n');
142 if phony_targets {
143 for file in files.iter().filter(|file| file.as_path() != source) {
144 rendered.push_str(&escape_make_word(&file.to_string_lossy()));
145 rendered.push_str(":\n");
146 }
147 }
148 rendered
149 }
150}
151
152fn escape_make_word(word: &str) -> String {
154 let mut escaped = String::with_capacity(word.len());
155 for character in word.chars() {
156 match character {
157 ' ' => escaped.push_str("\\ "),
158 '#' => escaped.push_str("\\#"),
159 '$' => escaped.push_str("$$"),
160 other => escaped.push(other),
161 }
162 }
163 escaped
164}
165
166fn join_continuations(contents: &str) -> Result<String, CcBypassReason> {
168 let mut joined = String::with_capacity(contents.len());
169 let mut continued = false;
170 for line in contents.lines() {
171 let trimmed = line.strip_suffix('\r').unwrap_or(line);
172 let (text, continues) = match trimmed.strip_suffix('\\') {
173 Some(text) => (text, true),
174 None => (trimmed, false),
175 };
176 if continued {
177 joined.push(' ');
178 }
179 joined.push_str(text.trim_end_matches(['\t']));
180 if !continues {
181 joined.push('\n');
182 }
183 continued = continues;
184 }
185 if continued {
186 return Err(CcBypassReason::MalformedDepfile(
187 "unterminated line continuation".into(),
188 ));
189 }
190 Ok(joined)
191}
192
193fn split_prerequisites(value: &str) -> Result<Vec<PathBuf>, CcBypassReason> {
198 let mut files = Vec::new();
199 let mut current = String::new();
200 let mut characters = value.chars().peekable();
201 while let Some(character) = characters.next() {
202 match character {
203 ' ' | '\t' => {
204 if !current.is_empty() {
205 files.push(PathBuf::from(std::mem::take(&mut current)));
206 }
207 }
208 '\\' => match characters.next() {
209 Some(' ') => current.push(' '),
210 Some('#') => current.push('#'),
211 Some(other) => {
212 return Err(CcBypassReason::MalformedDepfile(format!(
213 "unmodeled escape \\{other}"
214 )));
215 }
216 None => {
217 return Err(CcBypassReason::MalformedDepfile(
218 "trailing escape character".into(),
219 ));
220 }
221 },
222 '$' => match characters.next() {
223 Some('$') => current.push('$'),
224 Some(other) => {
225 return Err(CcBypassReason::MalformedDepfile(format!(
226 "unmodeled variable reference ${other}"
227 )));
228 }
229 None => {
230 return Err(CcBypassReason::MalformedDepfile(
231 "trailing variable reference".into(),
232 ));
233 }
234 },
235 other => current.push(other),
236 }
237 }
238 if !current.is_empty() {
239 files.push(PathBuf::from(current));
240 }
241 Ok(files)
242}
243
244#[derive(Debug, Clone, PartialEq, Eq)]
246pub struct CcDiscoveredInputs {
247 working_dir: PathBuf,
248 pub inputs: Vec<CcActionInput>,
250 identities: Vec<Option<FileIdentity>>,
254}
255
256impl CcDiscoveredInputs {
257 pub fn collect(
264 working_dir: &Path,
265 files: BTreeSet<PathBuf>,
266 directories: BTreeSet<PathBuf>,
267 digests: &dyn FileDigestCache,
268 ) -> Result<Self, CcBypassReason> {
269 if !working_dir.is_absolute() {
270 return Err(CcBypassReason::RelativeWorkingDirectory(
271 working_dir.to_path_buf(),
272 ));
273 }
274 let directories = minimal_manifest_directories(directories);
275 if files.len() + directories.len() > MAX_PREDICTED_INPUTS {
276 return Err(CcBypassReason::TooManyInputs);
277 }
278 let working_dir = normalize_components(working_dir);
279 let mut inputs = Vec::with_capacity(files.len() + directories.len());
280 let mut total_bytes = 0_u64;
281 let mut identified = Vec::with_capacity(files.len());
288 for path in files {
289 let metadata = std::fs::metadata(&path).map_err(|error| CcBypassReason::InputRead {
290 path: path.clone(),
291 message: error.to_string(),
292 })?;
293 if !metadata.is_file() {
294 return Err(CcBypassReason::InputRead {
295 path,
296 message: "input is not a regular file".into(),
297 });
298 }
299 total_bytes = total_bytes.saturating_add(metadata.len());
300 if total_bytes > MAX_INPUT_BYTES {
301 return Err(CcBypassReason::TooManyInputs);
302 }
303 let identity = FileIdentity::for_digest_cache(&path, &metadata).map_err(|error| {
304 CcBypassReason::InputRead {
305 path: path.clone(),
306 message: error.to_string(),
307 }
308 })?;
309 identified.push((path, identity));
310 }
311 let queries = identified
312 .iter()
313 .filter_map(|(_, identity)| identity.clone())
314 .collect::<Vec<_>>();
315 let mut recorded = digests
316 .resolve(FileDigestScope::CcInput, &queries)
317 .into_iter();
318 let mut identities = Vec::with_capacity(inputs.capacity());
319 let mut fresh = Vec::new();
320 for (path, identity) in identified {
321 identities.push(identity.clone());
322 let resolution = identity
323 .as_ref()
324 .and_then(|_| recorded.next())
325 .unwrap_or(FileDigestResolution::Unresolved);
326 let digest = match resolution {
327 FileDigestResolution::Digest(digest)
328 if identity
329 .as_ref()
330 .is_some_and(|identity| identity.len == digest.size) =>
331 {
332 digest
333 }
334 FileDigestResolution::EmbeddedTimestampMacro => {
335 return Err(CcBypassReason::EmbeddedTimestampMacro(path));
336 }
337 FileDigestResolution::Digest(_) | FileDigestResolution::Unresolved => {
338 let resolution =
339 digest_file(FileDigestScope::CcInput, &path).map_err(|error| {
340 CcBypassReason::InputRead {
341 path: path.clone(),
342 message: error.to_string(),
343 }
344 })?;
345 let FileDigestResolution::Digest(digest) = resolution else {
346 return Err(CcBypassReason::EmbeddedTimestampMacro(path));
347 };
348 if let Some(identity) = identity
349 && identity.len == digest.size
350 {
351 fresh.push(RecordedFileDigest {
352 file: identity,
353 digest: digest.clone(),
354 });
355 }
356 digest
357 }
358 };
359 inputs.push(CcActionInput { path, digest });
360 }
361 if !fresh.is_empty() {
362 digests.record(FileDigestScope::CcInput, fresh);
363 }
364 let mut manifest_entries = 0_usize;
365 for directory in directories {
366 let digest = include_manifest(&directory, &mut manifest_entries)?;
367 inputs.push(CcActionInput {
368 path: PathBuf::from(format!("{INCLUDE_MANIFEST_PREFIX}{}", directory.display())),
369 digest,
370 });
371 identities.push(None);
372 }
373 Ok(Self {
374 working_dir,
375 inputs,
376 identities,
377 })
378 }
379
380 pub fn files(&self) -> impl Iterator<Item = &CcActionInput> {
382 self.inputs
383 .iter()
384 .filter(|input| !is_manifest_input(&input.path))
385 }
386
387 pub fn verify_not_modified_since(&self, started_at: SystemTime) -> Result<(), CcBypassReason> {
394 self.verify_not_modified_since_with_snapshots(started_at, &BTreeMap::new())
395 }
396
397 pub fn verify_not_modified_since_with_snapshots(
400 &self,
401 started_at: SystemTime,
402 before: &BTreeMap<PathBuf, FileSnapshot>,
403 ) -> Result<(), CcBypassReason> {
404 for input in self.files() {
405 if let Some(previous) = before.get(&input.path)
406 && previous.proves_content_change()
407 {
408 let metadata =
409 std::fs::metadata(&input.path).map_err(|error| CcBypassReason::InputRead {
410 path: input.path.clone(),
411 message: error.to_string(),
412 })?;
413 let identity = FileIdentity::for_digest_cache(&input.path, &metadata)
414 .map_err(|error| CcBypassReason::InputRead {
415 path: input.path.clone(),
416 message: error.to_string(),
417 })?
418 .or_else(|| FileIdentity::describe(&input.path, &metadata));
419 if previous.matches(identity.as_ref(), &input.digest) {
420 continue;
421 }
422 return Err(CcBypassReason::InputModifiedDuringCompilation(
423 input.path.clone(),
424 ));
425 }
426 let metadata =
427 std::fs::metadata(&input.path).map_err(|error| CcBypassReason::InputRead {
428 path: input.path.clone(),
429 message: error.to_string(),
430 })?;
431 let modified = metadata
432 .modified()
433 .map_err(|error| CcBypassReason::InputRead {
434 path: input.path.clone(),
435 message: error.to_string(),
436 })?;
437 if modified >= started_at {
438 return Err(CcBypassReason::InputModifiedDuringCompilation(
439 input.path.clone(),
440 ));
441 }
442 }
443 Ok(())
444 }
445
446 pub fn verify_not_modified_since_with_identities(
448 &self,
449 started_at: SystemTime,
450 before: &BTreeMap<PathBuf, FileIdentity>,
451 ) -> Result<(), CcBypassReason> {
452 let snapshots = before
453 .iter()
454 .map(|(path, identity)| (path.clone(), identity.clone().into()))
455 .collect();
456 self.verify_not_modified_since_with_snapshots(started_at, &snapshots)
457 }
458
459 pub fn verify(&self) -> Result<(), CcBypassReason> {
468 for (index, input) in self.inputs.iter().enumerate() {
469 if is_manifest_input(&input.path) {
470 continue;
471 }
472 let read_error = |error: std::io::Error| CcBypassReason::InputRead {
473 path: input.path.clone(),
474 message: error.to_string(),
475 };
476 if let Some(Some(identity)) = self.identities.get(index)
477 && identity.can_skip_content_verification()
478 && identity.still_describes().map_err(read_error)?
479 {
480 continue;
481 }
482 let matches = input.digest.matches_file(&input.path).map_err(|error| {
483 CcBypassReason::InputRead {
484 path: input.path.clone(),
485 message: error.to_string(),
486 }
487 })?;
488 if !matches {
489 return Err(CcBypassReason::InputChanged(input.path.clone()));
490 }
491 }
492 Ok(())
493 }
494
495 pub fn apply_to(self, context: &mut CcActionContext) -> Result<(), CcBypassReason> {
498 if normalize_components(&context.working_dir) != self.working_dir {
499 return Err(CcBypassReason::DiscoveryWorkingDirectory);
500 }
501 context.inputs.extend(self.inputs);
502 Ok(())
503 }
504}
505
506fn minimal_manifest_directories(directories: BTreeSet<PathBuf>) -> Vec<PathBuf> {
514 let mut directories = directories
515 .into_iter()
516 .map(|directory| {
517 let normalized = normalize_components(&directory);
518 (directory, normalized)
519 })
520 .collect::<Vec<_>>();
521 directories.sort_by(|(left, left_normalized), (right, right_normalized)| {
522 left_normalized
523 .components()
524 .count()
525 .cmp(&right_normalized.components().count())
526 .then_with(|| left_normalized.cmp(right_normalized))
527 .then_with(|| left.cmp(right))
528 });
529
530 let mut minimal = Vec::<(PathBuf, PathBuf)>::new();
531 for (directory, normalized) in directories {
532 if !minimal
533 .iter()
534 .any(|(_, ancestor)| manifest_covers(ancestor, &normalized))
535 {
536 minimal.push((directory, normalized));
537 }
538 }
539 minimal
540 .into_iter()
541 .map(|(directory, _)| directory)
542 .collect()
543}
544
545fn manifest_covers(ancestor: &Path, descendant: &Path) -> bool {
552 let Ok(relative) = descendant.strip_prefix(ancestor) else {
553 return false;
554 };
555 if relative.as_os_str().is_empty() {
556 return false;
557 }
558 let mut current = ancestor.to_path_buf();
559 for component in relative.components() {
560 current.push(component);
561 let Ok(metadata) = std::fs::symlink_metadata(¤t) else {
562 return false;
563 };
564 if !metadata.is_dir() || metadata.file_type().is_symlink() {
565 return false;
566 }
567 }
568 true
569}
570
571fn is_manifest_input(path: &Path) -> bool {
572 path.to_str()
573 .is_some_and(|path| path.starts_with(INCLUDE_MANIFEST_PREFIX))
574}
575
576pub fn manifest_snapshot(
584 directories: &BTreeSet<PathBuf>,
585) -> Result<BTreeMap<PathBuf, CacheDigest>, CcBypassReason> {
586 let mut budget = 0_usize;
587 minimal_manifest_directories(directories.iter().cloned().collect())
588 .into_iter()
589 .map(|directory| {
590 include_manifest(&directory, &mut budget).map(|digest| (directory, digest))
591 })
592 .collect()
593}
594
595const INCLUDABLE_EXTENSIONS: &[&str] = &[
606 "c", "c++", "cc", "cpp", "cxx", "def", "gch", "h", "h++", "hh", "hpp", "hxx", "inc", "inl",
607 "ipp", "pch", "s", "tcc",
608];
609
610fn is_includable(name: &str) -> bool {
619 match name.rsplit_once('.') {
620 Some((stem, extension)) if !stem.is_empty() => INCLUDABLE_EXTENSIONS
621 .binary_search(&extension.to_ascii_lowercase().as_str())
622 .is_ok(),
623 _ => !name.starts_with('.'),
625 }
626}
627
628fn include_manifest(directory: &Path, budget: &mut usize) -> Result<CacheDigest, CcBypassReason> {
635 let mut names = Vec::new();
636 let mut pending = vec![(directory.to_path_buf(), String::new())];
637 while let Some((current, prefix)) = pending.pop() {
638 let entries = match std::fs::read_dir(¤t) {
639 Ok(entries) => entries,
640 Err(error) if error.kind() == std::io::ErrorKind::NotFound => continue,
641 Err(error) => {
642 return Err(CcBypassReason::InputRead {
643 path: current,
644 message: error.to_string(),
645 });
646 }
647 };
648 for entry in entries {
649 let entry = entry.map_err(|error| CcBypassReason::InputRead {
650 path: current.clone(),
651 message: error.to_string(),
652 })?;
653 let name = entry.file_name();
654 let Some(name) = name.to_str() else {
655 return Err(CcBypassReason::NonUtf8Path(entry.path()));
656 };
657 let relative = if prefix.is_empty() {
658 name.to_string()
659 } else {
660 format!("{prefix}/{name}")
661 };
662 let file_type = entry
663 .file_type()
664 .map_err(|error| CcBypassReason::InputRead {
665 path: entry.path(),
666 message: error.to_string(),
667 })?;
668 if file_type.is_dir() {
669 pending.push((entry.path(), relative));
670 continue;
671 }
672 if !is_includable(name) {
673 continue;
674 }
675 *budget += 1;
676 if *budget > MAX_MANIFEST_ENTRIES {
677 return Err(CcBypassReason::TooManyInputs);
678 }
679 names.push(relative);
680 }
681 }
682 names.sort();
683 Ok(CacheDigest::blake3(names.join("\n").as_bytes()))
684}
685
686pub(crate) fn contains_assembler_input_directive(path: &Path) -> Result<bool, CcBypassReason> {
693 contains_any(path, ASSEMBLER_INPUT_DIRECTIVES)
694}
695
696fn contains_any(path: &Path, needles: &[&[u8]]) -> Result<bool, CcBypassReason> {
697 let file = std::fs::File::open(path).map_err(|error| CcBypassReason::InputRead {
698 path: path.to_path_buf(),
699 message: error.to_string(),
700 })?;
701 let longest = needles
702 .iter()
703 .map(|needle| needle.len())
704 .max()
705 .unwrap_or_default();
706 let mut reader = std::io::BufReader::new(file);
707 let mut window = Vec::with_capacity(SCAN_CHUNK_BYTES + longest);
708 let mut chunk = vec![0_u8; SCAN_CHUNK_BYTES];
709 loop {
710 let read = reader
711 .read(&mut chunk)
712 .map_err(|error| CcBypassReason::InputRead {
713 path: path.to_path_buf(),
714 message: error.to_string(),
715 })?;
716 if read == 0 {
717 return Ok(false);
718 }
719 window.extend_from_slice(&chunk[..read]);
720 if needles
721 .iter()
722 .any(|needle| contains_subslice_ascii_case_insensitive(&window, needle))
723 {
724 return Ok(true);
725 }
726 let keep = window.len().saturating_sub(longest.saturating_sub(1));
727 window.drain(..keep);
728 }
729}
730
731fn contains_subslice_ascii_case_insensitive(haystack: &[u8], needle: &[u8]) -> bool {
732 if needle.is_empty() || haystack.len() < needle.len() {
733 return false;
734 }
735 haystack.windows(needle.len()).any(|window| {
736 window
737 .iter()
738 .zip(needle)
739 .all(|(left, right)| left.eq_ignore_ascii_case(right))
740 })
741}
742
743#[cfg(test)]
744#[path = "depfile_tests.rs"]
745mod tests;