1use crate::{
4 CcActionContext, CcActionInput, CcBypassReason, CcCompilerFamily, MAX_INPUT_BYTES,
5 MAX_MANIFEST_ENTRIES, MAX_PREDICTED_INPUTS, normalize_components,
6};
7use mbx_cache_core::{
8 CacheDigest, FileDigestCache, FileDigestScope, FileIdentity, FileSnapshot, RecordedFileDigest,
9};
10use std::collections::{BTreeMap, BTreeSet};
11use std::io::Read;
12use std::path::{Path, PathBuf};
13use std::time::SystemTime;
14
15pub const INCLUDE_MANIFEST_PREFIX: &str = "@include-manifest:";
17
18const TIMESTAMP_MACROS: &[&[u8]] = &[b"__DATE__", b"__TIME__", b"__TIMESTAMP__"];
20
21const ASSEMBLER_INPUT_DIRECTIVES: &[&[u8]] = &[b".include", b".incbin", b".sinclude"];
24
25const SCAN_CHUNK_BYTES: usize = 64 * 1024;
26
27#[derive(Debug, Clone, PartialEq, Eq, Default)]
29pub struct CcDepfile {
30 pub files: Vec<PathBuf>,
32}
33
34impl CcDepfile {
35 pub fn read(path: &Path) -> Result<Self, CcBypassReason> {
37 let contents =
38 std::fs::read_to_string(path).map_err(|error| CcBypassReason::DepfileRead {
39 path: path.to_path_buf(),
40 message: error.to_string(),
41 })?;
42 Self::parse(&contents)
43 }
44
45 pub fn read_for(path: &Path, family: CcCompilerFamily) -> Result<Self, CcBypassReason> {
47 if family.is_msvc() {
48 Self::read_msvc(path)
49 } else {
50 Self::read(path)
51 }
52 }
53
54 pub fn read_msvc(path: &Path) -> Result<Self, CcBypassReason> {
56 let contents = std::fs::read(path).map_err(|error| CcBypassReason::DepfileRead {
57 path: path.to_path_buf(),
58 message: error.to_string(),
59 })?;
60 let value: serde_json::Value = serde_json::from_slice(&contents)
61 .map_err(|error| CcBypassReason::MalformedDepfile(error.to_string()))?;
62 let data = value
63 .get("Data")
64 .and_then(serde_json::Value::as_object)
65 .ok_or_else(|| CcBypassReason::MalformedDepfile("missing Data object".into()))?;
66 if data
67 .get("ImportedModules")
68 .and_then(serde_json::Value::as_array)
69 .is_some_and(|modules| !modules.is_empty())
70 || data.get("ProvidedModule").is_some_and(|module| {
71 !module.is_null() && module.as_str().is_none_or(|s| !s.is_empty())
72 })
73 {
74 return Err(CcBypassReason::MalformedDepfile(
75 "C++ module dependencies are not modeled".into(),
76 ));
77 }
78 let includes = data
79 .get("Includes")
80 .and_then(serde_json::Value::as_array)
81 .ok_or_else(|| CcBypassReason::MalformedDepfile("missing Includes array".into()))?;
82 let files = includes
83 .iter()
84 .map(|entry| {
85 entry.as_str().map(PathBuf::from).ok_or_else(|| {
86 CcBypassReason::MalformedDepfile("non-string include path".into())
87 })
88 })
89 .collect::<Result<Vec<_>, _>>()?;
90 Ok(Self { files })
91 }
92
93 pub fn parse(contents: &str) -> Result<Self, CcBypassReason> {
99 let joined = join_continuations(contents)?;
100 let (_, prerequisites) = joined
101 .lines()
102 .find_map(|line| line.split_once(RULE_SEPARATOR))
103 .ok_or_else(|| CcBypassReason::MalformedDepfile("no dependency rule".into()))?;
104 let files = split_prerequisites(prerequisites)?;
105 Ok(Self { files })
106 }
107}
108
109const RULE_SEPARATOR: &str = ": ";
110
111impl CcDepfile {
112 pub fn render(
122 targets: &[crate::DepfileTarget],
123 files: &[PathBuf],
124 source: &Path,
125 phony_targets: bool,
126 ) -> String {
127 let mut rendered = String::new();
128 for (index, target) in targets.iter().enumerate() {
129 if index > 0 {
130 rendered.push(' ');
131 }
132 if target.quoted {
133 rendered.push_str(&escape_make_word(&target.name));
134 } else {
135 rendered.push_str(&target.name);
136 }
137 }
138 rendered.push(':');
139 for file in files {
140 rendered.push_str(" \\\n ");
141 rendered.push_str(&escape_make_word(&file.to_string_lossy()));
142 }
143 rendered.push('\n');
144 if phony_targets {
145 for file in files.iter().filter(|file| file.as_path() != source) {
146 rendered.push_str(&escape_make_word(&file.to_string_lossy()));
147 rendered.push_str(":\n");
148 }
149 }
150 rendered
151 }
152}
153
154fn escape_make_word(word: &str) -> String {
156 let mut escaped = String::with_capacity(word.len());
157 for character in word.chars() {
158 match character {
159 ' ' => escaped.push_str("\\ "),
160 '#' => escaped.push_str("\\#"),
161 '$' => escaped.push_str("$$"),
162 other => escaped.push(other),
163 }
164 }
165 escaped
166}
167
168fn join_continuations(contents: &str) -> Result<String, CcBypassReason> {
170 let mut joined = String::with_capacity(contents.len());
171 let mut continued = false;
172 for line in contents.lines() {
173 let trimmed = line.strip_suffix('\r').unwrap_or(line);
174 let (text, continues) = match trimmed.strip_suffix('\\') {
175 Some(text) => (text, true),
176 None => (trimmed, false),
177 };
178 if continued {
179 joined.push(' ');
180 }
181 joined.push_str(text.trim_end_matches(['\t']));
182 if !continues {
183 joined.push('\n');
184 }
185 continued = continues;
186 }
187 if continued {
188 return Err(CcBypassReason::MalformedDepfile(
189 "unterminated line continuation".into(),
190 ));
191 }
192 Ok(joined)
193}
194
195fn split_prerequisites(value: &str) -> Result<Vec<PathBuf>, CcBypassReason> {
200 let mut files = Vec::new();
201 let mut current = String::new();
202 let mut characters = value.chars().peekable();
203 while let Some(character) = characters.next() {
204 match character {
205 ' ' | '\t' => {
206 if !current.is_empty() {
207 files.push(PathBuf::from(std::mem::take(&mut current)));
208 }
209 }
210 '\\' => match characters.next() {
211 Some(' ') => current.push(' '),
212 Some('#') => current.push('#'),
213 Some(other) => {
214 return Err(CcBypassReason::MalformedDepfile(format!(
215 "unmodeled escape \\{other}"
216 )));
217 }
218 None => {
219 return Err(CcBypassReason::MalformedDepfile(
220 "trailing escape character".into(),
221 ));
222 }
223 },
224 '$' => match characters.next() {
225 Some('$') => current.push('$'),
226 Some(other) => {
227 return Err(CcBypassReason::MalformedDepfile(format!(
228 "unmodeled variable reference ${other}"
229 )));
230 }
231 None => {
232 return Err(CcBypassReason::MalformedDepfile(
233 "trailing variable reference".into(),
234 ));
235 }
236 },
237 other => current.push(other),
238 }
239 }
240 if !current.is_empty() {
241 files.push(PathBuf::from(current));
242 }
243 Ok(files)
244}
245
246#[derive(Debug, Clone, PartialEq, Eq)]
248pub struct CcDiscoveredInputs {
249 working_dir: PathBuf,
250 pub inputs: Vec<CcActionInput>,
252 identities: Vec<Option<FileIdentity>>,
256}
257
258impl CcDiscoveredInputs {
259 pub fn collect(
266 working_dir: &Path,
267 files: BTreeSet<PathBuf>,
268 directories: BTreeSet<PathBuf>,
269 digests: &dyn FileDigestCache,
270 ) -> Result<Self, CcBypassReason> {
271 if !working_dir.is_absolute() {
272 return Err(CcBypassReason::RelativeWorkingDirectory(
273 working_dir.to_path_buf(),
274 ));
275 }
276 let directories = minimal_manifest_directories(directories);
277 if files.len() + directories.len() > MAX_PREDICTED_INPUTS {
278 return Err(CcBypassReason::TooManyInputs);
279 }
280 let working_dir = normalize_components(working_dir);
281 let mut inputs = Vec::with_capacity(files.len() + directories.len());
282 let mut total_bytes = 0_u64;
283 let mut identified = Vec::with_capacity(files.len());
290 for path in files {
291 let metadata = std::fs::metadata(&path).map_err(|error| CcBypassReason::InputRead {
292 path: path.clone(),
293 message: error.to_string(),
294 })?;
295 if !metadata.is_file() {
296 return Err(CcBypassReason::InputRead {
297 path,
298 message: "input is not a regular file".into(),
299 });
300 }
301 total_bytes = total_bytes.saturating_add(metadata.len());
302 if total_bytes > MAX_INPUT_BYTES {
303 return Err(CcBypassReason::TooManyInputs);
304 }
305 let identity = FileIdentity::for_digest_cache(&path, &metadata).map_err(|error| {
306 CcBypassReason::InputRead {
307 path: path.clone(),
308 message: error.to_string(),
309 }
310 })?;
311 identified.push((path, identity));
312 }
313 let queries = identified
314 .iter()
315 .filter_map(|(_, identity)| identity.clone())
316 .collect::<Vec<_>>();
317 let mut recorded = digests.find(FileDigestScope::CcInput, &queries).into_iter();
318 let mut identities = Vec::with_capacity(inputs.capacity());
319 let mut fresh = Vec::new();
320 for (path, identity) in identified {
321 identities.push(identity.clone());
322 let remembered = identity
323 .as_ref()
324 .and_then(|_| recorded.next().flatten())
325 .filter(|digest| {
326 identity
327 .as_ref()
328 .is_some_and(|identity| identity.len == digest.size)
329 });
330 let digest = match remembered {
331 Some(digest) => digest,
332 None => {
333 if contains_timestamp_macro(&path)? {
334 return Err(CcBypassReason::EmbeddedTimestampMacro(path));
335 }
336 let digest = CacheDigest::blake3_file(&path).map_err(|error| {
337 CcBypassReason::InputRead {
338 path: path.clone(),
339 message: error.to_string(),
340 }
341 })?;
342 if let Some(identity) = identity
343 && identity.len == digest.size
344 {
345 fresh.push(RecordedFileDigest {
346 file: identity,
347 digest: digest.clone(),
348 });
349 }
350 digest
351 }
352 };
353 inputs.push(CcActionInput { path, digest });
354 }
355 if !fresh.is_empty() {
356 digests.record(FileDigestScope::CcInput, fresh);
357 }
358 let mut manifest_entries = 0_usize;
359 for directory in directories {
360 let digest = include_manifest(&directory, &mut manifest_entries)?;
361 inputs.push(CcActionInput {
362 path: PathBuf::from(format!("{INCLUDE_MANIFEST_PREFIX}{}", directory.display())),
363 digest,
364 });
365 identities.push(None);
366 }
367 Ok(Self {
368 working_dir,
369 inputs,
370 identities,
371 })
372 }
373
374 pub fn files(&self) -> impl Iterator<Item = &CcActionInput> {
376 self.inputs
377 .iter()
378 .filter(|input| !is_manifest_input(&input.path))
379 }
380
381 pub fn verify_not_modified_since(&self, started_at: SystemTime) -> Result<(), CcBypassReason> {
388 self.verify_not_modified_since_with_snapshots(started_at, &BTreeMap::new())
389 }
390
391 pub fn verify_not_modified_since_with_snapshots(
394 &self,
395 started_at: SystemTime,
396 before: &BTreeMap<PathBuf, FileSnapshot>,
397 ) -> Result<(), CcBypassReason> {
398 for input in self.files() {
399 if let Some(previous) = before.get(&input.path)
400 && previous.proves_content_change()
401 {
402 let metadata =
403 std::fs::metadata(&input.path).map_err(|error| CcBypassReason::InputRead {
404 path: input.path.clone(),
405 message: error.to_string(),
406 })?;
407 let identity = FileIdentity::describe(&input.path, &metadata);
408 if previous.matches(identity.as_ref(), &input.digest) {
409 continue;
410 }
411 return Err(CcBypassReason::InputModifiedDuringCompilation(
412 input.path.clone(),
413 ));
414 }
415 let metadata =
416 std::fs::metadata(&input.path).map_err(|error| CcBypassReason::InputRead {
417 path: input.path.clone(),
418 message: error.to_string(),
419 })?;
420 let modified = metadata
421 .modified()
422 .map_err(|error| CcBypassReason::InputRead {
423 path: input.path.clone(),
424 message: error.to_string(),
425 })?;
426 if modified >= started_at {
427 return Err(CcBypassReason::InputModifiedDuringCompilation(
428 input.path.clone(),
429 ));
430 }
431 }
432 Ok(())
433 }
434
435 pub fn verify_not_modified_since_with_identities(
437 &self,
438 started_at: SystemTime,
439 before: &BTreeMap<PathBuf, FileIdentity>,
440 ) -> Result<(), CcBypassReason> {
441 let snapshots = before
442 .iter()
443 .map(|(path, identity)| (path.clone(), identity.clone().into()))
444 .collect();
445 self.verify_not_modified_since_with_snapshots(started_at, &snapshots)
446 }
447
448 pub fn verify(&self) -> Result<(), CcBypassReason> {
457 for (index, input) in self.inputs.iter().enumerate() {
458 if is_manifest_input(&input.path) {
459 continue;
460 }
461 let read_error = |error: std::io::Error| CcBypassReason::InputRead {
462 path: input.path.clone(),
463 message: error.to_string(),
464 };
465 if let Some(Some(identity)) = self.identities.get(index)
466 && identity.changed.is_some()
467 && identity.still_describes().map_err(read_error)?
468 {
469 continue;
470 }
471 let matches = input.digest.matches_file(&input.path).map_err(|error| {
472 CcBypassReason::InputRead {
473 path: input.path.clone(),
474 message: error.to_string(),
475 }
476 })?;
477 if !matches {
478 return Err(CcBypassReason::InputChanged(input.path.clone()));
479 }
480 }
481 Ok(())
482 }
483
484 pub fn apply_to(self, context: &mut CcActionContext) -> Result<(), CcBypassReason> {
487 if normalize_components(&context.working_dir) != self.working_dir {
488 return Err(CcBypassReason::DiscoveryWorkingDirectory);
489 }
490 context.inputs.extend(self.inputs);
491 Ok(())
492 }
493}
494
495fn minimal_manifest_directories(directories: BTreeSet<PathBuf>) -> Vec<PathBuf> {
503 let mut directories = directories
504 .into_iter()
505 .map(|directory| {
506 let normalized = normalize_components(&directory);
507 (directory, normalized)
508 })
509 .collect::<Vec<_>>();
510 directories.sort_by(|(left, left_normalized), (right, right_normalized)| {
511 left_normalized
512 .components()
513 .count()
514 .cmp(&right_normalized.components().count())
515 .then_with(|| left_normalized.cmp(right_normalized))
516 .then_with(|| left.cmp(right))
517 });
518
519 let mut minimal = Vec::<(PathBuf, PathBuf)>::new();
520 for (directory, normalized) in directories {
521 if !minimal
522 .iter()
523 .any(|(_, ancestor)| manifest_covers(ancestor, &normalized))
524 {
525 minimal.push((directory, normalized));
526 }
527 }
528 minimal
529 .into_iter()
530 .map(|(directory, _)| directory)
531 .collect()
532}
533
534fn manifest_covers(ancestor: &Path, descendant: &Path) -> bool {
541 let Ok(relative) = descendant.strip_prefix(ancestor) else {
542 return false;
543 };
544 if relative.as_os_str().is_empty() {
545 return false;
546 }
547 let mut current = ancestor.to_path_buf();
548 for component in relative.components() {
549 current.push(component);
550 let Ok(metadata) = std::fs::symlink_metadata(¤t) else {
551 return false;
552 };
553 if !metadata.is_dir() || metadata.file_type().is_symlink() {
554 return false;
555 }
556 }
557 true
558}
559
560fn is_manifest_input(path: &Path) -> bool {
561 path.to_str()
562 .is_some_and(|path| path.starts_with(INCLUDE_MANIFEST_PREFIX))
563}
564
565pub fn manifest_snapshot(
573 directories: &BTreeSet<PathBuf>,
574) -> Result<BTreeMap<PathBuf, CacheDigest>, CcBypassReason> {
575 let mut budget = 0_usize;
576 minimal_manifest_directories(directories.iter().cloned().collect())
577 .into_iter()
578 .map(|directory| {
579 include_manifest(&directory, &mut budget).map(|digest| (directory, digest))
580 })
581 .collect()
582}
583
584const INCLUDABLE_EXTENSIONS: &[&str] = &[
595 "c", "c++", "cc", "cpp", "cxx", "def", "gch", "h", "h++", "hh", "hpp", "hxx", "inc", "inl",
596 "ipp", "pch", "s", "tcc",
597];
598
599fn is_includable(name: &str) -> bool {
608 match name.rsplit_once('.') {
609 Some((stem, extension)) if !stem.is_empty() => INCLUDABLE_EXTENSIONS
610 .binary_search(&extension.to_ascii_lowercase().as_str())
611 .is_ok(),
612 _ => !name.starts_with('.'),
614 }
615}
616
617fn include_manifest(directory: &Path, budget: &mut usize) -> Result<CacheDigest, CcBypassReason> {
624 let mut names = Vec::new();
625 let mut pending = vec![(directory.to_path_buf(), String::new())];
626 while let Some((current, prefix)) = pending.pop() {
627 let entries = match std::fs::read_dir(¤t) {
628 Ok(entries) => entries,
629 Err(error) if error.kind() == std::io::ErrorKind::NotFound => continue,
630 Err(error) => {
631 return Err(CcBypassReason::InputRead {
632 path: current,
633 message: error.to_string(),
634 });
635 }
636 };
637 for entry in entries {
638 let entry = entry.map_err(|error| CcBypassReason::InputRead {
639 path: current.clone(),
640 message: error.to_string(),
641 })?;
642 let name = entry.file_name();
643 let Some(name) = name.to_str() else {
644 return Err(CcBypassReason::NonUtf8Path(entry.path()));
645 };
646 let relative = if prefix.is_empty() {
647 name.to_string()
648 } else {
649 format!("{prefix}/{name}")
650 };
651 let file_type = entry
652 .file_type()
653 .map_err(|error| CcBypassReason::InputRead {
654 path: entry.path(),
655 message: error.to_string(),
656 })?;
657 if file_type.is_dir() {
658 pending.push((entry.path(), relative));
659 continue;
660 }
661 if !is_includable(name) {
662 continue;
663 }
664 *budget += 1;
665 if *budget > MAX_MANIFEST_ENTRIES {
666 return Err(CcBypassReason::TooManyInputs);
667 }
668 names.push(relative);
669 }
670 }
671 names.sort();
672 Ok(CacheDigest::blake3(names.join("\n").as_bytes()))
673}
674
675fn contains_timestamp_macro(path: &Path) -> Result<bool, CcBypassReason> {
682 let file = std::fs::File::open(path).map_err(|error| CcBypassReason::InputRead {
683 path: path.to_path_buf(),
684 message: error.to_string(),
685 })?;
686 let longest = TIMESTAMP_MACROS
687 .iter()
688 .map(|macro_name| macro_name.len())
689 .max()
690 .unwrap_or_default();
691 let mut reader = std::io::BufReader::new(file);
692 let mut window = Vec::with_capacity(SCAN_CHUNK_BYTES + longest);
693 let mut chunk = vec![0_u8; SCAN_CHUNK_BYTES];
694 loop {
695 let read = reader
696 .read(&mut chunk)
697 .map_err(|error| CcBypassReason::InputRead {
698 path: path.to_path_buf(),
699 message: error.to_string(),
700 })?;
701 if read == 0 {
702 return Ok(false);
703 }
704 window.extend_from_slice(&chunk[..read]);
705 if TIMESTAMP_MACROS
706 .iter()
707 .any(|macro_name| contains_subslice(&window, macro_name))
708 {
709 return Ok(true);
710 }
711 let keep = window.len().saturating_sub(longest.saturating_sub(1));
713 window.drain(..keep);
714 }
715}
716
717pub(crate) fn contains_assembler_input_directive(path: &Path) -> Result<bool, CcBypassReason> {
724 contains_any(path, ASSEMBLER_INPUT_DIRECTIVES)
725}
726
727fn contains_any(path: &Path, needles: &[&[u8]]) -> Result<bool, CcBypassReason> {
728 let file = std::fs::File::open(path).map_err(|error| CcBypassReason::InputRead {
729 path: path.to_path_buf(),
730 message: error.to_string(),
731 })?;
732 let longest = needles
733 .iter()
734 .map(|needle| needle.len())
735 .max()
736 .unwrap_or_default();
737 let mut reader = std::io::BufReader::new(file);
738 let mut window = Vec::with_capacity(SCAN_CHUNK_BYTES + longest);
739 let mut chunk = vec![0_u8; SCAN_CHUNK_BYTES];
740 loop {
741 let read = reader
742 .read(&mut chunk)
743 .map_err(|error| CcBypassReason::InputRead {
744 path: path.to_path_buf(),
745 message: error.to_string(),
746 })?;
747 if read == 0 {
748 return Ok(false);
749 }
750 window.extend_from_slice(&chunk[..read]);
751 if needles
752 .iter()
753 .any(|needle| contains_subslice_ascii_case_insensitive(&window, needle))
754 {
755 return Ok(true);
756 }
757 let keep = window.len().saturating_sub(longest.saturating_sub(1));
758 window.drain(..keep);
759 }
760}
761
762fn contains_subslice_ascii_case_insensitive(haystack: &[u8], needle: &[u8]) -> bool {
763 if needle.is_empty() || haystack.len() < needle.len() {
764 return false;
765 }
766 haystack.windows(needle.len()).any(|window| {
767 window
768 .iter()
769 .zip(needle)
770 .all(|(left, right)| left.eq_ignore_ascii_case(right))
771 })
772}
773
774fn contains_subslice(haystack: &[u8], needle: &[u8]) -> bool {
775 if needle.is_empty() || haystack.len() < needle.len() {
776 return false;
777 }
778 haystack
779 .windows(needle.len())
780 .any(|window| window == needle)
781}
782
783#[cfg(test)]
784#[path = "depfile_tests.rs"]
785mod tests;