1use crate::{
4 CcActionContext, CcActionInput, CcBypassReason, CcCompilerFamily, MAX_INPUT_BYTES,
5 MAX_MANIFEST_ENTRIES, MAX_PREDICTED_INPUTS, normalize_components,
6};
7use mbx_cache_core::{
8 CacheDigest, FileDigestCache, FileDigestScope, FileIdentity, RecordedFileDigest,
9};
10use std::collections::{BTreeMap, BTreeSet};
11use std::io::Read;
12use std::path::{Path, PathBuf};
13use std::time::SystemTime;
14
15pub const INCLUDE_MANIFEST_PREFIX: &str = "@include-manifest:";
17
18const TIMESTAMP_MACROS: &[&[u8]] = &[b"__DATE__", b"__TIME__", b"__TIMESTAMP__"];
20
21const ASSEMBLER_INPUT_DIRECTIVES: &[&[u8]] = &[b".include", b".incbin", b".sinclude"];
24
25const SCAN_CHUNK_BYTES: usize = 64 * 1024;
26
27#[derive(Debug, Clone, PartialEq, Eq, Default)]
29pub struct CcDepfile {
30 pub files: Vec<PathBuf>,
32}
33
34impl CcDepfile {
35 pub fn read(path: &Path) -> Result<Self, CcBypassReason> {
37 let contents =
38 std::fs::read_to_string(path).map_err(|error| CcBypassReason::DepfileRead {
39 path: path.to_path_buf(),
40 message: error.to_string(),
41 })?;
42 Self::parse(&contents)
43 }
44
45 pub fn read_for(path: &Path, family: CcCompilerFamily) -> Result<Self, CcBypassReason> {
47 if family.is_msvc() {
48 Self::read_msvc(path)
49 } else {
50 Self::read(path)
51 }
52 }
53
54 pub fn read_msvc(path: &Path) -> Result<Self, CcBypassReason> {
56 let contents = std::fs::read(path).map_err(|error| CcBypassReason::DepfileRead {
57 path: path.to_path_buf(),
58 message: error.to_string(),
59 })?;
60 let value: serde_json::Value = serde_json::from_slice(&contents)
61 .map_err(|error| CcBypassReason::MalformedDepfile(error.to_string()))?;
62 let data = value
63 .get("Data")
64 .and_then(serde_json::Value::as_object)
65 .ok_or_else(|| CcBypassReason::MalformedDepfile("missing Data object".into()))?;
66 if data
67 .get("ImportedModules")
68 .and_then(serde_json::Value::as_array)
69 .is_some_and(|modules| !modules.is_empty())
70 || data.get("ProvidedModule").is_some_and(|module| {
71 !module.is_null() && module.as_str().is_none_or(|s| !s.is_empty())
72 })
73 {
74 return Err(CcBypassReason::MalformedDepfile(
75 "C++ module dependencies are not modeled".into(),
76 ));
77 }
78 let includes = data
79 .get("Includes")
80 .and_then(serde_json::Value::as_array)
81 .ok_or_else(|| CcBypassReason::MalformedDepfile("missing Includes array".into()))?;
82 let files = includes
83 .iter()
84 .map(|entry| {
85 entry.as_str().map(PathBuf::from).ok_or_else(|| {
86 CcBypassReason::MalformedDepfile("non-string include path".into())
87 })
88 })
89 .collect::<Result<Vec<_>, _>>()?;
90 Ok(Self { files })
91 }
92
93 pub fn parse(contents: &str) -> Result<Self, CcBypassReason> {
99 let joined = join_continuations(contents)?;
100 let (_, prerequisites) = joined
101 .lines()
102 .find_map(|line| line.split_once(RULE_SEPARATOR))
103 .ok_or_else(|| CcBypassReason::MalformedDepfile("no dependency rule".into()))?;
104 let files = split_prerequisites(prerequisites)?;
105 Ok(Self { files })
106 }
107}
108
109const RULE_SEPARATOR: &str = ": ";
110
111impl CcDepfile {
112 pub fn render(
122 targets: &[crate::DepfileTarget],
123 files: &[PathBuf],
124 source: &Path,
125 phony_targets: bool,
126 ) -> String {
127 let mut rendered = String::new();
128 for (index, target) in targets.iter().enumerate() {
129 if index > 0 {
130 rendered.push(' ');
131 }
132 if target.quoted {
133 rendered.push_str(&escape_make_word(&target.name));
134 } else {
135 rendered.push_str(&target.name);
136 }
137 }
138 rendered.push(':');
139 for file in files {
140 rendered.push_str(" \\\n ");
141 rendered.push_str(&escape_make_word(&file.to_string_lossy()));
142 }
143 rendered.push('\n');
144 if phony_targets {
145 for file in files.iter().filter(|file| file.as_path() != source) {
146 rendered.push_str(&escape_make_word(&file.to_string_lossy()));
147 rendered.push_str(":\n");
148 }
149 }
150 rendered
151 }
152}
153
154fn escape_make_word(word: &str) -> String {
156 let mut escaped = String::with_capacity(word.len());
157 for character in word.chars() {
158 match character {
159 ' ' => escaped.push_str("\\ "),
160 '#' => escaped.push_str("\\#"),
161 '$' => escaped.push_str("$$"),
162 other => escaped.push(other),
163 }
164 }
165 escaped
166}
167
168fn join_continuations(contents: &str) -> Result<String, CcBypassReason> {
170 let mut joined = String::with_capacity(contents.len());
171 let mut continued = false;
172 for line in contents.lines() {
173 let trimmed = line.strip_suffix('\r').unwrap_or(line);
174 let (text, continues) = match trimmed.strip_suffix('\\') {
175 Some(text) => (text, true),
176 None => (trimmed, false),
177 };
178 if continued {
179 joined.push(' ');
180 }
181 joined.push_str(text.trim_end_matches(['\t']));
182 if !continues {
183 joined.push('\n');
184 }
185 continued = continues;
186 }
187 if continued {
188 return Err(CcBypassReason::MalformedDepfile(
189 "unterminated line continuation".into(),
190 ));
191 }
192 Ok(joined)
193}
194
195fn split_prerequisites(value: &str) -> Result<Vec<PathBuf>, CcBypassReason> {
200 let mut files = Vec::new();
201 let mut current = String::new();
202 let mut characters = value.chars().peekable();
203 while let Some(character) = characters.next() {
204 match character {
205 ' ' | '\t' => {
206 if !current.is_empty() {
207 files.push(PathBuf::from(std::mem::take(&mut current)));
208 }
209 }
210 '\\' => match characters.next() {
211 Some(' ') => current.push(' '),
212 Some('#') => current.push('#'),
213 Some(other) => {
214 return Err(CcBypassReason::MalformedDepfile(format!(
215 "unmodeled escape \\{other}"
216 )));
217 }
218 None => {
219 return Err(CcBypassReason::MalformedDepfile(
220 "trailing escape character".into(),
221 ));
222 }
223 },
224 '$' => match characters.next() {
225 Some('$') => current.push('$'),
226 Some(other) => {
227 return Err(CcBypassReason::MalformedDepfile(format!(
228 "unmodeled variable reference ${other}"
229 )));
230 }
231 None => {
232 return Err(CcBypassReason::MalformedDepfile(
233 "trailing variable reference".into(),
234 ));
235 }
236 },
237 other => current.push(other),
238 }
239 }
240 if !current.is_empty() {
241 files.push(PathBuf::from(current));
242 }
243 Ok(files)
244}
245
246#[derive(Debug, Clone, PartialEq, Eq)]
248pub struct CcDiscoveredInputs {
249 working_dir: PathBuf,
250 pub inputs: Vec<CcActionInput>,
252 identities: Vec<Option<FileIdentity>>,
256}
257
258impl CcDiscoveredInputs {
259 pub fn collect(
266 working_dir: &Path,
267 files: BTreeSet<PathBuf>,
268 directories: BTreeSet<PathBuf>,
269 digests: &dyn FileDigestCache,
270 ) -> Result<Self, CcBypassReason> {
271 if !working_dir.is_absolute() {
272 return Err(CcBypassReason::RelativeWorkingDirectory(
273 working_dir.to_path_buf(),
274 ));
275 }
276 let directories = minimal_manifest_directories(directories);
277 if files.len() + directories.len() > MAX_PREDICTED_INPUTS {
278 return Err(CcBypassReason::TooManyInputs);
279 }
280 let working_dir = normalize_components(working_dir);
281 let mut inputs = Vec::with_capacity(files.len() + directories.len());
282 let mut total_bytes = 0_u64;
283 let mut identified = Vec::with_capacity(files.len());
290 for path in files {
291 let metadata = std::fs::metadata(&path).map_err(|error| CcBypassReason::InputRead {
292 path: path.clone(),
293 message: error.to_string(),
294 })?;
295 if !metadata.is_file() {
296 return Err(CcBypassReason::InputRead {
297 path,
298 message: "input is not a regular file".into(),
299 });
300 }
301 total_bytes = total_bytes.saturating_add(metadata.len());
302 if total_bytes > MAX_INPUT_BYTES {
303 return Err(CcBypassReason::TooManyInputs);
304 }
305 let identity = FileIdentity::describe(&path, &metadata);
306 identified.push((path, identity));
307 }
308 let queries = identified
309 .iter()
310 .filter_map(|(_, identity)| identity.clone())
311 .collect::<Vec<_>>();
312 let mut recorded = digests.find(FileDigestScope::CcInput, &queries).into_iter();
313 let mut identities = Vec::with_capacity(inputs.capacity());
314 let mut fresh = Vec::new();
315 for (path, identity) in identified {
316 identities.push(identity.clone());
317 let remembered = identity
318 .as_ref()
319 .and_then(|_| recorded.next().flatten())
320 .filter(|digest| {
321 identity
322 .as_ref()
323 .is_some_and(|identity| identity.len == digest.size)
324 });
325 let digest = match remembered {
326 Some(digest) => digest,
327 None => {
328 if contains_timestamp_macro(&path)? {
329 return Err(CcBypassReason::EmbeddedTimestampMacro(path));
330 }
331 let digest = CacheDigest::blake3_file(&path).map_err(|error| {
332 CcBypassReason::InputRead {
333 path: path.clone(),
334 message: error.to_string(),
335 }
336 })?;
337 if let Some(identity) = identity
338 && identity.len == digest.size
339 {
340 fresh.push(RecordedFileDigest {
341 file: identity,
342 digest: digest.clone(),
343 });
344 }
345 digest
346 }
347 };
348 inputs.push(CcActionInput { path, digest });
349 }
350 if !fresh.is_empty() {
351 digests.record(FileDigestScope::CcInput, fresh);
352 }
353 let mut manifest_entries = 0_usize;
354 for directory in directories {
355 let digest = include_manifest(&directory, &mut manifest_entries)?;
356 inputs.push(CcActionInput {
357 path: PathBuf::from(format!("{INCLUDE_MANIFEST_PREFIX}{}", directory.display())),
358 digest,
359 });
360 identities.push(None);
361 }
362 Ok(Self {
363 working_dir,
364 inputs,
365 identities,
366 })
367 }
368
369 pub fn files(&self) -> impl Iterator<Item = &CcActionInput> {
371 self.inputs
372 .iter()
373 .filter(|input| !is_manifest_input(&input.path))
374 }
375
376 pub fn verify_not_modified_since(&self, started_at: SystemTime) -> Result<(), CcBypassReason> {
383 self.verify_not_modified_since_with_identities(started_at, &BTreeMap::new())
384 }
385
386 pub fn verify_not_modified_since_with_identities(
389 &self,
390 started_at: SystemTime,
391 before: &BTreeMap<PathBuf, FileIdentity>,
392 ) -> Result<(), CcBypassReason> {
393 for input in self.files() {
394 let metadata =
395 std::fs::metadata(&input.path).map_err(|error| CcBypassReason::InputRead {
396 path: input.path.clone(),
397 message: error.to_string(),
398 })?;
399 let identity = FileIdentity::describe(&input.path, &metadata);
400 if let Some(previous) = before.get(&input.path)
401 && previous.changed.is_some()
402 {
403 if identity.as_ref() == Some(previous) {
404 continue;
405 }
406 return Err(CcBypassReason::InputModifiedDuringCompilation(
407 input.path.clone(),
408 ));
409 }
410 let modified = metadata
411 .modified()
412 .map_err(|error| CcBypassReason::InputRead {
413 path: input.path.clone(),
414 message: error.to_string(),
415 })?;
416 if modified >= started_at {
417 return Err(CcBypassReason::InputModifiedDuringCompilation(
418 input.path.clone(),
419 ));
420 }
421 }
422 Ok(())
423 }
424
425 pub fn verify(&self) -> Result<(), CcBypassReason> {
434 for (index, input) in self.inputs.iter().enumerate() {
435 if is_manifest_input(&input.path) {
436 continue;
437 }
438 let read_error = |error: std::io::Error| CcBypassReason::InputRead {
439 path: input.path.clone(),
440 message: error.to_string(),
441 };
442 if let Some(Some(identity)) = self.identities.get(index)
443 && identity.changed.is_some()
444 && identity.still_describes().map_err(read_error)?
445 {
446 continue;
447 }
448 let matches = input.digest.matches_file(&input.path).map_err(|error| {
449 CcBypassReason::InputRead {
450 path: input.path.clone(),
451 message: error.to_string(),
452 }
453 })?;
454 if !matches {
455 return Err(CcBypassReason::InputChanged(input.path.clone()));
456 }
457 }
458 Ok(())
459 }
460
461 pub fn apply_to(self, context: &mut CcActionContext) -> Result<(), CcBypassReason> {
464 if normalize_components(&context.working_dir) != self.working_dir {
465 return Err(CcBypassReason::DiscoveryWorkingDirectory);
466 }
467 context.inputs.extend(self.inputs);
468 Ok(())
469 }
470}
471
472fn minimal_manifest_directories(directories: BTreeSet<PathBuf>) -> Vec<PathBuf> {
480 let mut directories = directories
481 .into_iter()
482 .map(|directory| {
483 let normalized = normalize_components(&directory);
484 (directory, normalized)
485 })
486 .collect::<Vec<_>>();
487 directories.sort_by(|(left, left_normalized), (right, right_normalized)| {
488 left_normalized
489 .components()
490 .count()
491 .cmp(&right_normalized.components().count())
492 .then_with(|| left_normalized.cmp(right_normalized))
493 .then_with(|| left.cmp(right))
494 });
495
496 let mut minimal = Vec::<(PathBuf, PathBuf)>::new();
497 for (directory, normalized) in directories {
498 if !minimal
499 .iter()
500 .any(|(_, ancestor)| manifest_covers(ancestor, &normalized))
501 {
502 minimal.push((directory, normalized));
503 }
504 }
505 minimal
506 .into_iter()
507 .map(|(directory, _)| directory)
508 .collect()
509}
510
511fn manifest_covers(ancestor: &Path, descendant: &Path) -> bool {
518 let Ok(relative) = descendant.strip_prefix(ancestor) else {
519 return false;
520 };
521 if relative.as_os_str().is_empty() {
522 return false;
523 }
524 let mut current = ancestor.to_path_buf();
525 for component in relative.components() {
526 current.push(component);
527 let Ok(metadata) = std::fs::symlink_metadata(¤t) else {
528 return false;
529 };
530 if !metadata.is_dir() || metadata.file_type().is_symlink() {
531 return false;
532 }
533 }
534 true
535}
536
537fn is_manifest_input(path: &Path) -> bool {
538 path.to_str()
539 .is_some_and(|path| path.starts_with(INCLUDE_MANIFEST_PREFIX))
540}
541
542pub fn manifest_snapshot(
550 directories: &BTreeSet<PathBuf>,
551) -> Result<BTreeMap<PathBuf, CacheDigest>, CcBypassReason> {
552 let mut budget = 0_usize;
553 minimal_manifest_directories(directories.iter().cloned().collect())
554 .into_iter()
555 .map(|directory| {
556 include_manifest(&directory, &mut budget).map(|digest| (directory, digest))
557 })
558 .collect()
559}
560
561const INCLUDABLE_EXTENSIONS: &[&str] = &[
572 "c", "c++", "cc", "cpp", "cxx", "def", "gch", "h", "h++", "hh", "hpp", "hxx", "inc", "inl",
573 "ipp", "pch", "s", "tcc",
574];
575
576fn is_includable(name: &str) -> bool {
585 match name.rsplit_once('.') {
586 Some((stem, extension)) if !stem.is_empty() => INCLUDABLE_EXTENSIONS
587 .binary_search(&extension.to_ascii_lowercase().as_str())
588 .is_ok(),
589 _ => !name.starts_with('.'),
591 }
592}
593
594fn include_manifest(directory: &Path, budget: &mut usize) -> Result<CacheDigest, CcBypassReason> {
601 let mut names = Vec::new();
602 let mut pending = vec![(directory.to_path_buf(), String::new())];
603 while let Some((current, prefix)) = pending.pop() {
604 let entries = match std::fs::read_dir(¤t) {
605 Ok(entries) => entries,
606 Err(error) if error.kind() == std::io::ErrorKind::NotFound => continue,
607 Err(error) => {
608 return Err(CcBypassReason::InputRead {
609 path: current,
610 message: error.to_string(),
611 });
612 }
613 };
614 for entry in entries {
615 let entry = entry.map_err(|error| CcBypassReason::InputRead {
616 path: current.clone(),
617 message: error.to_string(),
618 })?;
619 let name = entry.file_name();
620 let Some(name) = name.to_str() else {
621 return Err(CcBypassReason::NonUtf8Path(entry.path()));
622 };
623 let relative = if prefix.is_empty() {
624 name.to_string()
625 } else {
626 format!("{prefix}/{name}")
627 };
628 let file_type = entry
629 .file_type()
630 .map_err(|error| CcBypassReason::InputRead {
631 path: entry.path(),
632 message: error.to_string(),
633 })?;
634 if file_type.is_dir() {
635 pending.push((entry.path(), relative));
636 continue;
637 }
638 if !is_includable(name) {
639 continue;
640 }
641 *budget += 1;
642 if *budget > MAX_MANIFEST_ENTRIES {
643 return Err(CcBypassReason::TooManyInputs);
644 }
645 names.push(relative);
646 }
647 }
648 names.sort();
649 Ok(CacheDigest::blake3(names.join("\n").as_bytes()))
650}
651
652fn contains_timestamp_macro(path: &Path) -> Result<bool, CcBypassReason> {
659 let file = std::fs::File::open(path).map_err(|error| CcBypassReason::InputRead {
660 path: path.to_path_buf(),
661 message: error.to_string(),
662 })?;
663 let longest = TIMESTAMP_MACROS
664 .iter()
665 .map(|macro_name| macro_name.len())
666 .max()
667 .unwrap_or_default();
668 let mut reader = std::io::BufReader::new(file);
669 let mut window = Vec::with_capacity(SCAN_CHUNK_BYTES + longest);
670 let mut chunk = vec![0_u8; SCAN_CHUNK_BYTES];
671 loop {
672 let read = reader
673 .read(&mut chunk)
674 .map_err(|error| CcBypassReason::InputRead {
675 path: path.to_path_buf(),
676 message: error.to_string(),
677 })?;
678 if read == 0 {
679 return Ok(false);
680 }
681 window.extend_from_slice(&chunk[..read]);
682 if TIMESTAMP_MACROS
683 .iter()
684 .any(|macro_name| contains_subslice(&window, macro_name))
685 {
686 return Ok(true);
687 }
688 let keep = window.len().saturating_sub(longest.saturating_sub(1));
690 window.drain(..keep);
691 }
692}
693
694pub(crate) fn contains_assembler_input_directive(path: &Path) -> Result<bool, CcBypassReason> {
701 contains_any(path, ASSEMBLER_INPUT_DIRECTIVES)
702}
703
704fn contains_any(path: &Path, needles: &[&[u8]]) -> Result<bool, CcBypassReason> {
705 let file = std::fs::File::open(path).map_err(|error| CcBypassReason::InputRead {
706 path: path.to_path_buf(),
707 message: error.to_string(),
708 })?;
709 let longest = needles
710 .iter()
711 .map(|needle| needle.len())
712 .max()
713 .unwrap_or_default();
714 let mut reader = std::io::BufReader::new(file);
715 let mut window = Vec::with_capacity(SCAN_CHUNK_BYTES + longest);
716 let mut chunk = vec![0_u8; SCAN_CHUNK_BYTES];
717 loop {
718 let read = reader
719 .read(&mut chunk)
720 .map_err(|error| CcBypassReason::InputRead {
721 path: path.to_path_buf(),
722 message: error.to_string(),
723 })?;
724 if read == 0 {
725 return Ok(false);
726 }
727 window.extend_from_slice(&chunk[..read]);
728 if needles
729 .iter()
730 .any(|needle| contains_subslice_ascii_case_insensitive(&window, needle))
731 {
732 return Ok(true);
733 }
734 let keep = window.len().saturating_sub(longest.saturating_sub(1));
735 window.drain(..keep);
736 }
737}
738
739fn contains_subslice_ascii_case_insensitive(haystack: &[u8], needle: &[u8]) -> bool {
740 if needle.is_empty() || haystack.len() < needle.len() {
741 return false;
742 }
743 haystack.windows(needle.len()).any(|window| {
744 window
745 .iter()
746 .zip(needle)
747 .all(|(left, right)| left.eq_ignore_ascii_case(right))
748 })
749}
750
751fn contains_subslice(haystack: &[u8], needle: &[u8]) -> bool {
752 if needle.is_empty() || haystack.len() < needle.len() {
753 return false;
754 }
755 haystack
756 .windows(needle.len())
757 .any(|window| window == needle)
758}
759
760#[cfg(test)]
761#[path = "depfile_tests.rs"]
762mod tests;