1use crate::{
4 CcActionContext, CcActionInput, CcBypassReason, CcCompilerFamily, MAX_INPUT_BYTES,
5 MAX_MANIFEST_ENTRIES, MAX_PREDICTED_INPUTS, normalize_components,
6};
7use mbx_cache_core::{
8 CacheDigest, FileDigestCache, FileDigestScope, FileIdentity, RecordedFileDigest,
9};
10use std::collections::{BTreeMap, BTreeSet};
11use std::io::Read;
12use std::path::{Path, PathBuf};
13use std::time::SystemTime;
14
15pub const INCLUDE_MANIFEST_PREFIX: &str = "@include-manifest:";
17
18const TIMESTAMP_MACROS: &[&[u8]] = &[b"__DATE__", b"__TIME__", b"__TIMESTAMP__"];
20
21const ASSEMBLER_INPUT_DIRECTIVES: &[&[u8]] = &[b".include", b".incbin", b".sinclude"];
24
25const SCAN_CHUNK_BYTES: usize = 64 * 1024;
26
27#[derive(Debug, Clone, PartialEq, Eq, Default)]
29pub struct CcDepfile {
30 pub files: Vec<PathBuf>,
32}
33
34impl CcDepfile {
35 pub fn read(path: &Path) -> Result<Self, CcBypassReason> {
37 let contents =
38 std::fs::read_to_string(path).map_err(|error| CcBypassReason::DepfileRead {
39 path: path.to_path_buf(),
40 message: error.to_string(),
41 })?;
42 Self::parse(&contents)
43 }
44
45 pub fn read_for(path: &Path, family: CcCompilerFamily) -> Result<Self, CcBypassReason> {
47 if family.is_msvc() {
48 Self::read_msvc(path)
49 } else {
50 Self::read(path)
51 }
52 }
53
54 pub fn read_msvc(path: &Path) -> Result<Self, CcBypassReason> {
56 let contents = std::fs::read(path).map_err(|error| CcBypassReason::DepfileRead {
57 path: path.to_path_buf(),
58 message: error.to_string(),
59 })?;
60 let value: serde_json::Value = serde_json::from_slice(&contents)
61 .map_err(|error| CcBypassReason::MalformedDepfile(error.to_string()))?;
62 let data = value
63 .get("Data")
64 .and_then(serde_json::Value::as_object)
65 .ok_or_else(|| CcBypassReason::MalformedDepfile("missing Data object".into()))?;
66 if data
67 .get("ImportedModules")
68 .and_then(serde_json::Value::as_array)
69 .is_some_and(|modules| !modules.is_empty())
70 || data.get("ProvidedModule").is_some_and(|module| {
71 !module.is_null() && module.as_str().is_none_or(|s| !s.is_empty())
72 })
73 {
74 return Err(CcBypassReason::MalformedDepfile(
75 "C++ module dependencies are not modeled".into(),
76 ));
77 }
78 let includes = data
79 .get("Includes")
80 .and_then(serde_json::Value::as_array)
81 .ok_or_else(|| CcBypassReason::MalformedDepfile("missing Includes array".into()))?;
82 let files = includes
83 .iter()
84 .map(|entry| {
85 entry.as_str().map(PathBuf::from).ok_or_else(|| {
86 CcBypassReason::MalformedDepfile("non-string include path".into())
87 })
88 })
89 .collect::<Result<Vec<_>, _>>()?;
90 Ok(Self { files })
91 }
92
93 pub fn parse(contents: &str) -> Result<Self, CcBypassReason> {
99 let joined = join_continuations(contents)?;
100 let (_, prerequisites) = joined
101 .lines()
102 .find_map(|line| line.split_once(RULE_SEPARATOR))
103 .ok_or_else(|| CcBypassReason::MalformedDepfile("no dependency rule".into()))?;
104 let files = split_prerequisites(prerequisites)?;
105 Ok(Self { files })
106 }
107}
108
109const RULE_SEPARATOR: &str = ": ";
110
111impl CcDepfile {
112 pub fn render(
122 targets: &[crate::DepfileTarget],
123 files: &[PathBuf],
124 source: &Path,
125 phony_targets: bool,
126 ) -> String {
127 let mut rendered = String::new();
128 for (index, target) in targets.iter().enumerate() {
129 if index > 0 {
130 rendered.push(' ');
131 }
132 if target.quoted {
133 rendered.push_str(&escape_make_word(&target.name));
134 } else {
135 rendered.push_str(&target.name);
136 }
137 }
138 rendered.push(':');
139 for file in files {
140 rendered.push_str(" \\\n ");
141 rendered.push_str(&escape_make_word(&file.to_string_lossy()));
142 }
143 rendered.push('\n');
144 if phony_targets {
145 for file in files.iter().filter(|file| file.as_path() != source) {
146 rendered.push_str(&escape_make_word(&file.to_string_lossy()));
147 rendered.push_str(":\n");
148 }
149 }
150 rendered
151 }
152}
153
154fn escape_make_word(word: &str) -> String {
156 let mut escaped = String::with_capacity(word.len());
157 for character in word.chars() {
158 match character {
159 ' ' => escaped.push_str("\\ "),
160 '#' => escaped.push_str("\\#"),
161 '$' => escaped.push_str("$$"),
162 other => escaped.push(other),
163 }
164 }
165 escaped
166}
167
168fn join_continuations(contents: &str) -> Result<String, CcBypassReason> {
170 let mut joined = String::with_capacity(contents.len());
171 let mut continued = false;
172 for line in contents.lines() {
173 let trimmed = line.strip_suffix('\r').unwrap_or(line);
174 let (text, continues) = match trimmed.strip_suffix('\\') {
175 Some(text) => (text, true),
176 None => (trimmed, false),
177 };
178 if continued {
179 joined.push(' ');
180 }
181 joined.push_str(text.trim_end_matches(['\t']));
182 if !continues {
183 joined.push('\n');
184 }
185 continued = continues;
186 }
187 if continued {
188 return Err(CcBypassReason::MalformedDepfile(
189 "unterminated line continuation".into(),
190 ));
191 }
192 Ok(joined)
193}
194
195fn split_prerequisites(value: &str) -> Result<Vec<PathBuf>, CcBypassReason> {
200 let mut files = Vec::new();
201 let mut current = String::new();
202 let mut characters = value.chars().peekable();
203 while let Some(character) = characters.next() {
204 match character {
205 ' ' | '\t' => {
206 if !current.is_empty() {
207 files.push(PathBuf::from(std::mem::take(&mut current)));
208 }
209 }
210 '\\' => match characters.next() {
211 Some(' ') => current.push(' '),
212 Some('#') => current.push('#'),
213 Some(other) => {
214 return Err(CcBypassReason::MalformedDepfile(format!(
215 "unmodeled escape \\{other}"
216 )));
217 }
218 None => {
219 return Err(CcBypassReason::MalformedDepfile(
220 "trailing escape character".into(),
221 ));
222 }
223 },
224 '$' => match characters.next() {
225 Some('$') => current.push('$'),
226 Some(other) => {
227 return Err(CcBypassReason::MalformedDepfile(format!(
228 "unmodeled variable reference ${other}"
229 )));
230 }
231 None => {
232 return Err(CcBypassReason::MalformedDepfile(
233 "trailing variable reference".into(),
234 ));
235 }
236 },
237 other => current.push(other),
238 }
239 }
240 if !current.is_empty() {
241 files.push(PathBuf::from(current));
242 }
243 Ok(files)
244}
245
246#[derive(Debug, Clone, PartialEq, Eq)]
248pub struct CcDiscoveredInputs {
249 working_dir: PathBuf,
250 pub inputs: Vec<CcActionInput>,
252 identities: Vec<Option<FileIdentity>>,
256}
257
258impl CcDiscoveredInputs {
259 pub fn collect(
266 working_dir: &Path,
267 files: BTreeSet<PathBuf>,
268 directories: BTreeSet<PathBuf>,
269 digests: &dyn FileDigestCache,
270 ) -> Result<Self, CcBypassReason> {
271 if !working_dir.is_absolute() {
272 return Err(CcBypassReason::RelativeWorkingDirectory(
273 working_dir.to_path_buf(),
274 ));
275 }
276 let directories = minimal_manifest_directories(directories);
277 if files.len() + directories.len() > MAX_PREDICTED_INPUTS {
278 return Err(CcBypassReason::TooManyInputs);
279 }
280 let working_dir = normalize_components(working_dir);
281 let mut inputs = Vec::with_capacity(files.len() + directories.len());
282 let mut total_bytes = 0_u64;
283 let mut identified = Vec::with_capacity(files.len());
290 for path in files {
291 let metadata = std::fs::metadata(&path).map_err(|error| CcBypassReason::InputRead {
292 path: path.clone(),
293 message: error.to_string(),
294 })?;
295 if !metadata.is_file() {
296 return Err(CcBypassReason::InputRead {
297 path,
298 message: "input is not a regular file".into(),
299 });
300 }
301 total_bytes = total_bytes.saturating_add(metadata.len());
302 if total_bytes > MAX_INPUT_BYTES {
303 return Err(CcBypassReason::TooManyInputs);
304 }
305 let identity = FileIdentity::describe(&path, &metadata);
306 identified.push((path, identity));
307 }
308 let queries = identified
309 .iter()
310 .filter_map(|(_, identity)| identity.clone())
311 .collect::<Vec<_>>();
312 let mut recorded = digests.find(FileDigestScope::CcInput, &queries).into_iter();
313 let mut identities = Vec::with_capacity(inputs.capacity());
314 let mut fresh = Vec::new();
315 for (path, identity) in identified {
316 identities.push(identity.clone());
317 let remembered = identity
318 .as_ref()
319 .and_then(|_| recorded.next().flatten())
320 .filter(|digest| {
321 identity
322 .as_ref()
323 .is_some_and(|identity| identity.len == digest.size)
324 });
325 let digest = match remembered {
326 Some(digest) => digest,
327 None => {
328 if contains_timestamp_macro(&path)? {
329 return Err(CcBypassReason::EmbeddedTimestampMacro(path));
330 }
331 let digest = CacheDigest::blake3_file(&path).map_err(|error| {
332 CcBypassReason::InputRead {
333 path: path.clone(),
334 message: error.to_string(),
335 }
336 })?;
337 if let Some(identity) = identity
338 && identity.len == digest.size
339 {
340 fresh.push(RecordedFileDigest {
341 file: identity,
342 digest: digest.clone(),
343 });
344 }
345 digest
346 }
347 };
348 inputs.push(CcActionInput { path, digest });
349 }
350 if !fresh.is_empty() {
351 digests.record(FileDigestScope::CcInput, fresh);
352 }
353 let mut manifest_entries = 0_usize;
354 for directory in directories {
355 let digest = include_manifest(&directory, &mut manifest_entries)?;
356 inputs.push(CcActionInput {
357 path: PathBuf::from(format!("{INCLUDE_MANIFEST_PREFIX}{}", directory.display())),
358 digest,
359 });
360 identities.push(None);
361 }
362 Ok(Self {
363 working_dir,
364 inputs,
365 identities,
366 })
367 }
368
369 pub fn files(&self) -> impl Iterator<Item = &CcActionInput> {
371 self.inputs
372 .iter()
373 .filter(|input| !is_manifest_input(&input.path))
374 }
375
376 pub fn verify_not_modified_since(&self, started_at: SystemTime) -> Result<(), CcBypassReason> {
383 for input in self.files() {
384 let modified = std::fs::metadata(&input.path)
385 .and_then(|metadata| metadata.modified())
386 .map_err(|error| CcBypassReason::InputRead {
387 path: input.path.clone(),
388 message: error.to_string(),
389 })?;
390 if modified >= started_at {
391 return Err(CcBypassReason::InputModifiedDuringCompilation(
392 input.path.clone(),
393 ));
394 }
395 }
396 Ok(())
397 }
398
399 pub fn verify(&self) -> Result<(), CcBypassReason> {
408 for (index, input) in self.inputs.iter().enumerate() {
409 if is_manifest_input(&input.path) {
410 continue;
411 }
412 let read_error = |error: std::io::Error| CcBypassReason::InputRead {
413 path: input.path.clone(),
414 message: error.to_string(),
415 };
416 if let Some(Some(identity)) = self.identities.get(index)
417 && identity.changed.is_some()
418 && identity.still_describes().map_err(read_error)?
419 {
420 continue;
421 }
422 let matches = input.digest.matches_file(&input.path).map_err(|error| {
423 CcBypassReason::InputRead {
424 path: input.path.clone(),
425 message: error.to_string(),
426 }
427 })?;
428 if !matches {
429 return Err(CcBypassReason::InputChanged(input.path.clone()));
430 }
431 }
432 Ok(())
433 }
434
435 pub fn apply_to(self, context: &mut CcActionContext) -> Result<(), CcBypassReason> {
438 if normalize_components(&context.working_dir) != self.working_dir {
439 return Err(CcBypassReason::DiscoveryWorkingDirectory);
440 }
441 context.inputs.extend(self.inputs);
442 Ok(())
443 }
444}
445
446fn minimal_manifest_directories(directories: BTreeSet<PathBuf>) -> Vec<PathBuf> {
454 let mut directories = directories
455 .into_iter()
456 .map(|directory| {
457 let normalized = normalize_components(&directory);
458 (directory, normalized)
459 })
460 .collect::<Vec<_>>();
461 directories.sort_by(|(left, left_normalized), (right, right_normalized)| {
462 left_normalized
463 .components()
464 .count()
465 .cmp(&right_normalized.components().count())
466 .then_with(|| left_normalized.cmp(right_normalized))
467 .then_with(|| left.cmp(right))
468 });
469
470 let mut minimal = Vec::<(PathBuf, PathBuf)>::new();
471 for (directory, normalized) in directories {
472 if !minimal
473 .iter()
474 .any(|(_, ancestor)| manifest_covers(ancestor, &normalized))
475 {
476 minimal.push((directory, normalized));
477 }
478 }
479 minimal
480 .into_iter()
481 .map(|(directory, _)| directory)
482 .collect()
483}
484
485fn manifest_covers(ancestor: &Path, descendant: &Path) -> bool {
492 let Ok(relative) = descendant.strip_prefix(ancestor) else {
493 return false;
494 };
495 if relative.as_os_str().is_empty() {
496 return false;
497 }
498 let mut current = ancestor.to_path_buf();
499 for component in relative.components() {
500 current.push(component);
501 let Ok(metadata) = std::fs::symlink_metadata(¤t) else {
502 return false;
503 };
504 if !metadata.is_dir() || metadata.file_type().is_symlink() {
505 return false;
506 }
507 }
508 true
509}
510
511fn is_manifest_input(path: &Path) -> bool {
512 path.to_str()
513 .is_some_and(|path| path.starts_with(INCLUDE_MANIFEST_PREFIX))
514}
515
516pub fn manifest_snapshot(
524 directories: &BTreeSet<PathBuf>,
525) -> Result<BTreeMap<PathBuf, CacheDigest>, CcBypassReason> {
526 let mut budget = 0_usize;
527 minimal_manifest_directories(directories.iter().cloned().collect())
528 .into_iter()
529 .map(|directory| {
530 include_manifest(&directory, &mut budget).map(|digest| (directory, digest))
531 })
532 .collect()
533}
534
535const INCLUDABLE_EXTENSIONS: &[&str] = &[
546 "c", "c++", "cc", "cpp", "cxx", "def", "gch", "h", "h++", "hh", "hpp", "hxx", "inc", "inl",
547 "ipp", "pch", "s", "tcc",
548];
549
550fn is_includable(name: &str) -> bool {
559 match name.rsplit_once('.') {
560 Some((stem, extension)) if !stem.is_empty() => INCLUDABLE_EXTENSIONS
561 .binary_search(&extension.to_ascii_lowercase().as_str())
562 .is_ok(),
563 _ => !name.starts_with('.'),
565 }
566}
567
568fn include_manifest(directory: &Path, budget: &mut usize) -> Result<CacheDigest, CcBypassReason> {
575 let mut names = Vec::new();
576 let mut pending = vec![(directory.to_path_buf(), String::new())];
577 while let Some((current, prefix)) = pending.pop() {
578 let entries = match std::fs::read_dir(¤t) {
579 Ok(entries) => entries,
580 Err(error) if error.kind() == std::io::ErrorKind::NotFound => continue,
581 Err(error) => {
582 return Err(CcBypassReason::InputRead {
583 path: current,
584 message: error.to_string(),
585 });
586 }
587 };
588 for entry in entries {
589 let entry = entry.map_err(|error| CcBypassReason::InputRead {
590 path: current.clone(),
591 message: error.to_string(),
592 })?;
593 let name = entry.file_name();
594 let Some(name) = name.to_str() else {
595 return Err(CcBypassReason::NonUtf8Path(entry.path()));
596 };
597 let relative = if prefix.is_empty() {
598 name.to_string()
599 } else {
600 format!("{prefix}/{name}")
601 };
602 let file_type = entry
603 .file_type()
604 .map_err(|error| CcBypassReason::InputRead {
605 path: entry.path(),
606 message: error.to_string(),
607 })?;
608 if file_type.is_dir() {
609 pending.push((entry.path(), relative));
610 continue;
611 }
612 if !is_includable(name) {
613 continue;
614 }
615 *budget += 1;
616 if *budget > MAX_MANIFEST_ENTRIES {
617 return Err(CcBypassReason::TooManyInputs);
618 }
619 names.push(relative);
620 }
621 }
622 names.sort();
623 Ok(CacheDigest::blake3(names.join("\n").as_bytes()))
624}
625
626fn contains_timestamp_macro(path: &Path) -> Result<bool, CcBypassReason> {
633 let file = std::fs::File::open(path).map_err(|error| CcBypassReason::InputRead {
634 path: path.to_path_buf(),
635 message: error.to_string(),
636 })?;
637 let longest = TIMESTAMP_MACROS
638 .iter()
639 .map(|macro_name| macro_name.len())
640 .max()
641 .unwrap_or_default();
642 let mut reader = std::io::BufReader::new(file);
643 let mut window = Vec::with_capacity(SCAN_CHUNK_BYTES + longest);
644 let mut chunk = vec![0_u8; SCAN_CHUNK_BYTES];
645 loop {
646 let read = reader
647 .read(&mut chunk)
648 .map_err(|error| CcBypassReason::InputRead {
649 path: path.to_path_buf(),
650 message: error.to_string(),
651 })?;
652 if read == 0 {
653 return Ok(false);
654 }
655 window.extend_from_slice(&chunk[..read]);
656 if TIMESTAMP_MACROS
657 .iter()
658 .any(|macro_name| contains_subslice(&window, macro_name))
659 {
660 return Ok(true);
661 }
662 let keep = window.len().saturating_sub(longest.saturating_sub(1));
664 window.drain(..keep);
665 }
666}
667
668pub(crate) fn contains_assembler_input_directive(path: &Path) -> Result<bool, CcBypassReason> {
675 contains_any(path, ASSEMBLER_INPUT_DIRECTIVES)
676}
677
678fn contains_any(path: &Path, needles: &[&[u8]]) -> Result<bool, CcBypassReason> {
679 let file = std::fs::File::open(path).map_err(|error| CcBypassReason::InputRead {
680 path: path.to_path_buf(),
681 message: error.to_string(),
682 })?;
683 let longest = needles
684 .iter()
685 .map(|needle| needle.len())
686 .max()
687 .unwrap_or_default();
688 let mut reader = std::io::BufReader::new(file);
689 let mut window = Vec::with_capacity(SCAN_CHUNK_BYTES + longest);
690 let mut chunk = vec![0_u8; SCAN_CHUNK_BYTES];
691 loop {
692 let read = reader
693 .read(&mut chunk)
694 .map_err(|error| CcBypassReason::InputRead {
695 path: path.to_path_buf(),
696 message: error.to_string(),
697 })?;
698 if read == 0 {
699 return Ok(false);
700 }
701 window.extend_from_slice(&chunk[..read]);
702 if needles
703 .iter()
704 .any(|needle| contains_subslice_ascii_case_insensitive(&window, needle))
705 {
706 return Ok(true);
707 }
708 let keep = window.len().saturating_sub(longest.saturating_sub(1));
709 window.drain(..keep);
710 }
711}
712
713fn contains_subslice_ascii_case_insensitive(haystack: &[u8], needle: &[u8]) -> bool {
714 if needle.is_empty() || haystack.len() < needle.len() {
715 return false;
716 }
717 haystack.windows(needle.len()).any(|window| {
718 window
719 .iter()
720 .zip(needle)
721 .all(|(left, right)| left.eq_ignore_ascii_case(right))
722 })
723}
724
725fn contains_subslice(haystack: &[u8], needle: &[u8]) -> bool {
726 if needle.is_empty() || haystack.len() < needle.len() {
727 return false;
728 }
729 haystack
730 .windows(needle.len())
731 .any(|window| window == needle)
732}
733
734#[cfg(test)]
735#[path = "depfile_tests.rs"]
736mod tests;