mbx_cache_cc/
depfile.rs

1//! Dependency-list parsing and input discovery for C and C++ compiles.
2
3use crate::{
4    CcActionContext, CcActionInput, CcBypassReason, MAX_INPUT_BYTES, MAX_MANIFEST_ENTRIES,
5    MAX_PREDICTED_INPUTS, normalize_components,
6};
7use mbx_cache_core::CacheDigest;
8use std::collections::{BTreeMap, BTreeSet};
9use std::io::Read;
10use std::path::{Path, PathBuf};
11use std::time::SystemTime;
12
13/// Marker distinguishing an include-directory name manifest from a file input.
14pub const INCLUDE_MANIFEST_PREFIX: &str = "@include-manifest:";
15
16/// Macros whose expansion is not a function of the compilation's inputs.
17const TIMESTAMP_MACROS: &[&[u8]] = &[b"__DATE__", b"__TIME__", b"__TIMESTAMP__"];
18
19const SCAN_CHUNK_BYTES: usize = 64 * 1024;
20
21/// A parsed GNU-style dependency list.
22#[derive(Debug, Clone, PartialEq, Eq, Default)]
23pub struct CcDepfile {
24    /// Prerequisite files named by the first rule.
25    pub files: Vec<PathBuf>,
26}
27
28impl CcDepfile {
29    /// Read and parse the dependency list the compiler wrote.
30    pub fn read(path: &Path) -> Result<Self, CcBypassReason> {
31        let contents =
32            std::fs::read_to_string(path).map_err(|error| CcBypassReason::DepfileRead {
33                path: path.to_path_buf(),
34                message: error.to_string(),
35            })?;
36        Self::parse(&contents)
37    }
38
39    /// Parse a GNU-style dependency list.
40    ///
41    /// Only the first rule is read. The adapter never passes `-MP`, so a
42    /// well-formed file the adapter asked for has exactly one rule, and
43    /// anything further is ignored rather than guessed at.
44    pub fn parse(contents: &str) -> Result<Self, CcBypassReason> {
45        let joined = join_continuations(contents)?;
46        let (_, prerequisites) = joined
47            .lines()
48            .find_map(|line| line.split_once(RULE_SEPARATOR))
49            .ok_or_else(|| CcBypassReason::MalformedDepfile("no dependency rule".into()))?;
50        let files = split_prerequisites(prerequisites)?;
51        Ok(Self { files })
52    }
53}
54
55const RULE_SEPARATOR: &str = ": ";
56
57/// Join physical lines the compiler wrapped with a trailing backslash.
58fn join_continuations(contents: &str) -> Result<String, CcBypassReason> {
59    let mut joined = String::with_capacity(contents.len());
60    let mut continued = false;
61    for line in contents.lines() {
62        let trimmed = line.strip_suffix('\r').unwrap_or(line);
63        let (text, continues) = match trimmed.strip_suffix('\\') {
64            Some(text) => (text, true),
65            None => (trimmed, false),
66        };
67        if continued {
68            joined.push(' ');
69        }
70        joined.push_str(text.trim_end_matches(['\t']));
71        if !continues {
72            joined.push('\n');
73        }
74        continued = continues;
75    }
76    if continued {
77        return Err(CcBypassReason::MalformedDepfile(
78            "unterminated line continuation".into(),
79        ));
80    }
81    Ok(joined)
82}
83
84/// Split a prerequisite list, honoring exactly the escapes make defines.
85///
86/// Anything else escaped is a spelling this parser does not model, and a
87/// mis-parsed prerequisite would silently drop an input from the key.
88fn split_prerequisites(value: &str) -> Result<Vec<PathBuf>, CcBypassReason> {
89    let mut files = Vec::new();
90    let mut current = String::new();
91    let mut characters = value.chars().peekable();
92    while let Some(character) = characters.next() {
93        match character {
94            ' ' | '\t' => {
95                if !current.is_empty() {
96                    files.push(PathBuf::from(std::mem::take(&mut current)));
97                }
98            }
99            '\\' => match characters.next() {
100                Some(' ') => current.push(' '),
101                Some('#') => current.push('#'),
102                Some(other) => {
103                    return Err(CcBypassReason::MalformedDepfile(format!(
104                        "unmodeled escape \\{other}"
105                    )));
106                }
107                None => {
108                    return Err(CcBypassReason::MalformedDepfile(
109                        "trailing escape character".into(),
110                    ));
111                }
112            },
113            '$' => match characters.next() {
114                Some('$') => current.push('$'),
115                Some(other) => {
116                    return Err(CcBypassReason::MalformedDepfile(format!(
117                        "unmodeled variable reference ${other}"
118                    )));
119                }
120                None => {
121                    return Err(CcBypassReason::MalformedDepfile(
122                        "trailing variable reference".into(),
123                    ));
124                }
125            },
126            other => current.push(other),
127        }
128    }
129    if !current.is_empty() {
130        files.push(PathBuf::from(current));
131    }
132    Ok(files)
133}
134
135/// A complete, content-addressed compiler input manifest.
136#[derive(Debug, Clone, PartialEq, Eq)]
137pub struct CcDiscoveredInputs {
138    working_dir: PathBuf,
139    /// Content-addressed inputs, including include-directory manifests.
140    pub inputs: Vec<CcActionInput>,
141}
142
143impl CcDiscoveredInputs {
144    /// Digest every file the compilation read, and summarize the directories it
145    /// searched.
146    ///
147    /// Digesting the files answers "did any input change". The directory
148    /// manifests answer the question a dependency list cannot: whether a header
149    /// that was *not* read now exists somewhere that would shadow one that was.
150    pub fn collect(
151        working_dir: &Path,
152        files: BTreeSet<PathBuf>,
153        directories: BTreeSet<PathBuf>,
154    ) -> Result<Self, CcBypassReason> {
155        if !working_dir.is_absolute() {
156            return Err(CcBypassReason::RelativeWorkingDirectory(
157                working_dir.to_path_buf(),
158            ));
159        }
160        if files.len() + directories.len() > MAX_PREDICTED_INPUTS {
161            return Err(CcBypassReason::TooManyInputs);
162        }
163        let working_dir = normalize_components(working_dir);
164        let mut inputs = Vec::with_capacity(files.len() + directories.len());
165        let mut total_bytes = 0_u64;
166        for path in files {
167            let metadata = std::fs::metadata(&path).map_err(|error| CcBypassReason::InputRead {
168                path: path.clone(),
169                message: error.to_string(),
170            })?;
171            if !metadata.is_file() {
172                return Err(CcBypassReason::InputRead {
173                    path,
174                    message: "input is not a regular file".into(),
175                });
176            }
177            total_bytes = total_bytes.saturating_add(metadata.len());
178            if total_bytes > MAX_INPUT_BYTES {
179                return Err(CcBypassReason::TooManyInputs);
180            }
181            if contains_timestamp_macro(&path)? {
182                return Err(CcBypassReason::EmbeddedTimestampMacro(path));
183            }
184            let digest =
185                CacheDigest::blake3_file(&path).map_err(|error| CcBypassReason::InputRead {
186                    path: path.clone(),
187                    message: error.to_string(),
188                })?;
189            inputs.push(CcActionInput { path, digest });
190        }
191        let mut manifest_entries = 0_usize;
192        for directory in directories {
193            let digest = include_manifest(&directory, &mut manifest_entries)?;
194            inputs.push(CcActionInput {
195                path: PathBuf::from(format!("{INCLUDE_MANIFEST_PREFIX}{}", directory.display())),
196                digest,
197            });
198        }
199        Ok(Self {
200            working_dir,
201            inputs,
202        })
203    }
204
205    /// File inputs, excluding include-directory manifests.
206    pub fn files(&self) -> impl Iterator<Item = &CcActionInput> {
207        self.inputs
208            .iter()
209            .filter(|input| !is_manifest_input(&input.path))
210    }
211
212    /// Reject inputs whose modification time overlaps the compiler invocation.
213    ///
214    /// Contents are hashed after the compiler reports the paths it read. This
215    /// timestamp barrier prevents a write that landed during the compile from
216    /// being mistaken for the contents that produced the object; `verify`
217    /// closes the remaining race after hashing.
218    pub fn verify_not_modified_since(&self, started_at: SystemTime) -> Result<(), CcBypassReason> {
219        for input in self.files() {
220            let modified = std::fs::metadata(&input.path)
221                .and_then(|metadata| metadata.modified())
222                .map_err(|error| CcBypassReason::InputRead {
223                    path: input.path.clone(),
224                    message: error.to_string(),
225                })?;
226            if modified >= started_at {
227                return Err(CcBypassReason::InputModifiedDuringCompilation(
228                    input.path.clone(),
229                ));
230            }
231        }
232        Ok(())
233    }
234
235    /// Rehash every discovered file before publication, degrading a changed
236    /// input to a miss rather than storing an object under a stale key.
237    pub fn verify(&self) -> Result<(), CcBypassReason> {
238        for input in self.files() {
239            let matches = input.digest.matches_file(&input.path).map_err(|error| {
240                CcBypassReason::InputRead {
241                    path: input.path.clone(),
242                    message: error.to_string(),
243                }
244            })?;
245            if !matches {
246                return Err(CcBypassReason::InputChanged(input.path.clone()));
247            }
248        }
249        Ok(())
250    }
251
252    /// Merge the manifest into an action context after confirming both use the
253    /// same compiler working directory.
254    pub fn apply_to(self, context: &mut CcActionContext) -> Result<(), CcBypassReason> {
255        if normalize_components(&context.working_dir) != self.working_dir {
256            return Err(CcBypassReason::DiscoveryWorkingDirectory);
257        }
258        context.inputs.extend(self.inputs);
259        Ok(())
260    }
261}
262
263fn is_manifest_input(path: &Path) -> bool {
264    path.to_str()
265        .is_some_and(|path| path.starts_with(INCLUDE_MANIFEST_PREFIX))
266}
267
268/// Digest the includable names in each directory, reading no file contents.
269///
270/// Taken once before the compiler runs and again before publishing, this is
271/// what detects a header that appeared in a search directory *while* the
272/// compilation was in flight. The manifest recorded in the key is the one from
273/// after the compile, and without this check that later state would be claimed
274/// as the state the compiler saw.
275pub fn manifest_snapshot(
276    directories: &BTreeSet<PathBuf>,
277) -> Result<BTreeMap<PathBuf, CacheDigest>, CcBypassReason> {
278    let mut budget = 0_usize;
279    directories
280        .iter()
281        .map(|directory| {
282            include_manifest(directory, &mut budget).map(|digest| (directory.clone(), digest))
283        })
284        .collect()
285}
286
287/// Extensions a file must carry to be a plausible `#include` target.
288///
289/// An extensionless name also qualifies: C++ standard headers are spelled that
290/// way and projects ship their own.
291///
292/// `gch` and `pch` are here because a precompiled header answers an `#include`
293/// without being named by one. GCC prefers `foo.h.gch` over `foo.h` on its own,
294/// with nothing on the command line to say so, which is precisely the
295/// substitution these manifests exist to notice -- and the one case the
296/// adapter's explicit precompiled-header bypass cannot see.
297const INCLUDABLE_EXTENSIONS: &[&str] = &[
298    "c", "c++", "cc", "cpp", "cxx", "def", "gch", "h", "h++", "hh", "hpp", "hxx", "inc", "inl",
299    "ipp", "pch", "tcc",
300];
301
302/// Whether a file name could be what an `#include` directive names.
303///
304/// The manifest exists to notice a file appearing where it would shadow a
305/// header that was read. A build writes its own objects, dependency files, and
306/// archives into these directories -- often the very directory a generated
307/// header lives in -- and none of those can shadow an include. Counting them
308/// would make the key depend on how many sibling compilations had finished,
309/// which is not a property of this compilation at all.
310fn is_includable(name: &str) -> bool {
311    match name.rsplit_once('.') {
312        Some((stem, extension)) if !stem.is_empty() => INCLUDABLE_EXTENSIONS
313            .binary_search(&extension.to_ascii_lowercase().as_str())
314            .is_ok(),
315        // No extension, or a leading-dot name like `.keep`.
316        _ => !name.starts_with('.'),
317    }
318}
319
320/// Digest the sorted includable file names beneath a directory.
321///
322/// Names only: the contents of anything actually read are digested as inputs,
323/// so this exists purely to notice a file appearing where it could shadow one
324/// of them. A directory that does not exist has an empty manifest, which is
325/// what makes "the directory was created" a key change rather than an error.
326fn include_manifest(directory: &Path, budget: &mut usize) -> Result<CacheDigest, CcBypassReason> {
327    let mut names = Vec::new();
328    let mut pending = vec![(directory.to_path_buf(), String::new())];
329    while let Some((current, prefix)) = pending.pop() {
330        let entries = match std::fs::read_dir(&current) {
331            Ok(entries) => entries,
332            Err(error) if error.kind() == std::io::ErrorKind::NotFound => continue,
333            Err(error) => {
334                return Err(CcBypassReason::InputRead {
335                    path: current,
336                    message: error.to_string(),
337                });
338            }
339        };
340        for entry in entries {
341            let entry = entry.map_err(|error| CcBypassReason::InputRead {
342                path: current.clone(),
343                message: error.to_string(),
344            })?;
345            let name = entry.file_name();
346            let Some(name) = name.to_str() else {
347                return Err(CcBypassReason::NonUtf8Path(entry.path()));
348            };
349            let relative = if prefix.is_empty() {
350                name.to_string()
351            } else {
352                format!("{prefix}/{name}")
353            };
354            let file_type = entry
355                .file_type()
356                .map_err(|error| CcBypassReason::InputRead {
357                    path: entry.path(),
358                    message: error.to_string(),
359                })?;
360            if file_type.is_dir() {
361                pending.push((entry.path(), relative));
362                continue;
363            }
364            if !is_includable(name) {
365                continue;
366            }
367            *budget += 1;
368            if *budget > MAX_MANIFEST_ENTRIES {
369                return Err(CcBypassReason::TooManyInputs);
370            }
371            names.push(relative);
372        }
373    }
374    names.sort();
375    Ok(CacheDigest::blake3(names.join("\n").as_bytes()))
376}
377
378/// Whether a file mentions a macro whose expansion is not a function of the
379/// compilation's inputs.
380///
381/// The token is looked for rather than its expansion: a match inside a comment
382/// or a string literal bypasses a compilation that would in fact have been
383/// cacheable, which is the conservative direction.
384fn contains_timestamp_macro(path: &Path) -> Result<bool, CcBypassReason> {
385    let file = std::fs::File::open(path).map_err(|error| CcBypassReason::InputRead {
386        path: path.to_path_buf(),
387        message: error.to_string(),
388    })?;
389    let longest = TIMESTAMP_MACROS
390        .iter()
391        .map(|macro_name| macro_name.len())
392        .max()
393        .unwrap_or_default();
394    let mut reader = std::io::BufReader::new(file);
395    let mut window = Vec::with_capacity(SCAN_CHUNK_BYTES + longest);
396    let mut chunk = vec![0_u8; SCAN_CHUNK_BYTES];
397    loop {
398        let read = reader
399            .read(&mut chunk)
400            .map_err(|error| CcBypassReason::InputRead {
401                path: path.to_path_buf(),
402                message: error.to_string(),
403            })?;
404        if read == 0 {
405            return Ok(false);
406        }
407        window.extend_from_slice(&chunk[..read]);
408        if TIMESTAMP_MACROS
409            .iter()
410            .any(|macro_name| contains_subslice(&window, macro_name))
411        {
412            return Ok(true);
413        }
414        // Keep the tail so a token split across two reads is still found.
415        let keep = window.len().saturating_sub(longest.saturating_sub(1));
416        window.drain(..keep);
417    }
418}
419
420fn contains_subslice(haystack: &[u8], needle: &[u8]) -> bool {
421    if needle.is_empty() || haystack.len() < needle.len() {
422        return false;
423    }
424    haystack
425        .windows(needle.len())
426        .any(|window| window == needle)
427}
428
429#[cfg(test)]
430#[path = "depfile_tests.rs"]
431mod tests;
mbx_cache_cc/depfile.rs

mbx_cache_cc/
depfile.rs
