cuke_dedup/
modes.rs

1//! Changed-files and existing-findings baseline modes for incremental CI adoption.
2
3use crate::config::normalize_platform_path;
4use crate::model::{stable_fingerprint_parts, Finding, Suppression};
5use crate::resource_limits::{read_utf8, MAX_BASELINE_INPUT_BYTES};
6use anyhow::{bail, Context, Result};
7use serde::{Deserialize, Serialize};
8use std::collections::{BTreeMap, BTreeSet};
9use std::fs;
10use std::io::Read;
11use std::path::{Path, PathBuf};
12use std::process::Command;
13
14// Snapshotting is additional disk work, separate from the analyzer's per-source read limits.
15const MAX_BASELINE_CHECKOUT_BYTES: u64 = 512 * 1024 * 1024;
16
17/// Returns tracked changes since `base` plus untracked files beneath `root`.
18pub fn git_changed_files(root: &Path, base: &str) -> Result<BTreeSet<PathBuf>> {
19    let base_oid = resolve_commit(root, base)?;
20    let tracked = git_paths(
21        root,
22        &[
23            "diff",
24            "--relative",
25            "--name-only",
26            "-z",
27            "--diff-filter=ACMR",
28            &base_oid,
29            "--",
30        ],
31        &format!("git diff against `{base}` failed"),
32    )?;
33    let untracked = git_paths(
34        root,
35        &["ls-files", "--others", "--exclude-standard", "-z", "--"],
36        "git untracked-file discovery failed",
37    )?;
38    Ok(tracked
39        .into_iter()
40        .chain(untracked)
41        .map(|path| root.join(path))
42        .collect())
43}
44
45/// Git environment variables that a `git` process exports to the hooks it runs. They outrank a
46/// child invocation's `-C`, so leaving them in place would point changed-files mode at whichever
47/// repository invoked the hook instead of the analyzed root.
48const INHERITED_GIT_VARIABLES: [&str; 8] = [
49    "GIT_DIR",
50    "GIT_WORK_TREE",
51    "GIT_INDEX_FILE",
52    "GIT_OBJECT_DIRECTORY",
53    "GIT_ALTERNATE_OBJECT_DIRECTORIES",
54    "GIT_COMMON_DIR",
55    "GIT_PREFIX",
56    "GIT_CEILING_DIRECTORIES",
57];
58
59/// Builds a `git` invocation bound to `root` and nothing else, so results describe the analyzed
60/// tree whether CukeDedup runs from a shell, CI, or another repository's Git hook.
61fn git_at(root: &Path) -> Command {
62    let mut command = Command::new("git");
63    for variable in INHERITED_GIT_VARIABLES {
64        command.env_remove(variable);
65    }
66    command.arg("-C").arg(root);
67    command
68}
69
70fn resolve_commit(root: &Path, base: &str) -> Result<String> {
71    if base.trim().is_empty() || base.starts_with('-') {
72        bail!("invalid --changed-since revision `{base}`");
73    }
74    let revision = format!("{base}^{{commit}}");
75    let output = git_at(root)
76        .args(["rev-parse", "--verify", "--end-of-options"])
77        .arg(&revision)
78        .output()
79        .with_context(|| "failed to resolve the --changed-since revision")?;
80    if !output.status.success() {
81        let stderr = String::from_utf8_lossy(&output.stderr).trim().to_owned();
82        bail!("could not resolve --changed-since revision `{base}`: {stderr}");
83    }
84    let oid = String::from_utf8(output.stdout)
85        .context("git returned a non-UTF-8 commit object id")?
86        .trim()
87        .to_owned();
88    if oid.is_empty() || !oid.bytes().all(|byte| byte.is_ascii_hexdigit()) {
89        bail!("git returned an invalid commit object id for `{base}`");
90    }
91    Ok(oid)
92}
93
94/// Materializes a fetched commit without running checkout hooks or configured filters.
95/// An independent local clone avoids registering worktrees or modifying the caller's index.
96pub(crate) fn baseline_snapshot(
97    root: &Path,
98    revision: &str,
99) -> Result<(tempfile::TempDir, PathBuf)> {
100    if revision.trim().is_empty() || revision.starts_with('-') {
101        bail!("invalid baseline revision `{revision}`");
102    }
103    let temporary = tempfile::tempdir().context("cannot create temporary baseline checkout")?;
104    let git = |directory: &Path, args: &[&std::ffi::OsStr]| -> Result<Vec<u8>> {
105        let mut command = git_at(directory);
106        // Neither caller-supplied Git config nor global templates/filters may execute code.
107        // Clone does not copy the source repository's local config or hooks.
108        for (name, _) in std::env::vars_os() {
109            // Windows preserves spelling but treats environment names case-insensitively.
110            if name
111                .to_string_lossy()
112                .to_ascii_uppercase()
113                .starts_with("GIT_")
114            {
115                command.env_remove(name);
116            }
117        }
118        command
119            .env("GIT_CONFIG_NOSYSTEM", "1")
120            .env("GIT_NO_REPLACE_OBJECTS", "1")
121            .env("GIT_NO_LAZY_FETCH", "1")
122            .env("GIT_TERMINAL_PROMPT", "0")
123            .env("GIT_CONFIG_GLOBAL", temporary.path().join("no-config"))
124            .arg("-c")
125            .arg(format!(
126                "core.hooksPath={}",
127                temporary.path().join("no-hooks").display()
128            ))
129            .args(["-c", "core.fsmonitor=false"])
130            .args(args);
131        if args.first().is_some_and(|arg| *arg == "ls-tree") {
132            // Fixed format: six mode digits, space, at most 20 size digits, newline.
133            let limit = 28 * crate::resource_limits::MAX_WORKSPACE_SCAN_ENTRIES as u64;
134            let mut child = command
135                .stdout(std::process::Stdio::piped())
136                .stderr(std::process::Stdio::null())
137                .spawn()?;
138            let mut bytes = Vec::new();
139            let read = child
140                .stdout
141                .take()
142                .context("missing baseline Git stdout")
143                .and_then(|stdout| Ok(stdout.take(limit + 1).read_to_end(&mut bytes)?));
144            if read.is_err() || bytes.len() as u64 > limit {
145                let _ = child.kill();
146                let _ = child.wait();
147                read?;
148                bail!("baseline tree listing exceeds the bounded snapshot metadata limit");
149            }
150            if !child.wait()?.success() {
151                bail!("baseline Git tree enumeration failed");
152            }
153            return Ok(bytes);
154        }
155        let output = command.output().context("failed to run Git for baseline")?;
156        if !output.status.success() {
157            bail!(
158                "baseline Git operation failed: {}",
159                String::from_utf8_lossy(&output.stderr).trim()
160            );
161        }
162        Ok(output.stdout)
163    };
164    let oid = git(
165        root,
166        &[
167            "rev-parse".as_ref(),
168            "--verify".as_ref(),
169            "--end-of-options".as_ref(),
170            format!("{revision}^{{commit}}").as_ref(),
171        ],
172    )?;
173    let oid = String::from_utf8(oid).context("invalid baseline commit id")?;
174    let tree = git(
175        root,
176        &[
177            "ls-tree".as_ref(),
178            "--full-tree".as_ref(),
179            "-r".as_ref(),
180            "--format=%(objectmode) %(objectsize)".as_ref(),
181            oid.trim().as_ref(),
182        ],
183    )?;
184    let mut total_bytes = 0_u64;
185    for (index, entry) in String::from_utf8(tree)?.lines().enumerate() {
186        if entry.starts_with("160000 ") {
187            bail!("baseline checkout contains unsupported submodules");
188        }
189        let (_, size) = entry
190            .split_once(' ')
191            .context("invalid baseline tree entry")?;
192        total_bytes = total_bytes.saturating_add(size.parse::<u64>()?);
193        if total_bytes > MAX_BASELINE_CHECKOUT_BYTES
194            || index >= crate::resource_limits::MAX_WORKSPACE_SCAN_ENTRIES
195        {
196            bail!("baseline checkout exceeds the 512 MiB / 100000-file snapshot limit");
197        }
198    }
199    let repository = git(root, &["rev-parse".as_ref(), "--show-toplevel".as_ref()])?;
200    // Remove Git's one line terminator, not whitespace belonging to the path.
201    let repository = crate::config::normalize_platform_path(path_from_git_bytes(
202        repository.strip_suffix(b"\n").unwrap_or(&repository),
203    )?);
204    let relative = root
205        .strip_prefix(&repository)
206        .context("baseline root is outside repository")?;
207    let checkout = temporary.path().join("checkout");
208    git(
209        &repository,
210        &[
211            "clone".as_ref(),
212            "--shared".as_ref(),
213            "--no-checkout".as_ref(),
214            "--template=".as_ref(),
215            "--".as_ref(),
216            repository.as_os_str(),
217            checkout.as_os_str(),
218        ],
219    )?;
220    git(
221        &checkout,
222        &[
223            "checkout".as_ref(),
224            "--detach".as_ref(),
225            oid.trim().as_ref(),
226        ],
227    )?;
228    let scoped = checkout.join(relative);
229    if !scoped.is_dir() {
230        bail!("baseline revision does not contain the analysis directory");
231    }
232    let scoped = crate::config::normalize_platform_path(scoped.canonicalize()?);
233    if !scoped.starts_with(crate::config::normalize_platform_path(
234        checkout.canonicalize()?,
235    )) {
236        bail!("baseline analysis directory escapes its checkout");
237    }
238    Ok((temporary, scoped))
239}
240
241/// Rejects changed-file mode when the analyzed root itself is excluded by Git.
242pub fn ensure_changed_root_is_trackable(root: &Path) -> Result<()> {
243    let root = normalize_platform_path(root.to_path_buf());
244    let repository = git_at(&root)
245        .args(["rev-parse", "--show-toplevel"])
246        .output()
247        .with_context(|| "failed to locate the Git repository for changed-files mode")?;
248    if !repository.status.success() {
249        bail!(
250            "changed-files mode requires a Git repository: {}",
251            String::from_utf8_lossy(&repository.stderr).trim()
252        );
253    }
254    let repository = normalize_platform_path(
255        PathBuf::from(
256            String::from_utf8(repository.stdout)
257                .context("git returned a non-UTF-8 repository path")?
258                .trim(),
259        )
260        .canonicalize()
261        .with_context(|| "failed to canonicalize the Git repository root")?,
262    );
263    if root == repository {
264        return Ok(());
265    }
266    let relative = root.strip_prefix(&repository).with_context(|| {
267        format!(
268            "target {} is outside Git repository {}",
269            root.display(),
270            repository.display()
271        )
272    })?;
273    let ignored = git_at(&repository)
274        .args(["check-ignore", "--quiet", "--"])
275        .arg(relative)
276        .status()
277        .with_context(|| "failed to check whether the changed-files target is ignored")?;
278    match ignored.code() {
279        Some(0) => bail!(
280            "changed-files target {} is ignored by Git and cannot be analyzed incrementally",
281            root.display()
282        ),
283        Some(1) => Ok(()),
284        _ => bail!("git check-ignore failed for target {}", root.display()),
285    }
286}
287
288fn git_paths(root: &Path, arguments: &[&str], failure: &str) -> Result<Vec<PathBuf>> {
289    let output = git_at(root)
290        .args(["-c", "core.quotePath=false"])
291        .args(arguments)
292        .output()
293        .with_context(|| "failed to run git for changed-files mode")?;
294    if !output.status.success() {
295        let stderr = String::from_utf8_lossy(&output.stderr).trim().to_owned();
296        bail!("{failure}: {stderr}");
297    }
298    output
299        .stdout
300        .split(|byte| *byte == b'\0')
301        .filter(|path| !path.is_empty())
302        .map(path_from_git_bytes)
303        .collect()
304}
305
306#[cfg(unix)]
307fn path_from_git_bytes(bytes: &[u8]) -> Result<PathBuf> {
308    use std::ffi::OsStr;
309    use std::os::unix::ffi::OsStrExt;
310
311    Ok(PathBuf::from(OsStr::from_bytes(bytes)))
312}
313
314#[cfg(not(unix))]
315fn path_from_git_bytes(bytes: &[u8]) -> Result<PathBuf> {
316    Ok(PathBuf::from(
317        std::str::from_utf8(bytes).context("git returned a non-UTF-8 file name")?,
318    ))
319}
320
321/// Retains findings whose primary or related location belongs to `changed`.
322pub fn retain_changed_findings(findings: &mut Vec<Finding>, changed: &BTreeSet<PathBuf>) {
323    findings.retain(|finding| {
324        changed.contains(&finding.primary.path)
325            || finding
326                .related
327                .iter()
328                .any(|location| changed.contains(&location.path))
329    });
330}
331
332/// Schema version of the compact, reviewable baseline format.
333pub const BASELINE_SCHEMA_VERSION: u32 = 3;
334
335#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
336#[serde(rename_all = "camelCase")]
337/// Accepted semantic finding fingerprints and their multiplicities.
338#[non_exhaustive]
339pub struct BaselineFile {
340    /// Baseline format version.
341    pub schema_version: u32,
342    /// Stable fingerprint counts, sorted for deterministic pull-request diffs.
343    pub fingerprints: BTreeMap<String, usize>,
344}
345
346impl BaselineFile {
347    /// Creates a baseline using the current schema version.
348    pub fn new(fingerprints: BTreeMap<String, usize>) -> Self {
349        Self {
350            schema_version: BASELINE_SCHEMA_VERSION,
351            fingerprints,
352        }
353    }
354
355    fn empty() -> Self {
356        Self::new(BTreeMap::new())
357    }
358}
359
360impl Default for BaselineFile {
361    fn default() -> Self {
362        Self::new(BTreeMap::new())
363    }
364}
365
366#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
367/// Effect of applying or rewriting a baseline.
368#[non_exhaustive]
369pub struct BaselineOutcome {
370    /// Findings matched and suppressed by the baseline.
371    pub suppressed: usize,
372    /// Active findings beyond the baseline's recorded multiplicities.
373    pub new_findings: usize,
374    /// Fingerprint instances added while updating.
375    pub added: usize,
376    /// Fingerprint instances removed while updating.
377    pub removed: usize,
378}
379
380/// Suppresses findings found in a schema-compatible semantic baseline.
381///
382/// Multiplicity is consumed in finding order, so adding another instance of accepted
383/// duplication remains visible even when its content fingerprint is already present.
384pub fn apply_baseline(findings: &mut [Finding], baseline_path: &Path) -> Result<BaselineOutcome> {
385    let baseline = load_baseline(baseline_path)?;
386    apply_loaded_baseline(findings, baseline_path, &baseline)
387}
388
389pub(crate) fn apply_reference_baseline(
390    findings: &mut [Finding],
391    accepted: &[Finding],
392    revision: &str,
393) -> Result<BaselineOutcome> {
394    apply_loaded_baseline(findings, Path::new(revision), &build_baseline(accepted))
395}
396
397/// Rewrites a baseline from the current active findings and suppresses those findings.
398pub fn update_baseline(findings: &mut [Finding], baseline_path: &Path) -> Result<BaselineOutcome> {
399    let previous = if baseline_path.is_file() {
400        load_baseline_for_update(baseline_path)?
401    } else {
402        BaselineFile::empty()
403    };
404    let current = build_baseline(findings);
405    let added = count_added(&previous.fingerprints, &current.fingerprints);
406    let removed = count_added(&current.fingerprints, &previous.fingerprints);
407    save_baseline(baseline_path, &current)?;
408    let mut outcome = apply_loaded_baseline(findings, baseline_path, &current)?;
409    outcome.added = added;
410    outcome.removed = removed;
411    Ok(outcome)
412}
413
414fn load_baseline(baseline_path: &Path) -> Result<BaselineFile> {
415    let baseline = parse_baseline(baseline_path)?;
416    if baseline.schema_version != BASELINE_SCHEMA_VERSION {
417        bail!(
418            "unsupported baseline schema version `{}` (expected `{}`); regenerate it with --update-baseline",
419            baseline.schema_version,
420            BASELINE_SCHEMA_VERSION
421        );
422    }
423    Ok(baseline)
424}
425
426fn load_baseline_for_update(baseline_path: &Path) -> Result<BaselineFile> {
427    let baseline = parse_baseline(baseline_path)?;
428    if baseline.schema_version > BASELINE_SCHEMA_VERSION {
429        bail!(
430            "baseline schema version `{}` is newer than supported version `{}` and cannot be replaced safely",
431            baseline.schema_version,
432            BASELINE_SCHEMA_VERSION
433        );
434    }
435    Ok(baseline)
436}
437
438fn parse_baseline(baseline_path: &Path) -> Result<BaselineFile> {
439    let text = read_utf8(baseline_path, "baseline", MAX_BASELINE_INPUT_BYTES)?;
440    // serde_json's default recursion limit remains enabled for untrusted baseline input.
441    serde_json::from_str(&text)
442        .with_context(|| format!("failed to parse baseline {}", baseline_path.display()))
443}
444
445fn save_baseline(path: &Path, baseline: &BaselineFile) -> Result<()> {
446    if let Some(parent) = path.parent() {
447        fs::create_dir_all(parent)
448            .with_context(|| format!("failed to create baseline directory {}", parent.display()))?;
449    }
450    let text = serde_json::to_string_pretty(baseline).context("failed to serialize baseline")?;
451    fs::write(path, format!("{text}\n"))
452        .with_context(|| format!("failed to write baseline {}", path.display()))
453}
454
455fn build_baseline(findings: &[Finding]) -> BaselineFile {
456    let mut baseline = BaselineFile::empty();
457    for finding in findings.iter().filter(|finding| finding.is_active()) {
458        *baseline
459            .fingerprints
460            .entry(finding_fingerprint(finding))
461            .or_default() += 1;
462    }
463    baseline
464}
465
466fn apply_loaded_baseline(
467    findings: &mut [Finding],
468    baseline_path: &Path,
469    baseline: &BaselineFile,
470) -> Result<BaselineOutcome> {
471    let mut allowance = baseline.fingerprints.clone();
472    let mut outcome = BaselineOutcome::default();
473    for finding in findings {
474        if !finding.is_active() {
475            continue;
476        }
477        let fingerprint = finding_fingerprint(finding);
478        match allowance.get_mut(&fingerprint) {
479            Some(remaining) if *remaining > 0 => {
480                *remaining -= 1;
481                finding.suppression = Some(Suppression {
482                    reason: format!("present in baseline {}", baseline_path.display()),
483                });
484                outcome.suppressed += 1;
485            }
486            _ => outcome.new_findings += 1,
487        }
488    }
489    Ok(outcome)
490}
491
492/// Returns a location-independent fingerprint for baseline comparison.
493pub fn finding_fingerprint(finding: &Finding) -> String {
494    let rule = finding.rule.to_string();
495    if let Some(cluster) = &finding.evidence.cluster {
496        let mut fingerprints = cluster
497            .definition_fingerprints
498            .iter()
499            .map(String::as_str)
500            .collect::<Vec<_>>();
501        fingerprints.sort_unstable();
502        return stable_fingerprint_parts(
503            ["finding-cluster", rule.as_str()]
504                .into_iter()
505                .chain(fingerprints),
506        );
507    }
508    if let Some(comparison) = &finding.evidence.comparison {
509        let mut sides = [
510            comparison.left_fingerprint.as_str(),
511            comparison.right_fingerprint.as_str(),
512        ];
513        sides.sort_unstable();
514        return stable_fingerprint_parts(
515            ["finding-comparison", rule.as_str()]
516                .into_iter()
517                .chain(sides),
518        );
519    }
520    stable_fingerprint_parts([
521        "finding-fallback",
522        rule.as_str(),
523        &finding.message,
524        &finding.evidence.matcher_difference,
525        &finding.evidence.handler_evidence,
526    ])
527}
528
529fn count_added(previous: &BTreeMap<String, usize>, current: &BTreeMap<String, usize>) -> usize {
530    current
531        .iter()
532        .map(|(fingerprint, count)| {
533            count.saturating_sub(previous.get(fingerprint).copied().unwrap_or_default())
534        })
535        .sum()
536}
537
538#[cfg(test)]
539mod tests;
cuke_dedup/modes.rs

cuke_dedup/
modes.rs
