cuke_dedup/
modes.rs

1//! Changed-files and existing-findings baseline modes for incremental CI adoption.
2
3use crate::config::{canonical_platform_path, normalize_platform_path};
4use crate::model::{stable_fingerprint_parts, Finding, Suppression};
5use crate::resource_limits::{read_utf8, MAX_BASELINE_INPUT_BYTES};
6use anyhow::{bail, Context, Result};
7use serde::{Deserialize, Serialize};
8use std::collections::{BTreeMap, BTreeSet};
9use std::fs;
10use std::io::Read;
11use std::path::{Path, PathBuf};
12use std::process::Command;
13
14// Snapshotting is additional disk work, separate from the analyzer's per-source read limits.
15const MAX_BASELINE_CHECKOUT_BYTES: u64 = 512 * 1024 * 1024;
16
17/// Returns tracked changes since `base` plus untracked files beneath `root`.
18pub fn git_changed_files(root: &Path, base: &str) -> Result<BTreeSet<PathBuf>> {
19    let base_oid = resolve_commit(root, base)?;
20    let tracked = git_paths(
21        root,
22        &[
23            "diff",
24            "--relative",
25            "--name-only",
26            "-z",
27            "--diff-filter=ACMR",
28            &base_oid,
29            "--",
30        ],
31        &format!("git diff against `{base}` failed"),
32    )?;
33    let untracked = git_paths(
34        root,
35        &["ls-files", "--others", "--exclude-standard", "-z", "--"],
36        "git untracked-file discovery failed",
37    )?;
38    Ok(tracked
39        .into_iter()
40        .chain(untracked)
41        .map(|path| root.join(path))
42        .collect())
43}
44
45/// Git environment variables that a `git` process exports to the hooks it runs. They outrank a
46/// child invocation's `-C`, so leaving them in place would point changed-files mode at whichever
47/// repository invoked the hook instead of the analyzed root.
48const INHERITED_GIT_VARIABLES: [&str; 8] = [
49    "GIT_DIR",
50    "GIT_WORK_TREE",
51    "GIT_INDEX_FILE",
52    "GIT_OBJECT_DIRECTORY",
53    "GIT_ALTERNATE_OBJECT_DIRECTORIES",
54    "GIT_COMMON_DIR",
55    "GIT_PREFIX",
56    "GIT_CEILING_DIRECTORIES",
57];
58
59/// Builds a `git` invocation bound to `root` and nothing else, so results describe the analyzed
60/// tree whether CukeDedup runs from a shell, CI, or another repository's Git hook.
61fn git_at(root: &Path) -> Command {
62    let mut command = Command::new("git");
63    for variable in INHERITED_GIT_VARIABLES {
64        command.env_remove(variable);
65    }
66    command.arg("-C").arg(root);
67    command
68}
69
70/// Peels `base` to a commit and returns its object id.
71///
72/// Rejects a blank or `-`-prefixed `base` before running Git; errors when `rev-parse` fails or
73/// prints anything other than a hex object id.
74fn resolve_commit(root: &Path, base: &str) -> Result<String> {
75    if base.trim().is_empty() || base.starts_with('-') {
76        bail!("invalid --changed-since revision `{base}`");
77    }
78    let revision = format!("{base}^{{commit}}");
79    let output = git_at(root)
80        .args(["rev-parse", "--verify", "--end-of-options"])
81        .arg(&revision)
82        .output()
83        .with_context(|| "failed to resolve the --changed-since revision")?;
84    if !output.status.success() {
85        let stderr = String::from_utf8_lossy(&output.stderr).trim().to_owned();
86        bail!("could not resolve --changed-since revision `{base}`: {stderr}");
87    }
88    let oid = String::from_utf8(output.stdout)
89        .context("git returned a non-UTF-8 commit object id")?
90        .trim()
91        .to_owned();
92    if oid.is_empty() || !oid.bytes().all(|byte| byte.is_ascii_hexdigit()) {
93        bail!("git returned an invalid commit object id for `{base}`");
94    }
95    Ok(oid)
96}
97
98/// Materializes a fetched commit without running checkout hooks or configured filters.
99/// An independent local clone avoids registering worktrees or modifying the caller's index.
100pub(crate) fn baseline_snapshot(
101    root: &Path,
102    revision: &str,
103) -> Result<(tempfile::TempDir, PathBuf)> {
104    if revision.trim().is_empty() || revision.starts_with('-') {
105        bail!("invalid baseline revision `{revision}`");
106    }
107    let temporary = tempfile::tempdir().context("cannot create temporary baseline checkout")?;
108    let git = |directory: &Path, args: &[&std::ffi::OsStr]| -> Result<Vec<u8>> {
109        let mut command = git_at(directory);
110        // Neither caller-supplied Git config nor global templates/filters may execute code.
111        // Clone does not copy the source repository's local config or hooks.
112        for (name, _) in std::env::vars_os() {
113            // Windows preserves spelling but treats environment names case-insensitively.
114            if name
115                .to_string_lossy()
116                .to_ascii_uppercase()
117                .starts_with("GIT_")
118            {
119                command.env_remove(name);
120            }
121        }
122        command
123            .env("GIT_CONFIG_NOSYSTEM", "1")
124            .env("GIT_NO_REPLACE_OBJECTS", "1")
125            .env("GIT_NO_LAZY_FETCH", "1")
126            .env("GIT_TERMINAL_PROMPT", "0")
127            .env("GIT_CONFIG_GLOBAL", temporary.path().join("no-config"))
128            .arg("-c")
129            .arg(format!(
130                "core.hooksPath={}",
131                temporary.path().join("no-hooks").display()
132            ))
133            .args(["-c", "core.fsmonitor=false"])
134            .args(args);
135        if args.first().is_some_and(|arg| *arg == "ls-tree") {
136            // Fixed format: six mode digits, space, at most 20 size digits, newline.
137            let limit = 28 * crate::resource_limits::MAX_WORKSPACE_SCAN_ENTRIES as u64;
138            let mut child = command
139                .stdout(std::process::Stdio::piped())
140                .stderr(std::process::Stdio::null())
141                .spawn()?;
142            let mut bytes = Vec::new();
143            let read = child
144                .stdout
145                .take()
146                .context("missing baseline Git stdout")
147                .and_then(|stdout| Ok(stdout.take(limit + 1).read_to_end(&mut bytes)?));
148            if read.is_err() || bytes.len() as u64 > limit {
149                let _ = child.kill();
150                let _ = child.wait();
151                read?;
152                bail!("baseline tree listing exceeds the bounded snapshot metadata limit");
153            }
154            if !child.wait()?.success() {
155                bail!("baseline Git tree enumeration failed");
156            }
157            return Ok(bytes);
158        }
159        let output = command.output().context("failed to run Git for baseline")?;
160        if !output.status.success() {
161            bail!(
162                "baseline Git operation failed: {}",
163                String::from_utf8_lossy(&output.stderr).trim()
164            );
165        }
166        Ok(output.stdout)
167    };
168    let oid = git(
169        root,
170        &[
171            "rev-parse".as_ref(),
172            "--verify".as_ref(),
173            "--end-of-options".as_ref(),
174            format!("{revision}^{{commit}}").as_ref(),
175        ],
176    )?;
177    let oid = String::from_utf8(oid).context("invalid baseline commit id")?;
178    let tree = git(
179        root,
180        &[
181            "ls-tree".as_ref(),
182            "--full-tree".as_ref(),
183            "-r".as_ref(),
184            "--format=%(objectmode) %(objectsize)".as_ref(),
185            oid.trim().as_ref(),
186        ],
187    )?;
188    let mut total_bytes = 0_u64;
189    for (index, entry) in String::from_utf8(tree)?.lines().enumerate() {
190        if entry.starts_with("160000 ") {
191            bail!("baseline checkout contains unsupported submodules");
192        }
193        let (_, size) = entry
194            .split_once(' ')
195            .context("invalid baseline tree entry")?;
196        total_bytes = total_bytes.saturating_add(size.parse::<u64>()?);
197        if total_bytes > MAX_BASELINE_CHECKOUT_BYTES
198            || index >= crate::resource_limits::MAX_WORKSPACE_SCAN_ENTRIES
199        {
200            bail!("baseline checkout exceeds the 512 MiB / 100000-file snapshot limit");
201        }
202    }
203    let repository = git(root, &["rev-parse".as_ref(), "--show-toplevel".as_ref()])?;
204    // Remove Git's one line terminator, not whitespace belonging to the path.
205    let repository = crate::config::normalize_platform_path(path_from_git_bytes(
206        repository.strip_suffix(b"\n").unwrap_or(&repository),
207    )?);
208    let relative = root
209        .strip_prefix(&repository)
210        .context("baseline root is outside repository")?;
211    let checkout = temporary.path().join("checkout");
212    git(
213        &repository,
214        &[
215            "clone".as_ref(),
216            "--shared".as_ref(),
217            "--no-checkout".as_ref(),
218            "--template=".as_ref(),
219            "--".as_ref(),
220            repository.as_os_str(),
221            checkout.as_os_str(),
222        ],
223    )?;
224    git(
225        &checkout,
226        &[
227            "checkout".as_ref(),
228            "--detach".as_ref(),
229            oid.trim().as_ref(),
230        ],
231    )?;
232    let scoped = checkout.join(relative);
233    if !scoped.is_dir() {
234        bail!("baseline revision does not contain the analysis directory");
235    }
236    let scoped = crate::config::canonical_platform_path(&scoped)?;
237    if !scoped.starts_with(crate::config::canonical_platform_path(&checkout)?) {
238        bail!("baseline analysis directory escapes its checkout");
239    }
240    Ok((temporary, scoped))
241}
242
243/// Rejects changed-file mode when the analyzed root itself is excluded by Git.
244pub fn ensure_changed_root_is_trackable(root: &Path) -> Result<()> {
245    let root = normalize_platform_path(root.to_path_buf());
246    let repository = git_at(&root)
247        .args(["rev-parse", "--show-toplevel"])
248        .output()
249        .with_context(|| "failed to locate the Git repository for changed-files mode")?;
250    if !repository.status.success() {
251        bail!(
252            "changed-files mode requires a Git repository: {}",
253            String::from_utf8_lossy(&repository.stderr).trim()
254        );
255    }
256    let repository = canonical_platform_path(Path::new(
257        String::from_utf8(repository.stdout)
258            .context("git returned a non-UTF-8 repository path")?
259            .trim(),
260    ))
261    .with_context(|| "failed to canonicalize the Git repository root")?;
262    if root == repository {
263        return Ok(());
264    }
265    let relative = root.strip_prefix(&repository).with_context(|| {
266        format!(
267            "target {} is outside Git repository {}",
268            root.display(),
269            repository.display()
270        )
271    })?;
272    let ignored = git_at(&repository)
273        .args(["check-ignore", "--quiet", "--"])
274        .arg(relative)
275        .status()
276        .with_context(|| "failed to check whether the changed-files target is ignored")?;
277    match ignored.code() {
278        Some(0) => bail!(
279            "changed-files target {} is ignored by Git and cannot be analyzed incrementally",
280            root.display()
281        ),
282        Some(1) => Ok(()),
283        _ => bail!("git check-ignore failed for target {}", root.display()),
284    }
285}
286
287/// Runs `git <arguments>` in `root` with `core.quotePath=false` and splits stdout on NUL into
288/// paths as Git printed them, skipping empty entries. A non-zero exit errors as
289/// `{failure}: {stderr}`.
290fn git_paths(root: &Path, arguments: &[&str], failure: &str) -> Result<Vec<PathBuf>> {
291    let output = git_at(root)
292        .args(["-c", "core.quotePath=false"])
293        .args(arguments)
294        .output()
295        .with_context(|| "failed to run git for changed-files mode")?;
296    if !output.status.success() {
297        let stderr = String::from_utf8_lossy(&output.stderr).trim().to_owned();
298        bail!("{failure}: {stderr}");
299    }
300    output
301        .stdout
302        .split(|byte| *byte == b'\0')
303        .filter(|path| !path.is_empty())
304        .map(path_from_git_bytes)
305        .collect()
306}
307
308/// Converts a raw Git path to a `PathBuf` byte for byte; never fails on Unix.
309#[cfg(unix)]
310fn path_from_git_bytes(bytes: &[u8]) -> Result<PathBuf> {
311    use std::ffi::OsStr;
312    use std::os::unix::ffi::OsStrExt;
313
314    Ok(PathBuf::from(OsStr::from_bytes(bytes)))
315}
316
317/// Converts a Git path to a `PathBuf` on non-Unix targets; errors when the bytes are not UTF-8.
318#[cfg(not(unix))]
319fn path_from_git_bytes(bytes: &[u8]) -> Result<PathBuf> {
320    Ok(PathBuf::from(
321        std::str::from_utf8(bytes).context("git returned a non-UTF-8 file name")?,
322    ))
323}
324
325/// Retains findings whose primary or related location belongs to `changed`.
326pub fn retain_changed_findings(findings: &mut Vec<Finding>, changed: &BTreeSet<PathBuf>) {
327    findings.retain(|finding| {
328        changed.contains(&finding.primary.path)
329            || finding
330                .related
331                .iter()
332                .any(|location| changed.contains(&location.path))
333    });
334}
335
336/// Schema version of the compact, reviewable baseline format.
337pub const BASELINE_SCHEMA_VERSION: u32 = 3;
338
339#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
340#[serde(rename_all = "camelCase")]
341/// Accepted semantic finding fingerprints and their multiplicities.
342#[non_exhaustive]
343pub struct BaselineFile {
344    /// Baseline format version.
345    pub schema_version: u32,
346    /// Stable fingerprint counts, sorted for deterministic pull-request diffs.
347    pub fingerprints: BTreeMap<String, usize>,
348}
349
350impl BaselineFile {
351    /// Creates a baseline using the current schema version.
352    pub fn new(fingerprints: BTreeMap<String, usize>) -> Self {
353        Self {
354            schema_version: BASELINE_SCHEMA_VERSION,
355            fingerprints,
356        }
357    }
358
359    /// Returns a baseline with no fingerprints at the current schema version.
360    fn empty() -> Self {
361        Self::new(BTreeMap::new())
362    }
363}
364
365impl Default for BaselineFile {
366    /// Returns a baseline with no fingerprints at the current schema version.
367    fn default() -> Self {
368        Self::new(BTreeMap::new())
369    }
370}
371
372#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
373/// Effect of applying or rewriting a baseline.
374#[non_exhaustive]
375pub struct BaselineOutcome {
376    /// Findings matched and suppressed by the baseline.
377    pub suppressed: usize,
378    /// Active findings beyond the baseline's recorded multiplicities.
379    pub new_findings: usize,
380    /// Fingerprint instances added while updating.
381    pub added: usize,
382    /// Fingerprint instances removed while updating.
383    pub removed: usize,
384}
385
386/// Suppresses findings found in a schema-compatible semantic baseline.
387///
388/// Multiplicity is consumed in finding order, so adding another instance of accepted
389/// duplication remains visible even when its content fingerprint is already present.
390pub fn apply_baseline(findings: &mut [Finding], baseline_path: &Path) -> Result<BaselineOutcome> {
391    let baseline = load_baseline(baseline_path)?;
392    apply_loaded_baseline(findings, baseline_path, &baseline)
393}
394
395/// Suppresses `findings` against a baseline built from the active findings in `accepted`.
396///
397/// `revision` stands in for the baseline path, so suppression reasons read
398/// `present in baseline <revision>`.
399pub(crate) fn apply_reference_baseline(
400    findings: &mut [Finding],
401    accepted: &[Finding],
402    revision: &str,
403) -> Result<BaselineOutcome> {
404    apply_loaded_baseline(findings, Path::new(revision), &build_baseline(accepted))
405}
406
407/// Rewrites a baseline from the current active findings and suppresses those findings.
408pub fn update_baseline(findings: &mut [Finding], baseline_path: &Path) -> Result<BaselineOutcome> {
409    let previous = if baseline_path.is_file() {
410        load_baseline_for_update(baseline_path)?
411    } else {
412        BaselineFile::empty()
413    };
414    let current = build_baseline(findings);
415    let added = count_added(&previous.fingerprints, &current.fingerprints);
416    let removed = count_added(&current.fingerprints, &previous.fingerprints);
417    save_baseline(baseline_path, &current)?;
418    let mut outcome = apply_loaded_baseline(findings, baseline_path, &current)?;
419    outcome.added = added;
420    outcome.removed = removed;
421    Ok(outcome)
422}
423
424/// Parses a baseline to apply; errors unless its schema version equals `BASELINE_SCHEMA_VERSION`.
425fn load_baseline(baseline_path: &Path) -> Result<BaselineFile> {
426    let baseline = parse_baseline(baseline_path)?;
427    if baseline.schema_version != BASELINE_SCHEMA_VERSION {
428        bail!(
429            "unsupported baseline schema version `{}` (expected `{}`); regenerate it with --update-baseline",
430            baseline.schema_version,
431            BASELINE_SCHEMA_VERSION
432        );
433    }
434    Ok(baseline)
435}
436
437/// Parses a baseline that is about to be replaced; accepts older schema versions and errors on a
438/// newer one.
439fn load_baseline_for_update(baseline_path: &Path) -> Result<BaselineFile> {
440    let baseline = parse_baseline(baseline_path)?;
441    if baseline.schema_version > BASELINE_SCHEMA_VERSION {
442        bail!(
443            "baseline schema version `{}` is newer than supported version `{}` and cannot be replaced safely",
444            baseline.schema_version,
445            BASELINE_SCHEMA_VERSION
446        );
447    }
448    Ok(baseline)
449}
450
451/// Reads and deserializes a baseline without checking its schema version.
452///
453/// Errors on I/O failure, input over `MAX_BASELINE_INPUT_BYTES`, invalid UTF-8, or invalid JSON.
454fn parse_baseline(baseline_path: &Path) -> Result<BaselineFile> {
455    let text = read_utf8(baseline_path, "baseline", MAX_BASELINE_INPUT_BYTES)?;
456    // serde_json's default recursion limit remains enabled for untrusted baseline input.
457    serde_json::from_str(&text)
458        .with_context(|| format!("failed to parse baseline {}", baseline_path.display()))
459}
460
461/// Writes `baseline` as pretty JSON with a trailing newline, creating parent directories; the
462/// existing file is overwritten in place, not atomically.
463fn save_baseline(path: &Path, baseline: &BaselineFile) -> Result<()> {
464    if let Some(parent) = path.parent() {
465        fs::create_dir_all(parent)
466            .with_context(|| format!("failed to create baseline directory {}", parent.display()))?;
467    }
468    let text = serde_json::to_string_pretty(baseline).context("failed to serialize baseline")?;
469    fs::write(path, format!("{text}\n"))
470        .with_context(|| format!("failed to write baseline {}", path.display()))
471}
472
473/// Counts active findings per `finding_fingerprint`; suppressed and `Off` findings are left out.
474fn build_baseline(findings: &[Finding]) -> BaselineFile {
475    let mut baseline = BaselineFile::empty();
476    for finding in findings.iter().filter(|finding| finding.is_active()) {
477        *baseline
478            .fingerprints
479            .entry(finding_fingerprint(finding))
480            .or_default() += 1;
481    }
482    baseline
483}
484
485/// Suppresses each active finding whose fingerprint has multiplicity left in `baseline`,
486/// consuming one per match in finding order; unmatched active findings count as new.
487///
488/// Inactive findings are skipped. Never returns `Err`.
489fn apply_loaded_baseline(
490    findings: &mut [Finding],
491    baseline_path: &Path,
492    baseline: &BaselineFile,
493) -> Result<BaselineOutcome> {
494    let mut allowance = baseline.fingerprints.clone();
495    let mut outcome = BaselineOutcome::default();
496    for finding in findings {
497        if !finding.is_active() {
498            continue;
499        }
500        let fingerprint = finding_fingerprint(finding);
501        match allowance.get_mut(&fingerprint) {
502            Some(remaining) if *remaining > 0 => {
503                *remaining -= 1;
504                finding.suppression = Some(Suppression {
505                    reason: format!("present in baseline {}", baseline_path.display()),
506                });
507                outcome.suppressed += 1;
508            }
509            _ => outcome.new_findings += 1,
510        }
511    }
512    Ok(outcome)
513}
514
515/// Returns a location-independent fingerprint for baseline comparison.
516pub fn finding_fingerprint(finding: &Finding) -> String {
517    let rule = finding.rule.to_string();
518    if let Some(cluster) = &finding.evidence.cluster {
519        let mut fingerprints = cluster
520            .definition_fingerprints
521            .iter()
522            .map(String::as_str)
523            .collect::<Vec<_>>();
524        fingerprints.sort_unstable();
525        return stable_fingerprint_parts(
526            ["finding-cluster", rule.as_str()]
527                .into_iter()
528                .chain(fingerprints),
529        );
530    }
531    if let Some(comparison) = &finding.evidence.comparison {
532        let mut sides = [
533            comparison.left_fingerprint.as_str(),
534            comparison.right_fingerprint.as_str(),
535        ];
536        sides.sort_unstable();
537        return stable_fingerprint_parts(
538            ["finding-comparison", rule.as_str()]
539                .into_iter()
540                .chain(sides),
541        );
542    }
543    stable_fingerprint_parts([
544        "finding-fallback",
545        rule.as_str(),
546        &finding.message,
547        &finding.evidence.matcher_difference,
548        &finding.evidence.handler_evidence,
549    ])
550}
551
552/// Sums, per fingerprint, the instances `current` has beyond `previous`; fingerprints present
553/// only in `previous` contribute nothing.
554fn count_added(previous: &BTreeMap<String, usize>, current: &BTreeMap<String, usize>) -> usize {
555    current
556        .iter()
557        .map(|(fingerprint, count)| {
558            count.saturating_sub(previous.get(fingerprint).copied().unwrap_or_default())
559        })
560        .sum()
561}
562
563#[cfg(test)]
564mod tests;
cuke_dedup/modes.rs

cuke_dedup/
modes.rs
