cuke_dedup/
modes.rs

1//! Changed-files and existing-findings baseline modes for incremental CI adoption.
2
3use crate::config::normalize_platform_path;
4use crate::model::{
5    stable_fingerprint, stable_fingerprint_parts, DefinitionComparison, Finding, Suppression,
6};
7use crate::resource_limits::{read_utf8, MAX_BASELINE_INPUT_BYTES};
8use anyhow::{bail, Context, Result};
9use serde::{Deserialize, Serialize};
10use std::collections::{BTreeMap, BTreeSet};
11use std::fs;
12use std::io::Read;
13use std::path::{Path, PathBuf};
14use std::process::Command;
15
16// Snapshotting is additional disk work, separate from the analyzer's per-source read limits.
17const MAX_BASELINE_CHECKOUT_BYTES: u64 = 512 * 1024 * 1024;
18
19/// Returns tracked changes since `base` plus untracked files beneath `root`.
20pub fn git_changed_files(root: &Path, base: &str) -> Result<BTreeSet<PathBuf>> {
21    let base_oid = resolve_commit(root, base)?;
22    let tracked = git_paths(
23        root,
24        &[
25            "diff",
26            "--relative",
27            "--name-only",
28            "-z",
29            "--diff-filter=ACMR",
30            &base_oid,
31            "--",
32        ],
33        &format!("git diff against `{base}` failed"),
34    )?;
35    let untracked = git_paths(
36        root,
37        &["ls-files", "--others", "--exclude-standard", "-z", "--"],
38        "git untracked-file discovery failed",
39    )?;
40    Ok(tracked
41        .into_iter()
42        .chain(untracked)
43        .map(|path| root.join(path))
44        .collect())
45}
46
47/// Git environment variables that a `git` process exports to the hooks it runs. They outrank a
48/// child invocation's `-C`, so leaving them in place would point changed-files mode at whichever
49/// repository invoked the hook instead of the analyzed root.
50const INHERITED_GIT_VARIABLES: [&str; 8] = [
51    "GIT_DIR",
52    "GIT_WORK_TREE",
53    "GIT_INDEX_FILE",
54    "GIT_OBJECT_DIRECTORY",
55    "GIT_ALTERNATE_OBJECT_DIRECTORIES",
56    "GIT_COMMON_DIR",
57    "GIT_PREFIX",
58    "GIT_CEILING_DIRECTORIES",
59];
60
61/// Builds a `git` invocation bound to `root` and nothing else, so results describe the analyzed
62/// tree whether CukeDedup runs from a shell, CI, or another repository's Git hook.
63fn git_at(root: &Path) -> Command {
64    let mut command = Command::new("git");
65    for variable in INHERITED_GIT_VARIABLES {
66        command.env_remove(variable);
67    }
68    command.arg("-C").arg(root);
69    command
70}
71
72fn resolve_commit(root: &Path, base: &str) -> Result<String> {
73    if base.trim().is_empty() || base.starts_with('-') {
74        bail!("invalid --changed-since revision `{base}`");
75    }
76    let revision = format!("{base}^{{commit}}");
77    let output = git_at(root)
78        .args(["rev-parse", "--verify", "--end-of-options"])
79        .arg(&revision)
80        .output()
81        .with_context(|| "failed to resolve the --changed-since revision")?;
82    if !output.status.success() {
83        let stderr = String::from_utf8_lossy(&output.stderr).trim().to_owned();
84        bail!("could not resolve --changed-since revision `{base}`: {stderr}");
85    }
86    let oid = String::from_utf8(output.stdout)
87        .context("git returned a non-UTF-8 commit object id")?
88        .trim()
89        .to_owned();
90    if oid.is_empty() || !oid.bytes().all(|byte| byte.is_ascii_hexdigit()) {
91        bail!("git returned an invalid commit object id for `{base}`");
92    }
93    Ok(oid)
94}
95
96/// Materializes a fetched commit without running checkout hooks or configured filters.
97/// An independent local clone avoids registering worktrees or modifying the caller's index.
98pub(crate) fn baseline_snapshot(
99    root: &Path,
100    revision: &str,
101) -> Result<(tempfile::TempDir, PathBuf)> {
102    if revision.trim().is_empty() || revision.starts_with('-') {
103        bail!("invalid baseline revision `{revision}`");
104    }
105    let temporary = tempfile::tempdir().context("cannot create temporary baseline checkout")?;
106    let git = |directory: &Path, args: &[&std::ffi::OsStr]| -> Result<Vec<u8>> {
107        let mut command = git_at(directory);
108        // Neither caller-supplied Git config nor global templates/filters may execute code.
109        // Clone does not copy the source repository's local config or hooks.
110        for (name, _) in std::env::vars_os() {
111            // Windows preserves spelling but treats environment names case-insensitively.
112            if name
113                .to_string_lossy()
114                .to_ascii_uppercase()
115                .starts_with("GIT_")
116            {
117                command.env_remove(name);
118            }
119        }
120        command
121            .env("GIT_CONFIG_NOSYSTEM", "1")
122            .env("GIT_NO_REPLACE_OBJECTS", "1")
123            .env("GIT_NO_LAZY_FETCH", "1")
124            .env("GIT_TERMINAL_PROMPT", "0")
125            .env("GIT_CONFIG_GLOBAL", temporary.path().join("no-config"))
126            .arg("-c")
127            .arg(format!(
128                "core.hooksPath={}",
129                temporary.path().join("no-hooks").display()
130            ))
131            .args(["-c", "core.fsmonitor=false"])
132            .args(args);
133        if args.first().is_some_and(|arg| *arg == "ls-tree") {
134            // Fixed format: six mode digits, space, at most 20 size digits, newline.
135            let limit = 28 * crate::resource_limits::MAX_WORKSPACE_SCAN_ENTRIES as u64;
136            let mut child = command
137                .stdout(std::process::Stdio::piped())
138                .stderr(std::process::Stdio::null())
139                .spawn()?;
140            let mut bytes = Vec::new();
141            let read = child
142                .stdout
143                .take()
144                .context("missing baseline Git stdout")
145                .and_then(|stdout| Ok(stdout.take(limit + 1).read_to_end(&mut bytes)?));
146            if read.is_err() || bytes.len() as u64 > limit {
147                let _ = child.kill();
148                let _ = child.wait();
149                read?;
150                bail!("baseline tree listing exceeds the bounded snapshot metadata limit");
151            }
152            if !child.wait()?.success() {
153                bail!("baseline Git tree enumeration failed");
154            }
155            return Ok(bytes);
156        }
157        let output = command.output().context("failed to run Git for baseline")?;
158        if !output.status.success() {
159            bail!(
160                "baseline Git operation failed: {}",
161                String::from_utf8_lossy(&output.stderr).trim()
162            );
163        }
164        Ok(output.stdout)
165    };
166    let oid = git(
167        root,
168        &[
169            "rev-parse".as_ref(),
170            "--verify".as_ref(),
171            "--end-of-options".as_ref(),
172            format!("{revision}^{{commit}}").as_ref(),
173        ],
174    )?;
175    let oid = String::from_utf8(oid).context("invalid baseline commit id")?;
176    let tree = git(
177        root,
178        &[
179            "ls-tree".as_ref(),
180            "--full-tree".as_ref(),
181            "-r".as_ref(),
182            "--format=%(objectmode) %(objectsize)".as_ref(),
183            oid.trim().as_ref(),
184        ],
185    )?;
186    let mut total_bytes = 0_u64;
187    for (index, entry) in String::from_utf8(tree)?.lines().enumerate() {
188        if entry.starts_with("160000 ") {
189            bail!("baseline checkout contains unsupported submodules");
190        }
191        let (_, size) = entry
192            .split_once(' ')
193            .context("invalid baseline tree entry")?;
194        total_bytes = total_bytes.saturating_add(size.parse::<u64>()?);
195        if total_bytes > MAX_BASELINE_CHECKOUT_BYTES
196            || index >= crate::resource_limits::MAX_WORKSPACE_SCAN_ENTRIES
197        {
198            bail!("baseline checkout exceeds the 512 MiB / 100000-file snapshot limit");
199        }
200    }
201    let repository = git(root, &["rev-parse".as_ref(), "--show-toplevel".as_ref()])?;
202    // Remove Git's one line terminator, not whitespace belonging to the path.
203    let repository = crate::config::normalize_platform_path(path_from_git_bytes(
204        repository.strip_suffix(b"\n").unwrap_or(&repository),
205    )?);
206    let relative = root
207        .strip_prefix(&repository)
208        .context("baseline root is outside repository")?;
209    let checkout = temporary.path().join("checkout");
210    git(
211        &repository,
212        &[
213            "clone".as_ref(),
214            "--shared".as_ref(),
215            "--no-checkout".as_ref(),
216            "--template=".as_ref(),
217            "--".as_ref(),
218            repository.as_os_str(),
219            checkout.as_os_str(),
220        ],
221    )?;
222    git(
223        &checkout,
224        &[
225            "checkout".as_ref(),
226            "--detach".as_ref(),
227            oid.trim().as_ref(),
228        ],
229    )?;
230    let scoped = checkout.join(relative);
231    if !scoped.is_dir() {
232        bail!("baseline revision does not contain the analysis directory");
233    }
234    let scoped = crate::config::normalize_platform_path(scoped.canonicalize()?);
235    if !scoped.starts_with(crate::config::normalize_platform_path(
236        checkout.canonicalize()?,
237    )) {
238        bail!("baseline analysis directory escapes its checkout");
239    }
240    Ok((temporary, scoped))
241}
242
243/// Rejects changed-file mode when the analyzed root itself is excluded by Git.
244pub fn ensure_changed_root_is_trackable(root: &Path) -> Result<()> {
245    let root = normalize_platform_path(root.to_path_buf());
246    let repository = git_at(&root)
247        .args(["rev-parse", "--show-toplevel"])
248        .output()
249        .with_context(|| "failed to locate the Git repository for changed-files mode")?;
250    if !repository.status.success() {
251        bail!(
252            "changed-files mode requires a Git repository: {}",
253            String::from_utf8_lossy(&repository.stderr).trim()
254        );
255    }
256    let repository = normalize_platform_path(
257        PathBuf::from(
258            String::from_utf8(repository.stdout)
259                .context("git returned a non-UTF-8 repository path")?
260                .trim(),
261        )
262        .canonicalize()
263        .with_context(|| "failed to canonicalize the Git repository root")?,
264    );
265    if root == repository {
266        return Ok(());
267    }
268    let relative = root.strip_prefix(&repository).with_context(|| {
269        format!(
270            "target {} is outside Git repository {}",
271            root.display(),
272            repository.display()
273        )
274    })?;
275    let ignored = git_at(&repository)
276        .args(["check-ignore", "--quiet", "--"])
277        .arg(relative)
278        .status()
279        .with_context(|| "failed to check whether the changed-files target is ignored")?;
280    match ignored.code() {
281        Some(0) => bail!(
282            "changed-files target {} is ignored by Git and cannot be analyzed incrementally",
283            root.display()
284        ),
285        Some(1) => Ok(()),
286        _ => bail!("git check-ignore failed for target {}", root.display()),
287    }
288}
289
290fn git_paths(root: &Path, arguments: &[&str], failure: &str) -> Result<Vec<PathBuf>> {
291    let output = git_at(root)
292        .args(["-c", "core.quotePath=false"])
293        .args(arguments)
294        .output()
295        .with_context(|| "failed to run git for changed-files mode")?;
296    if !output.status.success() {
297        let stderr = String::from_utf8_lossy(&output.stderr).trim().to_owned();
298        bail!("{failure}: {stderr}");
299    }
300    output
301        .stdout
302        .split(|byte| *byte == b'\0')
303        .filter(|path| !path.is_empty())
304        .map(path_from_git_bytes)
305        .collect()
306}
307
308#[cfg(unix)]
309fn path_from_git_bytes(bytes: &[u8]) -> Result<PathBuf> {
310    use std::ffi::OsStr;
311    use std::os::unix::ffi::OsStrExt;
312
313    Ok(PathBuf::from(OsStr::from_bytes(bytes)))
314}
315
316#[cfg(not(unix))]
317fn path_from_git_bytes(bytes: &[u8]) -> Result<PathBuf> {
318    Ok(PathBuf::from(
319        std::str::from_utf8(bytes).context("git returned a non-UTF-8 file name")?,
320    ))
321}
322
323/// Retains findings whose primary or related location belongs to `changed`.
324pub fn retain_changed_findings(findings: &mut Vec<Finding>, changed: &BTreeSet<PathBuf>) {
325    findings.retain(|finding| {
326        changed.contains(&finding.primary.path)
327            || finding
328                .related
329                .iter()
330                .any(|location| changed.contains(&location.path))
331    });
332}
333
334/// Schema version of the compact, reviewable baseline format.
335pub const BASELINE_SCHEMA_VERSION: u32 = 2;
336
337#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
338#[serde(rename_all = "camelCase")]
339/// Accepted semantic finding fingerprints and their multiplicities.
340#[non_exhaustive]
341pub struct BaselineFile {
342    /// Baseline format version.
343    pub schema_version: u32,
344    /// Stable fingerprint counts, sorted for deterministic pull-request diffs.
345    pub fingerprints: BTreeMap<String, usize>,
346}
347
348impl BaselineFile {
349    /// Creates a baseline using the current schema version.
350    pub fn new(fingerprints: BTreeMap<String, usize>) -> Self {
351        Self {
352            schema_version: BASELINE_SCHEMA_VERSION,
353            fingerprints,
354        }
355    }
356
357    fn empty() -> Self {
358        Self::new(BTreeMap::new())
359    }
360}
361
362impl Default for BaselineFile {
363    fn default() -> Self {
364        Self::new(BTreeMap::new())
365    }
366}
367
368#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
369/// Effect of applying or rewriting a baseline.
370#[non_exhaustive]
371pub struct BaselineOutcome {
372    /// Findings matched and suppressed by the baseline.
373    pub suppressed: usize,
374    /// Active findings beyond the baseline's recorded multiplicities.
375    pub new_findings: usize,
376    /// Fingerprint instances added while updating.
377    pub added: usize,
378    /// Fingerprint instances removed while updating.
379    pub removed: usize,
380}
381
382/// Suppresses findings found in a schema-compatible semantic baseline.
383///
384/// Multiplicity is consumed in finding order, so adding another instance of accepted
385/// duplication remains visible even when its content fingerprint is already present.
386pub fn apply_baseline(findings: &mut [Finding], baseline_path: &Path) -> Result<BaselineOutcome> {
387    let baseline = load_baseline(baseline_path)?;
388    apply_loaded_baseline(findings, baseline_path, &baseline)
389}
390
391pub(crate) fn apply_reference_baseline(
392    findings: &mut [Finding],
393    accepted: &[Finding],
394    revision: &str,
395) -> Result<BaselineOutcome> {
396    apply_loaded_baseline(findings, Path::new(revision), &build_baseline(accepted))
397}
398
399/// Rewrites a baseline from the current active findings and suppresses those findings.
400pub fn update_baseline(findings: &mut [Finding], baseline_path: &Path) -> Result<BaselineOutcome> {
401    let previous = if baseline_path.is_file() {
402        load_baseline_for_update(baseline_path)?
403    } else {
404        BaselineFile::empty()
405    };
406    let current = build_baseline(findings);
407    let added = count_added(&previous.fingerprints, &current.fingerprints);
408    let removed = count_added(&current.fingerprints, &previous.fingerprints);
409    save_baseline(baseline_path, &current)?;
410    let mut outcome = apply_loaded_baseline(findings, baseline_path, &current)?;
411    outcome.added = added;
412    outcome.removed = removed;
413    Ok(outcome)
414}
415
416fn load_baseline(baseline_path: &Path) -> Result<BaselineFile> {
417    let baseline = parse_baseline(baseline_path)?;
418    if baseline.schema_version != BASELINE_SCHEMA_VERSION {
419        bail!(
420            "unsupported baseline schema version `{}` (expected `{}`); regenerate it with --update-baseline",
421            baseline.schema_version,
422            BASELINE_SCHEMA_VERSION
423        );
424    }
425    Ok(baseline)
426}
427
428fn load_baseline_for_update(baseline_path: &Path) -> Result<BaselineFile> {
429    let baseline = parse_baseline(baseline_path)?;
430    if baseline.schema_version > BASELINE_SCHEMA_VERSION {
431        bail!(
432            "baseline schema version `{}` is newer than supported version `{}` and cannot be replaced safely",
433            baseline.schema_version,
434            BASELINE_SCHEMA_VERSION
435        );
436    }
437    Ok(baseline)
438}
439
440fn parse_baseline(baseline_path: &Path) -> Result<BaselineFile> {
441    let text = read_utf8(baseline_path, "baseline", MAX_BASELINE_INPUT_BYTES)?;
442    // serde_json's default recursion limit remains enabled for untrusted baseline input.
443    serde_json::from_str(&text)
444        .with_context(|| format!("failed to parse baseline {}", baseline_path.display()))
445}
446
447fn save_baseline(path: &Path, baseline: &BaselineFile) -> Result<()> {
448    if let Some(parent) = path.parent() {
449        fs::create_dir_all(parent)
450            .with_context(|| format!("failed to create baseline directory {}", parent.display()))?;
451    }
452    let text = serde_json::to_string_pretty(baseline).context("failed to serialize baseline")?;
453    fs::write(path, format!("{text}\n"))
454        .with_context(|| format!("failed to write baseline {}", path.display()))
455}
456
457fn build_baseline(findings: &[Finding]) -> BaselineFile {
458    let mut baseline = BaselineFile::empty();
459    for finding in findings.iter().filter(|finding| finding.is_active()) {
460        *baseline
461            .fingerprints
462            .entry(finding_fingerprint(finding))
463            .or_default() += 1;
464    }
465    baseline
466}
467
468fn apply_loaded_baseline(
469    findings: &mut [Finding],
470    baseline_path: &Path,
471    baseline: &BaselineFile,
472) -> Result<BaselineOutcome> {
473    let mut allowance = baseline.fingerprints.clone();
474    let mut outcome = BaselineOutcome::default();
475    for finding in findings {
476        if !finding.is_active() {
477            continue;
478        }
479        let fingerprint = finding_fingerprint(finding);
480        match allowance.get_mut(&fingerprint) {
481            Some(remaining) if *remaining > 0 => {
482                *remaining -= 1;
483                finding.suppression = Some(Suppression {
484                    reason: format!("present in baseline {}", baseline_path.display()),
485                });
486                outcome.suppressed += 1;
487            }
488            _ => outcome.new_findings += 1,
489        }
490    }
491    Ok(outcome)
492}
493
494/// Returns a location-independent fingerprint for baseline comparison.
495pub fn finding_fingerprint(finding: &Finding) -> String {
496    if let Some(cluster) = &finding.evidence.cluster {
497        let rule = finding.rule.to_string();
498        let mut fingerprints = cluster
499            .definition_fingerprints
500            .iter()
501            .map(String::as_str)
502            .collect::<Vec<_>>();
503        fingerprints.sort_unstable();
504        return stable_fingerprint_parts(std::iter::once(rule.as_str()).chain(fingerprints));
505    }
506    let semantic = finding
507        .evidence
508        .comparison
509        .as_ref()
510        .map(comparison_fingerprint_input)
511        .unwrap_or_else(|| {
512            format!(
513                "{}\u{0}{}\u{0}{}",
514                finding.message,
515                finding.evidence.matcher_difference,
516                finding.evidence.handler_evidence
517            )
518        });
519    stable_fingerprint(&format!("{}\u{0}{semantic}", finding.rule))
520}
521
522fn comparison_fingerprint_input(comparison: &DefinitionComparison) -> String {
523    let mut sides = [
524        comparison.left_fingerprint.clone(),
525        comparison.right_fingerprint.clone(),
526    ];
527    sides.sort();
528    sides.join("\u{0}")
529}
530
531fn count_added(previous: &BTreeMap<String, usize>, current: &BTreeMap<String, usize>) -> usize {
532    current
533        .iter()
534        .map(|(fingerprint, count)| {
535            count.saturating_sub(previous.get(fingerprint).copied().unwrap_or_default())
536        })
537        .sum()
538}
539
540#[cfg(test)]
541mod tests;
cuke_dedup/modes.rs

cuke_dedup/
modes.rs
