cuke_dedup/
modes.rs

1//! Changed-files and existing-findings baseline modes for incremental CI adoption.
2
3use crate::config::normalize_platform_path;
4use crate::model::{
5    stable_fingerprint, stable_fingerprint_parts, DefinitionComparison, Finding, Suppression,
6};
7use crate::resource_limits::{read_utf8, MAX_BASELINE_INPUT_BYTES};
8use anyhow::{bail, Context, Result};
9use serde::{Deserialize, Serialize};
10use std::collections::{BTreeMap, BTreeSet};
11use std::fs;
12use std::path::{Path, PathBuf};
13use std::process::Command;
14
15/// Returns tracked changes since `base` plus untracked files beneath `root`.
16pub fn git_changed_files(root: &Path, base: &str) -> Result<BTreeSet<PathBuf>> {
17    let base_oid = resolve_commit(root, base)?;
18    let tracked = git_paths(
19        root,
20        &[
21            "diff",
22            "--relative",
23            "--name-only",
24            "-z",
25            "--diff-filter=ACMR",
26            &base_oid,
27            "--",
28        ],
29        &format!("git diff against `{base}` failed"),
30    )?;
31    let untracked = git_paths(
32        root,
33        &["ls-files", "--others", "--exclude-standard", "-z", "--"],
34        "git untracked-file discovery failed",
35    )?;
36    Ok(tracked
37        .into_iter()
38        .chain(untracked)
39        .map(|path| root.join(path))
40        .collect())
41}
42
43fn resolve_commit(root: &Path, base: &str) -> Result<String> {
44    if base.trim().is_empty() || base.starts_with('-') {
45        bail!("invalid --changed-since revision `{base}`");
46    }
47    let revision = format!("{base}^{{commit}}");
48    let output = Command::new("git")
49        .arg("-C")
50        .arg(root)
51        .args(["rev-parse", "--verify", "--end-of-options"])
52        .arg(&revision)
53        .output()
54        .with_context(|| "failed to resolve the --changed-since revision")?;
55    if !output.status.success() {
56        let stderr = String::from_utf8_lossy(&output.stderr).trim().to_owned();
57        bail!("could not resolve --changed-since revision `{base}`: {stderr}");
58    }
59    let oid = String::from_utf8(output.stdout)
60        .context("git returned a non-UTF-8 commit object id")?
61        .trim()
62        .to_owned();
63    if oid.is_empty() || !oid.bytes().all(|byte| byte.is_ascii_hexdigit()) {
64        bail!("git returned an invalid commit object id for `{base}`");
65    }
66    Ok(oid)
67}
68
69/// Rejects changed-file mode when the analyzed root itself is excluded by Git.
70pub fn ensure_changed_root_is_trackable(root: &Path) -> Result<()> {
71    let root = normalize_platform_path(root.to_path_buf());
72    let repository = Command::new("git")
73        .arg("-C")
74        .arg(&root)
75        .args(["rev-parse", "--show-toplevel"])
76        .output()
77        .with_context(|| "failed to locate the Git repository for changed-files mode")?;
78    if !repository.status.success() {
79        bail!(
80            "changed-files mode requires a Git repository: {}",
81            String::from_utf8_lossy(&repository.stderr).trim()
82        );
83    }
84    let repository = normalize_platform_path(
85        PathBuf::from(
86            String::from_utf8(repository.stdout)
87                .context("git returned a non-UTF-8 repository path")?
88                .trim(),
89        )
90        .canonicalize()
91        .with_context(|| "failed to canonicalize the Git repository root")?,
92    );
93    if root == repository {
94        return Ok(());
95    }
96    let relative = root.strip_prefix(&repository).with_context(|| {
97        format!(
98            "target {} is outside Git repository {}",
99            root.display(),
100            repository.display()
101        )
102    })?;
103    let ignored = Command::new("git")
104        .arg("-C")
105        .arg(&repository)
106        .args(["check-ignore", "--quiet", "--"])
107        .arg(relative)
108        .status()
109        .with_context(|| "failed to check whether the changed-files target is ignored")?;
110    match ignored.code() {
111        Some(0) => bail!(
112            "changed-files target {} is ignored by Git and cannot be analyzed incrementally",
113            root.display()
114        ),
115        Some(1) => Ok(()),
116        _ => bail!("git check-ignore failed for target {}", root.display()),
117    }
118}
119
120fn git_paths(root: &Path, arguments: &[&str], failure: &str) -> Result<Vec<PathBuf>> {
121    let output = Command::new("git")
122        .arg("-C")
123        .arg(root)
124        .args(["-c", "core.quotePath=false"])
125        .args(arguments)
126        .output()
127        .with_context(|| "failed to run git for changed-files mode")?;
128    if !output.status.success() {
129        let stderr = String::from_utf8_lossy(&output.stderr).trim().to_owned();
130        bail!("{failure}: {stderr}");
131    }
132    output
133        .stdout
134        .split(|byte| *byte == b'\0')
135        .filter(|path| !path.is_empty())
136        .map(path_from_git_bytes)
137        .collect()
138}
139
140#[cfg(unix)]
141fn path_from_git_bytes(bytes: &[u8]) -> Result<PathBuf> {
142    use std::ffi::OsStr;
143    use std::os::unix::ffi::OsStrExt;
144
145    Ok(PathBuf::from(OsStr::from_bytes(bytes)))
146}
147
148#[cfg(not(unix))]
149fn path_from_git_bytes(bytes: &[u8]) -> Result<PathBuf> {
150    Ok(PathBuf::from(
151        std::str::from_utf8(bytes).context("git returned a non-UTF-8 file name")?,
152    ))
153}
154
155/// Retains findings whose primary or related location belongs to `changed`.
156pub fn retain_changed_findings(findings: &mut Vec<Finding>, changed: &BTreeSet<PathBuf>) {
157    findings.retain(|finding| {
158        changed.contains(&finding.primary.path)
159            || finding
160                .related
161                .iter()
162                .any(|location| changed.contains(&location.path))
163    });
164}
165
166/// Schema version of the compact, reviewable baseline format.
167pub const BASELINE_SCHEMA_VERSION: u32 = 2;
168
169#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
170#[serde(rename_all = "camelCase")]
171/// Accepted semantic finding fingerprints and their multiplicities.
172#[non_exhaustive]
173pub struct BaselineFile {
174    /// Baseline format version.
175    pub schema_version: u32,
176    /// Stable fingerprint counts, sorted for deterministic pull-request diffs.
177    pub fingerprints: BTreeMap<String, usize>,
178}
179
180impl BaselineFile {
181    /// Creates a baseline using the current schema version.
182    pub fn new(fingerprints: BTreeMap<String, usize>) -> Self {
183        Self {
184            schema_version: BASELINE_SCHEMA_VERSION,
185            fingerprints,
186        }
187    }
188
189    fn empty() -> Self {
190        Self::new(BTreeMap::new())
191    }
192}
193
194impl Default for BaselineFile {
195    fn default() -> Self {
196        Self::new(BTreeMap::new())
197    }
198}
199
200#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
201/// Effect of applying or rewriting a baseline.
202#[non_exhaustive]
203pub struct BaselineOutcome {
204    /// Findings matched and suppressed by the baseline.
205    pub suppressed: usize,
206    /// Active findings beyond the baseline's recorded multiplicities.
207    pub new_findings: usize,
208    /// Fingerprint instances added while updating.
209    pub added: usize,
210    /// Fingerprint instances removed while updating.
211    pub removed: usize,
212}
213
214/// Suppresses findings found in a schema-compatible semantic baseline.
215///
216/// Multiplicity is consumed in finding order, so adding another instance of accepted
217/// duplication remains visible even when its content fingerprint is already present.
218pub fn apply_baseline(findings: &mut [Finding], baseline_path: &Path) -> Result<BaselineOutcome> {
219    let baseline = load_baseline(baseline_path)?;
220    apply_loaded_baseline(findings, baseline_path, &baseline)
221}
222
223/// Rewrites a baseline from the current active findings and suppresses those findings.
224pub fn update_baseline(findings: &mut [Finding], baseline_path: &Path) -> Result<BaselineOutcome> {
225    let previous = if baseline_path.is_file() {
226        load_baseline_for_update(baseline_path)?
227    } else {
228        BaselineFile::empty()
229    };
230    let current = build_baseline(findings);
231    let added = count_added(&previous.fingerprints, &current.fingerprints);
232    let removed = count_added(&current.fingerprints, &previous.fingerprints);
233    save_baseline(baseline_path, &current)?;
234    let mut outcome = apply_loaded_baseline(findings, baseline_path, &current)?;
235    outcome.added = added;
236    outcome.removed = removed;
237    Ok(outcome)
238}
239
240fn load_baseline(baseline_path: &Path) -> Result<BaselineFile> {
241    let baseline = parse_baseline(baseline_path)?;
242    if baseline.schema_version != BASELINE_SCHEMA_VERSION {
243        bail!(
244            "unsupported baseline schema version `{}` (expected `{}`); regenerate it with --update-baseline",
245            baseline.schema_version,
246            BASELINE_SCHEMA_VERSION
247        );
248    }
249    Ok(baseline)
250}
251
252fn load_baseline_for_update(baseline_path: &Path) -> Result<BaselineFile> {
253    let baseline = parse_baseline(baseline_path)?;
254    if baseline.schema_version > BASELINE_SCHEMA_VERSION {
255        bail!(
256            "baseline schema version `{}` is newer than supported version `{}` and cannot be replaced safely",
257            baseline.schema_version,
258            BASELINE_SCHEMA_VERSION
259        );
260    }
261    Ok(baseline)
262}
263
264fn parse_baseline(baseline_path: &Path) -> Result<BaselineFile> {
265    let text = read_utf8(baseline_path, "baseline", MAX_BASELINE_INPUT_BYTES)?;
266    // serde_json's default recursion limit remains enabled for untrusted baseline input.
267    serde_json::from_str(&text)
268        .with_context(|| format!("failed to parse baseline {}", baseline_path.display()))
269}
270
271fn save_baseline(path: &Path, baseline: &BaselineFile) -> Result<()> {
272    if let Some(parent) = path.parent() {
273        fs::create_dir_all(parent)
274            .with_context(|| format!("failed to create baseline directory {}", parent.display()))?;
275    }
276    let text = serde_json::to_string_pretty(baseline).context("failed to serialize baseline")?;
277    fs::write(path, format!("{text}\n"))
278        .with_context(|| format!("failed to write baseline {}", path.display()))
279}
280
281fn build_baseline(findings: &[Finding]) -> BaselineFile {
282    let mut baseline = BaselineFile::empty();
283    for finding in findings.iter().filter(|finding| finding.is_active()) {
284        *baseline
285            .fingerprints
286            .entry(finding_fingerprint(finding))
287            .or_default() += 1;
288    }
289    baseline
290}
291
292fn apply_loaded_baseline(
293    findings: &mut [Finding],
294    baseline_path: &Path,
295    baseline: &BaselineFile,
296) -> Result<BaselineOutcome> {
297    let mut allowance = baseline.fingerprints.clone();
298    let mut outcome = BaselineOutcome::default();
299    for finding in findings {
300        if !finding.is_active() {
301            continue;
302        }
303        let fingerprint = finding_fingerprint(finding);
304        match allowance.get_mut(&fingerprint) {
305            Some(remaining) if *remaining > 0 => {
306                *remaining -= 1;
307                finding.suppression = Some(Suppression {
308                    reason: format!("present in baseline {}", baseline_path.display()),
309                });
310                outcome.suppressed += 1;
311            }
312            _ => outcome.new_findings += 1,
313        }
314    }
315    Ok(outcome)
316}
317
318/// Returns a location-independent fingerprint for baseline comparison.
319pub fn finding_fingerprint(finding: &Finding) -> String {
320    if let Some(cluster) = &finding.evidence.cluster {
321        let rule = finding.rule.to_string();
322        let mut fingerprints = cluster
323            .definition_fingerprints
324            .iter()
325            .map(String::as_str)
326            .collect::<Vec<_>>();
327        fingerprints.sort_unstable();
328        return stable_fingerprint_parts(std::iter::once(rule.as_str()).chain(fingerprints));
329    }
330    let semantic = finding
331        .evidence
332        .comparison
333        .as_ref()
334        .map(comparison_fingerprint_input)
335        .unwrap_or_else(|| {
336            format!(
337                "{}\u{0}{}\u{0}{}",
338                finding.message,
339                finding.evidence.matcher_difference,
340                finding.evidence.handler_evidence
341            )
342        });
343    stable_fingerprint(&format!("{}\u{0}{semantic}", finding.rule))
344}
345
346fn comparison_fingerprint_input(comparison: &DefinitionComparison) -> String {
347    let mut sides = [
348        comparison.left_fingerprint.clone(),
349        comparison.right_fingerprint.clone(),
350    ];
351    sides.sort();
352    sides.join("\u{0}")
353}
354
355fn count_added(previous: &BTreeMap<String, usize>, current: &BTreeMap<String, usize>) -> usize {
356    current
357        .iter()
358        .map(|(fingerprint, count)| {
359            count.saturating_sub(previous.get(fingerprint).copied().unwrap_or_default())
360        })
361        .sum()
362}
363
364#[cfg(test)]
365mod tests {
366    use super::*;
367    use crate::model::{DefinitionCluster, FindingEvidence, Rule, Severity, SourceLocation};
368
369    fn initialize_repository(root: &Path) {
370        for arguments in [
371            ["init", "-q"].as_slice(),
372            ["config", "user.email", "test@example.com"].as_slice(),
373            ["config", "user.name", "Test"].as_slice(),
374            ["add", "."].as_slice(),
375            ["commit", "-qm", "initial"].as_slice(),
376        ] {
377            assert!(Command::new("git")
378                .args(arguments)
379                .current_dir(root)
380                .status()
381                .unwrap()
382                .success());
383        }
384    }
385
386    fn finding(root: &Path, file: &str) -> Finding {
387        Finding {
388            rule: Rule::DuplicateMatcher,
389            severity: Severity::Error,
390            message: "duplicate".to_owned(),
391            primary: SourceLocation::new(root.join(file), 2, 1, 2, 10),
392            related: vec![SourceLocation::new(
393                root.join("steps/shared.ts"),
394                4,
395                1,
396                4,
397                10,
398            )],
399            evidence: FindingEvidence {
400                matcher_similarity: Some(1.0),
401                handler_similarity: Some(0.5),
402                matcher_difference: "same".to_owned(),
403                handler_evidence: "different".to_owned(),
404                comparison: None,
405                cluster: None,
406            },
407            suggested_action: "consolidate".to_owned(),
408            suppression: None,
409        }
410    }
411
412    #[test]
413    fn changed_mode_keeps_findings_related_to_a_changed_file() {
414        let root = Path::new("/repo");
415        let mut findings = vec![
416            finding(root, "steps/changed.ts"),
417            finding(root, "steps/old.ts"),
418        ];
419        let changed = BTreeSet::from([root.join("steps/changed.ts")]);
420        retain_changed_findings(&mut findings, &changed);
421        assert_eq!(findings.len(), 1);
422        assert!(findings[0].primary.path.ends_with("changed.ts"));
423    }
424
425    #[test]
426    fn cluster_fingerprints_are_location_and_member_order_independent() {
427        let root = Path::new("/repo");
428        let mut original = finding(root, "steps/a.ts");
429        original.evidence.cluster = Some(DefinitionCluster {
430            member_count: 3,
431            definition_fingerprints: vec!["charlie".into(), "alpha".into(), "bravo".into()],
432            pair_findings_collapsed: 2,
433            members_truncated: false,
434        });
435        let mut moved = finding(root, "moved/renamed.ts");
436        moved.evidence.cluster = Some(DefinitionCluster {
437            member_count: 3,
438            definition_fingerprints: vec!["bravo".into(), "charlie".into(), "alpha".into()],
439            pair_findings_collapsed: 99,
440            members_truncated: false,
441        });
442
443        assert_eq!(finding_fingerprint(&original), finding_fingerprint(&moved));
444        moved
445            .evidence
446            .cluster
447            .as_mut()
448            .unwrap()
449            .definition_fingerprints
450            .push("delta".into());
451        assert_ne!(finding_fingerprint(&original), finding_fingerprint(&moved));
452    }
453
454    #[test]
455    fn changed_files_are_relative_to_a_subdirectory_and_include_untracked_files() {
456        let directory = tempfile::tempdir().unwrap();
457        let root = directory.path();
458        fs::create_dir_all(root.join("packages/e2e/steps")).unwrap();
459        fs::write(root.join("packages/e2e/steps/träcked.ts"), "before").unwrap();
460        initialize_repository(root);
461        fs::write(root.join("packages/e2e/steps/träcked.ts"), "after").unwrap();
462        fs::write(root.join("packages/e2e/steps/ new step.ts"), "new").unwrap();
463        #[cfg(unix)]
464        fs::write(root.join("packages/e2e/steps/trailing step.ts "), "new").unwrap();
465
466        let subdirectory = root.join("packages/e2e");
467        let changed = git_changed_files(&subdirectory, "HEAD").unwrap();
468        let mut expected = BTreeSet::from([
469            subdirectory.join("steps/ new step.ts"),
470            subdirectory.join("steps/träcked.ts"),
471        ]);
472        #[cfg(unix)]
473        expected.insert(subdirectory.join("steps/trailing step.ts "));
474        assert_eq!(changed, expected);
475    }
476
477    #[cfg(unix)]
478    #[test]
479    fn changed_files_preserve_newlines_in_unix_names() {
480        let directory = tempfile::tempdir().unwrap();
481        let root = directory.path();
482        let steps = root.join("steps");
483        fs::create_dir_all(&steps).unwrap();
484        let newline = steps.join("line\nbreak.ts");
485        fs::write(&newline, "before").unwrap();
486        initialize_repository(root);
487        fs::write(&newline, "after").unwrap();
488
489        let changed = git_changed_files(root, "HEAD").unwrap();
490        assert_eq!(changed, BTreeSet::from([newline]));
491    }
492
493    #[cfg(unix)]
494    #[test]
495    fn git_path_decoder_preserves_non_utf8_unix_bytes() {
496        use std::os::unix::ffi::OsStrExt;
497
498        let bytes = b"steps/non-utf8-\xff.ts";
499        let path = path_from_git_bytes(bytes).unwrap();
500        assert_eq!(path.as_os_str().as_bytes(), bytes);
501    }
502
503    #[cfg(target_os = "linux")]
504    #[test]
505    fn changed_files_preserve_non_utf8_linux_names() {
506        use std::ffi::OsString;
507        use std::os::unix::ffi::OsStringExt;
508
509        let directory = tempfile::tempdir().unwrap();
510        let root = directory.path();
511        let steps = root.join("steps");
512        fs::create_dir_all(&steps).unwrap();
513        let non_utf8 = steps.join(OsString::from_vec(b"non-utf8-\xff.ts".to_vec()));
514        fs::write(&non_utf8, "before").unwrap();
515        initialize_repository(root);
516        fs::write(&non_utf8, "after").unwrap();
517
518        let changed = git_changed_files(root, "HEAD").unwrap();
519        assert_eq!(changed, BTreeSet::from([non_utf8]));
520    }
521
522    #[test]
523    fn semantic_baseline_survives_location_changes_and_enforces_multiplicity() {
524        let directory = tempfile::tempdir().unwrap();
525        let root = directory.path();
526        let baseline_path = root.join("baseline.json");
527        let mut accepted = vec![finding(root, "steps/original.ts")];
528        let update = update_baseline(&mut accepted, &baseline_path).unwrap();
529        assert_eq!(update.added, 1);
530        assert_eq!(update.removed, 0);
531        assert_eq!(update.suppressed, 1);
532
533        let mut current = vec![
534            finding(root, "moved/renamed.ts"),
535            finding(root, "steps/third-copy.ts"),
536        ];
537        current[0].primary.line = 200;
538        let outcome = apply_baseline(&mut current, &baseline_path).unwrap();
539        assert_eq!(outcome.suppressed, 1);
540        assert_eq!(outcome.new_findings, 1);
541        assert!(!current[0].is_active());
542        assert!(current[1].is_active());
543
544        let baseline: BaselineFile =
545            serde_json::from_str(&fs::read_to_string(&baseline_path).unwrap()).unwrap();
546        assert_eq!(baseline.schema_version, BASELINE_SCHEMA_VERSION);
547        assert_eq!(baseline.fingerprints.values().sum::<usize>(), 1);
548    }
549
550    #[test]
551    fn changed_mode_validates_revisions_repositories_and_ignored_roots() {
552        let directory = tempfile::tempdir().unwrap();
553        fs::write(directory.path().join("tracked.ts"), "initial").unwrap();
554        initialize_repository(directory.path());
555        assert!(
556            ensure_changed_root_is_trackable(&directory.path().canonicalize().unwrap()).is_ok()
557        );
558        for revision in ["", " ", "--all", "missing-revision"] {
559            assert!(
560                resolve_commit(directory.path(), revision).is_err(),
561                "{revision}"
562            );
563        }
564        assert!(git_paths(directory.path(), &["not-a-command"], "expected failure").is_err());
565
566        let ignored = directory.path().join("ignored");
567        fs::create_dir(&ignored).unwrap();
568        fs::write(directory.path().join(".gitignore"), "ignored/\n").unwrap();
569        assert!(
570            ensure_changed_root_is_trackable(&ignored.canonicalize().unwrap())
571                .unwrap_err()
572                .to_string()
573                .contains("ignored by Git")
574        );
575
576        let outside = tempfile::tempdir().unwrap();
577        assert!(ensure_changed_root_is_trackable(outside.path()).is_err());
578    }
579
580    #[test]
581    fn changed_mode_keeps_findings_with_changed_related_locations() {
582        let root = Path::new("/repo");
583        let mut findings = vec![finding(root, "steps/old.ts")];
584        let changed = BTreeSet::from([root.join("steps/shared.ts")]);
585        retain_changed_findings(&mut findings, &changed);
586        assert_eq!(findings.len(), 1);
587    }
588
589    #[test]
590    fn baseline_validation_counts_removals_and_ignores_suppressed_findings() {
591        let directory = tempfile::tempdir().unwrap();
592        let baseline_path = directory.path().join("nested/baseline.json");
593        let mut initial = vec![
594            finding(directory.path(), "one.ts"),
595            finding(directory.path(), "two.ts"),
596        ];
597        initial[1].message = "second duplicate".to_owned();
598        update_baseline(&mut initial, &baseline_path).unwrap();
599
600        let mut current = vec![finding(directory.path(), "one.ts")];
601        current[0].suppression = Some(Suppression {
602            reason: "already accepted elsewhere".to_owned(),
603        });
604        let outcome = update_baseline(&mut current, &baseline_path).unwrap();
605        assert_eq!(outcome.added, 0);
606        assert_eq!(outcome.removed, 2);
607        assert_eq!(outcome.suppressed, 0);
608
609        fs::write(&baseline_path, r#"{"schemaVersion":99,"fingerprints":{}}"#).unwrap();
610        assert!(apply_baseline(&mut [], &baseline_path)
611            .unwrap_err()
612            .to_string()
613            .contains("unsupported baseline schema"));
614        fs::write(&baseline_path, "not json").unwrap();
615        assert!(apply_baseline(&mut [], &baseline_path)
616            .unwrap_err()
617            .to_string()
618            .contains("failed to parse baseline"));
619    }
620
621    #[test]
622    fn comparison_fingerprints_are_independent_of_pair_order() {
623        let comparison = DefinitionComparison {
624            left_fingerprint: "left".to_owned(),
625            right_fingerprint: "right".to_owned(),
626            left_matcher: String::new(),
627            right_matcher: String::new(),
628            left_handler: String::new(),
629            right_handler: String::new(),
630            matcher_diff: crate::model::MatcherDiff {
631                prefix: String::new(),
632                left_change: String::new(),
633                right_change: String::new(),
634                suffix: String::new(),
635            },
636        };
637        let mut reversed = comparison.clone();
638        std::mem::swap(
639            &mut reversed.left_fingerprint,
640            &mut reversed.right_fingerprint,
641        );
642        assert_eq!(
643            comparison_fingerprint_input(&comparison),
644            comparison_fingerprint_input(&reversed)
645        );
646        assert_eq!(
647            count_added(
648                &BTreeMap::from([("a".to_owned(), 3)]),
649                &BTreeMap::from([("a".to_owned(), 1)])
650            ),
651            0
652        );
653    }
654}
cuke_dedup/modes.rs

cuke_dedup/
modes.rs
