use anyhow::Result;
use sha2::{Digest, Sha256};
use std::collections::HashMap;
use std::path::{Path, PathBuf};
use crate::config::{CheckerConfig, output_config_hash, resolve_extra_inputs};
use crate::file_index::FileIndex;
use crate::graph::{BuildGraph, Product};
pub struct DuplicateFilesProcessor {
config: CheckerConfig,
}
impl DuplicateFilesProcessor {
pub const fn new(config: CheckerConfig) -> Self {
Self { config }
}
fn dependency_inputs(&self) -> Result<Vec<PathBuf>> {
let scan = &self.config.standard;
let mut names = scan.dep_inputs.clone();
for ai in &scan.dep_auto {
names.extend(crate::processor::config_file_inputs(ai));
}
resolve_extra_inputs(&names)
}
fn check_files(&self, files: &[&Path]) -> Result<()> {
let mut checksums: HashMap<Vec<u8>, Vec<&Path>> = HashMap::new();
for &file in files {
let bytes = crate::errors::ctx(
std::fs::read(file),
&format!("Failed to read {}", file.display()),
)?;
let hash = Sha256::digest(&bytes).to_vec();
checksums.entry(hash).or_default().push(file);
}
let mut duplicates: Vec<String> = Vec::new();
for paths in checksums.values() {
if paths.len() > 1 {
let names: Vec<String> = paths.iter().map(|p| p.display().to_string()).collect();
duplicates.push(format!(" {}", names.join(", ")));
}
}
if duplicates.is_empty() {
Ok(())
} else {
duplicates.sort();
anyhow::bail!(
"{} set(s) of duplicate files found:\n{}",
duplicates.len(),
duplicates.join("\n"),
)
}
}
}
impl crate::processor::Processor for DuplicateFilesProcessor {
fn scan_config(&self) -> &crate::config::StandardConfig {
&self.config.standard
}
fn required_tools(&self) -> Vec<String> {
Vec::new()
}
fn discover(
&self,
graph: &mut BuildGraph,
file_index: &FileIndex,
instance_name: &str,
) -> Result<()> {
let mut inputs = file_index.scan(&self.config.standard, true);
if inputs.is_empty() {
return Ok(());
}
for dep in self.dependency_inputs()? {
if !inputs.contains(&dep) {
inputs.push(dep);
}
}
let hash = Some(output_config_hash(
&self.config,
&crate::config::checksum_fields_of(instance_name),
));
graph.add_product(inputs, vec![], instance_name, hash)?;
Ok(())
}
fn execute(&self, _ctx: &crate::build_context::BuildContext, product: &Product) -> Result<()> {
let dependencies = self.dependency_inputs()?;
let mut files: Vec<&Path> = Vec::with_capacity(product.inputs.len());
for input in &product.inputs {
if !dependencies.contains(input) && !files.contains(&input.as_path()) {
files.push(input.as_path());
}
}
self.check_files(&files)
}
}
fn plugin_create(toml: &toml::Value) -> anyhow::Result<Box<dyn crate::processor::Processor>> {
crate::registries::deserialize_and_create(toml, |cfg| {
Box::new(DuplicateFilesProcessor::new(cfg))
})
}
inventory::submit! {
crate::registries::ProcessorPlugin {
version: 1,
name: "duplicate_files",
processor_type: crate::processor::ProcessorType::Checker,
create: plugin_create,
defconfig_json: crate::registries::default_config_json::<crate::config::CheckerConfig>,
fields: &[],
omit_standard_fields: &[
"command",
"formats",
"args",
"output_dir",
"required_tools",
"batch",
],
scan_defaults: Some(crate::config::ScanDefaultsData { src_dirs: &[], src_extensions: &[".py", ".rs", ".js", ".ts", ".c", ".cc", ".h", ".hh", ".java", ".rb", ".go", ".sh", ".md", ".yaml", ".yml", ".json", ".toml", ".xml", ".html", ".css"], src_exclude_dirs: &[] }),
defaults: None,
keywords: &["checker", "duplicates", "files"],
description: "Detect duplicate files by content (SHA-256)",
is_builtin: true,
is_rust: true,
supports_batch: false,
caches_outputs: true,
max_jobs_cap: None,
}
}