use crate::fs_utils::atomic_write_with_mode;
use crate::tool::command::CleanUselessCommentsArgs;
use crate::tool::config::Config;
use crate::tool::tree_sitter_processor::{TreeSitterProcessor, compile_preserve_regexes};
use anyhow::{Result, anyhow};
use glob::Pattern;
use ignore::WalkBuilder;
use regex::Regex;
use std::collections::HashSet;
#[cfg(unix)]
use std::ffi::OsString;
use std::fs;
use std::path::{Path, PathBuf};
use std::process::Command;
#[cfg(unix)]
fn pathbuf_from_git_bytes(bytes: &[u8]) -> PathBuf {
use std::os::unix::ffi::OsStringExt;
PathBuf::from(OsString::from_vec(bytes.to_vec()))
}
#[cfg(not(unix))]
fn pathbuf_from_git_bytes(bytes: &[u8]) -> PathBuf {
PathBuf::from(String::from_utf8_lossy(bytes).as_ref())
}
struct CompiledScopePatterns {
include: Vec<Pattern>,
exclude: Vec<Pattern>,
}
impl CompiledScopePatterns {
fn new(scope: &crate::tool::config::ScopeConfig) -> Self {
Self {
include: scope
.include
.iter()
.filter_map(|pattern| Pattern::new(pattern).ok())
.collect(),
exclude: scope
.exclude
.iter()
.filter_map(|pattern| Pattern::new(pattern).ok())
.collect(),
}
}
fn matches(&self, path: &Path) -> bool {
if self
.exclude
.iter()
.any(|pattern| pattern.matches_path(path))
{
return false;
}
self.include
.iter()
.any(|pattern| pattern.matches_path(path))
}
}
pub struct CommentProcessor {
config: Config,
args: CleanUselessCommentsArgs,
tree_sitter_processor: Option<TreeSitterProcessor>,
}
impl CommentProcessor {
pub fn new(config: Config, args: CleanUselessCommentsArgs) -> Self {
let tree_sitter_processor = match TreeSitterProcessor::new() {
Ok(processor) => Some(processor),
Err(e) => {
eprintln!(
"{}",
t!(
"tool.clean_comments.processor.init_tree_sitter_failed",
error = e
)
);
None
}
};
Self {
config,
args,
tree_sitter_processor,
}
}
pub fn process(&mut self) -> Result<ProcessingResult> {
let cwd = std::env::current_dir()?;
self.process_with_roots(Path::new("."), &cwd)
}
pub fn process_with_cwd(&mut self, cwd: &Path) -> Result<ProcessingResult> {
self.process_with_roots(cwd, cwd)
}
fn process_with_roots(&mut self, walk_root: &Path, cwd: &Path) -> Result<ProcessingResult> {
let clean_config = self
.config
.get_clean_comments_config()
.ok_or_else(|| anyhow!(t!("tool.clean_comments.processor.config_missing")))?;
println!("{}", t!("tool.clean_comments.processor.processing_files"));
let effective_dry_run = self.effective_dry_run();
let clean_config_clone = clean_config.clone();
let compiled_scope = CompiledScopePatterns::new(&clean_config_clone.scope);
let git_only = self.args.git_only
|| clean_config_clone
.safety
.as_ref()
.and_then(|safety| safety.git_aware)
.unwrap_or(false);
let tracked_files = if git_only {
self.load_git_tracked_files(cwd)?
} else {
None
};
let files_to_process = self.find_files(
&compiled_scope,
git_only,
tracked_files.as_ref(),
walk_root,
cwd,
)?;
let mut results = ProcessingResult::default();
for file_path in files_to_process {
if self.args.verbose {
println!(
"{}",
t!(
"tool.clean_comments.processor.processing_file",
path = file_path.display()
)
);
}
match self.process_file(&file_path, &clean_config_clone) {
Ok(file_result) => {
if file_result.has_changes() {
results.files_changed.push(file_path.clone());
results.comments_removed += file_result.comments_removed;
if self.args.verbose || !effective_dry_run {
println!(
"{}",
t!(
"tool.clean_comments.processor.file_removed_comments",
path = file_path.display(),
count = file_result.comments_removed
)
);
}
}
}
Err(e) => {
eprintln!(
"{}",
t!(
"tool.clean_comments.processor.error_processing",
path = file_path.display(),
error = e
)
);
results.errors += 1;
}
}
}
Ok(results)
}
fn find_files(
&self,
scope: &CompiledScopePatterns,
git_only: bool,
tracked_files: Option<&HashSet<PathBuf>>,
walk_root: &Path,
cwd: &Path,
) -> Result<Vec<PathBuf>> {
let mut files = Vec::new();
if git_only && tracked_files.is_none() {
eprintln!("{}", t!("tool.clean_comments.processor.git_only_no_repo"));
return Ok(files);
}
if !self.args.files.is_empty() {
for file in &self.args.files {
let candidate = if file.is_absolute() {
file.clone()
} else {
cwd.join(file)
};
let Some(candidate) = self.validate_explicit_file_path(&candidate)? else {
continue;
};
if !git_only
|| tracked_files
.map(|tracked| self.is_tracked(&candidate, cwd, tracked))
.unwrap_or(false)
{
files.push(candidate);
} else {
eprintln!(
"{}",
t!(
"tool.clean_comments.processor.file_not_tracked",
path = candidate.display()
)
);
}
}
return Ok(files);
}
let walker = WalkBuilder::new(walk_root);
for result in walker.build() {
match result {
Ok(entry) => {
if entry
.file_type()
.is_some_and(|file_type| file_type.is_symlink())
{
continue;
}
let path = entry.path();
if path.is_file() {
let relative = path.strip_prefix(walk_root).unwrap_or(path);
if scope.matches(relative)
&& (!git_only
|| tracked_files
.map(|tracked| self.is_tracked(path, cwd, tracked))
.unwrap_or(false))
{
files.push(path.to_path_buf());
}
}
}
Err(e) => {
eprintln!(
"{}",
t!("tool.clean_comments.processor.walk_error", error = e)
);
}
}
}
Ok(files)
}
fn validate_explicit_file_path(&self, candidate: &Path) -> Result<Option<PathBuf>> {
if !candidate.exists() {
eprintln!(
"{}",
t!(
"tool.clean_comments.processor.file_not_found",
path = candidate.display()
)
);
return Ok(None);
}
let metadata = fs::symlink_metadata(candidate)?;
if metadata.file_type().is_symlink() {
eprintln!(
"{}",
t!(
"tool.clean_comments.processor.path_symlink_skipped",
path = candidate.display()
)
);
return Ok(None);
}
if !metadata.is_file() {
eprintln!(
"{}",
t!(
"tool.clean_comments.processor.path_skipped_not_file",
path = candidate.display()
)
);
return Ok(None);
}
let normalized = match candidate.canonicalize() {
Ok(path) => path,
Err(error) => {
eprintln!(
"{}",
t!(
"tool.clean_comments.processor.path_resolve_failed",
path = candidate.display(),
error = error
)
);
return Ok(None);
}
};
Ok(Some(normalized))
}
fn is_tracked(&self, path: &Path, cwd: &Path, tracked_files: &HashSet<PathBuf>) -> bool {
let absolute = if path.is_absolute() {
path.to_path_buf()
} else {
cwd.join(path)
};
match absolute.canonicalize() {
Ok(normalized) => tracked_files.contains(&normalized),
Err(_) => false,
}
}
fn load_git_tracked_files(&self, cwd: &Path) -> Result<Option<HashSet<PathBuf>>> {
let repo_root = match self.git_repo_root(cwd)? {
Some(root) => root,
None => return Ok(None),
};
let output = match Command::new("git")
.args(["ls-files", "-z"])
.current_dir(&repo_root)
.output()
{
Ok(output) => output,
Err(_) => return Ok(None),
};
if !output.status.success() {
return Ok(None);
}
let mut tracked = HashSet::new();
for entry in output.stdout.split(|byte| *byte == 0) {
if entry.is_empty() {
continue;
}
let candidate = repo_root.join(pathbuf_from_git_bytes(entry));
if let Ok(canonical) = candidate.canonicalize() {
tracked.insert(canonical);
}
}
Ok(Some(tracked))
}
fn git_repo_root(&self, cwd: &Path) -> Result<Option<PathBuf>> {
let output = match Command::new("git")
.args(["rev-parse", "--show-toplevel"])
.current_dir(cwd)
.output()
{
Ok(output) => output,
Err(_) => return Ok(None),
};
if !output.status.success() {
return Ok(None);
}
#[cfg(unix)]
{
use std::os::unix::ffi::OsStringExt;
let mut bytes = output.stdout;
while bytes.last().is_some_and(|b| *b == b'\n' || *b == b'\r') {
bytes.pop();
}
if bytes.is_empty() {
return Ok(None);
}
Ok(Some(PathBuf::from(OsString::from_vec(bytes))))
}
#[cfg(not(unix))]
{
let root = String::from_utf8_lossy(&output.stdout).trim().to_string();
if root.is_empty() {
return Ok(None);
}
Ok(Some(PathBuf::from(root)))
}
}
fn process_file(
&mut self,
file_path: &Path,
clean_config: &crate::tool::config::CleanUselessCommentsConfig,
) -> Result<FileProcessingResult> {
let metadata = fs::symlink_metadata(file_path)?;
if metadata.file_type().is_symlink() {
return Err(anyhow!(
"refusing to modify symlink path: {}",
file_path.display()
));
}
let content = fs::read_to_string(file_path)?;
let language = self.detect_language(file_path);
if let Some(lang_rules) = self.get_language_rules(language, &clean_config.lang_rules) {
let (new_content, comments_removed) =
self.remove_comments_with_tree_sitter(&content, file_path, language, lang_rules)?;
let has_changes = new_content != content;
if !self.effective_dry_run() && has_changes {
atomic_write_with_mode(file_path, new_content.as_bytes(), None)?;
}
Ok(FileProcessingResult {
comments_removed,
has_changes,
})
} else {
Ok(FileProcessingResult {
comments_removed: 0,
has_changes: false,
})
}
}
fn detect_language(&self, path: &Path) -> Option<&'static str> {
match path.extension().and_then(|ext| ext.to_str()) {
Some("py") => Some("python"),
Some("js") => Some("javascript"),
Some("ts") => Some("typescript"),
Some("tsx") => Some("typescript"),
Some("rs") => Some("rust"),
Some("go") => Some("go"),
_ => None,
}
}
fn get_language_rules<'a>(
&self,
language: Option<&str>,
lang_rules: &'a crate::tool::config::LanguageRules,
) -> Option<&'a crate::tool::config::LanguageSpecificRules> {
match language {
Some("python") => lang_rules.python.as_ref(),
Some("javascript") => lang_rules.javascript.as_ref(),
Some("typescript") => lang_rules
.typescript
.as_ref()
.or(lang_rules.javascript.as_ref()),
Some("rust") => lang_rules.rust.as_ref(),
Some("go") => lang_rules.go.as_ref(),
_ => None,
}
}
#[allow(dead_code)]
fn remove_comments(
&self,
content: &str,
rules: &crate::tool::config::LanguageSpecificRules,
) -> Result<String> {
let mut result = content.to_string();
if rules.single_line_comments.unwrap_or(false) {
result = self.remove_single_line_comments(&result, rules)?;
}
if rules.multi_line_comments.unwrap_or(false) {
result = self.remove_multi_line_comments(&result, rules)?;
}
Ok(result)
}
#[allow(dead_code)]
fn remove_single_line_comments(
&self,
content: &str,
rules: &crate::tool::config::LanguageSpecificRules,
) -> Result<String> {
let empty_patterns = vec![];
let preserve_patterns = rules.preserve_patterns.as_ref().unwrap_or(&empty_patterns);
let preserve_regexes: Vec<_> = preserve_patterns
.iter()
.filter_map(|p| Regex::new(p).ok())
.collect();
if self.args.verbose {
println!(
"{}",
t!(
"tool.clean_comments.processor.preserve_patterns_found",
count = preserve_regexes.len()
)
);
}
let lines: Vec<&str> = content.lines().collect();
let mut result = Vec::new();
for line in lines {
let mut should_remove = false;
if let Some(comment_start) = line.find('#').or_else(|| line.find("//")) {
let before_comment = &line[..comment_start];
let comment = &line[comment_start..];
let should_preserve = preserve_regexes
.iter()
.any(|regex| regex.is_match(comment.trim()));
let min_length = rules.min_comment_length.unwrap_or(0);
let comment_too_short = comment.trim().len() < min_length;
let _mostly_comment = before_comment.trim().len() <= 10;
should_remove = !should_preserve && comment_too_short;
if self.args.verbose {
println!(
"{}",
t!("tool.clean_comments.processor.debug_line", line = line)
);
println!(
"{}",
t!(
"tool.clean_comments.processor.debug_comment",
comment = comment.trim()
)
);
println!(
"{}",
t!(
"tool.clean_comments.processor.debug_should_preserve",
preserve = should_preserve
)
);
println!(
"{}",
t!(
"tool.clean_comments.processor.debug_comment_too_short",
too_short = comment_too_short,
len = comment.trim().len(),
min = min_length
)
);
println!(
"{}",
t!(
"tool.clean_comments.processor.debug_would_remove",
remove = should_remove
)
);
}
}
if should_remove {
if let Some(comment_start) = line.find('#').or_else(|| line.find("//")) {
result.push(&line[..comment_start]);
}
} else {
result.push(line);
}
}
Ok(result.join("\n"))
}
#[allow(dead_code)]
fn remove_multi_line_comments(
&self,
content: &str,
_rules: &crate::tool::config::LanguageSpecificRules,
) -> Result<String> {
let mut result = content.to_string();
let re = Regex::new(r"/\*.*?\*/").map_err(|e| {
anyhow!(t!(
"tool.clean_comments.processor.regex_build_failed",
error = e
))
})?;
result = re.replace_all(&result, "").to_string();
Ok(result)
}
fn remove_comments_with_tree_sitter(
&mut self,
content: &str,
file_path: &Path,
language: Option<&str>,
rules: &crate::tool::config::LanguageSpecificRules,
) -> Result<(String, u32)> {
let tree_sitter_processor = self.tree_sitter_processor.as_mut().ok_or_else(|| {
anyhow!(t!(
"tool.clean_comments.processor.tree_sitter_unavailable",
path = file_path.display()
))
})?;
let (preserve_regexes, invalid_patterns) =
compile_preserve_regexes(rules.preserve_patterns.as_deref());
if self.args.verbose && !invalid_patterns.is_empty() {
let lang = language.unwrap_or("unknown");
let key = format!("tools.clean-useless-comments.lang-rules.{lang}.preserve-patterns");
for invalid in invalid_patterns {
eprintln!(
"Warning: invalid regex in {} for {}: {} ({})",
key,
file_path.display(),
invalid.pattern,
invalid.error
);
}
}
match tree_sitter_processor.remove_comments_from_content_with_regexes(
content,
file_path,
rules,
&preserve_regexes,
) {
Ok((new_content, removed_comments)) => {
let comments_removed = removed_comments.len() as u32;
Ok((new_content, comments_removed))
}
Err(e) => Err(anyhow!(t!(
"tool.clean_comments.processor.tree_sitter_failed",
path = file_path.display(),
error = e
))),
}
}
#[allow(dead_code)]
fn count_removed_comments(&self, original: &str, modified: &str) -> u32 {
let original_lines = original.lines().count();
let modified_lines = modified.lines().count();
(original_lines.saturating_sub(modified_lines)) as u32
}
fn effective_dry_run(&self) -> bool {
self.args.dry_run || !self.args.yes
}
}
#[derive(Debug, Default)]
pub struct ProcessingResult {
pub files_changed: Vec<PathBuf>,
pub comments_removed: u32,
pub errors: u32,
}
#[derive(Debug)]
pub struct FileProcessingResult {
pub comments_removed: u32,
pub has_changes: bool,
}
impl FileProcessingResult {
pub fn has_changes(&self) -> bool {
self.has_changes
}
}
#[cfg(test)]
mod tests {
use super::CommentProcessor;
use crate::tool::command::CleanUselessCommentsArgs;
use crate::tool::config::{
CleanUselessCommentsConfig, Config, LanguageRules, LanguageSpecificRules, ScopeConfig,
ToolsConfig,
};
use tempfile::TempDir;
#[test]
fn test_tree_sitter_unavailable_does_not_modify_files() {
let temp_dir = TempDir::new().unwrap();
let work_dir = temp_dir.path();
let file_path = work_dir.join("test.py");
let original = "# short\n\ndef test():\n pass\n";
std::fs::write(&file_path, original).unwrap();
let config = Config {
version: "0.1".to_string(),
tools: ToolsConfig {
rm_useless_dirs: None,
clean_useless_comments: Some(CleanUselessCommentsConfig {
scope: ScopeConfig {
include: vec!["**/*.py".to_string()],
exclude: Vec::new(),
},
lang_rules: LanguageRules {
python: Some(LanguageSpecificRules {
single_line_comments: Some(true),
min_comment_length: Some(100),
..Default::default()
}),
javascript: None,
typescript: None,
rust: None,
go: None,
},
global_rules: None,
safety: None,
output: None,
}),
},
};
let args = CleanUselessCommentsArgs {
config: None,
dry_run: false,
yes: true,
interactive: false,
force: true,
verbose: false,
git_only: false,
files: vec![file_path.clone()],
};
let mut processor = CommentProcessor {
config,
args,
tree_sitter_processor: None,
};
let result = processor.process_with_cwd(work_dir).unwrap();
assert_eq!(result.errors, 1);
assert!(result.files_changed.is_empty());
let actual = std::fs::read_to_string(&file_path).unwrap();
assert_eq!(actual, original);
}
#[cfg(unix)]
#[test]
fn test_symlink_input_is_skipped() {
use std::os::unix::fs as unix_fs;
let temp_dir = TempDir::new().unwrap();
let root = temp_dir.path();
let work_dir = root.join("work");
std::fs::create_dir_all(&work_dir).unwrap();
let target = root.join("target.py");
let original = "# short\n\ndef test():\n pass\n";
std::fs::write(&target, original).unwrap();
let link = work_dir.join("linked.py");
unix_fs::symlink(&target, &link).unwrap();
let config = Config {
version: "0.1".to_string(),
tools: ToolsConfig {
rm_useless_dirs: None,
clean_useless_comments: Some(CleanUselessCommentsConfig {
scope: ScopeConfig {
include: vec!["**/*.py".to_string()],
exclude: Vec::new(),
},
lang_rules: LanguageRules {
python: Some(LanguageSpecificRules {
single_line_comments: Some(true),
min_comment_length: Some(100),
..Default::default()
}),
javascript: None,
typescript: None,
rust: None,
go: None,
},
global_rules: None,
safety: None,
output: None,
}),
},
};
let args = CleanUselessCommentsArgs {
config: None,
dry_run: false,
yes: true,
interactive: false,
force: true,
verbose: false,
git_only: false,
files: vec![link],
};
let mut processor = CommentProcessor {
config,
args,
tree_sitter_processor: None,
};
let result = processor.process_with_cwd(&work_dir).unwrap();
assert_eq!(result.errors, 0);
assert!(result.files_changed.is_empty());
assert_eq!(std::fs::read_to_string(&target).unwrap(), original);
}
}