zzz_arc/
filter.rs

1//! file filtering system with comprehensive garbage file exclusion
2
3use crate::Result;
4use glob::Pattern;
5use once_cell::sync::Lazy;
6use std::path::Path;
7use walkdir::WalkDir;
8
9/// comprehensive list of garbage files to exclude by default
10pub const GARBAGE_FILES: &[&str] = &[
11    // macOS system files
12    "__MACOSX",
13    ".AppleDouble",
14    ".DS_Store",
15    "._*", // resource forks
16    ".LSOverride",
17    ".Spotlight-V100",         // spotlight index
18    ".Trashes",                // trash
19    ".fseventsd",              // filesystem events
20    ".VolumeIcon.icns",        // volume icons
21    ".DocumentRevisions-V100", // document revisions
22    ".TemporaryItems",         // temporary items
23    "Icon\r",                  // finder icon
24    // Windows system files
25    "thumbs.db",
26    "Thumbs.db",
27    "desktop.ini",
28    "Desktop.ini",
29    "ehthumbs.db",
30    "ehthumbs_vista.db",
31    "$RECYCLE.BIN",
32    "System Volume Information",
33    "hiberfil.sys",
34    "pagefile.sys",
35    "swapfile.sys",
36    // Linux/Unix system files
37    ".directory", // KDE folder metadata
38    ".trash",
39    ".Trash-*", // trash directories
40    ".nfs*",    // NFS lock files
41    "lost+found",
42    // Development artifacts
43    "__pycache__",
44    "*.pyc",
45    "*.pyo",
46    "*.pyd",
47    ".mypy_cache",
48    ".pytest_cache",
49    ".ruff_cache",
50    ".hypothesis",
51    ".coverage",
52    ".nox",
53    ".tox",
54    "__pypackages__",
55    ".venv",
56    "venv",
57    "node_modules",
58    ".npm",
59    ".pnpm-store",
60    ".yarn",
61    ".yarn-cache",
62    "bower_components",
63    ".git",
64    ".svn",
65    ".hg",
66    ".bzr",
67    "target",
68    "target/debug",
69    "target/release", // Rust build dirs
70    ".gradle",
71    ".maven",
72    ".vscode",
73    ".idea", // IDE files (configurable)
74    ".direnv",
75    ".cache",
76    ".parcel-cache",
77    ".turbo",
78    ".next",
79    ".nuxt",
80    ".svelte-kit",
81    "CMakeFiles",
82    "CMakeCache.txt",
83    "cmake-build-*",
84    "npm-debug.log",
85    "yarn-error.log",
86    "pnpm-debug.log",
87    // Temporary/backup files
88    "*.tmp",
89    "*.temp",
90    "*.bak",
91    "*.orig",
92    "*~",
93    ".#*",
94    "#*#", // editor backup files
95    "*.swp",
96    "*.swo",  // vim swap files
97    ".*.sw?", // vim swap file pattern
98];
99
100/// common sensitive files and directories for redact mode
101pub const SENSITIVE_FILES: &[&str] = &[
102    ".env",
103    ".env.*",
104    ".npmrc",
105    ".pypirc",
106    ".netrc",
107    ".aws",
108    ".ssh",
109    ".gnupg",
110    ".kube",
111    "id_rsa",
112    "id_dsa",
113    "id_ecdsa",
114    "id_ed25519",
115    "*.pem",
116    "*.key",
117    "*.p12",
118    "*.pfx",
119    "*.kdbx",
120    "*.gpg",
121    "*.pgp",
122    "*.age",
123    ".sops.yaml",
124    ".sops.yml",
125    ".sops.json",
126];
127
128static COMPILED_GARBAGE_PATTERNS: Lazy<Vec<Pattern>> = Lazy::new(|| {
129    GARBAGE_FILES
130        .iter()
131        .filter_map(|s| Pattern::new(s).ok())
132        .collect()
133});
134
135pub struct FileFilter {
136    use_defaults: bool,
137    custom_excludes: Vec<Pattern>,
138}
139
140impl FileFilter {
141    fn normalize_relative_path(path: &Path) -> String {
142        let mut parts = Vec::new();
143        for component in path.components() {
144            if let std::path::Component::Normal(part) = component {
145                parts.push(part.to_string_lossy());
146            }
147        }
148        parts.join("/")
149    }
150
151    fn matches_patterns(patterns: &[Pattern], filename: &str, relative_path: &Path) -> bool {
152        if !filename.is_empty() && patterns.iter().any(|pattern| pattern.matches(filename)) {
153            return true;
154        }
155
156        let components: Vec<String> = relative_path
157            .components()
158            .filter_map(|component| match component {
159                std::path::Component::Normal(part) => Some(part.to_string_lossy().to_string()),
160                _ => None,
161            })
162            .collect();
163
164        if components
165            .iter()
166            .any(|component| patterns.iter().any(|pattern| pattern.matches(component)))
167        {
168            return true;
169        }
170
171        for ancestor in relative_path.ancestors() {
172            let ancestor_str = Self::normalize_relative_path(ancestor);
173            if ancestor_str.is_empty() {
174                continue;
175            }
176            if patterns
177                .iter()
178                .any(|pattern| pattern.matches(&ancestor_str))
179            {
180                return true;
181            }
182        }
183
184        false
185    }
186
187    /// create new file filter with optional custom patterns
188    pub fn new(use_defaults: bool, custom_patterns: &[String]) -> Result<Self> {
189        let mut custom_excludes = Vec::new();
190        for pattern in custom_patterns {
191            custom_excludes.push(Pattern::new(pattern)?);
192        }
193        Ok(Self {
194            use_defaults,
195            custom_excludes,
196        })
197    }
198
199    /// check if a path should be excluded from archiving
200    pub fn should_exclude(&self, path: &Path) -> bool {
201        let filename = path.file_name().and_then(|n| n.to_str()).unwrap_or("");
202
203        // check custom patterns first (against filename only for consistency)
204        for pattern in &self.custom_excludes {
205            if pattern.matches(filename) {
206                return true;
207            }
208        }
209
210        // check default garbage files if enabled
211        if self.use_defaults {
212            for pattern in &*COMPILED_GARBAGE_PATTERNS {
213                if pattern.matches(filename) {
214                    return true;
215                }
216            }
217        }
218
219        false
220    }
221
222    /// check if a relative path should be excluded from archiving
223    pub fn should_exclude_relative(&self, relative_path: &Path) -> bool {
224        if relative_path.as_os_str().is_empty() {
225            return false;
226        }
227
228        let filename = relative_path
229            .file_name()
230            .and_then(|n| n.to_str())
231            .unwrap_or("");
232
233        if Self::matches_patterns(&self.custom_excludes, filename, relative_path) {
234            return true;
235        }
236
237        if self.use_defaults
238            && Self::matches_patterns(&COMPILED_GARBAGE_PATTERNS, filename, relative_path)
239        {
240            return true;
241        }
242
243        false
244    }
245
246    /// check if a path should be excluded, based on its relative path to a root
247    pub fn should_exclude_path(&self, root: &Path, path: &Path) -> bool {
248        let relative_path = path.strip_prefix(root).unwrap_or(path);
249        self.should_exclude_relative(relative_path)
250    }
251
252    /// check if a path should be included in archiving (inverse of should_exclude)
253    pub fn should_include(&self, path: &Path) -> bool {
254        !self.should_exclude(path)
255    }
256
257    /// check if a relative path should be included in archiving
258    pub fn should_include_relative(&self, relative_path: &Path) -> bool {
259        !self.should_exclude_relative(relative_path)
260    }
261
262    /// check if a path should be included, based on its relative path to a root
263    pub fn should_include_path(&self, root: &Path, path: &Path) -> bool {
264        !self.should_exclude_path(root, path)
265    }
266
267    /// walk a directory tree while applying filter pruning
268    pub fn walk_entries<'a>(
269        &'a self,
270        root: &'a Path,
271    ) -> impl Iterator<Item = walkdir::Result<walkdir::DirEntry>> + 'a {
272        self.walk_entries_with_follow(root, false)
273    }
274
275    /// walk a directory tree while applying filter pruning, optionally following symlinks
276    pub fn walk_entries_with_follow<'a>(
277        &'a self,
278        root: &'a Path,
279        follow_links: bool,
280    ) -> impl Iterator<Item = walkdir::Result<walkdir::DirEntry>> + 'a {
281        WalkDir::new(root)
282            .follow_links(follow_links)
283            .into_iter()
284            .filter_entry(move |entry| self.should_include_path(root, entry.path()))
285    }
286}
287
288#[cfg(test)]
289mod tests {
290    use super::*;
291    use std::path::Path;
292
293    #[test]
294    fn test_file_filter_with_defaults() -> Result<()> {
295        let filter = FileFilter::new(true, &[])?;
296
297        // Test default garbage files
298        assert!(filter.should_exclude(Path::new(".DS_Store")));
299        assert!(filter.should_exclude(Path::new("thumbs.db")));
300        assert!(filter.should_exclude(Path::new("__pycache__")));
301        assert!(filter.should_exclude(Path::new("node_modules")));
302        assert!(filter.should_exclude(Path::new(".git")));
303        assert!(filter.should_exclude(Path::new("__MACOSX")));
304        assert!(filter.should_exclude(Path::new(".AppleDouble")));
305        assert!(filter.should_exclude(Path::new("lost+found")));
306        assert!(filter.should_exclude(Path::new(".mypy_cache")));
307        assert!(filter.should_exclude(Path::new(".ruff_cache")));
308        assert!(filter.should_exclude(Path::new("npm-debug.log")));
309        assert!(filter.should_exclude(Path::new("target")));
310        assert!(filter.should_exclude(Path::new(".cache")));
311        assert!(filter.should_exclude(Path::new(".venv")));
312        assert!(filter.should_exclude(Path::new("__pypackages__")));
313        assert!(filter.should_exclude(Path::new("CMakeCache.txt")));
314
315        // Test pattern matching
316        assert!(filter.should_exclude(Path::new("._resource_fork")));
317        assert!(filter.should_exclude(Path::new("file.tmp")));
318        assert!(filter.should_exclude(Path::new("backup~")));
319        assert!(filter.should_exclude(Path::new(".#lockfile")));
320
321        // Test normal files are not excluded
322        assert!(!filter.should_exclude(Path::new("README.md")));
323        assert!(!filter.should_exclude(Path::new("src/main.rs")));
324        assert!(!filter.should_exclude(Path::new("Cargo.toml")));
325        assert!(!filter.should_exclude(Path::new(".env")));
326
327        Ok(())
328    }
329
330    #[test]
331    fn test_file_filter_without_defaults() -> Result<()> {
332        let filter = FileFilter::new(false, &[])?;
333
334        // Default garbage files should NOT be excluded when defaults are disabled
335        assert!(!filter.should_exclude(Path::new(".DS_Store")));
336        assert!(!filter.should_exclude(Path::new("thumbs.db")));
337        assert!(!filter.should_exclude(Path::new("__pycache__")));
338
339        Ok(())
340    }
341
342    #[test]
343    fn test_custom_excludes() -> Result<()> {
344        let custom_patterns = vec![
345            "*.log".to_string(),
346            "test_*".to_string(),
347            "secret.txt".to_string(),
348        ];
349        let filter = FileFilter::new(true, &custom_patterns)?;
350
351        // Test custom patterns
352        assert!(filter.should_exclude(Path::new("debug.log")));
353        assert!(filter.should_exclude(Path::new("test_file.txt")));
354        assert!(filter.should_exclude(Path::new("secret.txt")));
355
356        // Test files that should not be excluded
357        assert!(!filter.should_exclude(Path::new("important.txt")));
358        assert!(!filter.should_exclude(Path::new("production_config.yaml")));
359
360        Ok(())
361    }
362
363    #[test]
364    fn test_custom_excludes_override_defaults() -> Result<()> {
365        let custom_patterns = vec!["*.rs".to_string()];
366        let filter = FileFilter::new(true, &custom_patterns)?;
367
368        // Custom patterns should work
369        assert!(filter.should_exclude(Path::new("main.rs")));
370        assert!(filter.should_exclude(Path::new("lib.rs")));
371
372        // Default patterns should still work
373        assert!(filter.should_exclude(Path::new(".DS_Store")));
374
375        // Non-matching files should not be excluded
376        assert!(!filter.should_exclude(Path::new("Cargo.toml")));
377
378        Ok(())
379    }
380
381    #[test]
382    fn test_sensitive_patterns_match_paths() -> Result<()> {
383        let custom_patterns = SENSITIVE_FILES
384            .iter()
385            .map(|pattern| (*pattern).to_string())
386            .collect::<Vec<_>>();
387        let filter = FileFilter::new(true, &custom_patterns)?;
388
389        assert!(filter.should_exclude_relative(Path::new(".env")));
390        assert!(filter.should_exclude_relative(Path::new("config/.env.local")));
391        assert!(filter.should_exclude_relative(Path::new(".ssh/id_rsa")));
392        assert!(filter.should_exclude_relative(Path::new("config/.aws/credentials")));
393        assert!(filter.should_exclude_relative(Path::new("keys/server.pem")));
394        assert!(filter.should_exclude_relative(Path::new("keys/key.p12")));
395        assert!(filter.should_exclude_relative(Path::new("vault/passwords.kdbx")));
396
397        assert!(!filter.should_exclude_relative(Path::new("docs/notes.txt")));
398
399        Ok(())
400    }
401
402    #[test]
403    fn test_path_based_matching() -> Result<()> {
404        let filter = FileFilter::new(true, &[])?;
405        let root = Path::new("project");
406
407        assert!(filter.should_exclude_path(root, Path::new("project/.git/config")));
408        assert!(filter.should_exclude_path(root, Path::new("project/vendor/.git/config")));
409        assert!(filter.should_exclude_path(root, Path::new("project/target/debug/app")));
410        assert!(!filter.should_exclude_path(root, Path::new("project/src/main.rs")));
411
412        Ok(())
413    }
414
415    #[test]
416    fn test_invalid_pattern() {
417        // Test that invalid glob patterns return an error
418        let result = FileFilter::new(true, &["[".to_string()]);
419        assert!(result.is_err());
420    }
421
422    #[test]
423    fn test_path_vs_filename_matching() -> Result<()> {
424        let filter = FileFilter::new(true, &[])?;
425
426        // Test that patterns match on filename, not full path
427        assert!(filter.should_exclude(Path::new("some/path/.DS_Store")));
428        assert!(filter.should_exclude(Path::new("nested/dir/thumbs.db")));
429
430        // Test directory names in paths don't trigger false matches
431        assert!(!filter.should_exclude(Path::new(".DS_Store_backup/file.txt")));
432
433        Ok(())
434    }
435}
zzz_arc/filter.rs

zzz_arc/
filter.rs
