smugmug_cli/dated/
scan.rs

1//! Finding the files to upload: walking the sources without opening any
2//! file (just a `stat` each), so a re-run over a big, unchanged library
3//! costs a directory walk and nothing more.
4//!
5//! Excluded paths are pruned (an excluded directory is never entered), and
6//! so are hidden files and directories (`.DS_Store`, AppleDouble `._*`
7//! files, `.thumbnails`) and NAS metadata directories. A `.smugmugignore`
8//! file in any directory adds .gitignore-style patterns for it.
9
10use anyhow::{Context, Result};
11use ignore::WalkBuilder;
12use ignore::overrides::OverrideBuilder;
13use std::collections::{HashMap, HashSet};
14use std::path::{Path, PathBuf};
15use std::time::UNIX_EPOCH;
16
17use crate::config::{LivePhotoVideos, RawHandling};
18use crate::uploader::{RawSelection, select_raw_files};
19
20/// Name of the per-directory ignore file.
21pub const IGNORE_FILE: &str = ".smugmugignore";
22
23/// Always left out: NAS metadata and recycle bins, never photos of anyone's.
24const DEFAULT_EXCLUDES: &[&str] = &["@eaDir/", "#recycle/", "#snapshot/", "$RECYCLE.BIN/"];
25
26#[derive(Debug, Clone, PartialEq, Eq)]
27pub struct ScannedFile {
28    pub path: PathBuf,
29    pub size: u64,
30    /// Modification time in nanoseconds since the Unix epoch.
31    pub mtime_ns: i64,
32    /// For the video half of a Live Photo: its photo.
33    pub live_photo_of: Option<PathBuf>,
34}
35
36#[derive(Debug, Default)]
37pub struct ScanResult {
38    /// Sorted by path, so files are read directory by directory.
39    pub files: Vec<ScannedFile>,
40    /// Files of types SmugMug doesn't take (sidecars, catalogs, documents).
41    pub unsupported: usize,
42    /// Entries that couldn't be read (permissions, vanished mid-walk).
43    pub errors: Vec<String>,
44    pub raw_selection: RawSelection,
45    /// Live Photo videos left out (`LivePhotoVideos::Skip`).
46    pub live_photo_videos_skipped: usize,
47    /// Live Photo videos kept, dated by their photo.
48    pub live_photo_videos: usize,
49}
50
51/// Walk `sources`, leaving out `excludes` (.gitignore syntax, relative to
52/// each source).
53pub fn scan(
54    sources: &[PathBuf],
55    excludes: &[String],
56    raw_handling: RawHandling,
57    live_photo_videos: LivePhotoVideos,
58) -> Result<ScanResult> {
59    let mut result = ScanResult::default();
60    let mut found = Vec::new();
61
62    for source in sources {
63        if !source.exists() {
64            anyhow::bail!("{} doesn't exist", source.display());
65        }
66        // A single file is just that file.
67        if source.is_file() {
68            match stat(source) {
69                Ok(f) if crate::scanner::is_supported_file(source) => found.push(f),
70                Ok(_) => result.unsupported += 1,
71                Err(e) => result.errors.push(format!("{}: {}", source.display(), e)),
72            }
73            continue;
74        }
75
76        let mut overrides = OverrideBuilder::new(source);
77        for pattern in DEFAULT_EXCLUDES
78            .iter()
79            .copied()
80            .chain(excludes.iter().map(String::as_str))
81        {
82            // In an override, a plain glob whitelists; "!" makes it ignore.
83            overrides
84                .add(&format!("!{}", pattern))
85                .with_context(|| format!("Invalid exclude pattern '{}'", pattern))?;
86        }
87        let overrides = overrides.build().context("Invalid exclude patterns")?;
88
89        let walker = WalkBuilder::new(source)
90            .standard_filters(false)
91            .hidden(true)
92            .follow_links(true)
93            .add_custom_ignore_filename(IGNORE_FILE)
94            .overrides(overrides)
95            .build();
96
97        for entry in walker {
98            let entry = match entry {
99                Ok(entry) => entry,
100                Err(e) => {
101                    result.errors.push(e.to_string());
102                    continue;
103                }
104            };
105            if !entry
106                .file_type()
107                .is_some_and(|t| t.is_file() || t.is_symlink())
108            {
109                continue;
110            }
111            let path = entry.path();
112            if !crate::scanner::is_supported_file(path) {
113                result.unsupported += 1;
114                continue;
115            }
116            match stat(path) {
117                Ok(f) => found.push(f),
118                Err(e) => result.errors.push(format!("{}: {}", path.display(), e)),
119            }
120        }
121    }
122
123    // Overlapping sources would list a file twice.
124    found.sort_by(|a, b| a.path.cmp(&b.path));
125    found.dedup_by(|a, b| a.path == b.path);
126
127    // RAW+JPEG pairs keep the JPEG (see `select_raw_files`).
128    let paths: Vec<PathBuf> = found.iter().map(|f| f.path.clone()).collect();
129    let (kept, raw_selection) = select_raw_files(paths, raw_handling);
130    result.raw_selection = raw_selection;
131    let kept: HashSet<PathBuf> = kept.into_iter().collect();
132    found.retain(|f| kept.contains(&f.path));
133
134    pair_live_photos(&mut found);
135    if live_photo_videos == LivePhotoVideos::Skip {
136        let before = found.len();
137        found.retain(|f| f.live_photo_of.is_none());
138        result.live_photo_videos_skipped = before - found.len();
139    } else {
140        result.live_photo_videos = found.iter().filter(|f| f.live_photo_of.is_some()).count();
141    }
142
143    result.files = found;
144    Ok(result)
145}
146
147fn stat(path: &Path) -> std::io::Result<ScannedFile> {
148    let meta = std::fs::metadata(path)?;
149    let mtime_ns = meta
150        .modified()
151        .ok()
152        .and_then(|t| t.duration_since(UNIX_EPOCH).ok())
153        .map(|d| d.as_nanos().min(i64::MAX as u128) as i64)
154        .unwrap_or(0);
155    Ok(ScannedFile {
156        path: path.to_path_buf(),
157        size: meta.len(),
158        mtime_ns,
159        live_photo_of: None,
160    })
161}
162
163fn extension(path: &Path) -> String {
164    path.extension()
165        .map(|e| e.to_string_lossy().to_lowercase())
166        .unwrap_or_default()
167}
168
169fn is_still(path: &Path) -> bool {
170    matches!(
171        extension(path).as_str(),
172        "jpg" | "jpeg" | "heic" | "heif" | "png" | "dng"
173    ) || crate::scanner::is_raw_file(path)
174}
175
176fn is_live_video(path: &Path) -> bool {
177    matches!(extension(path).as_str(), "mov" | "mp4")
178}
179
180/// Mark each video that shares its directory and name (ignoring case and
181/// extension) with a photo as that photo's Live Photo video.
182fn pair_live_photos(files: &mut [ScannedFile]) {
183    let key = |p: &Path| {
184        (
185            p.parent().map(Path::to_path_buf),
186            p.file_stem().map(|s| s.to_string_lossy().to_lowercase()),
187        )
188    };
189    let stills: HashMap<_, PathBuf> = files
190        .iter()
191        .filter(|f| is_still(&f.path))
192        .map(|f| (key(&f.path), f.path.clone()))
193        .collect();
194    for f in files.iter_mut() {
195        if is_live_video(&f.path) {
196            f.live_photo_of = stills.get(&key(&f.path)).cloned();
197        }
198    }
199}
200
201#[cfg(test)]
202mod tests {
203    use super::*;
204    use std::fs;
205    use tempfile::TempDir;
206
207    fn touch(root: &Path, rel: &str) {
208        let path = root.join(rel);
209        fs::create_dir_all(path.parent().unwrap()).unwrap();
210        fs::write(path, b"x").unwrap();
211    }
212
213    fn rel(result: &ScanResult, root: &Path) -> Vec<String> {
214        result
215            .files
216            .iter()
217            .map(|f| {
218                f.path
219                    .strip_prefix(root)
220                    .unwrap()
221                    .to_string_lossy()
222                    .replace('\\', "/")
223            })
224            .collect()
225    }
226
227    fn scan_one(root: &Path, excludes: &[&str]) -> ScanResult {
228        let excludes: Vec<String> = excludes.iter().map(|s| s.to_string()).collect();
229        scan(
230            &[root.to_path_buf()],
231            &excludes,
232            RawHandling::Render,
233            LivePhotoVideos::Upload,
234        )
235        .unwrap()
236    }
237
238    #[test]
239    fn excluded_directories_hidden_files_and_nas_metadata_are_left_out() {
240        let dir = TempDir::new().unwrap();
241        let root = dir.path();
242        touch(root, "keep/a.jpg");
243        touch(root, "old_backup/b.jpg");
244        touch(root, "keep/deep/old_backup/c.jpg");
245        touch(root, "keep/._a.jpg");
246        touch(root, ".hidden/d.jpg");
247        touch(root, "keep/@eaDir/a.jpg/SYNOPHOTO_THUMB_XL.jpg");
248        touch(root, "keep/Thumbs.db");
249        touch(root, "keep/a.xmp");
250
251        let result = scan_one(root, &["/old_backup/"]);
252        assert_eq!(
253            rel(&result, root),
254            vec!["keep/a.jpg", "keep/deep/old_backup/c.jpg"]
255        );
256        assert_eq!(result.unsupported, 2);
257    }
258
259    #[test]
260    fn unanchored_patterns_match_at_any_depth() {
261        let dir = TempDir::new().unwrap();
262        let root = dir.path();
263        touch(root, "a/old_backup/b.jpg");
264        touch(root, "a/c.jpg");
265        touch(root, "a/screenshot.png");
266
267        let result = scan_one(root, &["old_backup/", "*.png"]);
268        assert_eq!(rel(&result, root), vec!["a/c.jpg"]);
269    }
270
271    #[test]
272    fn smugmugignore_files_add_patterns() {
273        let dir = TempDir::new().unwrap();
274        let root = dir.path();
275        touch(root, "a/keep.jpg");
276        touch(root, "a/skip/x.jpg");
277        fs::write(root.join("a").join(IGNORE_FILE), "skip/\n").unwrap();
278
279        let result = scan_one(root, &[]);
280        assert_eq!(rel(&result, root), vec!["a/keep.jpg"]);
281    }
282
283    #[test]
284    fn records_size_and_mtime() {
285        let dir = TempDir::new().unwrap();
286        fs::write(dir.path().join("a.jpg"), b"12345").unwrap();
287        let result = scan_one(dir.path(), &[]);
288        assert_eq!(result.files[0].size, 5);
289        assert!(result.files[0].mtime_ns > 0);
290    }
291
292    #[test]
293    fn live_photo_videos_are_paired_with_their_photo() {
294        let dir = TempDir::new().unwrap();
295        let root = dir.path();
296        touch(root, "IMG_1.HEIC");
297        touch(root, "IMG_1.MOV");
298        touch(root, "IMG_2.jpg");
299        touch(root, "img_2.mp4");
300        touch(root, "VID_3.mov");
301        touch(root, "other/IMG_1.MOV");
302
303        let result = scan_one(root, &[]);
304        let paired: Vec<(String, String)> = result
305            .files
306            .iter()
307            .filter_map(|f| {
308                let photo = f.live_photo_of.as_ref()?;
309                Some((
310                    f.path.file_name()?.to_string_lossy().into_owned(),
311                    photo.file_name()?.to_string_lossy().into_owned(),
312                ))
313            })
314            .collect();
315        assert_eq!(
316            paired,
317            vec![
318                ("IMG_1.MOV".to_string(), "IMG_1.HEIC".to_string()),
319                ("img_2.mp4".to_string(), "IMG_2.jpg".to_string()),
320            ]
321        );
322        assert_eq!(result.live_photo_videos, 2);
323
324        let skipped = scan(
325            &[root.to_path_buf()],
326            &[],
327            RawHandling::Render,
328            LivePhotoVideos::Skip,
329        )
330        .unwrap();
331        assert_eq!(skipped.live_photo_videos_skipped, 2);
332        assert_eq!(skipped.files.len(), 4);
333    }
334
335    #[test]
336    fn raw_files_with_a_jpeg_sibling_are_dropped() {
337        let dir = TempDir::new().unwrap();
338        let root = dir.path();
339        touch(root, "IMG_1.CR2");
340        touch(root, "IMG_1.JPG");
341        touch(root, "IMG_2.CR2");
342
343        let result = scan_one(root, &[]);
344        assert_eq!(rel(&result, root), vec!["IMG_1.JPG", "IMG_2.CR2"]);
345        assert_eq!(result.raw_selection.skipped_with_sibling, 1);
346    }
347
348    #[test]
349    fn overlapping_sources_list_files_once() {
350        let dir = TempDir::new().unwrap();
351        let root = dir.path();
352        touch(root, "a/b.jpg");
353        let result = scan(
354            &[root.to_path_buf(), root.join("a")],
355            &[],
356            RawHandling::Render,
357            LivePhotoVideos::Upload,
358        )
359        .unwrap();
360        assert_eq!(result.files.len(), 1);
361    }
362
363    #[test]
364    fn a_missing_source_is_an_error() {
365        let dir = TempDir::new().unwrap();
366        assert!(
367            scan(
368                &[dir.path().join("nope")],
369                &[],
370                RawHandling::Render,
371                LivePhotoVideos::Upload
372            )
373            .is_err()
374        );
375    }
376}
smugmug_cli/dated/scan.rs

smugmug_cli/dated/
scan.rs
