1use crate::{
8 AnalysisOptions, ResourceAnalysis,
9 filesystem::{hash, write_new},
10};
11use anyhow::{Context, Result, ensure};
12use serde::{Deserialize, Serialize};
13use std::{
14 fs,
15 path::{Path, PathBuf},
16 sync::OnceLock,
17 time::SystemTime,
18};
19
20const CACHE_SCHEMA: u32 = 6;
22const MAX_ENTRY_FILE_BYTES: u64 = 64 * 1024 * 1024;
23pub(crate) const DEFAULT_MAX_BYTES: u64 = 2 * 1024 * 1024 * 1024;
25
26pub(crate) struct Cache {
27 root: PathBuf,
28}
29
30#[derive(Serialize, Deserialize)]
31#[serde(deny_unknown_fields)]
32struct Entry {
33 schema: u32,
34 key: String,
35 analysis: ResourceAnalysis,
36 files: Vec<EntryFile>,
37}
38
39#[derive(Serialize, Deserialize)]
40#[serde(deny_unknown_fields)]
41struct EntryFile {
42 template: String,
44 name: String,
45 sha256: String,
46}
47
48#[derive(Serialize)]
51struct KeyMaterial<'a> {
52 schema: u32,
53 version: &'a str,
54 backend: &'a str,
55 source_sha256: &'a str,
56 policy: &'a str,
57 qualities: &'a [u8],
58 min_input_bytes: u64,
59 min_savings_bytes: u64,
60 max_alpha_error: u32,
61 min_score: u64,
62 max_pixels: usize,
63 png_level: u8,
64 png_reductions: bool,
65 webp: bool,
66 lossy_png: bool,
67 heic_near_lossless: bool,
68}
69
70pub(crate) fn backend_id() -> &'static str {
73 static ID: OnceLock<String> = OnceLock::new();
74 ID.get_or_init(|| {
75 if cfg!(target_os = "macos") {
76 let build = std::process::Command::new("/usr/bin/sw_vers")
77 .arg("-buildVersion")
78 .output()
79 .ok()
80 .filter(|o| o.status.success())
81 .and_then(|o| String::from_utf8(o.stdout).ok())
82 .map(|s| s.trim().to_string())
83 .filter(|s| !s.is_empty() && s.len() <= 32)
84 .unwrap_or_else(|| "unknown".into());
85 format!("macos-imageio-{build}")
86 } else {
87 format!("portable-{}", std::env::consts::OS)
88 }
89 })
90}
91
92pub fn default_directory() -> Option<PathBuf> {
94 let var = |name: &str| std::env::var_os(name).filter(|v| !v.is_empty());
95 if let Some(path) = var("RESOPT_CACHE_DIR") {
96 return Some(PathBuf::from(path));
97 }
98 if cfg!(target_os = "macos") {
99 var("HOME").map(|home| PathBuf::from(home).join("Library/Caches/resopt"))
100 } else if cfg!(windows) {
101 var("LOCALAPPDATA").map(|dir| PathBuf::from(dir).join("resopt").join("cache"))
102 } else {
103 var("XDG_CACHE_HOME")
104 .map(PathBuf::from)
105 .or_else(|| var("HOME").map(|home| PathBuf::from(home).join(".cache")))
106 .map(|dir| dir.join("resopt"))
107 }
108}
109
110impl Cache {
111 pub fn open(directory: &Path) -> Result<Self> {
112 let root = directory.join(format!("v{CACHE_SCHEMA}"));
113 fs::create_dir_all(&root)
114 .with_context(|| format!("creating cache directory {}", root.display()))?;
115 let root = fs::canonicalize(root)?;
116 Ok(Self { root })
117 }
118
119 pub fn key(source_sha256: &str, policy: &str, options: &AnalysisOptions) -> Result<String> {
120 Ok(hash(&serde_json::to_vec(&KeyMaterial {
121 schema: CACHE_SCHEMA,
122 version: env!("CARGO_PKG_VERSION"),
123 backend: backend_id(),
124 source_sha256,
125 policy,
126 qualities: &options.qualities,
127 min_input_bytes: options.min_input_bytes,
128 min_savings_bytes: options.min_savings_bytes,
129 max_alpha_error: options.max_alpha_error.to_bits(),
130 min_score: options.min_score.to_bits(),
131 max_pixels: options.max_pixels,
132 png_level: options.png_level,
133 png_reductions: options.png_reductions,
134 webp: options.webp,
135 lossy_png: options.lossy_png,
136 heic_near_lossless: options.heic_near_lossless,
137 })?))
138 }
139
140 fn entry_directory(&self, key: &str) -> Result<PathBuf> {
141 ensure!(
142 key.len() == 64 && key.bytes().all(|b| b.is_ascii_hexdigit()),
143 "invalid cache key"
144 );
145 Ok(self.root.join(&key[..2]).join(key))
146 }
147
148 pub fn load(&self, key: &str, out: &Path, index: usize) -> Option<ResourceAnalysis> {
151 let directory = self.entry_directory(key).ok()?;
152 if !directory.is_dir() {
153 return None;
154 }
155 match self.load_verified(&directory, key, out, index) {
156 Ok(analysis) => {
157 let _ = fs::File::options()
159 .write(true)
160 .open(directory.join("entry.json"))
161 .and_then(|file| file.set_modified(SystemTime::now()));
162 Some(analysis)
163 }
164 Err(_) => {
165 let _ = fs::remove_dir_all(&directory);
166 None
167 }
168 }
169 }
170
171 fn load_verified(
172 &self,
173 directory: &Path,
174 key: &str,
175 out: &Path,
176 index: usize,
177 ) -> Result<ResourceAnalysis> {
178 let manifest = directory.join("entry.json");
179 ensure!(
180 fs::symlink_metadata(&manifest)?.len() <= MAX_ENTRY_FILE_BYTES,
181 "cache manifest too large"
182 );
183 let entry: Entry = serde_json::from_slice(&fs::read(&manifest)?)?;
184 ensure!(
185 entry.schema == CACHE_SCHEMA && entry.key == key,
186 "cache entry mismatch"
187 );
188 let mut staged = Vec::with_capacity(entry.files.len());
189 for file in &entry.files {
190 ensure!(
191 file.name.len() <= 8
192 && file.name.bytes().all(|b| b.is_ascii_alphanumeric())
193 && safe_template(&file.template),
194 "unsafe cache file"
195 );
196 let path = directory.join(&file.name);
197 let metadata = fs::symlink_metadata(&path)?;
198 ensure!(
199 metadata.file_type().is_file() && metadata.len() <= MAX_ENTRY_FILE_BYTES,
200 "invalid cache file"
201 );
202 let bytes = fs::read(&path)?;
203 ensure!(hash(&bytes) == file.sha256, "cache file corrupted");
204 staged.push((file.template.replace("{i}", &index.to_string()), bytes));
205 }
206 let mut analysis = entry.analysis;
207 let rename = |path: &mut Option<PathBuf>| -> Result<()> {
208 if let Some(template) = path.as_ref().and_then(|p| p.to_str()) {
209 ensure!(
210 entry.files.iter().any(|f| f.template == template),
211 "cache entry references a missing file"
212 );
213 *path = Some(PathBuf::from(template.replace("{i}", &index.to_string())));
214 }
215 Ok(())
216 };
217 rename(&mut analysis.original_preview)?;
218 for candidate in &mut analysis.candidates {
219 rename(&mut candidate.artifact)?;
220 rename(&mut candidate.preview)?;
221 }
222 let mut written = Vec::with_capacity(staged.len());
223 for (relative, bytes) in staged {
224 let path = out.join(relative);
225 if let Err(error) = crate::filesystem::write_artifact(&path, &bytes) {
226 for path in written {
229 let _ = fs::remove_file(path);
230 }
231 return Err(error);
232 }
233 written.push(path);
234 }
235 Ok(analysis)
236 }
237
238 pub fn store(
242 &self,
243 key: &str,
244 analysis: &ResourceAnalysis,
245 out: &Path,
246 index: usize,
247 ) -> Result<()> {
248 let directory = self.entry_directory(key)?;
249 if directory.exists() {
250 return Ok(());
251 }
252 let parent = directory.parent().context("cache entry has no parent")?;
253 fs::create_dir_all(parent)?;
254 let staging = tempfile::Builder::new()
255 .prefix("tmp-")
256 .tempdir_in(&self.root)?;
257 let mut template_analysis = analysis.clone();
258 template_analysis.original_artifact = None;
259 let mut files = Vec::new();
260 let mut add = |path: &mut Option<PathBuf>| -> Result<()> {
261 let Some(relative) = path.clone() else {
262 return Ok(());
263 };
264 let template = template_for(&relative, index)?;
265 let bytes = fs::read(out.join(&relative))?;
266 let name = format!("f{}", files.len());
267 crate::filesystem::write_artifact(&staging.path().join(&name), &bytes)?;
269 files.push(EntryFile {
270 template: template.clone(),
271 name,
272 sha256: hash(&bytes),
273 });
274 *path = Some(PathBuf::from(template));
275 Ok(())
276 };
277 add(&mut template_analysis.original_preview)?;
278 for candidate in &mut template_analysis.candidates {
279 add(&mut candidate.artifact)?;
280 add(&mut candidate.preview)?;
281 }
282 let entry = Entry {
283 schema: CACHE_SCHEMA,
284 key: key.to_string(),
285 analysis: template_analysis,
286 files,
287 };
288 write_new(
290 &staging.path().join("entry.json"),
291 &serde_json::to_vec(&entry)?,
292 )?;
293 let staged = staging.keep();
294 if fs::rename(&staged, &directory).is_err() {
295 let _ = fs::remove_dir_all(&staged);
297 }
298 Ok(())
299 }
300
301 pub fn prune(&self, max_bytes: u64) -> Result<()> {
303 let mut entries = Vec::new();
304 let mut total = 0_u64;
305 for shard in fs::read_dir(&self.root)? {
306 let shard = shard?.path();
307 let name = shard.file_name().unwrap_or_default().to_string_lossy();
308 if name.starts_with("tmp-") {
309 let stale = fs::metadata(&shard)
310 .and_then(|m| m.modified())
311 .ok()
312 .and_then(|t| t.elapsed().ok())
313 .is_some_and(|age| age.as_secs() > 3600);
314 if stale {
315 let _ = fs::remove_dir_all(&shard);
316 }
317 continue;
318 }
319 if !shard.is_dir() {
320 continue;
321 }
322 for entry in fs::read_dir(&shard)? {
323 let entry = entry?.path();
324 let Ok(manifest) = fs::metadata(entry.join("entry.json")) else {
325 let _ = fs::remove_dir_all(&entry);
326 continue;
327 };
328 let bytes: u64 = fs::read_dir(&entry)?
329 .filter_map(|f| f.ok()?.metadata().ok())
330 .map(|m| m.len())
331 .sum();
332 total += bytes;
333 entries.push((
334 manifest.modified().unwrap_or(SystemTime::UNIX_EPOCH),
335 bytes,
336 entry,
337 ));
338 }
339 }
340 entries.sort();
341 for (_, bytes, entry) in entries {
342 if total <= max_bytes {
343 break;
344 }
345 if fs::remove_dir_all(&entry).is_ok() {
346 total = total.saturating_sub(bytes);
347 }
348 }
349 Ok(())
350 }
351}
352
353fn safe_template(template: &str) -> bool {
354 let mut parts = template.split('/');
355 matches!(parts.next(), Some("candidates" | "previews"))
356 && parts.next().is_some_and(|name| {
357 name.starts_with("{i}-")
358 && name[3..]
359 .bytes()
360 .all(|b| b.is_ascii_alphanumeric() || matches!(b, b'-' | b'.'))
361 && !name.contains("..")
362 })
363 && parts.next().is_none()
364}
365
366fn template_for(relative: &Path, index: usize) -> Result<String> {
367 let text = relative
368 .to_str()
369 .context("non-UTF8 artifact path")?
370 .replace('\\', "/");
371 let (folder, name) = text
372 .split_once('/')
373 .context("artifact path has no folder")?;
374 let rest = name
375 .strip_prefix(&format!("{index}-"))
376 .context("artifact name does not start with its resource index")?;
377 let template = format!("{folder}/{{i}}-{rest}");
378 ensure!(safe_template(&template), "unsupported artifact name {text}");
379 Ok(template)
380}
381
382#[cfg(test)]
383mod tests {
384 use super::*;
385 use crate::{ImageCandidate, Resource};
386
387 fn analysis(index: usize) -> ResourceAnalysis {
388 ResourceAnalysis {
389 resource: Resource::for_tests("a/picture.png", "png"),
390 sha256: Some("0".repeat(64)),
391 image: None,
392 status: "candidates_available".into(),
393 issues: vec![],
394 candidates: vec![ImageCandidate {
395 format: "png".into(),
396 quality: None,
397 lossy: false,
398 bytes: 3,
399 savings_bytes: 5,
400 valid: true,
401 rejection: None,
402 difference: None,
403 artifact: Some(format!("candidates/{index}-png-0.png").into()),
404 preview: Some(format!("previews/{index}-png-0.png").into()),
405 sha256: Some(hash(b"art")),
406 warnings: vec![],
407 notes: vec![],
408 }],
409 smallest_candidate: Some(0),
410 original_preview: Some(format!("previews/{index}-original.png").into()),
411 original_artifact: Some(format!("originals/{index}.png").into()),
412 fingerprint: None,
413 media: None,
414 archive: None,
415 vap: None,
416 pag: None,
417 animation: None,
418 }
419 }
420
421 fn report_directory(index: usize) -> tempfile::TempDir {
422 let out = tempfile::tempdir().unwrap();
423 for folder in ["candidates", "previews", "originals"] {
424 fs::create_dir(out.path().join(folder)).unwrap();
425 }
426 fs::write(
427 out.path().join(format!("candidates/{index}-png-0.png")),
428 b"art",
429 )
430 .unwrap();
431 fs::write(
432 out.path().join(format!("previews/{index}-png-0.png")),
433 b"pre",
434 )
435 .unwrap();
436 fs::write(
437 out.path().join(format!("previews/{index}-original.png")),
438 b"orig-preview",
439 )
440 .unwrap();
441 out
442 }
443
444 fn empty_report() -> tempfile::TempDir {
445 let out = tempfile::tempdir().unwrap();
446 for folder in ["candidates", "previews", "originals"] {
447 fs::create_dir(out.path().join(folder)).unwrap();
448 }
449 out
450 }
451
452 #[test]
453 fn round_trip_renames_files_for_the_new_resource_index() {
454 let home = tempfile::tempdir().unwrap();
455 let cache = Cache::open(home.path()).unwrap();
456 let key = Cache::key(&"1".repeat(64), "loose", &AnalysisOptions::default()).unwrap();
457 let first = report_directory(4);
458 cache.store(&key, &analysis(4), first.path(), 4).unwrap();
459 let second = empty_report();
460 let loaded = cache.load(&key, second.path(), 9).unwrap();
461 assert_eq!(
462 loaded.candidates[0].artifact.as_deref(),
463 Some(Path::new("candidates/9-png-0.png"))
464 );
465 assert_eq!(loaded.original_artifact, None);
466 assert_eq!(
467 fs::read(second.path().join("candidates/9-png-0.png")).unwrap(),
468 b"art"
469 );
470 assert_eq!(
471 fs::read(second.path().join("previews/9-original.png")).unwrap(),
472 b"orig-preview"
473 );
474 }
475
476 #[test]
477 fn key_changes_with_source_policy_and_relevant_options_only() {
478 let options = AnalysisOptions::default();
479 let base = Cache::key(&"1".repeat(64), "loose", &options).unwrap();
480 assert_ne!(
481 base,
482 Cache::key(&"2".repeat(64), "loose", &options).unwrap()
483 );
484 assert_ne!(
485 base,
486 Cache::key(&"1".repeat(64), "catalog", &options).unwrap()
487 );
488 for changed in [
489 AnalysisOptions {
490 qualities: vec![85],
491 ..options.clone()
492 },
493 AnalysisOptions {
494 webp: !options.webp,
495 ..options.clone()
496 },
497 AnalysisOptions {
498 lossy_png: !options.lossy_png,
499 ..options.clone()
500 },
501 AnalysisOptions {
502 heic_near_lossless: !options.heic_near_lossless,
503 ..options.clone()
504 },
505 AnalysisOptions {
506 min_score: 50.0,
507 ..options.clone()
508 },
509 AnalysisOptions {
510 png_reductions: true,
511 ..options.clone()
512 },
513 ] {
514 assert_ne!(
515 base,
516 Cache::key(&"1".repeat(64), "loose", &changed).unwrap()
517 );
518 }
519 let scheduling_only = AnalysisOptions {
520 jobs: 1,
521 include_ignored: true,
522 ..options.clone()
523 };
524 assert_eq!(
525 base,
526 Cache::key(&"1".repeat(64), "loose", &scheduling_only).unwrap()
527 );
528 }
529
530 #[test]
531 fn corrupted_truncated_and_interrupted_entries_are_discarded() {
532 let home = tempfile::tempdir().unwrap();
533 let cache = Cache::open(home.path()).unwrap();
534 let key = Cache::key(&"1".repeat(64), "loose", &AnalysisOptions::default()).unwrap();
535 let source = report_directory(0);
536 let directory = cache.entry_directory(&key).unwrap();
537
538 cache.store(&key, &analysis(0), source.path(), 0).unwrap();
540 fs::write(directory.join("f1"), b"tampered").unwrap();
541 let out = empty_report();
542 assert!(cache.load(&key, out.path(), 0).is_none());
543 assert!(!directory.exists());
544 assert!(
545 fs::read_dir(out.path().join("candidates"))
546 .unwrap()
547 .next()
548 .is_none()
549 );
550
551 fs::create_dir_all(&directory).unwrap();
553 fs::write(directory.join("f0"), b"art").unwrap();
554 assert!(cache.load(&key, empty_report().path(), 0).is_none());
555 assert!(!directory.exists());
556
557 cache.store(&key, &analysis(0), source.path(), 0).unwrap();
559 fs::write(directory.join("entry.json"), b"{\"schema\":1").unwrap();
560 assert!(cache.load(&key, empty_report().path(), 0).is_none());
561 cache.store(&key, &analysis(0), source.path(), 0).unwrap();
562 let other = Cache::key(&"3".repeat(64), "loose", &AnalysisOptions::default()).unwrap();
563 let other_directory = cache.entry_directory(&other).unwrap();
564 fs::create_dir_all(other_directory.parent().unwrap()).unwrap();
565 fs::rename(&directory, &other_directory).unwrap();
566 assert!(cache.load(&other, empty_report().path(), 0).is_none());
567 }
568
569 #[test]
570 fn traversal_templates_are_never_written() {
571 assert!(safe_template("candidates/{i}-png-0.png"));
572 for template in [
573 "candidates/../{i}-x.png",
574 "../{i}-x.png",
575 "originals/{i}-x.png",
576 "candidates/{i}-../x",
577 "candidates/x.png",
578 "candidates/{i}-a/b.png",
579 ] {
580 assert!(!safe_template(template), "{template}");
581 }
582 }
583
584 #[test]
585 fn prune_removes_oldest_entries_and_stale_staging() {
586 let home = tempfile::tempdir().unwrap();
587 let cache = Cache::open(home.path()).unwrap();
588 let source = report_directory(0);
589 let keys: Vec<_> = (0..3)
590 .map(|i| {
591 Cache::key(
592 &i.to_string().repeat(64),
593 "loose",
594 &AnalysisOptions::default(),
595 )
596 .unwrap()
597 })
598 .collect();
599 for (age, key) in keys.iter().enumerate() {
600 cache.store(key, &analysis(0), source.path(), 0).unwrap();
601 let manifest = cache.entry_directory(key).unwrap().join("entry.json");
602 fs::File::options()
603 .write(true)
604 .open(manifest)
605 .unwrap()
606 .set_modified(
607 SystemTime::now() - std::time::Duration::from_secs(1000 - age as u64 * 100),
608 )
609 .unwrap();
610 }
611 let entry_bytes: u64 = fs::read_dir(cache.entry_directory(&keys[0]).unwrap())
612 .unwrap()
613 .map(|f| f.unwrap().metadata().unwrap().len())
614 .sum();
615 cache.prune(entry_bytes * 2).unwrap();
616 assert!(!cache.entry_directory(&keys[0]).unwrap().exists());
617 assert!(cache.entry_directory(&keys[1]).unwrap().exists());
618 assert!(cache.entry_directory(&keys[2]).unwrap().exists());
619 }
620}