Skip to main content

wows_data_mgr/
dump.rs

1use std::collections::BTreeMap;
2use std::io::Read;
3use std::path::Path;
4use std::sync::Arc;
5
6use indicatif::ProgressBar;
7use indicatif::ProgressStyle;
8use rootcause::prelude::*;
9use wowsunpack::game_data;
10use wowsunpack::game_params::cache;
11use wowsunpack::game_params::provider::GameMetadataProvider;
12use wowsunpack::game_params::types::GameParamProvider;
13use wowsunpack::game_params::types::Param;
14use wowsunpack::vfs::VfsFileType;
15use wowsunpack::vfs::VfsPath;
16
17use crate::builds::BuildEntry;
18use crate::builds::BuildMetadata;
19use crate::builds::BuildsIndex;
20use crate::cas;
21
22/// VFS directories dumped in their entirety. `gui/battle_hud` is dumped whole
23/// (rather than enumerating subdirectories) so newly-added HUD icons are always
24/// captured without code changes.
25const VFS_DIRS: &[&str] = &[
26    "gui/fla/minimap",
27    "gui/battle_hud",
28    "gui/consumables",
29    "gui/powerups/drops",
30    "gui/fonts",
31    "gui/data/constants",
32    "gui/ships_silhouettes",
33    "gui/ribbons",
34    "gui/achievements",
35    "gui/nation_flags",
36    "gui/crew_commander/skills",
37    "gui/modernization_icons",
38    "gui/signal_flags",
39    "scripts/entity_defs",
40];
41
42/// Directories whose absence (zero files extracted) makes the dump unusable.
43/// Kept deliberately small: most icon directories are version-dependent (added
44/// in later game versions), so their absence is tolerated with a warning.
45const REQUIRED_NONEMPTY_DIRS: &[&str] = &["scripts/entity_defs"];
46
47/// Individual VFS files required beyond the directory dumps. A dump missing any
48/// of these can't parse replays, so their absence is fatal.
49const REQUIRED_VFS_FILES: &[&str] = &["content/GameParams.data", "scripts/entities.xml"];
50
51/// Files to extract per map from `spaces/<map>/`.
52const MAP_FILES_SPACES: &[&str] = &["minimap.png", "minimap_water.png", "space.settings"];
53
54/// Files to extract per map from `content/gameplay/<map>/`.
55const MAP_FILES_GAMEPLAY: &[&str] = &["space.settings"];
56
57/// Glob patterns covering every VFS path the dump extracts.
58///
59/// Feed these to `wowsunpack pkgs` to resolve the minimal set of `.pkg` files
60/// to download for a build — letting callers fetch all idx (small) first, then
61/// only the packages actually required, instead of the full multi-GiB depots.
62pub fn required_path_globs() -> Vec<String> {
63    let mut globs = Vec::new();
64    for dir in VFS_DIRS {
65        // `wowsunpack pkgs` matches with the glob crate's default options, where
66        // `*` spans `/`, so `{dir}/*` matches every file under the tree.
67        globs.push(format!("{dir}/*"));
68    }
69    for file in REQUIRED_VFS_FILES {
70        globs.push((*file).to_string());
71    }
72    for name in MAP_FILES_SPACES {
73        globs.push(format!("spaces/*/{name}"));
74    }
75    for name in MAP_FILES_GAMEPLAY {
76        globs.push(format!("content/gameplay/*/{name}"));
77    }
78    globs
79}
80
81/// Returns the dump directory path for a given version and build.
82pub fn dump_dir(output_base: &Path, version_str: &str, build: u32) -> std::path::PathBuf {
83    output_base.join(format!("{version_str}_{build}"))
84}
85
86/// Check if a valid dump exists for the given version and build.
87pub fn dump_exists(output_base: &Path, version_str: &str, build: u32) -> bool {
88    dump_dir(output_base, version_str, build).join("metadata.toml").exists()
89}
90
91/// Dump game data with content-addressed deduplication.
92///
93/// VFS files are stored in `{output_base}/common/` by hash, with symlinks
94/// in the build's `vfs/` directory. Non-VFS files (game_params.rkyv, translations)
95/// are stored directly in the build directory.
96///
97/// When `progress` is `Some`, a CLI progress bar is updated during extraction.
98/// When `allow_existing` is true and a complete dump already exists, returns immediately.
99pub fn dump_renderer_data(
100    game_dir: &Path,
101    build: u32,
102    version_str: &str,
103    output_base: &Path,
104    progress: Option<&ProgressBar>,
105    allow_existing: bool,
106) -> Result<(), Report> {
107    let output_dir = dump_dir(output_base, version_str, build);
108    let vfs_dir = output_dir.join("vfs");
109    let cas_root = cas::cas_root(output_base);
110
111    if output_dir.join("metadata.toml").exists() {
112        if allow_existing {
113            return Ok(());
114        }
115        bail!("Output directory already exists: {}", output_dir.display());
116    }
117
118    // Clean up partial dumps
119    if output_dir.exists() {
120        std::fs::remove_dir_all(&output_dir)
121            .attach_with(|| format!("Failed to clean up partial dump at {}", output_dir.display()))?;
122    }
123
124    let vfs = game_data::build_game_vfs_for_build(game_dir, build).attach_with(|| "Failed to build game VFS")?;
125
126    // Extract VFS files through CAS
127    let mut file_hashes: BTreeMap<String, String> = BTreeMap::new();
128
129    let mut dir_counts: BTreeMap<&str, usize> = BTreeMap::new();
130    for dir in VFS_DIRS {
131        let count = extract_vfs_dir_cas(&vfs, dir, &vfs_dir, &cas_root, &mut file_hashes, progress)?;
132        dir_counts.insert(dir, count);
133    }
134    let mut missing_files = Vec::new();
135    for file in REQUIRED_VFS_FILES {
136        if !extract_vfs_file_cas(&vfs, file, &vfs_dir, &cas_root, &mut file_hashes)? {
137            missing_files.push(*file);
138        }
139        if let Some(pb) = progress {
140            pb.inc(1);
141        }
142    }
143    let map_count =
144        extract_map_files_cas(&vfs, "spaces", MAP_FILES_SPACES, &vfs_dir, &cas_root, &mut file_hashes, progress)?;
145    extract_map_files_cas(
146        &vfs,
147        "content/gameplay",
148        MAP_FILES_GAMEPLAY,
149        &vfs_dir,
150        &cas_root,
151        &mut file_hashes,
152        progress,
153    )?;
154
155    if let Some(pb) = progress {
156        pb.finish_and_clear();
157    }
158
159    // Fail loudly on an incomplete dump rather than silently shipping one that
160    // renders blank maps or can't parse replays. CAS objects already written
161    // are shared and harmless; only the (unregistered) build dir is discarded.
162    let mut problems = Vec::new();
163    if !missing_files.is_empty() {
164        problems.push(format!("missing required file(s): {}", missing_files.join(", ")));
165    }
166    for dir in REQUIRED_NONEMPTY_DIRS {
167        if dir_counts.get(dir).copied().unwrap_or(0) == 0 {
168            problems.push(format!("required directory '{dir}' extracted no files"));
169        }
170    }
171    if map_count == 0 {
172        problems.push(
173            "no map data extracted (spaces/*/minimap.png); the content depot's spaces packages are likely missing"
174                .to_string(),
175        );
176    }
177    if !problems.is_empty() {
178        let _ = std::fs::remove_dir_all(&output_dir);
179        bail!("Incomplete dump for build {build} ({version_str}): {}", problems.join("; "));
180    }
181    for dir in VFS_DIRS {
182        if dir_counts.get(dir).copied().unwrap_or(0) == 0 {
183            tracing::warn!("dump for build {build}: directory '{dir}' extracted no files");
184        }
185    }
186
187    std::fs::create_dir_all(&output_dir)
188        .attach_with(|| format!("Failed to create output directory {}", output_dir.display()))?;
189
190    dump_all_translations(game_dir, build, &output_dir)?;
191
192    // Fetch and store versioned constants (non-fatal)
193    #[cfg(feature = "constants")]
194    match crate::constants::ConstantsFetcher::new() {
195        Ok(fetcher) => {
196            write_constants_for_build(&output_dir, build, Some(version_str), &fetcher);
197        }
198        Err(e) => {
199            tracing::warn!("Could not initialize constants fetcher for build {build}: {e:?}");
200        }
201    }
202
203    // Write enhanced metadata with file hashes. The derived artifacts (rkyv
204    // blob, compressed copies) are generated and content-addressed by the same
205    // step the refresh-derived command uses, so dumps and refreshes agree.
206    let mut metadata =
207        BuildMetadata { version: version_str.to_string(), build, files: file_hashes, derived: BTreeMap::new() };
208    refresh_build_derived(&output_dir, &cas_root, &mut metadata)?;
209    metadata.save(&output_dir.join("metadata.toml"))?;
210
211    // Update master builds index
212    let builds_path = output_base.join("builds.toml");
213    let mut index = BuildsIndex::load(&builds_path);
214    index.upsert(BuildEntry {
215        version: version_str.to_string(),
216        build,
217        dir: format!("{version_str}_{build}"),
218        dumped_at: jiff::Zoned::now().to_string(),
219    });
220    index.save(&builds_path)?;
221
222    Ok(())
223}
224
225/// Add the assets an existing build is missing without re-extracting the data it
226/// already has. Extracts maps (and, with `with_gui`, the `gui/` asset dirs) from
227/// `game_dir` into the build's `vfs/`, then regenerates derived artifacts (the
228/// rkyv game-params blob, with the current parser) from the build's existing
229/// `GameParams.data`.
230///
231/// Unlike [`dump_renderer_data`], this never reads `content/GameParams.data`,
232/// `scripts/`, or other already-present data from the game install, so the caller
233/// only needs the `gui` and `spaces_*` packages on disk -- not the multi-gigabyte
234/// `basecontent` package whose `GameParams.data` the build already holds.
235///
236/// The build must already exist in `builds.toml`. Returns the number of maps
237/// extracted; errors if no maps were found.
238pub fn complete_build(game_dir: &Path, build: u32, output_base: &Path, with_gui: bool) -> Result<usize, Report> {
239    let index = BuildsIndex::load(&output_base.join("builds.toml"));
240    let entry = index
241        .find_by_build(build)
242        .ok_or_else(|| report!("build {build} is not in builds.toml; dump it normally first"))?
243        .clone();
244    let output_dir = output_base.join(&entry.dir);
245    let vfs_dir = output_dir.join("vfs");
246    let cas_root = cas::cas_root(output_base);
247    let meta_path = output_dir.join("metadata.toml");
248    let mut metadata =
249        BuildMetadata::load(&meta_path).ok_or_else(|| report!("{} has no readable metadata.toml", entry.dir))?;
250
251    let vfs = game_data::build_game_vfs_for_build(game_dir, build).attach_with(|| "Failed to build game VFS")?;
252
253    if with_gui {
254        // Only the `gui/` dirs live in the gui package; re-extracting other
255        // VFS_DIRS (e.g. scripts/entity_defs) would need packages we deliberately
256        // skip, and that data is already present in the build.
257        for dir in VFS_DIRS.iter().filter(|d| d.starts_with("gui")) {
258            extract_vfs_dir_cas(&vfs, dir, &vfs_dir, &cas_root, &mut metadata.files, None)?;
259        }
260    }
261
262    let map_count =
263        extract_map_files_cas(&vfs, "spaces", MAP_FILES_SPACES, &vfs_dir, &cas_root, &mut metadata.files, None)?;
264    extract_map_files_cas(
265        &vfs,
266        "content/gameplay",
267        MAP_FILES_GAMEPLAY,
268        &vfs_dir,
269        &cas_root,
270        &mut metadata.files,
271        None,
272    )?;
273    if map_count == 0 {
274        bail!("no maps extracted for build {build}; refusing to record an incomplete build");
275    }
276
277    // Regenerate derived artifacts from the build's existing GameParams.data,
278    // picking up the current parser. Needs no package downloads.
279    refresh_build_derived(&output_dir, &cas_root, &mut metadata)?;
280    metadata.save(&meta_path)?;
281    Ok(map_count)
282}
283
284/// Create a configured progress bar for CLI use.
285pub fn create_progress_bar(game_dir: &Path) -> Option<ProgressBar> {
286    let vfs = game_data::build_game_vfs(game_dir).ok()?;
287    let mut total_files = 0u64;
288    for dir in VFS_DIRS {
289        total_files += count_vfs_dir_files(&vfs, dir);
290    }
291    total_files += REQUIRED_VFS_FILES.len() as u64;
292    total_files += count_map_files(&vfs, "spaces", MAP_FILES_SPACES);
293    total_files += count_map_files(&vfs, "content/gameplay", MAP_FILES_GAMEPLAY);
294
295    let pb = ProgressBar::new(total_files);
296    pb.set_style(
297        ProgressStyle::default_bar()
298            .template("{msg} [{bar:40}] {pos}/{len}")
299            .expect("valid template")
300            .progress_chars("=> "),
301    );
302    pb.set_message("Extracting VFS");
303    Some(pb)
304}
305
306/// Remove a dumped build, cleaning up orphaned CAS objects.
307pub fn remove_build(output_base: &Path, target_build: u32) -> Result<(), Report> {
308    let builds_path = output_base.join("builds.toml");
309    let mut index = BuildsIndex::load(&builds_path);
310    let entry = index
311        .find_by_build(target_build)
312        .ok_or_else(|| report!("Build {target_build} not found in builds.toml"))?
313        .clone();
314
315    let target_dir = output_base.join(&entry.dir);
316    let target_meta = BuildMetadata::load(&target_dir.join("metadata.toml"));
317
318    // Collect hashes still in use by other builds (vfs tree + derived data)
319    let mut live_hashes = std::collections::HashSet::new();
320    for other in &index.builds {
321        if other.build == target_build {
322            continue;
323        }
324        if let Some(meta) = BuildMetadata::load(&output_base.join(&other.dir).join("metadata.toml")) {
325            live_hashes.extend(meta.referenced_hashes());
326        }
327    }
328
329    // Delete orphaned CAS objects
330    if let Some(meta) = target_meta {
331        let cas_root = cas::cas_root(output_base);
332        for hash in meta.referenced_hashes() {
333            if !live_hashes.contains(&hash) {
334                let path = cas::cas_path(&cas_root, &hash);
335                let _ = std::fs::remove_file(&path);
336            }
337        }
338        // Clean up empty fanout directories
339        let _ = cas::gc(&cas_root, &live_hashes);
340    }
341
342    // Remove build directory
343    if target_dir.exists() {
344        std::fs::remove_dir_all(&target_dir)
345            .attach_with(|| format!("Failed to remove build directory {}", target_dir.display()))?;
346    }
347
348    // Update builds index
349    index.remove_build(target_build);
350    index.save(&builds_path)?;
351
352    Ok(())
353}
354
355// -- Local sync (offline mirror of download_repo) --
356
357/// Which builds to copy when syncing from a local source dump base.
358pub enum SyncSelector {
359    /// Every build listed in the source `builds.toml`.
360    All,
361    /// Only the highest build number in the source.
362    Latest,
363    /// A single exact build number.
364    Build(u32),
365    /// All builds matching a `major.minor.patch` version string.
366    Version(String),
367}
368
369/// Outcome of copying one build during a sync.
370pub struct SyncedBuild {
371    pub build: u32,
372    pub version: String,
373    /// Whether content was copied (`false` means it was already present and skipped).
374    pub copied: bool,
375}
376
377/// Copy builds from a local source dump base into `output_base`, deduplicating
378/// against content already present in the destination CAS.
379///
380/// This is the offline analog of `download_repo::download_build`: it reconstructs
381/// each selected build's directory (vfs/derived symlinks, constants, metadata)
382/// from the source's content-addressed store with no network access, copying only
383/// the content objects the destination is missing. Useful for promoting a
384/// freshly-dumped build into the toolkit's data cache without publishing it.
385pub fn sync_from_local(
386    source_base: &Path,
387    output_base: &Path,
388    selector: &SyncSelector,
389    force: bool,
390) -> Result<Vec<SyncedBuild>, Report> {
391    if source_base == output_base {
392        bail!("source and destination are the same directory");
393    }
394    let source_index = BuildsIndex::load(&source_base.join("builds.toml"));
395    if source_index.builds.is_empty() {
396        bail!("no builds.toml entries found in source {}", source_base.display());
397    }
398
399    let entries: Vec<BuildEntry> = match selector {
400        SyncSelector::All => source_index.builds.clone(),
401        SyncSelector::Latest => {
402            let latest =
403                source_index.builds.iter().max_by_key(|e| e.build).ok_or_else(|| report!("source has no builds"))?;
404            vec![latest.clone()]
405        }
406        SyncSelector::Build(b) => {
407            let entry = source_index
408                .find_by_build(*b)
409                .ok_or_else(|| report!("build {b} not found in source {}", source_base.display()))?;
410            vec![entry.clone()]
411        }
412        SyncSelector::Version(v) => {
413            let matches: Vec<BuildEntry> = source_index.find_by_version(v).into_iter().cloned().collect();
414            if matches.is_empty() {
415                bail!("no builds matching version '{v}' in source {}", source_base.display());
416            }
417            matches
418        }
419    };
420
421    let mut synced = Vec::new();
422    for entry in &entries {
423        let copied = copy_build_from_local(source_base, output_base, entry, force)?;
424        synced.push(SyncedBuild { build: entry.build, version: entry.version.clone(), copied });
425    }
426    Ok(synced)
427}
428
429/// Copy one build's data from `source_base` into `output_base`. Returns `true`
430/// when content was copied, `false` when an existing complete copy was reused.
431fn copy_build_from_local(
432    source_base: &Path,
433    output_base: &Path,
434    entry: &BuildEntry,
435    force: bool,
436) -> Result<bool, Report> {
437    let src_cas = cas::cas_root(source_base);
438    let dst_cas = cas::cas_root(output_base);
439    let output_dir = output_base.join(&entry.dir);
440
441    // A complete copy already on disk only needs registering, unless forced.
442    if !force && output_dir.join("metadata.toml").exists() {
443        register_build(output_base, entry)?;
444        return Ok(false);
445    }
446
447    let src_dir = source_base.join(&entry.dir);
448    let meta_path = src_dir.join("metadata.toml");
449    let metadata = BuildMetadata::load(&meta_path)
450        .ok_or_else(|| report!("source build {} has no readable metadata.toml", entry.dir))?;
451
452    // Copy every referenced content object the destination doesn't already have,
453    // verifying each against its hash. This happens before touching the existing
454    // build directory so a failed copy (e.g. an inconsistent source) never
455    // destroys a good local copy. Objects land in the shared store; nothing is
456    // wired into the build until every object is present.
457    for hash in metadata.referenced_hashes() {
458        if cas::object_exists(&dst_cas, &hash) {
459            continue;
460        }
461        let src_obj = cas::cas_path(&src_cas, &hash);
462        let data =
463            std::fs::read(&src_obj).attach_with(|| format!("source content object {} missing", src_obj.display()))?;
464        let actual = cas::hash_bytes(&data);
465        if actual != hash {
466            bail!("source content object {hash} hashed to {actual}");
467        }
468        cas::store(&dst_cas, &data)?;
469    }
470
471    // All content is present; now clear any partial/stale directory and rebuild.
472    if output_dir.exists() {
473        std::fs::remove_dir_all(&output_dir)
474            .attach_with(|| format!("failed to clear destination build dir {}", output_dir.display()))?;
475    }
476
477    // Recreate the extracted vfs tree and derived artifacts as symlinks into the
478    // destination CAS.
479    let vfs_dir = output_dir.join("vfs");
480    for (rel, hash) in &metadata.files {
481        cas::link_file(&dst_cas, hash, &vfs_dir.join(rel))?;
482    }
483    for (rel, hash) in &metadata.derived {
484        cas::link_file(&dst_cas, hash, &output_dir.join(rel))?;
485    }
486
487    // Versioned constants, when present alongside the source build.
488    let src_constants = src_dir.join("constants.json");
489    if src_constants.exists() {
490        let bytes =
491            std::fs::read(&src_constants).attach_with(|| format!("failed to read {}", src_constants.display()))?;
492        let dest = output_dir.join("constants.json");
493        std::fs::create_dir_all(dest.parent().unwrap())?;
494        std::fs::write(&dest, &bytes).attach_with(|| format!("failed to write {}", dest.display()))?;
495    }
496
497    metadata.save(&output_dir.join("metadata.toml"))?;
498    register_build(output_base, entry)?;
499    Ok(true)
500}
501
502/// Add or update the build's entry in the destination `builds.toml`.
503fn register_build(output_base: &Path, entry: &BuildEntry) -> Result<(), Report> {
504    let builds_path = output_base.join("builds.toml");
505    let mut index = BuildsIndex::load(&builds_path);
506    index.upsert(entry.clone());
507    index.save(&builds_path)
508}
509
510// -- Translation dumping --
511
512fn dump_all_translations(game_dir: &Path, build: u32, output_dir: &Path) -> Result<(), Report> {
513    let texts_dir = game_dir.join("bin").join(build.to_string()).join("res/texts");
514    if !texts_dir.exists() {
515        tracing::warn!("Translations directory not found: {}", texts_dir.display());
516        return Ok(());
517    }
518    for entry in std::fs::read_dir(&texts_dir)
519        .attach_with(|| format!("Failed to read translations directory {}", texts_dir.display()))?
520        .flatten()
521    {
522        if !entry.file_type().map(|t| t.is_dir()).unwrap_or(false) {
523            continue;
524        }
525        let lang = entry.file_name();
526        let mo_src = entry.path().join("LC_MESSAGES/global.mo");
527        if mo_src.exists() {
528            let mo_dest = output_dir.join("translations").join(&lang).join("LC_MESSAGES/global.mo");
529            std::fs::create_dir_all(mo_dest.parent().unwrap())?;
530            std::fs::copy(&mo_src, &mo_dest)?;
531        }
532    }
533    Ok(())
534}
535
536// -- CAS-aware extraction helpers --
537
538/// Read a VFS file into a buffer, store in CAS, and create a link in the build's vfs dir.
539fn store_and_link(
540    data: &[u8],
541    rel_path: &str,
542    vfs_dir: &Path,
543    cas_root: &Path,
544    file_hashes: &mut BTreeMap<String, String>,
545) -> Result<(), Report> {
546    let hash = cas::store(cas_root, data)?;
547    let link_path = vfs_dir.join(rel_path.trim_start_matches('/'));
548    cas::link_file(cas_root, &hash, &link_path)?;
549    file_hashes.insert(rel_path.trim_start_matches('/').to_string(), hash);
550    Ok(())
551}
552
553// -- Derived artifact generation (shared by dump and refresh-derived) --
554
555/// Convert `vfs_dir/content/GameParams.data` into a rkyv-encoded `Vec<Param>`
556/// using the current `wowsunpack` schema. Returns `None` when the source file
557/// is missing or the conversion fails (panic from a layout-incompatible older
558/// pickle, or serialization error). Diagnostics are logged via stderr.
559fn derive_game_params_rkyv(vfs_dir: &Path) -> Option<Vec<u8>> {
560    if !vfs_dir.join("content/GameParams.data").exists() {
561        return None;
562    }
563    let vfs = VfsPath::new(wowsunpack::vfs::PhysicalFS::new(vfs_dir));
564    let result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
565        let gmp = GameMetadataProvider::from_vfs(&vfs)?;
566        let params: Vec<Param> = gmp.params().iter().map(|p| Arc::unwrap_or_clone(Arc::clone(p))).collect();
567        cache::encode(&params).map_err(|e| report!("Failed to serialize: {e}"))
568    }));
569    match result {
570        Ok(Ok(bytes)) => Some(bytes),
571        Ok(Err(e)) => {
572            eprintln!("WARN: GameParams re-derivation failed for {}: {e:?}", vfs_dir.display());
573            None
574        }
575        Err(_) => {
576            eprintln!("WARN: GameParams re-derivation panicked for {} (incompatible pickle format)", vfs_dir.display(),);
577            None
578        }
579    }
580}
581
582/// Store `data` in the CAS and point `link_path` at it, replacing any file or
583/// symlink already there. Returns the content hash.
584fn store_and_relink(data: &[u8], link_path: &Path, cas_root: &Path) -> Result<String, Report> {
585    let hash = cas::store(cas_root, data)?;
586    let _ = std::fs::remove_file(link_path);
587    cas::link_file(cas_root, &hash, link_path)?;
588    Ok(hash)
589}
590
591/// Generate and content-address a build's derived artifacts: the rkyv game
592/// params blob, its zstd copy, and the English translation catalog's zstd copy.
593/// The rkyv blob is derived from `vfs/content/GameParams.data` against the
594/// current `wowsunpack::game_params::types` schema; the on-disk rkyv is only
595/// consulted as a fallback when the extracted vfs is missing or conversion
596/// fails. Each artifact is stored in the CAS, linked back into `build_dir`,
597/// and recorded in `metadata.derived`. Idempotent.
598pub fn refresh_build_derived(build_dir: &Path, cas_root: &Path, metadata: &mut BuildMetadata) -> Result<(), Report> {
599    metadata.derived.clear();
600
601    let rkyv_path = build_dir.join("game_params.rkyv");
602    let rkyv_bytes = derive_game_params_rkyv(&build_dir.join("vfs"));
603    let rkyv_bytes = match rkyv_bytes {
604        Some(b) => Some(b),
605        None if rkyv_path.exists() => {
606            Some(std::fs::read(&rkyv_path).attach_with(|| format!("Failed to read {}", rkyv_path.display()))?)
607        }
608        None => None,
609    };
610    if let Some(rkyv_bytes) = rkyv_bytes {
611        let hash = store_and_relink(&rkyv_bytes, &rkyv_path, cas_root)?;
612        metadata.derived.insert("game_params.rkyv".to_string(), hash);
613
614        let compressed =
615            ruzstd::encoding::compress_to_vec(rkyv_bytes.as_slice(), ruzstd::encoding::CompressionLevel::Fastest);
616        let zst_path = build_dir.join("game_params.rkyv.zst");
617        let hash = store_and_relink(&compressed, &zst_path, cas_root)?;
618        metadata.derived.insert("game_params.rkyv.zst".to_string(), hash);
619    }
620
621    // Content-address the per-locale translation catalogs (raw .mo files copied
622    // from the game install). They are identical across many builds, so this
623    // deduplicates them into the shared store like every other asset.
624    let translations_dir = build_dir.join("translations");
625    if translations_dir.exists() {
626        for lang_entry in std::fs::read_dir(&translations_dir)
627            .attach_with(|| format!("Failed to read {}", translations_dir.display()))?
628            .flatten()
629        {
630            if !lang_entry.file_type().map(|t| t.is_dir()).unwrap_or(false) {
631                continue;
632            }
633            let lang = lang_entry.file_name().to_string_lossy().into_owned();
634            let mo_rel = format!("translations/{lang}/LC_MESSAGES/global.mo");
635            let mo_path = build_dir.join(&mo_rel);
636            if !mo_path.exists() {
637                continue;
638            }
639            let bytes = std::fs::read(&mo_path).attach_with(|| format!("Failed to read {}", mo_path.display()))?;
640            let hash = store_and_relink(&bytes, &mo_path, cas_root)?;
641            metadata.derived.insert(mo_rel, hash);
642        }
643    }
644
645    // The web client fetches only the English catalog, zstd-compressed.
646    let mo_rel = "translations/en/LC_MESSAGES/global.mo";
647    let mo_path = build_dir.join(mo_rel);
648    if mo_path.exists() {
649        let mo_bytes = std::fs::read(&mo_path).attach_with(|| format!("Failed to read {}", mo_path.display()))?;
650        let compressed =
651            ruzstd::encoding::compress_to_vec(mo_bytes.as_slice(), ruzstd::encoding::CompressionLevel::Fastest);
652        let zst_path = build_dir.join(format!("{mo_rel}.zst"));
653        let hash = store_and_relink(&compressed, &zst_path, cas_root)?;
654        metadata.derived.insert(format!("{mo_rel}.zst"), hash);
655    }
656
657    Ok(())
658}
659
660/// Regenerate derived artifacts for every dumped build (or one build when
661/// `only_build` is given), then garbage-collect CAS objects no longer
662/// referenced by any build.
663pub fn refresh_derived(output_base: &Path, only_build: Option<u32>) -> Result<(), Report> {
664    let index = BuildsIndex::load(&output_base.join("builds.toml"));
665    let cas_root = cas::cas_root(output_base);
666
667    let targets: Vec<&BuildEntry> = match only_build {
668        Some(b) => index.builds.iter().filter(|e| e.build == b).collect(),
669        None => index.builds.iter().collect(),
670    };
671    if targets.is_empty() {
672        bail!("No matching builds found in {}", output_base.join("builds.toml").display());
673    }
674
675    // Build a single fetcher up front so the GitHub listing only runs once
676    // even when backfilling constants for many builds. `None` here just skips
677    // the constants step rather than failing the whole refresh.
678    #[cfg(feature = "constants")]
679    let constants_fetcher = match crate::constants::ConstantsFetcher::new() {
680        Ok(f) => Some(f),
681        Err(e) => {
682            eprintln!("WARN: Could not initialize constants fetcher: {e:?}");
683            None
684        }
685    };
686
687    for entry in &targets {
688        let build_dir = output_base.join(&entry.dir);
689        let meta_path = build_dir.join("metadata.toml");
690        let mut metadata = BuildMetadata::load(&meta_path).unwrap_or(BuildMetadata {
691            version: entry.version.clone(),
692            build: entry.build,
693            ..Default::default()
694        });
695
696        #[cfg(feature = "constants")]
697        let constants_added = if let Some(fetcher) = constants_fetcher.as_ref() {
698            update_constants_if_missing(&build_dir, entry.build, Some(entry.version.as_str()), fetcher)
699        } else {
700            false
701        };
702
703        match refresh_build_derived(&build_dir, &cas_root, &mut metadata) {
704            Ok(()) => {
705                metadata.save(&meta_path)?;
706                #[cfg(feature = "constants")]
707                let constants_note = if constants_added { " + constants" } else { "" };
708                #[cfg(not(feature = "constants"))]
709                let constants_note = "";
710                println!("  {} - {} derived artifact(s){}", entry.dir, metadata.derived.len(), constants_note);
711            }
712            Err(e) => eprintln!("WARN: {} - failed to refresh derived data: {e:?}", entry.dir),
713        }
714    }
715
716    println!("Refreshed {} build(s).", targets.len());
717    Ok(())
718}
719
720/// Write `constants.json` into `build_dir` if upstream has constants for this
721/// build. Logs a warning and leaves the build alone when nothing is published
722/// (e.g. very old builds the wows-constants repo doesn't cover).
723#[cfg(feature = "constants")]
724fn write_constants_for_build(
725    build_dir: &Path,
726    build: u32,
727    version: Option<&str>,
728    fetcher: &crate::constants::ConstantsFetcher,
729) -> bool {
730    let Some((data, actual_build)) = fetcher.fetch(build, version) else {
731        tracing::warn!("No upstream constants available for build {build}");
732        return false;
733    };
734    let bytes = match serde_json::to_vec_pretty(&data) {
735        Ok(b) => b,
736        Err(e) => {
737            tracing::warn!("Failed to serialize constants for build {build}: {e}");
738            return false;
739        }
740    };
741    if let Err(e) = std::fs::write(build_dir.join("constants.json"), &bytes) {
742        tracing::warn!("Failed to write constants.json for build {build}: {e}");
743        return false;
744    }
745    if actual_build != build {
746        tracing::info!("Stored constants from build {actual_build} (fallback for {build})");
747    }
748    true
749}
750
751/// Fetch and write `constants.json` only when the build doesn't already have
752/// one. Returns `true` if a new file was written. Constants for already-shipped
753/// builds don't change upstream, so leaving existing files alone keeps repeat
754/// refreshes idempotent and fast.
755#[cfg(feature = "constants")]
756fn update_constants_if_missing(
757    build_dir: &Path,
758    build: u32,
759    version: Option<&str>,
760    fetcher: &crate::constants::ConstantsFetcher,
761) -> bool {
762    if build_dir.join("constants.json").exists() {
763        return false;
764    }
765    write_constants_for_build(build_dir, build, version, fetcher)
766}
767
768/// Remove content-addressed objects no longer referenced by any build. An
769/// object is live if it appears in some build's metadata (the extracted vfs
770/// tree or the derived artifacts). Aborts without deleting anything if any
771/// build's metadata cannot be read, so in-use objects are never removed.
772pub fn gc_cas(output_base: &Path) -> Result<(), Report> {
773    let index = BuildsIndex::load(&output_base.join("builds.toml"));
774    let cas_root = cas::cas_root(output_base);
775
776    let mut live = std::collections::HashSet::new();
777    for entry in &index.builds {
778        let meta_path = output_base.join(&entry.dir).join("metadata.toml");
779        let meta = BuildMetadata::load(&meta_path)
780            .ok_or_else(|| report!("{} has no readable metadata.toml; aborting GC", entry.dir))?;
781        live.extend(meta.referenced_hashes());
782    }
783
784    let removed = cas::gc(&cas_root, &live)?;
785    println!("GC removed {removed} orphaned CAS object(s); {} still referenced.", live.len());
786    Ok(())
787}
788
789/// Consistency report for one build in a dump base.
790pub struct BuildVerification {
791    pub dir: String,
792    pub build: u32,
793    pub version: String,
794    /// Total unique content hashes the build references.
795    pub referenced: usize,
796    /// Referenced hashes with no object in the shared store.
797    pub missing_objects: Vec<String>,
798    /// VFS-relative paths whose symlink target does not resolve to a file.
799    pub broken_links: Vec<String>,
800    /// True when `metadata.toml` was missing or unparseable.
801    pub metadata_unreadable: bool,
802}
803
804impl BuildVerification {
805    pub fn is_ok(&self) -> bool {
806        !self.metadata_unreadable && self.missing_objects.is_empty() && self.broken_links.is_empty()
807    }
808}
809
810/// Verify that every build in `builds.toml` is internally consistent: its
811/// `metadata.toml` parses, every referenced content object exists in the shared
812/// store, and (when `check_links` is set) every reconstructed symlink resolves
813/// to a readable file. Returns a report per build; the caller decides how to act.
814pub fn verify_builds(output_base: &Path, check_links: bool) -> Result<Vec<BuildVerification>, Report> {
815    let index = BuildsIndex::load(&output_base.join("builds.toml"));
816    let cas_root = cas::cas_root(output_base);
817
818    let mut reports = Vec::new();
819    for entry in &index.builds {
820        let build_dir = output_base.join(&entry.dir);
821        let Some(meta) = BuildMetadata::load(&build_dir.join("metadata.toml")) else {
822            reports.push(BuildVerification {
823                dir: entry.dir.clone(),
824                build: entry.build,
825                version: entry.version.clone(),
826                referenced: 0,
827                missing_objects: Vec::new(),
828                broken_links: Vec::new(),
829                metadata_unreadable: true,
830            });
831            continue;
832        };
833
834        let referenced = meta.referenced_hashes();
835        let mut missing_objects: Vec<String> =
836            referenced.iter().filter(|h| !cas::object_exists(&cas_root, h)).cloned().collect();
837        missing_objects.sort();
838
839        let mut broken_links = Vec::new();
840        if check_links {
841            for (rel, _) in meta.files.iter().chain(meta.derived.iter()) {
842                let path = build_dir.join("vfs").join(rel);
843                // Derived artifacts live at the build root, not under vfs/.
844                let candidate = if path.exists() { path } else { build_dir.join(rel) };
845                if !candidate.exists() {
846                    broken_links.push(rel.clone());
847                }
848            }
849            broken_links.sort();
850        }
851
852        reports.push(BuildVerification {
853            dir: entry.dir.clone(),
854            build: entry.build,
855            version: entry.version.clone(),
856            referenced: referenced.len(),
857            missing_objects,
858            broken_links,
859            metadata_unreadable: false,
860        });
861    }
862    Ok(reports)
863}
864
865/// Remove content-addressed objects no longer referenced by any build present
866/// on disk. Scans every directory under `output_base` that contains a
867/// `metadata.toml`, so it stays correct even when `builds.toml` is out of sync
868/// (e.g. a build directory was deleted manually without GC). Aborts without
869/// removing anything if any metadata file cannot be read, so in-use objects are
870/// never deleted. Returns the number of objects removed.
871pub fn gc_unreferenced(output_base: &Path) -> Result<usize, Report> {
872    let cas_root = cas::cas_root(output_base);
873    if !cas_root.exists() {
874        return Ok(0);
875    }
876
877    let mut live = std::collections::HashSet::new();
878    for entry in
879        std::fs::read_dir(output_base).attach_with(|| format!("Failed to read {}", output_base.display()))?.flatten()
880    {
881        let meta_path = entry.path().join("metadata.toml");
882        if !meta_path.exists() {
883            continue;
884        }
885        match BuildMetadata::load(&meta_path) {
886            Some(meta) => live.extend(meta.referenced_hashes()),
887            None => bail!("unreadable metadata at {}; aborting GC", meta_path.display()),
888        }
889    }
890
891    cas::gc(&cas_root, &live)
892}
893
894/// Migrate a dump base from the legacy `vfs_common/` CAS directory to `common/`,
895/// rewriting every build's symlinks to point at the new store. Handles both a
896/// clean rename (when `common/` doesn't exist yet) and a merge (when a redump
897/// has already created `common/` while old builds still reference `vfs_common/`).
898/// No-op when the legacy directory is absent. Returns whether a migration ran.
899pub fn migrate_cas_dir_name(output_base: &Path) -> Result<bool, Report> {
900    let legacy = output_base.join(cas::LEGACY_CAS_DIR);
901    let current = cas::cas_root(output_base);
902    if !legacy.exists() {
903        return Ok(false);
904    }
905
906    if !current.exists() {
907        // Fast path: nothing in the new store yet, so move it wholesale.
908        std::fs::rename(&legacy, &current)
909            .attach_with(|| format!("failed to rename {} to {}", legacy.display(), current.display()))?;
910    } else {
911        // Both stores exist: fold legacy objects into `common/`. Objects are
912        // content-addressed, so a name collision means identical bytes — keep
913        // the existing copy and drop the duplicate.
914        merge_cas_objects(&legacy, &current)?;
915    }
916
917    // Relative symlinks under each build may still name the old store, so
918    // re-create every build's links against `common/`.
919    relink_all_builds(output_base, &current)?;
920
921    // Drop whatever remains of the legacy tree (emptied by the merge, or already
922    // gone after the rename).
923    if legacy.exists() {
924        std::fs::remove_dir_all(&legacy)
925            .attach_with(|| format!("failed to remove emptied legacy store {}", legacy.display()))?;
926    }
927
928    Ok(true)
929}
930
931/// Move every object from a legacy CAS tree into `dest`, deduplicating by hash.
932/// A collision (same fanout/name) is identical content, so the source copy is
933/// simply removed. Empties the source fanout directories as it goes.
934fn merge_cas_objects(legacy: &Path, dest: &Path) -> Result<(), Report> {
935    for fanout in std::fs::read_dir(legacy).attach_with(|| format!("Failed to read {}", legacy.display()))?.flatten() {
936        if !fanout.file_type().map(|t| t.is_dir()).unwrap_or(false) {
937            continue;
938        }
939        let dest_fanout = dest.join(fanout.file_name());
940        for obj in std::fs::read_dir(fanout.path())?.flatten() {
941            if !obj.file_type().map(|t| t.is_file()).unwrap_or(false) {
942                continue;
943            }
944            let dest_path = dest_fanout.join(obj.file_name());
945            if dest_path.exists() {
946                std::fs::remove_file(obj.path())?;
947                continue;
948            }
949            std::fs::create_dir_all(&dest_fanout)?;
950            // rename works within a volume; fall back to copy+delete across volumes.
951            if std::fs::rename(obj.path(), &dest_path).is_err() {
952                std::fs::copy(obj.path(), &dest_path)?;
953                std::fs::remove_file(obj.path())?;
954            }
955        }
956    }
957    Ok(())
958}
959
960/// Re-create every build's `vfs/` and derived symlinks against `cas_root`,
961/// using the hashes recorded in each `metadata.toml`. Idempotent.
962fn relink_all_builds(output_base: &Path, cas_root: &Path) -> Result<(), Report> {
963    for entry in
964        std::fs::read_dir(output_base).attach_with(|| format!("Failed to read {}", output_base.display()))?.flatten()
965    {
966        let build_dir = entry.path();
967        let Some(meta) = BuildMetadata::load(&build_dir.join("metadata.toml")) else {
968            continue;
969        };
970        let vfs_dir = build_dir.join("vfs");
971        let relink = |rel: &str, hash: &str, base: &Path| {
972            let link = base.join(rel);
973            let _ = std::fs::remove_file(&link);
974            if let Err(e) = cas::link_file(cas_root, hash, &link) {
975                tracing::warn!("failed to relink {}: {e}", link.display());
976            }
977        };
978        for (rel, hash) in &meta.files {
979            relink(rel, hash, &vfs_dir);
980        }
981        for (rel, hash) in &meta.derived {
982            relink(rel, hash, &build_dir);
983        }
984    }
985    Ok(())
986}
987
988/// Migrate any pre-CAS dumps in `output_base` into content-addressed storage.
989///
990/// Older dumps stored the extracted `vfs/` tree as plain files with no entries
991/// in `metadata.files`. This rehashes those files into `common/`, replaces
992/// them with symlinks, records the hashes, and regenerates derived artifacts so
993/// the dump deduplicates against every other build. Returns the number of
994/// builds migrated. Builds already in CAS format are left untouched.
995pub fn migrate_to_cas(output_base: &Path) -> Result<usize, Report> {
996    let cas_root = cas::cas_root(output_base);
997    let mut migrated = 0;
998    for entry in
999        std::fs::read_dir(output_base).attach_with(|| format!("Failed to read {}", output_base.display()))?.flatten()
1000    {
1001        let build_dir = entry.path();
1002        let meta_path = build_dir.join("metadata.toml");
1003        let Some(mut metadata) = BuildMetadata::load(&meta_path) else {
1004            continue;
1005        };
1006        if metadata.has_file_hashes() || !build_dir.join("vfs").exists() {
1007            continue;
1008        }
1009        match migrate_build_to_cas(&build_dir, &cas_root, &mut metadata) {
1010            Ok(()) => {
1011                metadata.save(&meta_path)?;
1012                migrated += 1;
1013            }
1014            Err(e) => tracing::warn!("failed to migrate {} to CAS: {e}", build_dir.display()),
1015        }
1016    }
1017    Ok(migrated)
1018}
1019
1020/// Rehash a single pre-CAS build's `vfs/` tree into the CAS, replacing each
1021/// plain file with a symlink and recording its hash in `metadata.files`.
1022fn migrate_build_to_cas(build_dir: &Path, cas_root: &Path, metadata: &mut BuildMetadata) -> Result<(), Report> {
1023    let vfs_dir = build_dir.join("vfs");
1024    let mut stack = vec![vfs_dir.clone()];
1025    while let Some(dir) = stack.pop() {
1026        for entry in std::fs::read_dir(&dir).attach_with(|| format!("Failed to read {}", dir.display()))?.flatten() {
1027            let path = entry.path();
1028            let file_type = entry.file_type().attach_with(|| format!("Failed to stat {}", path.display()))?;
1029            if file_type.is_dir() {
1030                stack.push(path);
1031                continue;
1032            }
1033            // Symlinks are already-migrated CAS references; leave them alone.
1034            if file_type.is_symlink() {
1035                continue;
1036            }
1037            let rel =
1038                path.strip_prefix(&vfs_dir).expect("walked path is under vfs_dir").to_string_lossy().replace('\\', "/");
1039            let data = std::fs::read(&path).attach_with(|| format!("Failed to read {}", path.display()))?;
1040            let hash = cas::store(cas_root, &data)?;
1041            std::fs::remove_file(&path).attach_with(|| format!("Failed to remove {}", path.display()))?;
1042            cas::link_file(cas_root, &hash, &path)?;
1043            metadata.files.insert(rel, hash);
1044        }
1045    }
1046    refresh_build_derived(build_dir, cas_root, metadata)
1047}
1048
1049fn extract_vfs_dir_cas(
1050    vfs: &VfsPath,
1051    vfs_path: &str,
1052    vfs_dir: &Path,
1053    cas_root: &Path,
1054    file_hashes: &mut BTreeMap<String, String>,
1055    progress: Option<&ProgressBar>,
1056) -> Result<usize, Report> {
1057    let dir = match vfs.join(vfs_path) {
1058        Ok(d) => d,
1059        Err(_) => return Ok(0),
1060    };
1061    let walker = match dir.walk_dir() {
1062        Ok(w) => w,
1063        Err(_) => return Ok(0),
1064    };
1065
1066    let mut count = 0;
1067    for entry in walker.flatten() {
1068        let metadata = match entry.metadata() {
1069            Ok(m) => m,
1070            Err(_) => continue,
1071        };
1072        if metadata.file_type != VfsFileType::File {
1073            continue;
1074        }
1075        let rel = entry.as_str();
1076        let mut buf = Vec::new();
1077        match entry.open_file() {
1078            Ok(mut f) => f.read_to_end(&mut buf)?,
1079            Err(e) => {
1080                tracing::warn!("Failed to open VFS file {rel}: {e}");
1081                continue;
1082            }
1083        };
1084        store_and_link(&buf, rel, vfs_dir, cas_root, file_hashes)?;
1085        count += 1;
1086        if let Some(pb) = progress {
1087            pb.inc(1);
1088        }
1089    }
1090    Ok(count)
1091}
1092
1093/// Extract a single VFS file. Returns `true` if it was found and stored.
1094fn extract_vfs_file_cas(
1095    vfs: &VfsPath,
1096    vfs_path: &str,
1097    vfs_dir: &Path,
1098    cas_root: &Path,
1099    file_hashes: &mut BTreeMap<String, String>,
1100) -> Result<bool, Report> {
1101    let file = match vfs.join(vfs_path) {
1102        Ok(f) => f,
1103        Err(_) => {
1104            tracing::warn!("VFS path not found (skipping): {vfs_path}");
1105            return Ok(false);
1106        }
1107    };
1108    let mut buf = Vec::new();
1109    match file.open_file() {
1110        Ok(mut f) => f.read_to_end(&mut buf)?,
1111        Err(_) => {
1112            tracing::warn!("Could not open VFS file (skipping): {vfs_path}");
1113            return Ok(false);
1114        }
1115    };
1116    store_and_link(&buf, vfs_path, vfs_dir, cas_root, file_hashes)?;
1117    Ok(true)
1118}
1119
1120/// Extract the named files from each subdirectory of `parent_dir`. Returns the
1121/// number of files extracted.
1122fn extract_map_files_cas(
1123    vfs: &VfsPath,
1124    parent_dir: &str,
1125    filenames: &[&str],
1126    vfs_dir: &Path,
1127    cas_root: &Path,
1128    file_hashes: &mut BTreeMap<String, String>,
1129    progress: Option<&ProgressBar>,
1130) -> Result<usize, Report> {
1131    let parent = match vfs.join(parent_dir) {
1132        Ok(d) => d,
1133        Err(_) => return Ok(0),
1134    };
1135    let entries = match parent.read_dir() {
1136        Ok(e) => e,
1137        Err(_) => return Ok(0),
1138    };
1139
1140    let mut count = 0;
1141    for entry in entries {
1142        if !entry.metadata().map(|m| m.file_type == VfsFileType::Directory).unwrap_or(false) {
1143            continue;
1144        }
1145        for filename in filenames {
1146            let file_path = match entry.join(filename) {
1147                Ok(f) => f,
1148                Err(_) => continue,
1149            };
1150            if !file_path.exists().unwrap_or(false) {
1151                continue;
1152            }
1153            let rel = file_path.as_str();
1154            let mut buf = Vec::new();
1155            match file_path.open_file() {
1156                Ok(mut f) => f.read_to_end(&mut buf)?,
1157                Err(e) => {
1158                    tracing::warn!("Failed to open VFS file {rel}: {e}");
1159                    continue;
1160                }
1161            };
1162            store_and_link(&buf, rel, vfs_dir, cas_root, file_hashes)?;
1163            count += 1;
1164            if let Some(pb) = progress {
1165                pb.inc(1);
1166            }
1167        }
1168    }
1169    Ok(count)
1170}
1171
1172// -- Counting helpers (for progress bar) --
1173
1174fn count_vfs_dir_files(vfs: &VfsPath, dir: &str) -> u64 {
1175    let mut count = 0;
1176    if let Ok(vfs_dir_path) = vfs.join(dir)
1177        && let Ok(walker) = vfs_dir_path.walk_dir()
1178    {
1179        for entry in walker.flatten() {
1180            if entry.metadata().map(|m| m.file_type == VfsFileType::File).unwrap_or(false) {
1181                count += 1;
1182            }
1183        }
1184    }
1185    count
1186}
1187
1188fn count_map_files(vfs: &VfsPath, parent_dir: &str, filenames: &[&str]) -> u64 {
1189    let mut count = 0;
1190    if let Ok(parent) = vfs.join(parent_dir)
1191        && let Ok(entries) = parent.read_dir()
1192    {
1193        for entry in entries {
1194            if entry.metadata().map(|m| m.file_type == VfsFileType::Directory).unwrap_or(false) {
1195                for filename in filenames {
1196                    if entry.join(filename).is_ok_and(|f: VfsPath| f.exists().unwrap_or(false)) {
1197                        count += 1;
1198                    }
1199                }
1200            }
1201        }
1202    }
1203    count
1204}
1205
1206#[cfg(test)]
1207mod maintenance_tests {
1208    use super::*;
1209
1210    fn write_build_metadata(build_dir: &Path, version: &str, build: u32, files: &[(&str, &str)]) {
1211        std::fs::create_dir_all(build_dir).unwrap();
1212        let mut meta = BuildMetadata { version: version.to_string(), build, ..Default::default() };
1213        for (rel, hash) in files {
1214            meta.files.insert((*rel).to_string(), (*hash).to_string());
1215        }
1216        meta.save(&build_dir.join("metadata.toml")).unwrap();
1217    }
1218
1219    #[test]
1220    fn gc_unreferenced_removes_orphans_and_keeps_live() {
1221        let dir = tempfile::tempdir().unwrap();
1222        let base = dir.path();
1223        let cas_root = base.join("common");
1224
1225        let live_hash = cas::store(&cas_root, b"live object").unwrap();
1226        let orphan_hash = cas::store(&cas_root, b"orphan object").unwrap();
1227        write_build_metadata(&base.join("1.0.0_100"), "1.0.0", 100, &[("gui/a.png", &live_hash)]);
1228
1229        let removed = gc_unreferenced(base).unwrap();
1230        assert_eq!(removed, 1);
1231        assert!(cas::object_exists(&cas_root, &live_hash));
1232        assert!(!cas::object_exists(&cas_root, &orphan_hash));
1233    }
1234
1235    #[test]
1236    fn gc_unreferenced_aborts_on_unreadable_metadata() {
1237        let dir = tempfile::tempdir().unwrap();
1238        let base = dir.path();
1239        let cas_root = base.join("common");
1240        let orphan_hash = cas::store(&cas_root, b"orphan object").unwrap();
1241
1242        let build_dir = base.join("1.0.0_100");
1243        std::fs::create_dir_all(&build_dir).unwrap();
1244        std::fs::write(build_dir.join("metadata.toml"), b"this is not valid toml = =").unwrap();
1245
1246        assert!(gc_unreferenced(base).is_err());
1247        // Nothing was removed because GC aborted.
1248        assert!(cas::object_exists(&cas_root, &orphan_hash));
1249    }
1250
1251    #[test]
1252    fn migrate_to_cas_dedups_plain_files() {
1253        let dir = tempfile::tempdir().unwrap();
1254        let base = dir.path();
1255        let build_dir = base.join("1.0.0_100");
1256
1257        // Old-format dump: plain files in vfs/, no file hashes in metadata.
1258        write_build_metadata(&build_dir, "1.0.0", 100, &[]);
1259        let file_path = build_dir.join("vfs/gui/a.png");
1260        std::fs::create_dir_all(file_path.parent().unwrap()).unwrap();
1261        std::fs::write(&file_path, b"some asset bytes").unwrap();
1262
1263        let migrated = migrate_to_cas(base).unwrap();
1264        assert_eq!(migrated, 1);
1265
1266        // The plain file is now a symlink whose content still reads back.
1267        assert!(std::fs::symlink_metadata(&file_path).unwrap().file_type().is_symlink());
1268        assert_eq!(std::fs::read(&file_path).unwrap(), b"some asset bytes");
1269
1270        // Metadata now records the hash, and a second pass is a no-op.
1271        let meta = BuildMetadata::load(&build_dir.join("metadata.toml")).unwrap();
1272        assert!(meta.has_file_hashes());
1273        assert!(meta.files.contains_key("gui/a.png"));
1274        assert_eq!(migrate_to_cas(base).unwrap(), 0);
1275    }
1276
1277    #[test]
1278    fn migrate_cas_dir_name_renames_and_relinks() {
1279        let dir = tempfile::tempdir().unwrap();
1280        let base = dir.path();
1281
1282        // Legacy layout: vfs_common/ store + a build whose vfs file symlinks into it.
1283        let legacy = base.join(cas::LEGACY_CAS_DIR);
1284        let hash = cas::store(&legacy, b"icon bytes").unwrap();
1285        let build_dir = base.join("1.0.0_100");
1286        let link = build_dir.join("vfs/gui/x.png");
1287        cas::link_file(&legacy, &hash, &link).unwrap();
1288        write_build_metadata(&build_dir, "1.0.0", 100, &[("gui/x.png", &hash)]);
1289
1290        assert!(migrate_cas_dir_name(base).unwrap());
1291        assert!(base.join(cas::CAS_DIR).exists());
1292        assert!(!base.join(cas::LEGACY_CAS_DIR).exists());
1293        // The symlink now resolves through common/ and still reads back.
1294        assert!(std::fs::symlink_metadata(&link).unwrap().file_type().is_symlink());
1295        assert_eq!(std::fs::read(&link).unwrap(), b"icon bytes");
1296        // Idempotent: nothing to migrate the second time.
1297        assert!(!migrate_cas_dir_name(base).unwrap());
1298    }
1299
1300    #[test]
1301    fn refresh_derived_content_addresses_translations() {
1302        let dir = tempfile::tempdir().unwrap();
1303        let base = dir.path();
1304        let cas_root = base.join(cas::CAS_DIR);
1305        let build_dir = base.join("1.0.0_100");
1306        // A raw per-locale catalog as a plain file (as copied from the game install).
1307        let mo = build_dir.join("translations/ru/LC_MESSAGES/global.mo");
1308        std::fs::create_dir_all(mo.parent().unwrap()).unwrap();
1309        std::fs::write(&mo, b"catalog bytes").unwrap();
1310
1311        let mut meta = BuildMetadata { version: "1.0.0".into(), build: 100, ..Default::default() };
1312        refresh_build_derived(&build_dir, &cas_root, &mut meta).unwrap();
1313
1314        // The catalog is now a symlink into the shared store, recorded in derived.
1315        assert!(std::fs::symlink_metadata(&mo).unwrap().file_type().is_symlink());
1316        assert_eq!(std::fs::read(&mo).unwrap(), b"catalog bytes");
1317        assert!(meta.derived.contains_key("translations/ru/LC_MESSAGES/global.mo"));
1318    }
1319}