1use std::collections::BTreeMap;
2use std::io::Read;
3use std::path::Path;
4use std::sync::Arc;
5
6use indicatif::ProgressBar;
7use indicatif::ProgressStyle;
8use rootcause::prelude::*;
9use wowsunpack::game_data;
10use wowsunpack::game_params::cache;
11use wowsunpack::game_params::provider::GameMetadataProvider;
12use wowsunpack::game_params::types::GameParamProvider;
13use wowsunpack::game_params::types::Param;
14use wowsunpack::vfs::VfsFileType;
15use wowsunpack::vfs::VfsPath;
16
17use crate::builds::BuildEntry;
18use crate::builds::BuildMetadata;
19use crate::builds::BuildsIndex;
20use crate::cas;
21
22const VFS_DIRS: &[&str] = &[
26 "gui/fla/minimap",
27 "gui/battle_hud",
28 "gui/consumables",
29 "gui/powerups/drops",
30 "gui/fonts",
31 "gui/data/constants",
32 "gui/ships_silhouettes",
33 "gui/ribbons",
34 "gui/achievements",
35 "gui/nation_flags",
36 "gui/crew_commander/skills",
37 "gui/modernization_icons",
38 "gui/signal_flags",
39 "scripts/entity_defs",
40];
41
42const REQUIRED_NONEMPTY_DIRS: &[&str] = &["scripts/entity_defs"];
46
47const REQUIRED_VFS_FILES: &[&str] = &["content/GameParams.data", "scripts/entities.xml"];
50
51const MAP_FILES_SPACES: &[&str] = &["minimap.png", "minimap_water.png", "space.settings"];
53
54const MAP_FILES_GAMEPLAY: &[&str] = &["space.settings"];
56
57pub fn required_path_globs() -> Vec<String> {
63 let mut globs = Vec::new();
64 for dir in VFS_DIRS {
65 globs.push(format!("{dir}/*"));
68 }
69 for file in REQUIRED_VFS_FILES {
70 globs.push((*file).to_string());
71 }
72 for name in MAP_FILES_SPACES {
73 globs.push(format!("spaces/*/{name}"));
74 }
75 for name in MAP_FILES_GAMEPLAY {
76 globs.push(format!("content/gameplay/*/{name}"));
77 }
78 globs
79}
80
81pub fn dump_dir(output_base: &Path, version_str: &str, build: u32) -> std::path::PathBuf {
83 output_base.join(format!("{version_str}_{build}"))
84}
85
86pub fn dump_exists(output_base: &Path, version_str: &str, build: u32) -> bool {
88 dump_dir(output_base, version_str, build).join("metadata.toml").exists()
89}
90
91pub fn dump_renderer_data(
100 game_dir: &Path,
101 build: u32,
102 version_str: &str,
103 output_base: &Path,
104 progress: Option<&ProgressBar>,
105 allow_existing: bool,
106) -> Result<(), Report> {
107 let output_dir = dump_dir(output_base, version_str, build);
108 let vfs_dir = output_dir.join("vfs");
109 let cas_root = cas::cas_root(output_base);
110
111 if output_dir.join("metadata.toml").exists() {
112 if allow_existing {
113 return Ok(());
114 }
115 bail!("Output directory already exists: {}", output_dir.display());
116 }
117
118 if output_dir.exists() {
120 std::fs::remove_dir_all(&output_dir)
121 .attach_with(|| format!("Failed to clean up partial dump at {}", output_dir.display()))?;
122 }
123
124 let vfs = game_data::build_game_vfs_for_build(game_dir, build).attach_with(|| "Failed to build game VFS")?;
125
126 let mut file_hashes: BTreeMap<String, String> = BTreeMap::new();
128
129 let mut dir_counts: BTreeMap<&str, usize> = BTreeMap::new();
130 for dir in VFS_DIRS {
131 let count = extract_vfs_dir_cas(&vfs, dir, &vfs_dir, &cas_root, &mut file_hashes, progress)?;
132 dir_counts.insert(dir, count);
133 }
134 let mut missing_files = Vec::new();
135 for file in REQUIRED_VFS_FILES {
136 if !extract_vfs_file_cas(&vfs, file, &vfs_dir, &cas_root, &mut file_hashes)? {
137 missing_files.push(*file);
138 }
139 if let Some(pb) = progress {
140 pb.inc(1);
141 }
142 }
143 let map_count =
144 extract_map_files_cas(&vfs, "spaces", MAP_FILES_SPACES, &vfs_dir, &cas_root, &mut file_hashes, progress)?;
145 extract_map_files_cas(
146 &vfs,
147 "content/gameplay",
148 MAP_FILES_GAMEPLAY,
149 &vfs_dir,
150 &cas_root,
151 &mut file_hashes,
152 progress,
153 )?;
154
155 if let Some(pb) = progress {
156 pb.finish_and_clear();
157 }
158
159 let mut problems = Vec::new();
163 if !missing_files.is_empty() {
164 problems.push(format!("missing required file(s): {}", missing_files.join(", ")));
165 }
166 for dir in REQUIRED_NONEMPTY_DIRS {
167 if dir_counts.get(dir).copied().unwrap_or(0) == 0 {
168 problems.push(format!("required directory '{dir}' extracted no files"));
169 }
170 }
171 if map_count == 0 {
172 problems.push(
173 "no map data extracted (spaces/*/minimap.png); the content depot's spaces packages are likely missing"
174 .to_string(),
175 );
176 }
177 if !problems.is_empty() {
178 let _ = std::fs::remove_dir_all(&output_dir);
179 bail!("Incomplete dump for build {build} ({version_str}): {}", problems.join("; "));
180 }
181 for dir in VFS_DIRS {
182 if dir_counts.get(dir).copied().unwrap_or(0) == 0 {
183 tracing::warn!("dump for build {build}: directory '{dir}' extracted no files");
184 }
185 }
186
187 std::fs::create_dir_all(&output_dir)
188 .attach_with(|| format!("Failed to create output directory {}", output_dir.display()))?;
189
190 dump_all_translations(game_dir, build, &output_dir)?;
191
192 #[cfg(feature = "constants")]
194 match crate::constants::ConstantsFetcher::new() {
195 Ok(fetcher) => {
196 write_constants_for_build(&output_dir, build, Some(version_str), &fetcher);
197 }
198 Err(e) => {
199 tracing::warn!("Could not initialize constants fetcher for build {build}: {e:?}");
200 }
201 }
202
203 let mut metadata =
207 BuildMetadata { version: version_str.to_string(), build, files: file_hashes, derived: BTreeMap::new() };
208 refresh_build_derived(&output_dir, &cas_root, &mut metadata)?;
209 metadata.save(&output_dir.join("metadata.toml"))?;
210
211 let builds_path = output_base.join("builds.toml");
213 let mut index = BuildsIndex::load(&builds_path);
214 index.upsert(BuildEntry {
215 version: version_str.to_string(),
216 build,
217 dir: format!("{version_str}_{build}"),
218 dumped_at: jiff::Zoned::now().to_string(),
219 });
220 index.save(&builds_path)?;
221
222 Ok(())
223}
224
225pub fn complete_build(game_dir: &Path, build: u32, output_base: &Path, with_gui: bool) -> Result<usize, Report> {
239 let index = BuildsIndex::load(&output_base.join("builds.toml"));
240 let entry = index
241 .find_by_build(build)
242 .ok_or_else(|| report!("build {build} is not in builds.toml; dump it normally first"))?
243 .clone();
244 let output_dir = output_base.join(&entry.dir);
245 let vfs_dir = output_dir.join("vfs");
246 let cas_root = cas::cas_root(output_base);
247 let meta_path = output_dir.join("metadata.toml");
248 let mut metadata =
249 BuildMetadata::load(&meta_path).ok_or_else(|| report!("{} has no readable metadata.toml", entry.dir))?;
250
251 let vfs = game_data::build_game_vfs_for_build(game_dir, build).attach_with(|| "Failed to build game VFS")?;
252
253 if with_gui {
254 for dir in VFS_DIRS.iter().filter(|d| d.starts_with("gui")) {
258 extract_vfs_dir_cas(&vfs, dir, &vfs_dir, &cas_root, &mut metadata.files, None)?;
259 }
260 }
261
262 let map_count =
263 extract_map_files_cas(&vfs, "spaces", MAP_FILES_SPACES, &vfs_dir, &cas_root, &mut metadata.files, None)?;
264 extract_map_files_cas(
265 &vfs,
266 "content/gameplay",
267 MAP_FILES_GAMEPLAY,
268 &vfs_dir,
269 &cas_root,
270 &mut metadata.files,
271 None,
272 )?;
273 if map_count == 0 {
274 bail!("no maps extracted for build {build}; refusing to record an incomplete build");
275 }
276
277 refresh_build_derived(&output_dir, &cas_root, &mut metadata)?;
280 metadata.save(&meta_path)?;
281 Ok(map_count)
282}
283
284pub fn create_progress_bar(game_dir: &Path) -> Option<ProgressBar> {
286 let vfs = game_data::build_game_vfs(game_dir).ok()?;
287 let mut total_files = 0u64;
288 for dir in VFS_DIRS {
289 total_files += count_vfs_dir_files(&vfs, dir);
290 }
291 total_files += REQUIRED_VFS_FILES.len() as u64;
292 total_files += count_map_files(&vfs, "spaces", MAP_FILES_SPACES);
293 total_files += count_map_files(&vfs, "content/gameplay", MAP_FILES_GAMEPLAY);
294
295 let pb = ProgressBar::new(total_files);
296 pb.set_style(
297 ProgressStyle::default_bar()
298 .template("{msg} [{bar:40}] {pos}/{len}")
299 .expect("valid template")
300 .progress_chars("=> "),
301 );
302 pb.set_message("Extracting VFS");
303 Some(pb)
304}
305
306pub fn remove_build(output_base: &Path, target_build: u32) -> Result<(), Report> {
308 let builds_path = output_base.join("builds.toml");
309 let mut index = BuildsIndex::load(&builds_path);
310 let entry = index
311 .find_by_build(target_build)
312 .ok_or_else(|| report!("Build {target_build} not found in builds.toml"))?
313 .clone();
314
315 let target_dir = output_base.join(&entry.dir);
316 let target_meta = BuildMetadata::load(&target_dir.join("metadata.toml"));
317
318 let mut live_hashes = std::collections::HashSet::new();
320 for other in &index.builds {
321 if other.build == target_build {
322 continue;
323 }
324 if let Some(meta) = BuildMetadata::load(&output_base.join(&other.dir).join("metadata.toml")) {
325 live_hashes.extend(meta.referenced_hashes());
326 }
327 }
328
329 if let Some(meta) = target_meta {
331 let cas_root = cas::cas_root(output_base);
332 for hash in meta.referenced_hashes() {
333 if !live_hashes.contains(&hash) {
334 let path = cas::cas_path(&cas_root, &hash);
335 let _ = std::fs::remove_file(&path);
336 }
337 }
338 let _ = cas::gc(&cas_root, &live_hashes);
340 }
341
342 if target_dir.exists() {
344 std::fs::remove_dir_all(&target_dir)
345 .attach_with(|| format!("Failed to remove build directory {}", target_dir.display()))?;
346 }
347
348 index.remove_build(target_build);
350 index.save(&builds_path)?;
351
352 Ok(())
353}
354
355pub enum SyncSelector {
359 All,
361 Latest,
363 Build(u32),
365 Version(String),
367}
368
369pub struct SyncedBuild {
371 pub build: u32,
372 pub version: String,
373 pub copied: bool,
375}
376
377pub fn sync_from_local(
386 source_base: &Path,
387 output_base: &Path,
388 selector: &SyncSelector,
389 force: bool,
390) -> Result<Vec<SyncedBuild>, Report> {
391 if source_base == output_base {
392 bail!("source and destination are the same directory");
393 }
394 let source_index = BuildsIndex::load(&source_base.join("builds.toml"));
395 if source_index.builds.is_empty() {
396 bail!("no builds.toml entries found in source {}", source_base.display());
397 }
398
399 let entries: Vec<BuildEntry> = match selector {
400 SyncSelector::All => source_index.builds.clone(),
401 SyncSelector::Latest => {
402 let latest =
403 source_index.builds.iter().max_by_key(|e| e.build).ok_or_else(|| report!("source has no builds"))?;
404 vec![latest.clone()]
405 }
406 SyncSelector::Build(b) => {
407 let entry = source_index
408 .find_by_build(*b)
409 .ok_or_else(|| report!("build {b} not found in source {}", source_base.display()))?;
410 vec![entry.clone()]
411 }
412 SyncSelector::Version(v) => {
413 let matches: Vec<BuildEntry> = source_index.find_by_version(v).into_iter().cloned().collect();
414 if matches.is_empty() {
415 bail!("no builds matching version '{v}' in source {}", source_base.display());
416 }
417 matches
418 }
419 };
420
421 let mut synced = Vec::new();
422 for entry in &entries {
423 let copied = copy_build_from_local(source_base, output_base, entry, force)?;
424 synced.push(SyncedBuild { build: entry.build, version: entry.version.clone(), copied });
425 }
426 Ok(synced)
427}
428
429fn copy_build_from_local(
432 source_base: &Path,
433 output_base: &Path,
434 entry: &BuildEntry,
435 force: bool,
436) -> Result<bool, Report> {
437 let src_cas = cas::cas_root(source_base);
438 let dst_cas = cas::cas_root(output_base);
439 let output_dir = output_base.join(&entry.dir);
440
441 if !force && output_dir.join("metadata.toml").exists() {
443 register_build(output_base, entry)?;
444 return Ok(false);
445 }
446
447 let src_dir = source_base.join(&entry.dir);
448 let meta_path = src_dir.join("metadata.toml");
449 let metadata = BuildMetadata::load(&meta_path)
450 .ok_or_else(|| report!("source build {} has no readable metadata.toml", entry.dir))?;
451
452 for hash in metadata.referenced_hashes() {
458 if cas::object_exists(&dst_cas, &hash) {
459 continue;
460 }
461 let src_obj = cas::cas_path(&src_cas, &hash);
462 let data =
463 std::fs::read(&src_obj).attach_with(|| format!("source content object {} missing", src_obj.display()))?;
464 let actual = cas::hash_bytes(&data);
465 if actual != hash {
466 bail!("source content object {hash} hashed to {actual}");
467 }
468 cas::store(&dst_cas, &data)?;
469 }
470
471 if output_dir.exists() {
473 std::fs::remove_dir_all(&output_dir)
474 .attach_with(|| format!("failed to clear destination build dir {}", output_dir.display()))?;
475 }
476
477 let vfs_dir = output_dir.join("vfs");
480 for (rel, hash) in &metadata.files {
481 cas::link_file(&dst_cas, hash, &vfs_dir.join(rel))?;
482 }
483 for (rel, hash) in &metadata.derived {
484 cas::link_file(&dst_cas, hash, &output_dir.join(rel))?;
485 }
486
487 let src_constants = src_dir.join("constants.json");
489 if src_constants.exists() {
490 let bytes =
491 std::fs::read(&src_constants).attach_with(|| format!("failed to read {}", src_constants.display()))?;
492 let dest = output_dir.join("constants.json");
493 std::fs::create_dir_all(dest.parent().unwrap())?;
494 std::fs::write(&dest, &bytes).attach_with(|| format!("failed to write {}", dest.display()))?;
495 }
496
497 metadata.save(&output_dir.join("metadata.toml"))?;
498 register_build(output_base, entry)?;
499 Ok(true)
500}
501
502fn register_build(output_base: &Path, entry: &BuildEntry) -> Result<(), Report> {
504 let builds_path = output_base.join("builds.toml");
505 let mut index = BuildsIndex::load(&builds_path);
506 index.upsert(entry.clone());
507 index.save(&builds_path)
508}
509
510fn dump_all_translations(game_dir: &Path, build: u32, output_dir: &Path) -> Result<(), Report> {
513 let texts_dir = game_dir.join("bin").join(build.to_string()).join("res/texts");
514 if !texts_dir.exists() {
515 tracing::warn!("Translations directory not found: {}", texts_dir.display());
516 return Ok(());
517 }
518 for entry in std::fs::read_dir(&texts_dir)
519 .attach_with(|| format!("Failed to read translations directory {}", texts_dir.display()))?
520 .flatten()
521 {
522 if !entry.file_type().map(|t| t.is_dir()).unwrap_or(false) {
523 continue;
524 }
525 let lang = entry.file_name();
526 let mo_src = entry.path().join("LC_MESSAGES/global.mo");
527 if mo_src.exists() {
528 let mo_dest = output_dir.join("translations").join(&lang).join("LC_MESSAGES/global.mo");
529 std::fs::create_dir_all(mo_dest.parent().unwrap())?;
530 std::fs::copy(&mo_src, &mo_dest)?;
531 }
532 }
533 Ok(())
534}
535
536fn store_and_link(
540 data: &[u8],
541 rel_path: &str,
542 vfs_dir: &Path,
543 cas_root: &Path,
544 file_hashes: &mut BTreeMap<String, String>,
545) -> Result<(), Report> {
546 let hash = cas::store(cas_root, data)?;
547 let link_path = vfs_dir.join(rel_path.trim_start_matches('/'));
548 cas::link_file(cas_root, &hash, &link_path)?;
549 file_hashes.insert(rel_path.trim_start_matches('/').to_string(), hash);
550 Ok(())
551}
552
553fn derive_game_params_rkyv(vfs_dir: &Path) -> Option<Vec<u8>> {
560 if !vfs_dir.join("content/GameParams.data").exists() {
561 return None;
562 }
563 let vfs = VfsPath::new(wowsunpack::vfs::PhysicalFS::new(vfs_dir));
564 let result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
565 let gmp = GameMetadataProvider::from_vfs(&vfs)?;
566 let params: Vec<Param> = gmp.params().iter().map(|p| Arc::unwrap_or_clone(Arc::clone(p))).collect();
567 cache::encode(¶ms).map_err(|e| report!("Failed to serialize: {e}"))
568 }));
569 match result {
570 Ok(Ok(bytes)) => Some(bytes),
571 Ok(Err(e)) => {
572 eprintln!("WARN: GameParams re-derivation failed for {}: {e:?}", vfs_dir.display());
573 None
574 }
575 Err(_) => {
576 eprintln!("WARN: GameParams re-derivation panicked for {} (incompatible pickle format)", vfs_dir.display(),);
577 None
578 }
579 }
580}
581
582fn store_and_relink(data: &[u8], link_path: &Path, cas_root: &Path) -> Result<String, Report> {
585 let hash = cas::store(cas_root, data)?;
586 let _ = std::fs::remove_file(link_path);
587 cas::link_file(cas_root, &hash, link_path)?;
588 Ok(hash)
589}
590
591pub fn refresh_build_derived(build_dir: &Path, cas_root: &Path, metadata: &mut BuildMetadata) -> Result<(), Report> {
599 metadata.derived.clear();
600
601 let rkyv_path = build_dir.join("game_params.rkyv");
602 let rkyv_bytes = derive_game_params_rkyv(&build_dir.join("vfs"));
603 let rkyv_bytes = match rkyv_bytes {
604 Some(b) => Some(b),
605 None if rkyv_path.exists() => {
606 Some(std::fs::read(&rkyv_path).attach_with(|| format!("Failed to read {}", rkyv_path.display()))?)
607 }
608 None => None,
609 };
610 if let Some(rkyv_bytes) = rkyv_bytes {
611 let hash = store_and_relink(&rkyv_bytes, &rkyv_path, cas_root)?;
612 metadata.derived.insert("game_params.rkyv".to_string(), hash);
613
614 let compressed =
615 ruzstd::encoding::compress_to_vec(rkyv_bytes.as_slice(), ruzstd::encoding::CompressionLevel::Fastest);
616 let zst_path = build_dir.join("game_params.rkyv.zst");
617 let hash = store_and_relink(&compressed, &zst_path, cas_root)?;
618 metadata.derived.insert("game_params.rkyv.zst".to_string(), hash);
619 }
620
621 let translations_dir = build_dir.join("translations");
625 if translations_dir.exists() {
626 for lang_entry in std::fs::read_dir(&translations_dir)
627 .attach_with(|| format!("Failed to read {}", translations_dir.display()))?
628 .flatten()
629 {
630 if !lang_entry.file_type().map(|t| t.is_dir()).unwrap_or(false) {
631 continue;
632 }
633 let lang = lang_entry.file_name().to_string_lossy().into_owned();
634 let mo_rel = format!("translations/{lang}/LC_MESSAGES/global.mo");
635 let mo_path = build_dir.join(&mo_rel);
636 if !mo_path.exists() {
637 continue;
638 }
639 let bytes = std::fs::read(&mo_path).attach_with(|| format!("Failed to read {}", mo_path.display()))?;
640 let hash = store_and_relink(&bytes, &mo_path, cas_root)?;
641 metadata.derived.insert(mo_rel, hash);
642 }
643 }
644
645 let mo_rel = "translations/en/LC_MESSAGES/global.mo";
647 let mo_path = build_dir.join(mo_rel);
648 if mo_path.exists() {
649 let mo_bytes = std::fs::read(&mo_path).attach_with(|| format!("Failed to read {}", mo_path.display()))?;
650 let compressed =
651 ruzstd::encoding::compress_to_vec(mo_bytes.as_slice(), ruzstd::encoding::CompressionLevel::Fastest);
652 let zst_path = build_dir.join(format!("{mo_rel}.zst"));
653 let hash = store_and_relink(&compressed, &zst_path, cas_root)?;
654 metadata.derived.insert(format!("{mo_rel}.zst"), hash);
655 }
656
657 Ok(())
658}
659
660pub fn refresh_derived(output_base: &Path, only_build: Option<u32>) -> Result<(), Report> {
664 let index = BuildsIndex::load(&output_base.join("builds.toml"));
665 let cas_root = cas::cas_root(output_base);
666
667 let targets: Vec<&BuildEntry> = match only_build {
668 Some(b) => index.builds.iter().filter(|e| e.build == b).collect(),
669 None => index.builds.iter().collect(),
670 };
671 if targets.is_empty() {
672 bail!("No matching builds found in {}", output_base.join("builds.toml").display());
673 }
674
675 #[cfg(feature = "constants")]
679 let constants_fetcher = match crate::constants::ConstantsFetcher::new() {
680 Ok(f) => Some(f),
681 Err(e) => {
682 eprintln!("WARN: Could not initialize constants fetcher: {e:?}");
683 None
684 }
685 };
686
687 for entry in &targets {
688 let build_dir = output_base.join(&entry.dir);
689 let meta_path = build_dir.join("metadata.toml");
690 let mut metadata = BuildMetadata::load(&meta_path).unwrap_or(BuildMetadata {
691 version: entry.version.clone(),
692 build: entry.build,
693 ..Default::default()
694 });
695
696 #[cfg(feature = "constants")]
697 let constants_added = if let Some(fetcher) = constants_fetcher.as_ref() {
698 update_constants_if_missing(&build_dir, entry.build, Some(entry.version.as_str()), fetcher)
699 } else {
700 false
701 };
702
703 match refresh_build_derived(&build_dir, &cas_root, &mut metadata) {
704 Ok(()) => {
705 metadata.save(&meta_path)?;
706 #[cfg(feature = "constants")]
707 let constants_note = if constants_added { " + constants" } else { "" };
708 #[cfg(not(feature = "constants"))]
709 let constants_note = "";
710 println!(" {} - {} derived artifact(s){}", entry.dir, metadata.derived.len(), constants_note);
711 }
712 Err(e) => eprintln!("WARN: {} - failed to refresh derived data: {e:?}", entry.dir),
713 }
714 }
715
716 println!("Refreshed {} build(s).", targets.len());
717 Ok(())
718}
719
720#[cfg(feature = "constants")]
724fn write_constants_for_build(
725 build_dir: &Path,
726 build: u32,
727 version: Option<&str>,
728 fetcher: &crate::constants::ConstantsFetcher,
729) -> bool {
730 let Some((data, actual_build)) = fetcher.fetch(build, version) else {
731 tracing::warn!("No upstream constants available for build {build}");
732 return false;
733 };
734 let bytes = match serde_json::to_vec_pretty(&data) {
735 Ok(b) => b,
736 Err(e) => {
737 tracing::warn!("Failed to serialize constants for build {build}: {e}");
738 return false;
739 }
740 };
741 if let Err(e) = std::fs::write(build_dir.join("constants.json"), &bytes) {
742 tracing::warn!("Failed to write constants.json for build {build}: {e}");
743 return false;
744 }
745 if actual_build != build {
746 tracing::info!("Stored constants from build {actual_build} (fallback for {build})");
747 }
748 true
749}
750
751#[cfg(feature = "constants")]
756fn update_constants_if_missing(
757 build_dir: &Path,
758 build: u32,
759 version: Option<&str>,
760 fetcher: &crate::constants::ConstantsFetcher,
761) -> bool {
762 if build_dir.join("constants.json").exists() {
763 return false;
764 }
765 write_constants_for_build(build_dir, build, version, fetcher)
766}
767
768pub fn gc_cas(output_base: &Path) -> Result<(), Report> {
773 let index = BuildsIndex::load(&output_base.join("builds.toml"));
774 let cas_root = cas::cas_root(output_base);
775
776 let mut live = std::collections::HashSet::new();
777 for entry in &index.builds {
778 let meta_path = output_base.join(&entry.dir).join("metadata.toml");
779 let meta = BuildMetadata::load(&meta_path)
780 .ok_or_else(|| report!("{} has no readable metadata.toml; aborting GC", entry.dir))?;
781 live.extend(meta.referenced_hashes());
782 }
783
784 let removed = cas::gc(&cas_root, &live)?;
785 println!("GC removed {removed} orphaned CAS object(s); {} still referenced.", live.len());
786 Ok(())
787}
788
789pub struct BuildVerification {
791 pub dir: String,
792 pub build: u32,
793 pub version: String,
794 pub referenced: usize,
796 pub missing_objects: Vec<String>,
798 pub broken_links: Vec<String>,
800 pub metadata_unreadable: bool,
802}
803
804impl BuildVerification {
805 pub fn is_ok(&self) -> bool {
806 !self.metadata_unreadable && self.missing_objects.is_empty() && self.broken_links.is_empty()
807 }
808}
809
810pub fn verify_builds(output_base: &Path, check_links: bool) -> Result<Vec<BuildVerification>, Report> {
815 let index = BuildsIndex::load(&output_base.join("builds.toml"));
816 let cas_root = cas::cas_root(output_base);
817
818 let mut reports = Vec::new();
819 for entry in &index.builds {
820 let build_dir = output_base.join(&entry.dir);
821 let Some(meta) = BuildMetadata::load(&build_dir.join("metadata.toml")) else {
822 reports.push(BuildVerification {
823 dir: entry.dir.clone(),
824 build: entry.build,
825 version: entry.version.clone(),
826 referenced: 0,
827 missing_objects: Vec::new(),
828 broken_links: Vec::new(),
829 metadata_unreadable: true,
830 });
831 continue;
832 };
833
834 let referenced = meta.referenced_hashes();
835 let mut missing_objects: Vec<String> =
836 referenced.iter().filter(|h| !cas::object_exists(&cas_root, h)).cloned().collect();
837 missing_objects.sort();
838
839 let mut broken_links = Vec::new();
840 if check_links {
841 for (rel, _) in meta.files.iter().chain(meta.derived.iter()) {
842 let path = build_dir.join("vfs").join(rel);
843 let candidate = if path.exists() { path } else { build_dir.join(rel) };
845 if !candidate.exists() {
846 broken_links.push(rel.clone());
847 }
848 }
849 broken_links.sort();
850 }
851
852 reports.push(BuildVerification {
853 dir: entry.dir.clone(),
854 build: entry.build,
855 version: entry.version.clone(),
856 referenced: referenced.len(),
857 missing_objects,
858 broken_links,
859 metadata_unreadable: false,
860 });
861 }
862 Ok(reports)
863}
864
865pub fn gc_unreferenced(output_base: &Path) -> Result<usize, Report> {
872 let cas_root = cas::cas_root(output_base);
873 if !cas_root.exists() {
874 return Ok(0);
875 }
876
877 let mut live = std::collections::HashSet::new();
878 for entry in
879 std::fs::read_dir(output_base).attach_with(|| format!("Failed to read {}", output_base.display()))?.flatten()
880 {
881 let meta_path = entry.path().join("metadata.toml");
882 if !meta_path.exists() {
883 continue;
884 }
885 match BuildMetadata::load(&meta_path) {
886 Some(meta) => live.extend(meta.referenced_hashes()),
887 None => bail!("unreadable metadata at {}; aborting GC", meta_path.display()),
888 }
889 }
890
891 cas::gc(&cas_root, &live)
892}
893
894pub fn migrate_cas_dir_name(output_base: &Path) -> Result<bool, Report> {
900 let legacy = output_base.join(cas::LEGACY_CAS_DIR);
901 let current = cas::cas_root(output_base);
902 if !legacy.exists() {
903 return Ok(false);
904 }
905
906 if !current.exists() {
907 std::fs::rename(&legacy, ¤t)
909 .attach_with(|| format!("failed to rename {} to {}", legacy.display(), current.display()))?;
910 } else {
911 merge_cas_objects(&legacy, ¤t)?;
915 }
916
917 relink_all_builds(output_base, ¤t)?;
920
921 if legacy.exists() {
924 std::fs::remove_dir_all(&legacy)
925 .attach_with(|| format!("failed to remove emptied legacy store {}", legacy.display()))?;
926 }
927
928 Ok(true)
929}
930
931fn merge_cas_objects(legacy: &Path, dest: &Path) -> Result<(), Report> {
935 for fanout in std::fs::read_dir(legacy).attach_with(|| format!("Failed to read {}", legacy.display()))?.flatten() {
936 if !fanout.file_type().map(|t| t.is_dir()).unwrap_or(false) {
937 continue;
938 }
939 let dest_fanout = dest.join(fanout.file_name());
940 for obj in std::fs::read_dir(fanout.path())?.flatten() {
941 if !obj.file_type().map(|t| t.is_file()).unwrap_or(false) {
942 continue;
943 }
944 let dest_path = dest_fanout.join(obj.file_name());
945 if dest_path.exists() {
946 std::fs::remove_file(obj.path())?;
947 continue;
948 }
949 std::fs::create_dir_all(&dest_fanout)?;
950 if std::fs::rename(obj.path(), &dest_path).is_err() {
952 std::fs::copy(obj.path(), &dest_path)?;
953 std::fs::remove_file(obj.path())?;
954 }
955 }
956 }
957 Ok(())
958}
959
960fn relink_all_builds(output_base: &Path, cas_root: &Path) -> Result<(), Report> {
963 for entry in
964 std::fs::read_dir(output_base).attach_with(|| format!("Failed to read {}", output_base.display()))?.flatten()
965 {
966 let build_dir = entry.path();
967 let Some(meta) = BuildMetadata::load(&build_dir.join("metadata.toml")) else {
968 continue;
969 };
970 let vfs_dir = build_dir.join("vfs");
971 let relink = |rel: &str, hash: &str, base: &Path| {
972 let link = base.join(rel);
973 let _ = std::fs::remove_file(&link);
974 if let Err(e) = cas::link_file(cas_root, hash, &link) {
975 tracing::warn!("failed to relink {}: {e}", link.display());
976 }
977 };
978 for (rel, hash) in &meta.files {
979 relink(rel, hash, &vfs_dir);
980 }
981 for (rel, hash) in &meta.derived {
982 relink(rel, hash, &build_dir);
983 }
984 }
985 Ok(())
986}
987
988pub fn migrate_to_cas(output_base: &Path) -> Result<usize, Report> {
996 let cas_root = cas::cas_root(output_base);
997 let mut migrated = 0;
998 for entry in
999 std::fs::read_dir(output_base).attach_with(|| format!("Failed to read {}", output_base.display()))?.flatten()
1000 {
1001 let build_dir = entry.path();
1002 let meta_path = build_dir.join("metadata.toml");
1003 let Some(mut metadata) = BuildMetadata::load(&meta_path) else {
1004 continue;
1005 };
1006 if metadata.has_file_hashes() || !build_dir.join("vfs").exists() {
1007 continue;
1008 }
1009 match migrate_build_to_cas(&build_dir, &cas_root, &mut metadata) {
1010 Ok(()) => {
1011 metadata.save(&meta_path)?;
1012 migrated += 1;
1013 }
1014 Err(e) => tracing::warn!("failed to migrate {} to CAS: {e}", build_dir.display()),
1015 }
1016 }
1017 Ok(migrated)
1018}
1019
1020fn migrate_build_to_cas(build_dir: &Path, cas_root: &Path, metadata: &mut BuildMetadata) -> Result<(), Report> {
1023 let vfs_dir = build_dir.join("vfs");
1024 let mut stack = vec![vfs_dir.clone()];
1025 while let Some(dir) = stack.pop() {
1026 for entry in std::fs::read_dir(&dir).attach_with(|| format!("Failed to read {}", dir.display()))?.flatten() {
1027 let path = entry.path();
1028 let file_type = entry.file_type().attach_with(|| format!("Failed to stat {}", path.display()))?;
1029 if file_type.is_dir() {
1030 stack.push(path);
1031 continue;
1032 }
1033 if file_type.is_symlink() {
1035 continue;
1036 }
1037 let rel =
1038 path.strip_prefix(&vfs_dir).expect("walked path is under vfs_dir").to_string_lossy().replace('\\', "/");
1039 let data = std::fs::read(&path).attach_with(|| format!("Failed to read {}", path.display()))?;
1040 let hash = cas::store(cas_root, &data)?;
1041 std::fs::remove_file(&path).attach_with(|| format!("Failed to remove {}", path.display()))?;
1042 cas::link_file(cas_root, &hash, &path)?;
1043 metadata.files.insert(rel, hash);
1044 }
1045 }
1046 refresh_build_derived(build_dir, cas_root, metadata)
1047}
1048
1049fn extract_vfs_dir_cas(
1050 vfs: &VfsPath,
1051 vfs_path: &str,
1052 vfs_dir: &Path,
1053 cas_root: &Path,
1054 file_hashes: &mut BTreeMap<String, String>,
1055 progress: Option<&ProgressBar>,
1056) -> Result<usize, Report> {
1057 let dir = match vfs.join(vfs_path) {
1058 Ok(d) => d,
1059 Err(_) => return Ok(0),
1060 };
1061 let walker = match dir.walk_dir() {
1062 Ok(w) => w,
1063 Err(_) => return Ok(0),
1064 };
1065
1066 let mut count = 0;
1067 for entry in walker.flatten() {
1068 let metadata = match entry.metadata() {
1069 Ok(m) => m,
1070 Err(_) => continue,
1071 };
1072 if metadata.file_type != VfsFileType::File {
1073 continue;
1074 }
1075 let rel = entry.as_str();
1076 let mut buf = Vec::new();
1077 match entry.open_file() {
1078 Ok(mut f) => f.read_to_end(&mut buf)?,
1079 Err(e) => {
1080 tracing::warn!("Failed to open VFS file {rel}: {e}");
1081 continue;
1082 }
1083 };
1084 store_and_link(&buf, rel, vfs_dir, cas_root, file_hashes)?;
1085 count += 1;
1086 if let Some(pb) = progress {
1087 pb.inc(1);
1088 }
1089 }
1090 Ok(count)
1091}
1092
1093fn extract_vfs_file_cas(
1095 vfs: &VfsPath,
1096 vfs_path: &str,
1097 vfs_dir: &Path,
1098 cas_root: &Path,
1099 file_hashes: &mut BTreeMap<String, String>,
1100) -> Result<bool, Report> {
1101 let file = match vfs.join(vfs_path) {
1102 Ok(f) => f,
1103 Err(_) => {
1104 tracing::warn!("VFS path not found (skipping): {vfs_path}");
1105 return Ok(false);
1106 }
1107 };
1108 let mut buf = Vec::new();
1109 match file.open_file() {
1110 Ok(mut f) => f.read_to_end(&mut buf)?,
1111 Err(_) => {
1112 tracing::warn!("Could not open VFS file (skipping): {vfs_path}");
1113 return Ok(false);
1114 }
1115 };
1116 store_and_link(&buf, vfs_path, vfs_dir, cas_root, file_hashes)?;
1117 Ok(true)
1118}
1119
1120fn extract_map_files_cas(
1123 vfs: &VfsPath,
1124 parent_dir: &str,
1125 filenames: &[&str],
1126 vfs_dir: &Path,
1127 cas_root: &Path,
1128 file_hashes: &mut BTreeMap<String, String>,
1129 progress: Option<&ProgressBar>,
1130) -> Result<usize, Report> {
1131 let parent = match vfs.join(parent_dir) {
1132 Ok(d) => d,
1133 Err(_) => return Ok(0),
1134 };
1135 let entries = match parent.read_dir() {
1136 Ok(e) => e,
1137 Err(_) => return Ok(0),
1138 };
1139
1140 let mut count = 0;
1141 for entry in entries {
1142 if !entry.metadata().map(|m| m.file_type == VfsFileType::Directory).unwrap_or(false) {
1143 continue;
1144 }
1145 for filename in filenames {
1146 let file_path = match entry.join(filename) {
1147 Ok(f) => f,
1148 Err(_) => continue,
1149 };
1150 if !file_path.exists().unwrap_or(false) {
1151 continue;
1152 }
1153 let rel = file_path.as_str();
1154 let mut buf = Vec::new();
1155 match file_path.open_file() {
1156 Ok(mut f) => f.read_to_end(&mut buf)?,
1157 Err(e) => {
1158 tracing::warn!("Failed to open VFS file {rel}: {e}");
1159 continue;
1160 }
1161 };
1162 store_and_link(&buf, rel, vfs_dir, cas_root, file_hashes)?;
1163 count += 1;
1164 if let Some(pb) = progress {
1165 pb.inc(1);
1166 }
1167 }
1168 }
1169 Ok(count)
1170}
1171
1172fn count_vfs_dir_files(vfs: &VfsPath, dir: &str) -> u64 {
1175 let mut count = 0;
1176 if let Ok(vfs_dir_path) = vfs.join(dir)
1177 && let Ok(walker) = vfs_dir_path.walk_dir()
1178 {
1179 for entry in walker.flatten() {
1180 if entry.metadata().map(|m| m.file_type == VfsFileType::File).unwrap_or(false) {
1181 count += 1;
1182 }
1183 }
1184 }
1185 count
1186}
1187
1188fn count_map_files(vfs: &VfsPath, parent_dir: &str, filenames: &[&str]) -> u64 {
1189 let mut count = 0;
1190 if let Ok(parent) = vfs.join(parent_dir)
1191 && let Ok(entries) = parent.read_dir()
1192 {
1193 for entry in entries {
1194 if entry.metadata().map(|m| m.file_type == VfsFileType::Directory).unwrap_or(false) {
1195 for filename in filenames {
1196 if entry.join(filename).is_ok_and(|f: VfsPath| f.exists().unwrap_or(false)) {
1197 count += 1;
1198 }
1199 }
1200 }
1201 }
1202 }
1203 count
1204}
1205
1206#[cfg(test)]
1207mod maintenance_tests {
1208 use super::*;
1209
1210 fn write_build_metadata(build_dir: &Path, version: &str, build: u32, files: &[(&str, &str)]) {
1211 std::fs::create_dir_all(build_dir).unwrap();
1212 let mut meta = BuildMetadata { version: version.to_string(), build, ..Default::default() };
1213 for (rel, hash) in files {
1214 meta.files.insert((*rel).to_string(), (*hash).to_string());
1215 }
1216 meta.save(&build_dir.join("metadata.toml")).unwrap();
1217 }
1218
1219 #[test]
1220 fn gc_unreferenced_removes_orphans_and_keeps_live() {
1221 let dir = tempfile::tempdir().unwrap();
1222 let base = dir.path();
1223 let cas_root = base.join("common");
1224
1225 let live_hash = cas::store(&cas_root, b"live object").unwrap();
1226 let orphan_hash = cas::store(&cas_root, b"orphan object").unwrap();
1227 write_build_metadata(&base.join("1.0.0_100"), "1.0.0", 100, &[("gui/a.png", &live_hash)]);
1228
1229 let removed = gc_unreferenced(base).unwrap();
1230 assert_eq!(removed, 1);
1231 assert!(cas::object_exists(&cas_root, &live_hash));
1232 assert!(!cas::object_exists(&cas_root, &orphan_hash));
1233 }
1234
1235 #[test]
1236 fn gc_unreferenced_aborts_on_unreadable_metadata() {
1237 let dir = tempfile::tempdir().unwrap();
1238 let base = dir.path();
1239 let cas_root = base.join("common");
1240 let orphan_hash = cas::store(&cas_root, b"orphan object").unwrap();
1241
1242 let build_dir = base.join("1.0.0_100");
1243 std::fs::create_dir_all(&build_dir).unwrap();
1244 std::fs::write(build_dir.join("metadata.toml"), b"this is not valid toml = =").unwrap();
1245
1246 assert!(gc_unreferenced(base).is_err());
1247 assert!(cas::object_exists(&cas_root, &orphan_hash));
1249 }
1250
1251 #[test]
1252 fn migrate_to_cas_dedups_plain_files() {
1253 let dir = tempfile::tempdir().unwrap();
1254 let base = dir.path();
1255 let build_dir = base.join("1.0.0_100");
1256
1257 write_build_metadata(&build_dir, "1.0.0", 100, &[]);
1259 let file_path = build_dir.join("vfs/gui/a.png");
1260 std::fs::create_dir_all(file_path.parent().unwrap()).unwrap();
1261 std::fs::write(&file_path, b"some asset bytes").unwrap();
1262
1263 let migrated = migrate_to_cas(base).unwrap();
1264 assert_eq!(migrated, 1);
1265
1266 assert!(std::fs::symlink_metadata(&file_path).unwrap().file_type().is_symlink());
1268 assert_eq!(std::fs::read(&file_path).unwrap(), b"some asset bytes");
1269
1270 let meta = BuildMetadata::load(&build_dir.join("metadata.toml")).unwrap();
1272 assert!(meta.has_file_hashes());
1273 assert!(meta.files.contains_key("gui/a.png"));
1274 assert_eq!(migrate_to_cas(base).unwrap(), 0);
1275 }
1276
1277 #[test]
1278 fn migrate_cas_dir_name_renames_and_relinks() {
1279 let dir = tempfile::tempdir().unwrap();
1280 let base = dir.path();
1281
1282 let legacy = base.join(cas::LEGACY_CAS_DIR);
1284 let hash = cas::store(&legacy, b"icon bytes").unwrap();
1285 let build_dir = base.join("1.0.0_100");
1286 let link = build_dir.join("vfs/gui/x.png");
1287 cas::link_file(&legacy, &hash, &link).unwrap();
1288 write_build_metadata(&build_dir, "1.0.0", 100, &[("gui/x.png", &hash)]);
1289
1290 assert!(migrate_cas_dir_name(base).unwrap());
1291 assert!(base.join(cas::CAS_DIR).exists());
1292 assert!(!base.join(cas::LEGACY_CAS_DIR).exists());
1293 assert!(std::fs::symlink_metadata(&link).unwrap().file_type().is_symlink());
1295 assert_eq!(std::fs::read(&link).unwrap(), b"icon bytes");
1296 assert!(!migrate_cas_dir_name(base).unwrap());
1298 }
1299
1300 #[test]
1301 fn refresh_derived_content_addresses_translations() {
1302 let dir = tempfile::tempdir().unwrap();
1303 let base = dir.path();
1304 let cas_root = base.join(cas::CAS_DIR);
1305 let build_dir = base.join("1.0.0_100");
1306 let mo = build_dir.join("translations/ru/LC_MESSAGES/global.mo");
1308 std::fs::create_dir_all(mo.parent().unwrap()).unwrap();
1309 std::fs::write(&mo, b"catalog bytes").unwrap();
1310
1311 let mut meta = BuildMetadata { version: "1.0.0".into(), build: 100, ..Default::default() };
1312 refresh_build_derived(&build_dir, &cas_root, &mut meta).unwrap();
1313
1314 assert!(std::fs::symlink_metadata(&mo).unwrap().file_type().is_symlink());
1316 assert_eq!(std::fs::read(&mo).unwrap(), b"catalog bytes");
1317 assert!(meta.derived.contains_key("translations/ru/LC_MESSAGES/global.mo"));
1318 }
1319}