wows_data_mgr/
dump.rs

1use std::collections::BTreeMap;
2use std::io::Read;
3use std::path::Path;
4use std::sync::Arc;
5
6use indicatif::ProgressBar;
7use indicatif::ProgressStyle;
8use rootcause::prelude::*;
9use wowsunpack::game_data;
10use wowsunpack::game_params::provider::GameMetadataProvider;
11use wowsunpack::game_params::types::GameParamProvider;
12use wowsunpack::game_params::types::Param;
13use wowsunpack::vfs::VfsFileType;
14use wowsunpack::vfs::VfsPath;
15
16use crate::builds::BuildEntry;
17use crate::builds::BuildMetadata;
18use crate::builds::BuildsIndex;
19use crate::cas;
20
21/// VFS directories dumped in their entirety.
22const REQUIRED_VFS_DIRS: &[&str] = &[
23    "gui/fla/minimap",
24    "gui/battle_hud/markers_minimap",
25    "gui/battle_hud/icon_frag",
26    "gui/battle_hud/markers/capture_point",
27    "gui/battle_hud/markers/building_icons",
28    "gui/consumables",
29    "gui/powerups/drops",
30    "gui/fonts",
31    "gui/data/constants",
32    "gui/ships_silhouettes",
33    "scripts/entity_defs",
34];
35
36/// Individual VFS files required beyond the directory dumps.
37const REQUIRED_VFS_FILES: &[&str] = &["content/GameParams.data", "scripts/entities.xml"];
38
39/// Files to extract per map from `spaces/<map>/`.
40const MAP_FILES_SPACES: &[&str] = &["minimap.png", "minimap_water.png", "space.settings"];
41
42/// Files to extract per map from `content/gameplay/<map>/`.
43const MAP_FILES_GAMEPLAY: &[&str] = &["space.settings"];
44
45/// Returns the dump directory path for a given version and build.
46pub fn dump_dir(output_base: &Path, version_str: &str, build: u32) -> std::path::PathBuf {
47    output_base.join(format!("{version_str}_{build}"))
48}
49
50/// Check if a valid dump exists for the given version and build.
51pub fn dump_exists(output_base: &Path, version_str: &str, build: u32) -> bool {
52    dump_dir(output_base, version_str, build).join("metadata.toml").exists()
53}
54
55/// Dump game data with content-addressed deduplication.
56///
57/// VFS files are stored in `{output_base}/vfs_common/` by hash, with symlinks
58/// in the build's `vfs/` directory. Non-VFS files (game_params.rkyv, translations)
59/// are stored directly in the build directory.
60///
61/// When `progress` is `Some`, a CLI progress bar is updated during extraction.
62/// When `allow_existing` is true and a complete dump already exists, returns immediately.
63pub fn dump_renderer_data(
64    game_dir: &Path,
65    build: u32,
66    version_str: &str,
67    output_base: &Path,
68    progress: Option<&ProgressBar>,
69    allow_existing: bool,
70) -> Result<(), Report> {
71    let output_dir = dump_dir(output_base, version_str, build);
72    let vfs_dir = output_dir.join("vfs");
73    let cas_root = output_base.join("vfs_common");
74
75    if output_dir.join("metadata.toml").exists() {
76        if allow_existing {
77            return Ok(());
78        }
79        bail!("Output directory already exists: {}", output_dir.display());
80    }
81
82    // Clean up partial dumps
83    if output_dir.exists() {
84        std::fs::remove_dir_all(&output_dir)
85            .attach_with(|| format!("Failed to clean up partial dump at {}", output_dir.display()))?;
86    }
87
88    let vfs = game_data::build_game_vfs_for_build(game_dir, build).attach_with(|| "Failed to build game VFS")?;
89
90    // Extract VFS files through CAS
91    let mut file_hashes: BTreeMap<String, String> = BTreeMap::new();
92
93    for dir in REQUIRED_VFS_DIRS {
94        extract_vfs_dir_cas(&vfs, dir, &vfs_dir, &cas_root, &mut file_hashes, progress)?;
95    }
96    for file in REQUIRED_VFS_FILES {
97        extract_vfs_file_cas(&vfs, file, &vfs_dir, &cas_root, &mut file_hashes)?;
98        if let Some(pb) = progress {
99            pb.inc(1);
100        }
101    }
102    extract_map_files_cas(&vfs, "spaces", MAP_FILES_SPACES, &vfs_dir, &cas_root, &mut file_hashes, progress)?;
103    extract_map_files_cas(
104        &vfs,
105        "content/gameplay",
106        MAP_FILES_GAMEPLAY,
107        &vfs_dir,
108        &cas_root,
109        &mut file_hashes,
110        progress,
111    )?;
112
113    if let Some(pb) = progress {
114        pb.finish_and_clear();
115    }
116
117    std::fs::create_dir_all(&output_dir)
118        .attach_with(|| format!("Failed to create output directory {}", output_dir.display()))?;
119
120    // Serialize GameParams via rkyv (stored directly, not in CAS).
121    // Wrap in catch_unwind because old game data may have missing fields that cause panics.
122    let vfs_clone = vfs.clone();
123    let game_params_result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(move || {
124        let gmp = GameMetadataProvider::from_vfs(&vfs_clone)?;
125        let params: Vec<Param> = gmp.params().iter().map(|p| Arc::unwrap_or_clone(Arc::clone(p))).collect();
126        let bytes = rkyv::to_bytes::<rkyv::rancor::Error>(&params).map_err(|e| report!("Failed to serialize: {e}"))?;
127        Ok::<_, rootcause::Report>(bytes)
128    }));
129    match game_params_result {
130        Ok(Ok(bytes)) => {
131            std::fs::write(output_dir.join("game_params.rkyv"), &bytes)
132                .attach_with(|| "Failed to write game_params.rkyv")?;
133        }
134        Ok(Err(e)) => {
135            eprintln!("WARN: GameParams conversion failed for build {build}: {e:?}");
136        }
137        Err(_) => {
138            eprintln!("WARN: GameParams conversion panicked for build {build} (incompatible format)");
139        }
140    }
141
142    // Copy all language translations (stored directly)
143    dump_all_translations(game_dir, build, &output_dir)?;
144
145    // Fetch and store versioned constants (non-fatal)
146    #[cfg(feature = "constants")]
147    {
148        match crate::constants::fetch_versioned_constants_blocking(build) {
149            Ok((data, actual_build)) => {
150                if let Ok(bytes) = serde_json::to_vec_pretty(&data) {
151                    let _ = std::fs::write(output_dir.join("constants.json"), &bytes);
152                    if actual_build != build {
153                        tracing::info!("Stored constants from build {actual_build} (fallback for {build})");
154                    }
155                }
156            }
157            Err(e) => {
158                tracing::warn!("Could not fetch constants for build {build}: {e}");
159            }
160        }
161    }
162
163    // Write enhanced metadata with file hashes
164    let metadata = BuildMetadata { version: version_str.to_string(), build, files: file_hashes };
165    metadata.save(&output_dir.join("metadata.toml"))?;
166
167    // Update master builds index
168    let builds_path = output_base.join("builds.toml");
169    let mut index = BuildsIndex::load(&builds_path);
170    index.upsert(BuildEntry {
171        version: version_str.to_string(),
172        build,
173        dir: format!("{version_str}_{build}"),
174        dumped_at: jiff::Zoned::now().to_string(),
175    });
176    index.save(&builds_path)?;
177
178    Ok(())
179}
180
181/// Create a configured progress bar for CLI use.
182pub fn create_progress_bar(game_dir: &Path) -> Option<ProgressBar> {
183    let vfs = game_data::build_game_vfs(game_dir).ok()?;
184    let mut total_files = 0u64;
185    for dir in REQUIRED_VFS_DIRS {
186        total_files += count_vfs_dir_files(&vfs, dir);
187    }
188    total_files += REQUIRED_VFS_FILES.len() as u64;
189    total_files += count_map_files(&vfs, "spaces", MAP_FILES_SPACES);
190    total_files += count_map_files(&vfs, "content/gameplay", MAP_FILES_GAMEPLAY);
191
192    let pb = ProgressBar::new(total_files);
193    pb.set_style(
194        ProgressStyle::default_bar()
195            .template("{msg} [{bar:40}] {pos}/{len}")
196            .expect("valid template")
197            .progress_chars("=> "),
198    );
199    pb.set_message("Extracting VFS");
200    Some(pb)
201}
202
203/// Remove a dumped build, cleaning up orphaned CAS objects.
204pub fn remove_build(output_base: &Path, target_build: u32) -> Result<(), Report> {
205    let builds_path = output_base.join("builds.toml");
206    let mut index = BuildsIndex::load(&builds_path);
207    let entry = index
208        .find_by_build(target_build)
209        .ok_or_else(|| report!("Build {target_build} not found in builds.toml"))?
210        .clone();
211
212    let target_dir = output_base.join(&entry.dir);
213    let target_meta = BuildMetadata::load(&target_dir.join("metadata.toml"));
214
215    // Collect hashes still in use by other builds
216    let mut live_hashes = std::collections::HashSet::new();
217    for other in &index.builds {
218        if other.build == target_build {
219            continue;
220        }
221        if let Some(meta) = BuildMetadata::load(&output_base.join(&other.dir).join("metadata.toml")) {
222            live_hashes.extend(meta.files.values().cloned());
223        }
224    }
225
226    // Delete orphaned CAS objects
227    if let Some(meta) = target_meta {
228        let cas_root = output_base.join("vfs_common");
229        for hash in meta.files.values() {
230            if !live_hashes.contains(hash) {
231                let path = cas::cas_path(&cas_root, hash);
232                let _ = std::fs::remove_file(&path);
233            }
234        }
235        // Clean up empty fanout directories
236        let _ = cas::gc(&cas_root, &live_hashes);
237    }
238
239    // Remove build directory
240    if target_dir.exists() {
241        std::fs::remove_dir_all(&target_dir)
242            .attach_with(|| format!("Failed to remove build directory {}", target_dir.display()))?;
243    }
244
245    // Update builds index
246    index.remove_build(target_build);
247    index.save(&builds_path)?;
248
249    Ok(())
250}
251
252// -- Translation dumping --
253
254fn dump_all_translations(game_dir: &Path, build: u32, output_dir: &Path) -> Result<(), Report> {
255    let texts_dir = game_dir.join("bin").join(build.to_string()).join("res/texts");
256    if !texts_dir.exists() {
257        tracing::warn!("Translations directory not found: {}", texts_dir.display());
258        return Ok(());
259    }
260    for entry in std::fs::read_dir(&texts_dir)
261        .attach_with(|| format!("Failed to read translations directory {}", texts_dir.display()))?
262        .flatten()
263    {
264        if !entry.file_type().map(|t| t.is_dir()).unwrap_or(false) {
265            continue;
266        }
267        let lang = entry.file_name();
268        let mo_src = entry.path().join("LC_MESSAGES/global.mo");
269        if mo_src.exists() {
270            let mo_dest = output_dir.join("translations").join(&lang).join("LC_MESSAGES/global.mo");
271            std::fs::create_dir_all(mo_dest.parent().unwrap())?;
272            std::fs::copy(&mo_src, &mo_dest)?;
273        }
274    }
275    Ok(())
276}
277
278// -- CAS-aware extraction helpers --
279
280/// Read a VFS file into a buffer, store in CAS, and create a link in the build's vfs dir.
281fn store_and_link(
282    data: &[u8],
283    rel_path: &str,
284    vfs_dir: &Path,
285    cas_root: &Path,
286    file_hashes: &mut BTreeMap<String, String>,
287) -> Result<(), Report> {
288    let hash = cas::store(cas_root, data)?;
289    let link_path = vfs_dir.join(rel_path.trim_start_matches('/'));
290    cas::link_file(cas_root, &hash, &link_path)?;
291    file_hashes.insert(rel_path.trim_start_matches('/').to_string(), hash);
292    Ok(())
293}
294
295fn extract_vfs_dir_cas(
296    vfs: &VfsPath,
297    vfs_path: &str,
298    vfs_dir: &Path,
299    cas_root: &Path,
300    file_hashes: &mut BTreeMap<String, String>,
301    progress: Option<&ProgressBar>,
302) -> Result<(), Report> {
303    let dir = match vfs.join(vfs_path) {
304        Ok(d) => d,
305        Err(_) => return Ok(()),
306    };
307    let walker = match dir.walk_dir() {
308        Ok(w) => w,
309        Err(_) => return Ok(()),
310    };
311
312    for entry in walker.flatten() {
313        let metadata = match entry.metadata() {
314            Ok(m) => m,
315            Err(_) => continue,
316        };
317        if metadata.file_type != VfsFileType::File {
318            continue;
319        }
320        let rel = entry.as_str();
321        let mut buf = Vec::new();
322        match entry.open_file() {
323            Ok(mut f) => f.read_to_end(&mut buf)?,
324            Err(e) => {
325                tracing::warn!("Failed to open VFS file {rel}: {e}");
326                continue;
327            }
328        };
329        store_and_link(&buf, rel, vfs_dir, cas_root, file_hashes)?;
330        if let Some(pb) = progress {
331            pb.inc(1);
332        }
333    }
334    Ok(())
335}
336
337fn extract_vfs_file_cas(
338    vfs: &VfsPath,
339    vfs_path: &str,
340    vfs_dir: &Path,
341    cas_root: &Path,
342    file_hashes: &mut BTreeMap<String, String>,
343) -> Result<(), Report> {
344    let file = match vfs.join(vfs_path) {
345        Ok(f) => f,
346        Err(_) => {
347            tracing::warn!("VFS path not found (skipping): {vfs_path}");
348            return Ok(());
349        }
350    };
351    let mut buf = Vec::new();
352    match file.open_file() {
353        Ok(mut f) => f.read_to_end(&mut buf)?,
354        Err(_) => {
355            tracing::warn!("Could not open VFS file (skipping): {vfs_path}");
356            return Ok(());
357        }
358    };
359    store_and_link(&buf, vfs_path, vfs_dir, cas_root, file_hashes)?;
360    Ok(())
361}
362
363fn extract_map_files_cas(
364    vfs: &VfsPath,
365    parent_dir: &str,
366    filenames: &[&str],
367    vfs_dir: &Path,
368    cas_root: &Path,
369    file_hashes: &mut BTreeMap<String, String>,
370    progress: Option<&ProgressBar>,
371) -> Result<(), Report> {
372    let parent = match vfs.join(parent_dir) {
373        Ok(d) => d,
374        Err(_) => return Ok(()),
375    };
376    let entries = match parent.read_dir() {
377        Ok(e) => e,
378        Err(_) => return Ok(()),
379    };
380
381    for entry in entries {
382        if !entry.metadata().map(|m| m.file_type == VfsFileType::Directory).unwrap_or(false) {
383            continue;
384        }
385        for filename in filenames {
386            let file_path = match entry.join(filename) {
387                Ok(f) => f,
388                Err(_) => continue,
389            };
390            if !file_path.exists().unwrap_or(false) {
391                continue;
392            }
393            let rel = file_path.as_str();
394            let mut buf = Vec::new();
395            match file_path.open_file() {
396                Ok(mut f) => f.read_to_end(&mut buf)?,
397                Err(e) => {
398                    tracing::warn!("Failed to open VFS file {rel}: {e}");
399                    continue;
400                }
401            };
402            store_and_link(&buf, rel, vfs_dir, cas_root, file_hashes)?;
403            if let Some(pb) = progress {
404                pb.inc(1);
405            }
406        }
407    }
408    Ok(())
409}
410
411// -- Counting helpers (for progress bar) --
412
413fn count_vfs_dir_files(vfs: &VfsPath, dir: &str) -> u64 {
414    let mut count = 0;
415    if let Ok(vfs_dir_path) = vfs.join(dir)
416        && let Ok(walker) = vfs_dir_path.walk_dir()
417    {
418        for entry in walker.flatten() {
419            if entry.metadata().map(|m| m.file_type == VfsFileType::File).unwrap_or(false) {
420                count += 1;
421            }
422        }
423    }
424    count
425}
426
427fn count_map_files(vfs: &VfsPath, parent_dir: &str, filenames: &[&str]) -> u64 {
428    let mut count = 0;
429    if let Ok(parent) = vfs.join(parent_dir)
430        && let Ok(entries) = parent.read_dir()
431    {
432        for entry in entries {
433            if entry.metadata().map(|m| m.file_type == VfsFileType::Directory).unwrap_or(false) {
434                for filename in filenames {
435                    if entry.join(filename).is_ok_and(|f: VfsPath| f.exists().unwrap_or(false)) {
436                        count += 1;
437                    }
438                }
439            }
440        }
441    }
442    count
443}
wows_data_mgr/dump.rs

wows_data_mgr/
dump.rs
