1use std::borrow::ToOwned;
4use std::cmp::Ordering;
5use std::ffi::OsStr;
6use std::fmt::{Display, Formatter};
7use std::path::{Path, PathBuf};
8use std::string::ToString;
9use std::sync::RwLock;
10
11use anyhow::{bail, Result};
12use rayon::prelude::*;
13use walkdir::WalkDir;
14
15use crate::block::Block;
16use crate::everything::{Everything, FilesError};
17use crate::game::Game;
18use crate::helpers::TigerHashSet;
19use crate::item::Item;
20#[cfg(feature = "vic3")]
21use crate::mod_metadata::ModMetadata;
22#[cfg(any(feature = "ck3", feature = "imperator", feature = "hoi4"))]
23use crate::modfile::ModFile;
24use crate::parse::ParserMemory;
25use crate::pathtable::{PathTable, PathTableIndex};
26use crate::report::{
27 add_loaded_dlc_root, add_loaded_mod_root, err, fatal, report, warn_abbreviated, warn_header,
28 will_maybe_log, ErrorKey, Severity,
29};
30use crate::token::Token;
31
32#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)]
37pub enum FileKind {
38 Internal,
40 Clausewitz,
42 Jomini,
43 Vanilla,
45 Dlc(u8),
47 LoadedMod(u8),
49 Mod,
51}
52
53impl FileKind {
54 pub fn counts_as_vanilla(&self) -> bool {
55 match self {
56 FileKind::Clausewitz | FileKind::Jomini | FileKind::Vanilla | FileKind::Dlc(_) => true,
57 FileKind::Internal | FileKind::LoadedMod(_) | FileKind::Mod => false,
58 }
59 }
60}
61
62#[derive(Clone, Debug, PartialEq, Eq)]
63pub struct FileEntry {
64 path: PathBuf,
67 kind: FileKind,
69 idx: Option<PathTableIndex>,
73 fullpath: PathBuf,
75}
76
77impl FileEntry {
78 pub fn new(path: PathBuf, kind: FileKind, fullpath: PathBuf) -> Self {
79 assert!(path.file_name().is_some());
80 Self { path, kind, idx: None, fullpath }
81 }
82
83 pub fn kind(&self) -> FileKind {
84 self.kind
85 }
86
87 pub fn path(&self) -> &Path {
88 &self.path
89 }
90
91 pub fn fullpath(&self) -> &Path {
92 &self.fullpath
93 }
94
95 #[allow(clippy::missing_panics_doc)]
98 pub fn filename(&self) -> &OsStr {
99 self.path.file_name().unwrap()
100 }
101
102 fn store_in_pathtable(&mut self) {
103 assert!(self.idx.is_none());
104 self.idx = Some(PathTable::store(self.path.clone(), self.fullpath.clone()));
105 }
106
107 pub fn path_idx(&self) -> Option<PathTableIndex> {
108 self.idx
109 }
110}
111
112impl Display for FileEntry {
113 fn fmt(&self, fmt: &mut Formatter) -> Result<(), std::fmt::Error> {
114 write!(fmt, "{}", self.path.display())
115 }
116}
117
118impl PartialOrd for FileEntry {
119 fn partial_cmp(&self, other: &Self) -> Option<Ordering> {
120 Some(self.cmp(other))
121 }
122}
123
124impl Ord for FileEntry {
125 fn cmp(&self, other: &Self) -> Ordering {
126 let path_ord = if self.idx.is_some() && other.idx.is_some() {
128 self.idx.unwrap().cmp(&other.idx.unwrap())
129 } else {
130 self.path.cmp(&other.path)
131 };
132
133 if path_ord == Ordering::Equal {
135 self.kind.cmp(&other.kind)
136 } else {
137 path_ord
138 }
139 }
140}
141
142pub trait FileHandler<T: Send>: Sync + Send {
144 fn config(&mut self, _config: &Block) {}
146
147 fn subpath(&self) -> PathBuf;
151
152 fn load_file(&self, entry: &FileEntry, parser: &ParserMemory) -> Option<T>;
157
158 fn handle_file(&mut self, entry: &FileEntry, loaded: T);
161
162 fn finalize(&mut self) {}
165}
166
167#[derive(Clone, Debug)]
168pub struct LoadedMod {
169 kind: FileKind,
171
172 #[allow(dead_code)]
174 label: String,
175
176 root: PathBuf,
178
179 replace_paths: Vec<PathBuf>,
181}
182
183impl LoadedMod {
184 fn new_main_mod(root: PathBuf, replace_paths: Vec<PathBuf>) -> Self {
185 Self { kind: FileKind::Mod, label: "MOD".to_string(), root, replace_paths }
186 }
187
188 fn new(kind: FileKind, label: String, root: PathBuf, replace_paths: Vec<PathBuf>) -> Self {
189 Self { kind, label, root, replace_paths }
190 }
191
192 pub fn root(&self) -> &Path {
193 &self.root
194 }
195
196 pub fn kind(&self) -> FileKind {
197 self.kind
198 }
199
200 pub fn should_replace(&self, path: &Path) -> bool {
201 self.replace_paths.iter().any(|p| p == path)
202 }
203}
204
205#[derive(Debug)]
206pub struct Fileset {
207 vanilla_root: Option<PathBuf>,
209
210 #[cfg(feature = "jomini")]
212 clausewitz_root: Option<PathBuf>,
213
214 #[cfg(feature = "jomini")]
216 jomini_root: Option<PathBuf>,
217
218 the_mod: LoadedMod,
220
221 pub loaded_mods: Vec<LoadedMod>,
223
224 loaded_dlcs: Vec<LoadedMod>,
226
227 config: Option<Block>,
229
230 files: Vec<FileEntry>,
232
233 ordered_files: Vec<FileEntry>,
235
236 filename_tokens: Vec<Token>,
239
240 filenames: TigerHashSet<PathBuf>,
242
243 directories: RwLock<TigerHashSet<PathBuf>>,
245
246 used: RwLock<TigerHashSet<String>>,
248}
249
250impl Fileset {
251 pub fn new(vanilla_dir: Option<&Path>, mod_root: PathBuf, replace_paths: Vec<PathBuf>) -> Self {
252 let vanilla_root = if Game::is_jomini() {
253 vanilla_dir.map(|dir| dir.join("game"))
254 } else {
255 vanilla_dir.map(ToOwned::to_owned)
256 };
257 #[cfg(feature = "jomini")]
258 let clausewitz_root = vanilla_dir.map(|dir| dir.join("clausewitz"));
259 #[cfg(feature = "jomini")]
260 let jomini_root = vanilla_dir.map(|dir| dir.join("jomini"));
261
262 Fileset {
263 vanilla_root,
264 #[cfg(feature = "jomini")]
265 clausewitz_root,
266 #[cfg(feature = "jomini")]
267 jomini_root,
268 the_mod: LoadedMod::new_main_mod(mod_root, replace_paths),
269 loaded_mods: Vec::new(),
270 loaded_dlcs: Vec::new(),
271 config: None,
272 files: Vec::new(),
273 ordered_files: Vec::new(),
274 filename_tokens: Vec::new(),
275 filenames: TigerHashSet::default(),
276 directories: RwLock::new(TigerHashSet::default()),
277 used: RwLock::new(TigerHashSet::default()),
278 }
279 }
280
281 pub fn config(&mut self, config: Block) -> Result<()> {
282 let config_path = config.loc.fullpath();
283 for block in config.get_field_blocks("load_mod") {
284 let mod_idx;
285 if let Ok(idx) = u8::try_from(self.loaded_mods.len()) {
286 mod_idx = idx;
287 } else {
288 bail!("too many loaded mods, cannot process more");
289 }
290
291 let default_label = || format!("MOD{mod_idx}");
292 let label =
293 block.get_field_value("label").map_or_else(default_label, ToString::to_string);
294 if Game::is_ck3() || Game::is_imperator() || Game::is_hoi4() {
295 #[cfg(any(feature = "ck3", feature = "imperator", feature = "hoi4"))]
296 if let Some(path) = block.get_field_value("modfile") {
297 let path = config_path
298 .parent()
299 .unwrap() .join(path.as_str());
301 let modfile = ModFile::read(&path)?;
302 eprintln!(
303 "Loading secondary mod {label} from: {}{}",
304 modfile.modpath().display(),
305 modfile
306 .display_name()
307 .map_or_else(String::new, |name| format!(" \"{name}\"")),
308 );
309 let kind = FileKind::LoadedMod(mod_idx);
310 let loaded_mod = LoadedMod::new(
311 kind,
312 label.clone(),
313 modfile.modpath().clone(),
314 modfile.replace_paths(),
315 );
316 add_loaded_mod_root(label);
317 self.loaded_mods.push(loaded_mod);
318 } else {
319 bail!("could not load secondary mod from config; missing `modfile` field");
320 }
321 } else if Game::is_vic3() {
322 #[cfg(feature = "vic3")]
323 if let Some(path) = block.get_field_value("mod") {
324 let pathdir = config_path
325 .parent()
326 .unwrap() .join(path.as_str());
328 if let Ok(metadata) = ModMetadata::read(&pathdir) {
329 eprintln!(
330 "Loading secondary mod {label} from: {}{}",
331 pathdir.display(),
332 metadata
333 .display_name()
334 .map_or_else(String::new, |name| format!(" \"{name}\"")),
335 );
336 let kind = FileKind::LoadedMod(mod_idx);
337 let loaded_mod =
338 LoadedMod::new(kind, label.clone(), pathdir, metadata.replace_paths());
339 add_loaded_mod_root(label);
340 self.loaded_mods.push(loaded_mod);
341 } else {
342 bail!("does not look like a mod dir: {}", pathdir.display());
343 }
344 } else {
345 bail!("could not load secondary mod from config; missing `mod` field");
346 }
347 }
348 }
349 self.config = Some(config);
350 Ok(())
351 }
352
353 fn should_replace(&self, path: &Path, kind: FileKind) -> bool {
354 if kind == FileKind::Mod {
355 return false;
356 }
357 if kind < FileKind::Mod && self.the_mod.should_replace(path) {
358 return true;
359 }
360 for loaded_mod in &self.loaded_mods {
361 if kind < loaded_mod.kind && loaded_mod.should_replace(path) {
362 return true;
363 }
364 }
365 false
366 }
367
368 fn scan(&mut self, path: &Path, kind: FileKind) -> Result<(), walkdir::Error> {
369 for entry in WalkDir::new(path) {
370 let entry = entry?;
371 if entry.depth() == 0 || !entry.file_type().is_file() {
372 continue;
373 }
374 let inner_path = entry.path().strip_prefix(path).unwrap();
376 if inner_path.starts_with(".git") {
377 continue;
378 }
379 let inner_dir = inner_path.parent().unwrap_or_else(|| Path::new(""));
380 if self.should_replace(inner_dir, kind) {
381 continue;
382 }
383 self.files.push(FileEntry::new(
384 inner_path.to_path_buf(),
385 kind,
386 entry.path().to_path_buf(),
387 ));
388 }
389 Ok(())
390 }
391
392 pub fn scan_all(&mut self) -> Result<(), FilesError> {
393 #[cfg(feature = "jomini")]
394 if let Some(clausewitz_root) = self.clausewitz_root.clone() {
395 self.scan(&clausewitz_root.clone(), FileKind::Clausewitz).map_err(|e| {
396 FilesError::VanillaUnreadable { path: clausewitz_root.clone(), source: e }
397 })?;
398 }
399 #[cfg(feature = "jomini")]
400 if let Some(jomini_root) = &self.jomini_root.clone() {
401 self.scan(&jomini_root.clone(), FileKind::Jomini).map_err(|e| {
402 FilesError::VanillaUnreadable { path: jomini_root.clone(), source: e }
403 })?;
404 }
405 if let Some(vanilla_root) = &self.vanilla_root.clone() {
406 self.scan(&vanilla_root.clone(), FileKind::Vanilla).map_err(|e| {
407 FilesError::VanillaUnreadable { path: vanilla_root.clone(), source: e }
408 })?;
409 #[cfg(feature = "hoi4")]
410 if Game::is_hoi4() {
411 self.load_dlcs(&vanilla_root.join("integrated_dlc"))?;
412 }
413 self.load_dlcs(&vanilla_root.join("dlc"))?;
414 }
415 for loaded_mod in &self.loaded_mods.clone() {
417 self.scan(loaded_mod.root(), loaded_mod.kind()).map_err(|e| {
418 FilesError::ModUnreadable { path: loaded_mod.root().to_path_buf(), source: e }
419 })?;
420 }
421 #[allow(clippy::unnecessary_to_owned)] self.scan(&self.the_mod.root().to_path_buf(), FileKind::Mod).map_err(|e| {
423 FilesError::ModUnreadable { path: self.the_mod.root().to_path_buf(), source: e }
424 })?;
425 Ok(())
426 }
427
428 pub fn load_dlcs(&mut self, dlc_root: &Path) -> Result<(), FilesError> {
429 for entry in WalkDir::new(dlc_root).max_depth(1).sort_by_file_name().into_iter().flatten() {
430 if entry.depth() == 1 && entry.file_type().is_dir() {
431 let label = entry.file_name().to_string_lossy().to_string();
432 let idx =
433 u8::try_from(self.loaded_dlcs.len()).expect("more than 256 DLCs installed");
434 let dlc = LoadedMod::new(
435 FileKind::Dlc(idx),
436 label.clone(),
437 entry.path().to_path_buf(),
438 Vec::new(),
439 );
440 self.scan(dlc.root(), dlc.kind()).map_err(|e| FilesError::VanillaUnreadable {
441 path: dlc.root().to_path_buf(),
442 source: e,
443 })?;
444 self.loaded_dlcs.push(dlc);
445 add_loaded_dlc_root(label);
446 }
447 }
448 Ok(())
449 }
450
451 pub fn finalize(&mut self) {
452 self.files.sort();
455
456 for entry in self.files.drain(..) {
458 if let Some(prev) = self.ordered_files.last_mut() {
459 if entry.path == prev.path {
460 *prev = entry;
461 } else {
462 self.ordered_files.push(entry);
463 }
464 } else {
465 self.ordered_files.push(entry);
466 }
467 }
468
469 for entry in &mut self.ordered_files {
470 let token = Token::new(&entry.filename().to_string_lossy(), (&*entry).into());
471 self.filename_tokens.push(token);
472 entry.store_in_pathtable();
473 self.filenames.insert(entry.path.clone());
474 }
475 }
476
477 pub fn get_files_under<'a>(&'a self, subpath: &'a Path) -> &'a [FileEntry] {
478 let start = self.ordered_files.partition_point(|entry| entry.path < subpath);
479 let end = start
480 + self.ordered_files[start..].partition_point(|entry| entry.path.starts_with(subpath));
481 &self.ordered_files[start..end]
482 }
483
484 pub fn filter_map_under<F, T>(&self, subpath: &Path, f: F) -> Vec<T>
485 where
486 F: Fn(&FileEntry) -> Option<T> + Sync + Send,
487 T: Send,
488 {
489 self.get_files_under(subpath).par_iter().filter_map(f).collect()
490 }
491
492 pub fn handle<T: Send, H: FileHandler<T>>(&self, handler: &mut H, parser: &ParserMemory) {
493 if let Some(config) = &self.config {
494 handler.config(config);
495 }
496 let subpath = handler.subpath();
497 let entries = self.filter_map_under(&subpath, |entry| {
498 handler.load_file(entry, parser).map(|loaded| (entry.clone(), loaded))
499 });
500 for (entry, loaded) in entries {
501 handler.handle_file(&entry, loaded);
502 }
503 handler.finalize();
504 }
505
506 pub fn mark_used(&self, file: &str) {
507 let file = file.strip_prefix('/').unwrap_or(file);
508 self.used.write().unwrap().insert(file.to_string());
509 }
510
511 pub fn exists(&self, key: &str) -> bool {
512 let key = key.strip_prefix('/').unwrap_or(key);
513 let filepath = if Game::is_hoi4() && key.contains('\\') {
514 PathBuf::from(key.replace('\\', "/"))
515 } else {
516 PathBuf::from(key)
517 };
518 self.filenames.contains(&filepath)
519 }
520
521 pub fn iter_keys(&self) -> impl Iterator<Item = &Token> {
522 self.filename_tokens.iter()
523 }
524
525 pub fn entry_exists(&self, key: &str) -> bool {
526 if self.exists(key) {
528 return true;
529 }
530
531 let dir = key.strip_prefix('/').unwrap_or(key);
533 let dirpath = Path::new(dir);
534
535 if self.directories.read().unwrap().contains(dirpath) {
536 return true;
537 }
538
539 match self.ordered_files.binary_search_by_key(&dirpath, |fe| fe.path.as_path()) {
540 Ok(_) => unreachable!(),
542 Err(idx) => {
543 if self.ordered_files[idx].path.starts_with(dirpath) {
545 self.directories.write().unwrap().insert(dirpath.to_path_buf());
546 return true;
547 }
548 }
549 }
550 false
551 }
552
553 pub fn verify_entry_exists(&self, entry: &str, token: &Token, max_sev: Severity) {
554 self.mark_used(&entry.replace("//", "/"));
555 if !self.entry_exists(entry) {
556 let msg = format!("file or directory {entry} does not exist");
557 report(ErrorKey::MissingFile, Item::File.severity().at_most(max_sev))
558 .msg(msg)
559 .loc(token)
560 .push();
561 }
562 }
563
564 #[cfg(feature = "ck3")] pub fn verify_exists(&self, file: &Token) {
566 self.mark_used(&file.as_str().replace("//", "/"));
567 if !self.exists(file.as_str()) {
568 let msg = "referenced file does not exist";
569 report(ErrorKey::MissingFile, Item::File.severity()).msg(msg).loc(file).push();
570 }
571 }
572
573 pub fn verify_exists_implied(&self, file: &str, t: &Token, max_sev: Severity) {
574 self.mark_used(&file.replace("//", "/"));
575 if !self.exists(file) {
576 let msg = format!("file {file} does not exist");
577 report(ErrorKey::MissingFile, Item::File.severity().at_most(max_sev))
578 .msg(msg)
579 .loc(t)
580 .push();
581 }
582 }
583
584 pub fn verify_exists_implied_crashes(&self, file: &str, t: &Token) {
585 self.mark_used(&file.replace("//", "/"));
586 if !self.exists(file) {
587 let msg = format!("file {file} does not exist");
588 fatal(ErrorKey::Crash).msg(msg).loc(t).push();
589 }
590 }
591
592 pub fn validate(&self, _data: &Everything) {
593 let common_dirs = match Game::game() {
594 #[cfg(feature = "ck3")]
595 Game::Ck3 => crate::ck3::tables::misc::COMMON_DIRS,
596 #[cfg(feature = "vic3")]
597 Game::Vic3 => crate::vic3::tables::misc::COMMON_DIRS,
598 #[cfg(feature = "imperator")]
599 Game::Imperator => crate::imperator::tables::misc::COMMON_DIRS,
600 #[cfg(feature = "hoi4")]
601 Game::Hoi4 => crate::hoi4::tables::misc::COMMON_DIRS,
602 };
603 let common_subdirs_ok = match Game::game() {
604 #[cfg(feature = "ck3")]
605 Game::Ck3 => crate::ck3::tables::misc::COMMON_SUBDIRS_OK,
606 #[cfg(feature = "vic3")]
607 Game::Vic3 => crate::vic3::tables::misc::COMMON_SUBDIRS_OK,
608 #[cfg(feature = "imperator")]
609 Game::Imperator => crate::imperator::tables::misc::COMMON_SUBDIRS_OK,
610 #[cfg(feature = "hoi4")]
611 Game::Hoi4 => crate::hoi4::tables::misc::COMMON_SUBDIRS_OK,
612 };
613 let mut warned: Vec<&Path> = Vec::new();
615 'outer: for entry in &self.ordered_files {
616 if !entry.path.to_string_lossy().ends_with(".txt") {
617 continue;
618 }
619 if entry.path == PathBuf::from("common/achievement_groups.txt") {
620 continue;
621 }
622 #[cfg(feature = "hoi4")]
623 if Game::is_hoi4() {
624 for valid in crate::hoi4::tables::misc::COMMON_FILES {
625 if <&str as AsRef<Path>>::as_ref(valid) == entry.path {
626 continue 'outer;
627 }
628 }
629 }
630 let dirname = entry.path.parent().unwrap();
631 if warned.contains(&dirname) {
632 continue;
633 }
634 if !entry.path.starts_with("common") {
635 let joined = Path::new("common").join(&entry.path);
637 for valid in common_dirs {
638 if joined.starts_with(valid) {
639 let msg = format!("file in unexpected directory {}", dirname.display());
640 let info = format!("did you mean common/{} ?", dirname.display());
641 err(ErrorKey::Filename).msg(msg).info(info).loc(entry).push();
642 warned.push(dirname);
643 continue 'outer;
644 }
645 }
646 continue;
647 }
648
649 for valid in common_subdirs_ok {
650 if entry.path.starts_with(valid) {
651 continue 'outer;
652 }
653 }
654
655 for valid in common_dirs {
656 if <&str as AsRef<Path>>::as_ref(valid) == dirname {
657 continue 'outer;
658 }
659 }
660
661 if entry.path.starts_with("common/scripted_values") {
662 let msg = "file should be in common/script_values/";
663 err(ErrorKey::Filename).msg(msg).loc(entry).push();
664 } else if (Game::is_ck3() || Game::is_imperator())
665 && entry.path.starts_with("common/on_actions")
666 {
667 let msg = "file should be in common/on_action/";
668 err(ErrorKey::Filename).msg(msg).loc(entry).push();
669 } else if (Game::is_vic3() || Game::is_hoi4())
670 && entry.path.starts_with("common/on_action")
671 {
672 let msg = "file should be in common/on_actions/";
673 err(ErrorKey::Filename).msg(msg).loc(entry).push();
674 } else if Game::is_vic3() && entry.path.starts_with("common/modifiers") {
675 let msg = "file should be in common/static_modifiers since 1.7";
676 err(ErrorKey::Filename).msg(msg).loc(entry).push();
677 } else if Game::is_ck3() && entry.path.starts_with("common/vassal_contracts") {
678 let msg = "common/vassal_contracts was replaced with common/subject_contracts/contracts/ in 1.16";
679 err(ErrorKey::Filename).msg(msg).loc(entry).push();
680 } else {
681 let msg = format!("file in unexpected directory `{}`", dirname.display());
682 err(ErrorKey::Filename).msg(msg).loc(entry).push();
683 }
684 warned.push(dirname);
685 }
686 }
687
688 pub fn check_unused_dds(&self, _data: &Everything) {
689 let mut vec = Vec::new();
690 for entry in &self.ordered_files {
691 let pathname = entry.path.to_string_lossy();
692 if entry.path.extension().is_some_and(|ext| ext.eq_ignore_ascii_case("dds"))
693 && !entry.path.starts_with("gfx/interface/illustrations/loading_screens")
694 && !self.used.read().unwrap().contains(pathname.as_ref())
695 {
696 vec.push(entry);
697 }
698 }
699 let mut printed_header = false;
700 for entry in vec {
701 if !printed_header && will_maybe_log(entry, ErrorKey::UnusedFile) {
702 warn_header(ErrorKey::UnusedFile, "Unused DDS files:\n");
703 printed_header = true;
704 }
705 warn_abbreviated(entry, ErrorKey::UnusedFile);
706 }
707 if printed_header {
708 warn_header(ErrorKey::UnusedFile, "");
709 }
710 }
711}