1use std::cmp::Ordering;
4use std::ffi::OsStr;
5use std::fmt::{Display, Formatter};
6use std::path::{Path, PathBuf};
7use std::string::ToString;
8use std::sync::RwLock;
9
10use anyhow::{bail, Result};
11use rayon::prelude::*;
12use walkdir::WalkDir;
13
14use crate::block::Block;
15use crate::everything::{Everything, FilesError};
16use crate::game::Game;
17use crate::helpers::TigerHashSet;
18use crate::item::Item;
19#[cfg(feature = "vic3")]
20use crate::mod_metadata::ModMetadata;
21#[cfg(any(feature = "ck3", feature = "imperator"))]
22use crate::modfile::ModFile;
23use crate::parse::ParserMemory;
24use crate::pathtable::{PathTable, PathTableIndex};
25use crate::report::{
26 add_loaded_dlc_root, add_loaded_mod_root, err, fatal, report, warn_abbreviated, warn_header,
27 will_maybe_log, ErrorKey, Severity,
28};
29use crate::token::Token;
30
31#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)]
36pub enum FileKind {
37 Internal,
39 Clausewitz,
41 Jomini,
42 Vanilla,
44 Dlc(u8),
46 LoadedMod(u8),
48 Mod,
50}
51
52impl FileKind {
53 pub fn counts_as_vanilla(&self) -> bool {
54 match self {
55 FileKind::Clausewitz | FileKind::Jomini | FileKind::Vanilla | FileKind::Dlc(_) => true,
56 FileKind::Internal | FileKind::LoadedMod(_) | FileKind::Mod => false,
57 }
58 }
59}
60
61#[derive(Clone, Debug, PartialEq, Eq)]
62pub struct FileEntry {
63 path: PathBuf,
66 kind: FileKind,
68 idx: Option<PathTableIndex>,
72 fullpath: PathBuf,
74}
75
76impl FileEntry {
77 pub fn new(path: PathBuf, kind: FileKind, fullpath: PathBuf) -> Self {
78 assert!(path.file_name().is_some());
79 Self { path, kind, idx: None, fullpath }
80 }
81
82 pub fn kind(&self) -> FileKind {
83 self.kind
84 }
85
86 pub fn path(&self) -> &Path {
87 &self.path
88 }
89
90 pub fn fullpath(&self) -> &Path {
91 &self.fullpath
92 }
93
94 #[allow(clippy::missing_panics_doc)]
97 pub fn filename(&self) -> &OsStr {
98 self.path.file_name().unwrap()
99 }
100
101 fn store_in_pathtable(&mut self) {
102 assert!(self.idx.is_none());
103 self.idx = Some(PathTable::store(self.path.clone(), self.fullpath.clone()));
104 }
105
106 pub fn path_idx(&self) -> Option<PathTableIndex> {
107 self.idx
108 }
109}
110
111impl Display for FileEntry {
112 fn fmt(&self, fmt: &mut Formatter) -> Result<(), std::fmt::Error> {
113 write!(fmt, "{}", self.path.display())
114 }
115}
116
117impl PartialOrd for FileEntry {
118 fn partial_cmp(&self, other: &Self) -> Option<Ordering> {
119 Some(self.cmp(other))
120 }
121}
122
123impl Ord for FileEntry {
124 fn cmp(&self, other: &Self) -> Ordering {
125 let path_ord = if self.idx.is_some() && other.idx.is_some() {
127 self.idx.unwrap().cmp(&other.idx.unwrap())
128 } else {
129 self.path.cmp(&other.path)
130 };
131
132 if path_ord == Ordering::Equal {
134 self.kind.cmp(&other.kind)
135 } else {
136 path_ord
137 }
138 }
139}
140
141pub trait FileHandler<T: Send>: Sync + Send {
143 fn config(&mut self, _config: &Block) {}
145
146 fn subpath(&self) -> PathBuf;
150
151 fn load_file(&self, entry: &FileEntry, parser: &ParserMemory) -> Option<T>;
156
157 fn handle_file(&mut self, entry: &FileEntry, loaded: T);
160
161 fn finalize(&mut self) {}
164}
165
166#[derive(Clone, Debug)]
167pub struct LoadedMod {
168 kind: FileKind,
170
171 #[allow(dead_code)]
173 label: String,
174
175 root: PathBuf,
177
178 replace_paths: Vec<PathBuf>,
180}
181
182impl LoadedMod {
183 fn new_main_mod(root: PathBuf, replace_paths: Vec<PathBuf>) -> Self {
184 Self { kind: FileKind::Mod, label: "MOD".to_string(), root, replace_paths }
185 }
186
187 fn new(kind: FileKind, label: String, root: PathBuf, replace_paths: Vec<PathBuf>) -> Self {
188 Self { kind, label, root, replace_paths }
189 }
190
191 pub fn root(&self) -> &Path {
192 &self.root
193 }
194
195 pub fn kind(&self) -> FileKind {
196 self.kind
197 }
198
199 pub fn should_replace(&self, path: &Path) -> bool {
200 self.replace_paths.iter().any(|p| p == path)
201 }
202}
203
204#[derive(Debug)]
205pub struct Fileset {
206 vanilla_root: Option<PathBuf>,
208
209 clausewitz_root: Option<PathBuf>,
211
212 jomini_root: Option<PathBuf>,
214
215 the_mod: LoadedMod,
217
218 pub loaded_mods: Vec<LoadedMod>,
220
221 loaded_dlcs: Vec<LoadedMod>,
223
224 config: Option<Block>,
226
227 files: Vec<FileEntry>,
229
230 ordered_files: Vec<FileEntry>,
232
233 filename_tokens: Vec<Token>,
236
237 filenames: TigerHashSet<PathBuf>,
239
240 directories: RwLock<TigerHashSet<PathBuf>>,
242
243 used: RwLock<TigerHashSet<String>>,
245}
246
247impl Fileset {
248 pub fn new(vanilla_dir: Option<&Path>, mod_root: PathBuf, replace_paths: Vec<PathBuf>) -> Self {
249 let vanilla_root = vanilla_dir.map(|dir| dir.join("game"));
250 let clausewitz_root = vanilla_dir.map(|dir| dir.join("clausewitz"));
251 let jomini_root = vanilla_dir.map(|dir| dir.join("jomini"));
252
253 Fileset {
254 vanilla_root,
255 clausewitz_root,
256 jomini_root,
257 the_mod: LoadedMod::new_main_mod(mod_root, replace_paths),
258 loaded_mods: Vec::new(),
259 loaded_dlcs: Vec::new(),
260 config: None,
261 files: Vec::new(),
262 ordered_files: Vec::new(),
263 filename_tokens: Vec::new(),
264 filenames: TigerHashSet::default(),
265 directories: RwLock::new(TigerHashSet::default()),
266 used: RwLock::new(TigerHashSet::default()),
267 }
268 }
269
270 pub fn config(&mut self, config: Block) -> Result<()> {
271 for block in config.get_field_blocks("load_mod") {
272 let mod_idx;
273 if let Ok(idx) = u8::try_from(self.loaded_mods.len()) {
274 mod_idx = idx;
275 } else {
276 bail!("too many loaded mods, cannot process more");
277 }
278
279 let default_label = || format!("MOD{mod_idx}");
280 let label =
281 block.get_field_value("label").map_or_else(default_label, ToString::to_string);
282 if Game::is_ck3() || Game::is_imperator() {
283 #[cfg(any(feature = "ck3", feature = "imperator"))]
284 if let Some(path) = block.get_field_value("modfile") {
285 let path = PathBuf::from(path.as_str());
286 let modfile = ModFile::read(&path)?;
287 eprintln!(
288 "Loading secondary mod {label} from: {}{}",
289 modfile.modpath().display(),
290 modfile
291 .display_name()
292 .map_or_else(String::new, |name| format!(" \"{name}\"")),
293 );
294 let kind = FileKind::LoadedMod(mod_idx);
295 let loaded_mod = LoadedMod::new(
296 kind,
297 label.clone(),
298 modfile.modpath().clone(),
299 modfile.replace_paths(),
300 );
301 add_loaded_mod_root(label);
302 self.loaded_mods.push(loaded_mod);
303 } else {
304 bail!("could not load secondary mod from config; missing `modfile` field");
305 }
306 } else if Game::is_vic3() {
307 #[cfg(feature = "vic3")]
308 if let Some(path) = block.get_field_value("mod") {
309 let pathdir = PathBuf::from(path.as_str());
310 if let Ok(metadata) = ModMetadata::read(&pathdir) {
311 eprintln!(
312 "Loading secondary mod {label} from: {}{}",
313 pathdir.display(),
314 metadata
315 .display_name()
316 .map_or_else(String::new, |name| format!(" \"{name}\"")),
317 );
318 let kind = FileKind::LoadedMod(mod_idx);
319 let loaded_mod =
320 LoadedMod::new(kind, label.clone(), pathdir, metadata.replace_paths());
321 add_loaded_mod_root(label);
322 self.loaded_mods.push(loaded_mod);
323 } else {
324 bail!("does not look like a mod dir: {}", pathdir.display());
325 }
326 } else {
327 bail!("could not load secondary mod from config; missing `mod` field");
328 }
329 }
330 }
331 self.config = Some(config);
332 Ok(())
333 }
334
335 fn should_replace(&self, path: &Path, kind: FileKind) -> bool {
336 if kind == FileKind::Mod {
337 return false;
338 }
339 if kind < FileKind::Mod && self.the_mod.should_replace(path) {
340 return true;
341 }
342 for loaded_mod in &self.loaded_mods {
343 if kind < loaded_mod.kind && loaded_mod.should_replace(path) {
344 return true;
345 }
346 }
347 false
348 }
349
350 fn scan(&mut self, path: &Path, kind: FileKind) -> Result<(), walkdir::Error> {
351 for entry in WalkDir::new(path) {
352 let entry = entry?;
353 if entry.depth() == 0 || !entry.file_type().is_file() {
354 continue;
355 }
356 let inner_path = entry.path().strip_prefix(path).unwrap();
358 if inner_path.starts_with(".git") {
359 continue;
360 }
361 let inner_dir = inner_path.parent().unwrap_or_else(|| Path::new(""));
362 if self.should_replace(inner_dir, kind) {
363 continue;
364 }
365 self.files.push(FileEntry::new(
366 inner_path.to_path_buf(),
367 kind,
368 entry.path().to_path_buf(),
369 ));
370 }
371 Ok(())
372 }
373
374 pub fn scan_all(&mut self) -> Result<(), FilesError> {
375 if let Some(clausewitz_root) = self.clausewitz_root.clone() {
376 self.scan(&clausewitz_root.clone(), FileKind::Clausewitz).map_err(|e| {
377 FilesError::VanillaUnreadable { path: clausewitz_root.clone(), source: e }
378 })?;
379 }
380 if let Some(jomini_root) = &self.jomini_root.clone() {
381 self.scan(&jomini_root.clone(), FileKind::Jomini).map_err(|e| {
382 FilesError::VanillaUnreadable { path: jomini_root.clone(), source: e }
383 })?;
384 }
385 if let Some(vanilla_root) = &self.vanilla_root.clone() {
386 self.scan(&vanilla_root.clone(), FileKind::Vanilla).map_err(|e| {
387 FilesError::VanillaUnreadable { path: vanilla_root.clone(), source: e }
388 })?;
389 let dlc_root = vanilla_root.join("dlc");
390 for entry in
391 WalkDir::new(dlc_root).max_depth(1).sort_by_file_name().into_iter().flatten()
392 {
393 if entry.depth() == 1 && entry.file_type().is_dir() {
394 let label = entry.file_name().to_string_lossy().to_string();
395 let idx =
396 u8::try_from(self.loaded_dlcs.len()).expect("more than 256 DLCs installed");
397 let dlc = LoadedMod::new(
398 FileKind::Dlc(idx),
399 label.clone(),
400 entry.path().to_path_buf(),
401 Vec::new(),
402 );
403 self.scan(dlc.root(), dlc.kind()).map_err(|e| {
404 FilesError::VanillaUnreadable { path: dlc.root().to_path_buf(), source: e }
405 })?;
406 self.loaded_dlcs.push(dlc);
407 add_loaded_dlc_root(label);
408 }
409 }
410 }
411 for loaded_mod in &self.loaded_mods.clone() {
413 self.scan(loaded_mod.root(), loaded_mod.kind()).map_err(|e| {
414 FilesError::ModUnreadable { path: loaded_mod.root().to_path_buf(), source: e }
415 })?;
416 }
417 #[allow(clippy::unnecessary_to_owned)] self.scan(&self.the_mod.root().to_path_buf(), FileKind::Mod).map_err(|e| {
419 FilesError::ModUnreadable { path: self.the_mod.root().to_path_buf(), source: e }
420 })?;
421 Ok(())
422 }
423
424 pub fn finalize(&mut self) {
425 self.files.sort();
428
429 for entry in self.files.drain(..) {
431 if let Some(prev) = self.ordered_files.last_mut() {
432 if entry.path == prev.path {
433 *prev = entry;
434 } else {
435 self.ordered_files.push(entry);
436 }
437 } else {
438 self.ordered_files.push(entry);
439 }
440 }
441
442 for entry in &mut self.ordered_files {
443 let token = Token::new(&entry.filename().to_string_lossy(), (&*entry).into());
444 self.filename_tokens.push(token);
445 entry.store_in_pathtable();
446 self.filenames.insert(entry.path.clone());
447 }
448 }
449
450 pub fn get_files_under<'a>(&'a self, subpath: &'a Path) -> &'a [FileEntry] {
451 let start = self.ordered_files.partition_point(|entry| entry.path < subpath);
452 let end = start
453 + self.ordered_files[start..].partition_point(|entry| entry.path.starts_with(subpath));
454 &self.ordered_files[start..end]
455 }
456
457 pub fn filter_map_under<F, T>(&self, subpath: &Path, f: F) -> Vec<T>
458 where
459 F: Fn(&FileEntry) -> Option<T> + Sync + Send,
460 T: Send,
461 {
462 self.get_files_under(subpath).par_iter().filter_map(f).collect()
463 }
464
465 pub fn handle<T: Send, H: FileHandler<T>>(&self, handler: &mut H, parser: &ParserMemory) {
466 if let Some(config) = &self.config {
467 handler.config(config);
468 }
469 let subpath = handler.subpath();
470 let entries = self.filter_map_under(&subpath, |entry| {
471 handler.load_file(entry, parser).map(|loaded| (entry.clone(), loaded))
472 });
473 for (entry, loaded) in entries {
474 handler.handle_file(&entry, loaded);
475 }
476 handler.finalize();
477 }
478
479 pub fn mark_used(&self, file: &str) {
480 let file = file.strip_prefix('/').unwrap_or(file);
481 self.used.write().unwrap().insert(file.to_string());
482 }
483
484 pub fn exists(&self, key: &str) -> bool {
485 let key = key.strip_prefix('/').unwrap_or(key);
486 let filepath = PathBuf::from(key);
487 self.filenames.contains(&filepath)
488 }
489
490 pub fn iter_keys(&self) -> impl Iterator<Item = &Token> {
491 self.filename_tokens.iter()
492 }
493
494 pub fn entry_exists(&self, key: &str) -> bool {
495 if self.exists(key) {
497 return true;
498 }
499
500 let dir = key.strip_prefix('/').unwrap_or(key);
502 let dirpath = Path::new(dir);
503
504 if self.directories.read().unwrap().contains(dirpath) {
505 return true;
506 }
507
508 match self.ordered_files.binary_search_by_key(&dirpath, |fe| fe.path.as_path()) {
509 Ok(_) => unreachable!(),
511 Err(idx) => {
512 if self.ordered_files[idx].path.starts_with(dirpath) {
514 self.directories.write().unwrap().insert(dirpath.to_path_buf());
515 return true;
516 }
517 }
518 }
519 false
520 }
521
522 pub fn verify_entry_exists(&self, entry: &str, token: &Token, max_sev: Severity) {
523 self.mark_used(&entry.replace("//", "/"));
524 if !self.entry_exists(entry) {
525 let msg = format!("file or directory {entry} does not exist");
526 report(ErrorKey::MissingFile, Item::File.severity().at_most(max_sev))
527 .msg(msg)
528 .loc(token)
529 .push();
530 }
531 }
532
533 #[cfg(feature = "ck3")] pub fn verify_exists(&self, file: &Token) {
535 self.mark_used(&file.as_str().replace("//", "/"));
536 if !self.exists(file.as_str()) {
537 let msg = "referenced file does not exist";
538 report(ErrorKey::MissingFile, Item::File.severity()).msg(msg).loc(file).push();
539 }
540 }
541
542 pub fn verify_exists_implied(&self, file: &str, t: &Token, max_sev: Severity) {
543 self.mark_used(&file.replace("//", "/"));
544 if !self.exists(file) {
545 let msg = format!("file {file} does not exist");
546 report(ErrorKey::MissingFile, Item::File.severity().at_most(max_sev))
547 .msg(msg)
548 .loc(t)
549 .push();
550 }
551 }
552
553 pub fn verify_exists_implied_crashes(&self, file: &str, t: &Token) {
554 self.mark_used(&file.replace("//", "/"));
555 if !self.exists(file) {
556 let msg = format!("file {file} does not exist");
557 fatal(ErrorKey::Crash).msg(msg).loc(t).push();
558 }
559 }
560
561 pub fn validate(&self, _data: &Everything) {
562 let common_dirs = match Game::game() {
563 #[cfg(feature = "ck3")]
564 Game::Ck3 => crate::ck3::tables::misc::COMMON_DIRS,
565 #[cfg(feature = "vic3")]
566 Game::Vic3 => crate::vic3::tables::misc::COMMON_DIRS,
567 #[cfg(feature = "imperator")]
568 Game::Imperator => crate::imperator::tables::misc::COMMON_DIRS,
569 };
570 let common_subdirs_ok = match Game::game() {
571 #[cfg(feature = "ck3")]
572 Game::Ck3 => crate::ck3::tables::misc::COMMON_SUBDIRS_OK,
573 #[cfg(feature = "vic3")]
574 Game::Vic3 => crate::vic3::tables::misc::COMMON_SUBDIRS_OK,
575 #[cfg(feature = "imperator")]
576 Game::Imperator => crate::imperator::tables::misc::COMMON_SUBDIRS_OK,
577 };
578 let mut warned: Vec<&Path> = Vec::new();
580 'outer: for entry in &self.ordered_files {
581 if !entry.path.to_string_lossy().ends_with(".txt") {
582 continue;
583 }
584 if entry.path == PathBuf::from("common/achievement_groups.txt") {
585 continue;
586 }
587 let dirname = entry.path.parent().unwrap();
588 if warned.contains(&dirname) {
589 continue;
590 }
591 if !entry.path.starts_with("common") {
592 let joined = Path::new("common").join(&entry.path);
594 for valid in common_dirs {
595 if joined.starts_with(valid) {
596 let msg = format!("file in unexpected directory {}", dirname.display());
597 let info = format!("did you mean common/{} ?", dirname.display());
598 err(ErrorKey::Filename).msg(msg).info(info).loc(entry).push();
599 warned.push(dirname);
600 continue 'outer;
601 }
602 }
603 continue;
604 }
605
606 for valid in common_subdirs_ok {
607 if entry.path.starts_with(valid) {
608 continue 'outer;
609 }
610 }
611
612 for valid in common_dirs {
613 if <&str as AsRef<Path>>::as_ref(valid) == dirname {
614 continue 'outer;
615 }
616 }
617
618 if entry.path.starts_with("common/scripted_values") {
619 let msg = "file should be in common/script_values/";
620 err(ErrorKey::Filename).msg(msg).loc(entry).push();
621 } else if (Game::is_ck3() || Game::is_imperator())
622 && entry.path.starts_with("common/on_actions")
623 {
624 let msg = "file should be in common/on_action/";
625 err(ErrorKey::Filename).msg(msg).loc(entry).push();
626 } else if Game::is_vic3() && entry.path.starts_with("common/on_action") {
627 let msg = "file should be in common/on_actions/";
628 err(ErrorKey::Filename).msg(msg).loc(entry).push();
629 } else if Game::is_vic3() && entry.path.starts_with("common/modifiers") {
630 let msg = "file should be in common/static_modifiers since 1.7";
631 err(ErrorKey::Filename).msg(msg).loc(entry).push();
632 } else {
633 let msg = format!("file in unexpected directory `{}`", dirname.display());
634 err(ErrorKey::Filename).msg(msg).loc(entry).push();
635 }
636 warned.push(dirname);
637 }
638 }
639
640 pub fn check_unused_dds(&self, _data: &Everything) {
641 let mut vec = Vec::new();
642 for entry in &self.ordered_files {
643 let pathname = entry.path.to_string_lossy();
644 if entry.path.extension().is_some_and(|ext| ext.eq_ignore_ascii_case("dds"))
645 && !entry.path.starts_with("gfx/interface/illustrations/loading_screens")
646 && !self.used.read().unwrap().contains(pathname.as_ref())
647 {
648 vec.push(entry);
649 }
650 }
651 let mut printed_header = false;
652 for entry in vec {
653 if !printed_header && will_maybe_log(entry, ErrorKey::UnusedFile) {
654 warn_header(ErrorKey::UnusedFile, "Unused DDS files:\n");
655 printed_header = true;
656 }
657 warn_abbreviated(entry, ErrorKey::UnusedFile);
658 }
659 if printed_header {
660 warn_header(ErrorKey::UnusedFile, "");
661 }
662 }
663}
664
665#[derive(Clone, Debug)]
666pub struct Files<'a> {
667 iter: std::iter::Skip<std::slice::Iter<'a, FileEntry>>,
668 subpath: &'a Path,
669}
670
671impl<'a> Iterator for Files<'a> {
672 type Item = &'a FileEntry;
673
674 fn next(&mut self) -> Option<Self::Item> {
675 if let Some(entry) = self.iter.next() {
676 if entry.path.starts_with(self.subpath) {
677 return Some(entry);
678 }
679 }
680 None
681 }
682}