use std::borrow::ToOwned;
use std::cmp::Ordering;
use std::ffi::OsStr;
use std::fmt::{Display, Formatter};
use std::path::{Path, PathBuf};
use std::string::ToString;
use std::sync::RwLock;
use anyhow::{bail, Result};
use rayon::prelude::*;
use walkdir::WalkDir;
use crate::block::Block;
use crate::everything::{Everything, FilesError};
use crate::game::Game;
use crate::helpers::TigerHashSet;
use crate::item::Item;
#[cfg(feature = "vic3")]
use crate::mod_metadata::ModMetadata;
#[cfg(any(feature = "ck3", feature = "imperator", feature = "hoi4"))]
use crate::modfile::ModFile;
use crate::parse::ParserMemory;
use crate::pathtable::{PathTable, PathTableIndex};
use crate::report::{
add_loaded_dlc_root, add_loaded_mod_root, err, fatal, report, ErrorKey, Severity,
};
use crate::token::Token;
use crate::util::fix_slashes_for_target_platform;
#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)]
pub enum FileKind {
Internal,
Clausewitz,
Jomini,
Vanilla,
Dlc(u8),
LoadedMod(u8),
Mod,
}
impl FileKind {
pub fn counts_as_vanilla(&self) -> bool {
match self {
FileKind::Clausewitz | FileKind::Jomini | FileKind::Vanilla | FileKind::Dlc(_) => true,
FileKind::Internal | FileKind::LoadedMod(_) | FileKind::Mod => false,
}
}
}
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct FileEntry {
path: PathBuf,
kind: FileKind,
idx: Option<PathTableIndex>,
fullpath: PathBuf,
}
impl FileEntry {
pub fn new(path: PathBuf, kind: FileKind, fullpath: PathBuf) -> Self {
assert!(path.file_name().is_some());
Self { path, kind, idx: None, fullpath }
}
pub fn kind(&self) -> FileKind {
self.kind
}
pub fn path(&self) -> &Path {
&self.path
}
pub fn fullpath(&self) -> &Path {
&self.fullpath
}
#[allow(clippy::missing_panics_doc)]
pub fn filename(&self) -> &OsStr {
self.path.file_name().unwrap()
}
fn store_in_pathtable(&mut self) {
assert!(self.idx.is_none());
self.idx = Some(PathTable::store(self.path.clone(), self.fullpath.clone()));
}
pub fn path_idx(&self) -> Option<PathTableIndex> {
self.idx
}
}
impl Display for FileEntry {
fn fmt(&self, fmt: &mut Formatter) -> Result<(), std::fmt::Error> {
write!(fmt, "{}", self.path.display())
}
}
impl PartialOrd for FileEntry {
fn partial_cmp(&self, other: &Self) -> Option<Ordering> {
Some(self.cmp(other))
}
}
impl Ord for FileEntry {
fn cmp(&self, other: &Self) -> Ordering {
let path_ord = if self.idx.is_some() && other.idx.is_some() {
self.idx.unwrap().cmp(&other.idx.unwrap())
} else {
self.path.cmp(&other.path)
};
if path_ord == Ordering::Equal {
self.kind.cmp(&other.kind)
} else {
path_ord
}
}
}
pub trait FileHandler<T: Send>: Sync + Send {
fn config(&mut self, _config: &Block) {}
fn subpath(&self) -> PathBuf;
fn load_file(&self, entry: &FileEntry, parser: &ParserMemory) -> Option<T>;
fn handle_file(&mut self, entry: &FileEntry, loaded: T);
fn finalize(&mut self) {}
}
#[derive(Clone, Debug)]
pub struct LoadedMod {
kind: FileKind,
#[allow(dead_code)]
label: String,
root: PathBuf,
replace_paths: Vec<PathBuf>,
}
impl LoadedMod {
fn new_main_mod(root: PathBuf, replace_paths: Vec<PathBuf>) -> Self {
Self { kind: FileKind::Mod, label: "MOD".to_string(), root, replace_paths }
}
fn new(kind: FileKind, label: String, root: PathBuf, replace_paths: Vec<PathBuf>) -> Self {
Self { kind, label, root, replace_paths }
}
pub fn root(&self) -> &Path {
&self.root
}
pub fn kind(&self) -> FileKind {
self.kind
}
pub fn should_replace(&self, path: &Path) -> bool {
self.replace_paths.iter().any(|p| p == path)
}
}
#[derive(Debug)]
pub struct Fileset {
vanilla_root: Option<PathBuf>,
#[cfg(feature = "jomini")]
clausewitz_root: Option<PathBuf>,
#[cfg(feature = "jomini")]
jomini_root: Option<PathBuf>,
the_mod: LoadedMod,
pub loaded_mods: Vec<LoadedMod>,
loaded_dlcs: Vec<LoadedMod>,
config: Option<Block>,
files: Vec<FileEntry>,
ordered_files: Vec<FileEntry>,
filename_tokens: Vec<Token>,
filenames: TigerHashSet<PathBuf>,
directories: RwLock<TigerHashSet<PathBuf>>,
used: RwLock<TigerHashSet<String>>,
}
impl Fileset {
pub fn new(vanilla_dir: Option<&Path>, mod_root: PathBuf, replace_paths: Vec<PathBuf>) -> Self {
let vanilla_root = if Game::is_jomini() {
vanilla_dir.map(|dir| dir.join("game"))
} else {
vanilla_dir.map(ToOwned::to_owned)
};
#[cfg(feature = "jomini")]
let clausewitz_root = vanilla_dir.map(|dir| dir.join("clausewitz"));
#[cfg(feature = "jomini")]
let jomini_root = vanilla_dir.map(|dir| dir.join("jomini"));
Fileset {
vanilla_root,
#[cfg(feature = "jomini")]
clausewitz_root,
#[cfg(feature = "jomini")]
jomini_root,
the_mod: LoadedMod::new_main_mod(mod_root, replace_paths),
loaded_mods: Vec::new(),
loaded_dlcs: Vec::new(),
config: None,
files: Vec::new(),
ordered_files: Vec::new(),
filename_tokens: Vec::new(),
filenames: TigerHashSet::default(),
directories: RwLock::new(TigerHashSet::default()),
used: RwLock::new(TigerHashSet::default()),
}
}
pub fn config(
&mut self,
config: Block,
#[allow(unused_variables)] workshop_dir: Option<&Path>,
#[allow(unused_variables)] paradox_dir: Option<&Path>,
) -> Result<()> {
let config_path = config.loc.fullpath();
for block in config.get_field_blocks("load_mod") {
let mod_idx;
if let Ok(idx) = u8::try_from(self.loaded_mods.len()) {
mod_idx = idx;
} else {
bail!("too many loaded mods, cannot process more");
}
let default_label = || format!("MOD{mod_idx}");
let label =
block.get_field_value("label").map_or_else(default_label, ToString::to_string);
if Game::is_ck3() || Game::is_imperator() || Game::is_hoi4() {
#[cfg(any(feature = "ck3", feature = "imperator", feature = "hoi4"))]
if let Some(path) = get_modfile(&label, config_path, block, paradox_dir) {
let modfile = ModFile::read(&path)?;
eprintln!(
"Loading secondary mod {label} from: {}{}",
modfile.modpath().display(),
modfile
.display_name()
.map_or_else(String::new, |name| format!(" \"{name}\"")),
);
let kind = FileKind::LoadedMod(mod_idx);
let loaded_mod = LoadedMod::new(
kind,
label.clone(),
modfile.modpath().clone(),
modfile.replace_paths(),
);
add_loaded_mod_root(label);
self.loaded_mods.push(loaded_mod);
} else {
bail!("could not load secondary mod from config; missing valid `modfile` or `workshop_id` field");
}
} else if Game::is_vic3() {
#[cfg(feature = "vic3")]
if let Some(pathdir) = get_mod(&label, config_path, block, workshop_dir) {
match ModMetadata::read(&pathdir) {
Ok(metadata) => {
eprintln!(
"Loading secondary mod {label} from: {}{}",
pathdir.display(),
metadata
.display_name()
.map_or_else(String::new, |name| format!(" \"{name}\"")),
);
let kind = FileKind::LoadedMod(mod_idx);
let loaded_mod = LoadedMod::new(
kind,
label.clone(),
pathdir,
metadata.replace_paths(),
);
add_loaded_mod_root(label);
self.loaded_mods.push(loaded_mod);
}
Err(e) => {
eprintln!(
"could not load secondary mod {label} from: {}",
pathdir.display()
);
eprintln!(" because: {e}");
}
}
} else {
bail!("could not load secondary mod from config; missing valid `mod` or `workshop_id` field");
}
}
}
self.config = Some(config);
Ok(())
}
fn should_replace(&self, path: &Path, kind: FileKind) -> bool {
if kind == FileKind::Mod {
return false;
}
if kind < FileKind::Mod && self.the_mod.should_replace(path) {
return true;
}
for loaded_mod in &self.loaded_mods {
if kind < loaded_mod.kind && loaded_mod.should_replace(path) {
return true;
}
}
false
}
fn scan(&mut self, path: &Path, kind: FileKind) -> Result<(), walkdir::Error> {
for entry in WalkDir::new(path) {
let entry = entry?;
if entry.depth() == 0 || !entry.file_type().is_file() {
continue;
}
let inner_path = entry.path().strip_prefix(path).unwrap();
if inner_path.starts_with(".git") {
continue;
}
let inner_dir = inner_path.parent().unwrap_or_else(|| Path::new(""));
if self.should_replace(inner_dir, kind) {
continue;
}
self.files.push(FileEntry::new(
inner_path.to_path_buf(),
kind,
entry.path().to_path_buf(),
));
}
Ok(())
}
pub fn scan_all(&mut self) -> Result<(), FilesError> {
#[cfg(feature = "jomini")]
if let Some(clausewitz_root) = self.clausewitz_root.clone() {
self.scan(&clausewitz_root.clone(), FileKind::Clausewitz).map_err(|e| {
FilesError::VanillaUnreadable { path: clausewitz_root.clone(), source: e }
})?;
}
#[cfg(feature = "jomini")]
if let Some(jomini_root) = &self.jomini_root.clone() {
self.scan(&jomini_root.clone(), FileKind::Jomini).map_err(|e| {
FilesError::VanillaUnreadable { path: jomini_root.clone(), source: e }
})?;
}
if let Some(vanilla_root) = &self.vanilla_root.clone() {
self.scan(&vanilla_root.clone(), FileKind::Vanilla).map_err(|e| {
FilesError::VanillaUnreadable { path: vanilla_root.clone(), source: e }
})?;
#[cfg(feature = "hoi4")]
if Game::is_hoi4() {
self.load_dlcs(&vanilla_root.join("integrated_dlc"))?;
}
self.load_dlcs(&vanilla_root.join("dlc"))?;
}
for loaded_mod in &self.loaded_mods.clone() {
self.scan(loaded_mod.root(), loaded_mod.kind()).map_err(|e| {
FilesError::ModUnreadable { path: loaded_mod.root().to_path_buf(), source: e }
})?;
}
#[allow(clippy::unnecessary_to_owned)] self.scan(&self.the_mod.root().to_path_buf(), FileKind::Mod).map_err(|e| {
FilesError::ModUnreadable { path: self.the_mod.root().to_path_buf(), source: e }
})?;
Ok(())
}
pub fn load_dlcs(&mut self, dlc_root: &Path) -> Result<(), FilesError> {
for entry in WalkDir::new(dlc_root).max_depth(1).sort_by_file_name().into_iter().flatten() {
if entry.depth() == 1 && entry.file_type().is_dir() {
let label = entry.file_name().to_string_lossy().to_string();
let idx =
u8::try_from(self.loaded_dlcs.len()).expect("more than 256 DLCs installed");
let dlc = LoadedMod::new(
FileKind::Dlc(idx),
label.clone(),
entry.path().to_path_buf(),
Vec::new(),
);
self.scan(dlc.root(), dlc.kind()).map_err(|e| FilesError::VanillaUnreadable {
path: dlc.root().to_path_buf(),
source: e,
})?;
self.loaded_dlcs.push(dlc);
add_loaded_dlc_root(label);
}
}
Ok(())
}
pub fn finalize(&mut self) {
self.files.sort();
for entry in self.files.drain(..) {
if let Some(prev) = self.ordered_files.last_mut() {
if entry.path == prev.path {
*prev = entry;
} else {
self.ordered_files.push(entry);
}
} else {
self.ordered_files.push(entry);
}
}
for entry in &mut self.ordered_files {
let token = Token::new(&entry.filename().to_string_lossy(), (&*entry).into());
self.filename_tokens.push(token);
entry.store_in_pathtable();
self.filenames.insert(entry.path.clone());
}
}
pub fn get_files_under<'a>(&'a self, subpath: &'a Path) -> &'a [FileEntry] {
let start = self.ordered_files.partition_point(|entry| entry.path < subpath);
let end = start
+ self.ordered_files[start..].partition_point(|entry| entry.path.starts_with(subpath));
&self.ordered_files[start..end]
}
pub fn filter_map_under<F, T>(&self, subpath: &Path, f: F) -> Vec<T>
where
F: Fn(&FileEntry) -> Option<T> + Sync + Send,
T: Send,
{
self.get_files_under(subpath).par_iter().filter_map(f).collect()
}
pub fn handle<T: Send, H: FileHandler<T>>(&self, handler: &mut H, parser: &ParserMemory) {
if let Some(config) = &self.config {
handler.config(config);
}
let subpath = handler.subpath();
let entries = self.filter_map_under(&subpath, |entry| {
handler.load_file(entry, parser).map(|loaded| (entry.clone(), loaded))
});
for (entry, loaded) in entries {
handler.handle_file(&entry, loaded);
}
handler.finalize();
}
pub fn mark_used(&self, file: &str) {
let file = file.strip_prefix('/').unwrap_or(file);
self.used.write().unwrap().insert(file.to_string());
}
pub fn exists(&self, key: &str) -> bool {
let key = key.strip_prefix('/').unwrap_or(key);
let filepath = if Game::is_hoi4() && key.contains('\\') {
PathBuf::from(key.replace('\\', "/"))
} else {
PathBuf::from(key)
};
self.filenames.contains(&filepath)
}
pub fn iter_keys(&self) -> impl Iterator<Item = &Token> {
self.filename_tokens.iter()
}
pub fn entry_exists(&self, key: &str) -> bool {
if self.exists(key) {
return true;
}
let dir = key.strip_prefix('/').unwrap_or(key);
let dirpath = Path::new(dir);
if self.directories.read().unwrap().contains(dirpath) {
return true;
}
match self.ordered_files.binary_search_by_key(&dirpath, |fe| fe.path.as_path()) {
Ok(_) => unreachable!(),
Err(idx) => {
if self.ordered_files[idx].path.starts_with(dirpath) {
self.directories.write().unwrap().insert(dirpath.to_path_buf());
return true;
}
}
}
false
}
pub fn verify_entry_exists(&self, entry: &str, token: &Token, max_sev: Severity) {
self.mark_used(&entry.replace("//", "/"));
if !self.entry_exists(entry) {
let msg = format!("file or directory {entry} does not exist");
report(ErrorKey::MissingFile, Item::File.severity().at_most(max_sev))
.msg(msg)
.loc(token)
.push();
}
}
#[cfg(feature = "ck3")] pub fn verify_exists(&self, file: &Token) {
self.mark_used(&file.as_str().replace("//", "/"));
if !self.exists(file.as_str()) {
let msg = "referenced file does not exist";
report(ErrorKey::MissingFile, Item::File.severity()).msg(msg).loc(file).push();
}
}
pub fn verify_exists_implied(&self, file: &str, t: &Token, max_sev: Severity) {
self.mark_used(&file.replace("//", "/"));
if !self.exists(file) {
let msg = format!("file {file} does not exist");
report(ErrorKey::MissingFile, Item::File.severity().at_most(max_sev))
.msg(msg)
.loc(t)
.push();
}
}
pub fn verify_exists_implied_crashes(&self, file: &str, t: &Token) {
self.mark_used(&file.replace("//", "/"));
if !self.exists(file) {
let msg = format!("file {file} does not exist");
fatal(ErrorKey::Crash).msg(msg).loc(t).push();
}
}
pub fn validate(&self, _data: &Everything) {
let common_dirs = match Game::game() {
#[cfg(feature = "ck3")]
Game::Ck3 => crate::ck3::tables::misc::COMMON_DIRS,
#[cfg(feature = "vic3")]
Game::Vic3 => crate::vic3::tables::misc::COMMON_DIRS,
#[cfg(feature = "imperator")]
Game::Imperator => crate::imperator::tables::misc::COMMON_DIRS,
#[cfg(feature = "hoi4")]
Game::Hoi4 => crate::hoi4::tables::misc::COMMON_DIRS,
};
let common_subdirs_ok = match Game::game() {
#[cfg(feature = "ck3")]
Game::Ck3 => crate::ck3::tables::misc::COMMON_SUBDIRS_OK,
#[cfg(feature = "vic3")]
Game::Vic3 => crate::vic3::tables::misc::COMMON_SUBDIRS_OK,
#[cfg(feature = "imperator")]
Game::Imperator => crate::imperator::tables::misc::COMMON_SUBDIRS_OK,
#[cfg(feature = "hoi4")]
Game::Hoi4 => crate::hoi4::tables::misc::COMMON_SUBDIRS_OK,
};
let mut warned: Vec<&Path> = Vec::new();
'outer: for entry in &self.ordered_files {
if !entry.path.to_string_lossy().ends_with(".txt") {
continue;
}
if entry.path == PathBuf::from("common/achievement_groups.txt") {
continue;
}
#[cfg(feature = "hoi4")]
if Game::is_hoi4() {
for valid in crate::hoi4::tables::misc::COMMON_FILES {
if <&str as AsRef<Path>>::as_ref(valid) == entry.path {
continue 'outer;
}
}
}
let dirname = entry.path.parent().unwrap();
if warned.contains(&dirname) {
continue;
}
if !entry.path.starts_with("common") {
let joined = Path::new("common").join(&entry.path);
for valid in common_dirs {
if joined.starts_with(valid) {
let msg = format!("file in unexpected directory {}", dirname.display());
let info = format!("did you mean common/{} ?", dirname.display());
err(ErrorKey::Filename).msg(msg).info(info).loc(entry).push();
warned.push(dirname);
continue 'outer;
}
}
continue;
}
for valid in common_subdirs_ok {
if entry.path.starts_with(valid) {
continue 'outer;
}
}
for valid in common_dirs {
if <&str as AsRef<Path>>::as_ref(valid) == dirname {
continue 'outer;
}
}
if entry.path.starts_with("common/scripted_values") {
let msg = "file should be in common/script_values/";
err(ErrorKey::Filename).msg(msg).loc(entry).push();
} else if (Game::is_ck3() || Game::is_imperator())
&& entry.path.starts_with("common/on_actions")
{
let msg = "file should be in common/on_action/";
err(ErrorKey::Filename).msg(msg).loc(entry).push();
} else if (Game::is_vic3() || Game::is_hoi4())
&& entry.path.starts_with("common/on_action")
{
let msg = "file should be in common/on_actions/";
err(ErrorKey::Filename).msg(msg).loc(entry).push();
} else if Game::is_vic3() && entry.path.starts_with("common/modifiers") {
let msg = "file should be in common/static_modifiers since 1.7";
err(ErrorKey::Filename).msg(msg).loc(entry).push();
} else if Game::is_ck3() && entry.path.starts_with("common/vassal_contracts") {
let msg = "common/vassal_contracts was replaced with common/subject_contracts/contracts/ in 1.16";
err(ErrorKey::Filename).msg(msg).loc(entry).push();
} else {
let msg = format!("file in unexpected directory `{}`", dirname.display());
err(ErrorKey::Filename).msg(msg).loc(entry).push();
}
warned.push(dirname);
}
}
pub fn check_unused_dds(&self, _data: &Everything) {
let mut vec = Vec::new();
for entry in &self.ordered_files {
let pathname = entry.path.to_string_lossy();
if entry.path.extension().is_some_and(|ext| ext.eq_ignore_ascii_case("dds"))
&& !entry.path.starts_with("gfx/interface/illustrations/loading_screens")
&& !self.used.read().unwrap().contains(pathname.as_ref())
{
vec.push(entry);
}
}
for entry in vec {
report(ErrorKey::UnusedFile, Severity::Untidy)
.msg("Unused DDS files")
.abbreviated(entry)
.push();
}
}
}
#[cfg(any(feature = "ck3", feature = "imperator", feature = "hoi4"))]
fn get_modfile(
label: &String,
config_path: &Path,
block: &Block,
paradox_dir: Option<&Path>,
) -> Option<PathBuf> {
let mut path: Option<PathBuf> = None;
if let Some(modfile) = block.get_field_value("modfile") {
let modfile_path = fix_slashes_for_target_platform(
config_path
.parent()
.unwrap() .join(modfile.as_str()),
);
if modfile_path.exists() {
path = Some(modfile_path);
} else {
eprintln!("Could not find mod {label} at: {}", modfile_path.display());
}
}
if path.is_none() {
if let Some(workshop_id) = block.get_field_value("workshop_id") {
match paradox_dir {
Some(p) => {
path = Some(fix_slashes_for_target_platform(
p.join(format!("mod/ugc_{workshop_id}.mod")),
));
}
None => eprintln!("workshop_id defined, but could not find paradox directory"),
}
}
}
path
}
#[cfg(feature = "vic3")]
fn get_mod(
label: &String,
config_path: &Path,
block: &Block,
workshop_dir: Option<&Path>,
) -> Option<PathBuf> {
let mut path: Option<PathBuf> = None;
if let Some(modfile) = block.get_field_value("mod") {
let mod_path = fix_slashes_for_target_platform(
config_path
.parent()
.unwrap() .join(modfile.as_str()),
);
if mod_path.exists() {
path = Some(mod_path);
} else {
eprintln!("Could not find mod {label} at: {}", mod_path.display());
}
}
if path.is_none() {
if let Some(workshop_id) = block.get_field_value("workshop_id") {
match workshop_dir {
Some(w) => {
path = Some(fix_slashes_for_target_platform(w.join(workshop_id.as_str())));
}
None => eprintln!("workshop_id defined, but could not find workshop"),
}
}
}
path
}