use std::{
ffi::OsStr,
fs,
path::{Path, PathBuf},
sync::OnceLock,
};
pub type L10nResMap = HashMap<KString, Vec<L10nMapEntry>>;
use anyhow::bail;
use dashmap::DashSet;
use getset::{Getters, WithSetters};
use glossa_dsl::Resolver;
use glossa_shared::{
ToCompactString,
small_list::SmallList,
tap::{Pipe, TapFallible, TryConv},
type_aliases::ahash::HashMap,
};
use kstring::KString;
use rayon::iter::{ParallelBridge, ParallelIterator};
use serde::{Deserialize, Serialize};
use walkdir::{DirEntry, WalkDir};
use crate::{AnyResult, MiniStr, to_kstr};
#[derive(Getters, WithSetters, Debug, Clone)]
#[getset(get = "pub with_prefix", set_with = "pub")]
pub struct L10nResources {
dir: PathBuf,
dsl_suffix: MiniStr,
#[getset(skip)]
#[getset(get = "pub")]
include_languages: SmallList<3>,
#[getset(skip)]
#[getset(get = "pub")]
include_map_names: SmallList<2>,
#[getset(skip)]
#[getset(get = "pub")]
exclude_languages: SmallList<1>,
#[getset(skip)]
#[getset(get = "pub")]
exclude_map_names: SmallList<1>,
#[getset(get)]
lazy_data: OnceLock<L10nResMap>,
}
impl Default for L10nResources {
fn default() -> Self {
Self {
dsl_suffix: ".dsl".into(),
dir: Default::default(),
include_languages: Default::default(),
include_map_names: Default::default(),
exclude_languages: Default::default(),
exclude_map_names: Default::default(),
lazy_data: Default::default(),
}
}
}
fn walk_file<P: AsRef<Path>>(dir: P) -> Option<impl Iterator<Item = DirEntry>> {
dir
.pipe(WalkDir::new)
.follow_links(true)
.into_iter()
.filter_map(Result::ok)
.filter(is_supported_config_file)
.pipe(Some)
}
fn is_supported_config_file(e: &DirEntry) -> bool {
let f = e.path();
f.is_file()
&& f
.extension()
.is_some_and(is_supported_cfg_format)
}
fn is_supported_cfg_format<S: AsRef<OsStr>>(o: S) -> bool {
["toml", "ron", "json", "json5", "yml", "yaml"]
.iter()
.map(OsStr::new)
.any(|a| o.as_ref() == a)
}
fn dir_name_to_opt_lang(dir: &Path) -> Option<KString> {
dir
.file_name()?
.to_str()?
.pipe(KString::from_ref)
.pipe(Some)
}
impl L10nResources {
pub fn new<P: Into<PathBuf>>(dir: P) -> Self {
Self {
dir: dir.into(),
..Default::default()
}
}
pub fn get_or_init_data(&self) -> &L10nResMap {
self
.get_lazy_data()
.get_or_init(|| {
self
.collect_localized_files()
.expect("Failed to init L10nResources Data")
})
}
fn walk_dir(&self) -> Option<impl ParallelIterator<Item = PathBuf>> {
self
.get_dir()
.pipe(fs::read_dir)
.ok()?
.par_bridge()
.filter_map(Result::ok)
.map(|e| e.path())
.filter(|d| d.is_dir())
.into()
}
fn process_file<P: AsRef<Path> + core::fmt::Debug>(
&self,
file: P,
file_stem: &MiniStr,
stem_set: &DashSet<MiniStr>,
) -> Option<L10nMapEntry> {
let data = deser_config_file(&file)
.tap_err(|e| eprintln!("[WARN] Deserialization error for {file:?}: {e}"))
.ok()?;
(!data.is_empty()).then_some(())?;
stem_set
.insert(file_stem.clone())
.then_some(())?;
let suffix = self.get_dsl_suffix().as_str();
let (tmpl_data, data) = match file_stem.ends_with(suffix) && !suffix.is_empty() {
true => (data.try_conv::<Resolver>().ok(), None),
_ => (None, Some(data)),
};
let map_name = file_stem
.trim_end_matches(suffix)
.into();
L10nMapEntry {
map_name,
data,
tmpl_data,
}
.pipe(Some)
}
fn collect_localized_files(&self) -> Option<L10nResMap> {
self
.walk_dir()?
.filter(|dir| self.filter_include_languages(dir))
.filter(|dir| self.filter_exclude_languages(dir))
.filter_map(|ref dir| {
let entries = self.parallel_collect_l10n_entries(dir)?;
let lang = dir_name_to_opt_lang(dir)?;
(!entries.is_empty()).then_some((lang, entries))
})
.collect::<HashMap<_, _>>()
.into()
}
fn parallel_collect_l10n_entries(&self, dir: &Path) -> Option<Vec<L10nMapEntry>> {
let stem_set = DashSet::with_capacity(64);
dir
.pipe(walk_file)?
.par_bridge()
.filter_map(annotate_entry_with_stem)
.filter(|(_, map_name)| self.filter_include_map_names(map_name))
.filter(|(_, map_name)| self.filter_exclude_map_names(map_name))
.filter_map(|(file, file_stem)| {
self.process_file(file.path(), &file_stem, &stem_set)
})
.collect::<Vec<_>>()
.into()
}
fn filter_include_map_names(&self, map_name: &MiniStr) -> bool {
match self.include_map_names.as_ref() {
[] => true,
list => contain_map_name(list, map_name),
}
}
fn filter_exclude_map_names(&self, map_name: &MiniStr) -> bool {
match self.exclude_map_names.as_ref() {
[] => true,
list => !contain_map_name(list, map_name),
}
}
fn filter_exclude_languages(&self, dir: &Path) -> bool {
match self.exclude_languages.as_ref() {
[] => true,
list => match dir.file_name() {
Some(dirname) => !contain_language(list, dirname),
_ => true,
},
}
}
fn filter_include_languages(&self, dir: &Path) -> bool {
match self.include_languages.as_ref() {
[] => true,
list => dir
.file_name()
.is_some_and(|dirname| contain_language(list, dirname)),
}
}
pub fn with_include_languages<S: Into<MiniStr>>(
mut self,
include_languages: impl IntoIterator<Item = S>,
) -> Self {
self.include_languages = include_languages
.into_iter()
.collect();
self
}
pub fn with_include_map_names<S: Into<MiniStr>>(
mut self,
include_map_names: impl IntoIterator<Item = S>,
) -> Self {
self.include_map_names = include_map_names
.into_iter()
.collect();
self
}
pub fn with_exclude_languages<S: Into<MiniStr>>(
mut self,
exclude_languages: impl IntoIterator<Item = S>,
) -> Self {
self.exclude_languages = exclude_languages
.into_iter()
.collect();
self
}
pub fn with_exclude_map_names<S: Into<MiniStr>>(
mut self,
exclude_map_names: impl IntoIterator<Item = S>,
) -> Self {
self.exclude_map_names = exclude_map_names
.into_iter()
.collect();
self
}
}
fn contain_language(list: &[MiniStr], language: &OsStr) -> bool {
list
.iter()
.any(|item| language.eq_ignore_ascii_case(item))
}
fn contain_map_name(list: &[MiniStr], map_name: &MiniStr) -> bool {
list
.iter()
.any(|item| map_name.eq_ignore_ascii_case(item))
}
fn annotate_entry_with_stem(p: DirEntry) -> Option<(DirEntry, MiniStr)> {
p.path()
.pipe(get_file_stem)
.map(|stem| (p, stem))
}
fn get_file_stem<P: AsRef<Path>>(file: P) -> Option<MiniStr> {
file
.as_ref()
.file_stem()?
.to_str()?
.to_compact_string()
.pipe(Some)
}
fn deser_config_file<P: AsRef<Path>>(
file: P,
) -> AnyResult<HashMap<KString, MiniStr>> {
let cfg_text = file.pipe_ref(fs::read_to_string)?;
if cfg_text.trim().is_empty() {
bail!("Empty File Content")
}
let new_err = || "Failed to deserialize config file.".pipe(anyhow::Error::msg);
let data = match file
.as_ref()
.extension()
.map(|x| x.to_string_lossy())
.ok_or_else(new_err)?
.as_ref()
{
#[cfg(feature = "json")]
"json" => match serde_json::from_str(&cfg_text) {
Ok(m) => m,
#[cfg(not(feature = "json5"))]
e => e?,
#[cfg(feature = "json5")]
_ => serde_json5::from_str(&cfg_text)?,
},
#[cfg(feature = "json5")]
"json5" => serde_json5::from_str(&cfg_text)?,
#[cfg(feature = "ron")]
"ron" => ron::from_str(&cfg_text)?,
#[cfg(feature = "toml")]
"toml" => toml::from_str(&cfg_text)?,
#[cfg(feature = "yaml")]
"yaml" | "yml" => serde_yml::from_str(&cfg_text)?,
_ => bail!("Skip unsupported file"),
};
Ok(data)
}
#[derive(Getters, WithSetters, Debug, Clone, Default, Serialize, Deserialize)]
#[getset(get = "pub(crate) with_prefix", set_with = "pub(crate)")]
pub struct L10nMapEntry {
map_name: MiniStr,
data: Option<HashMap<KString, MiniStr>>,
tmpl_data: Option<Resolver>,
}
impl L10nMapEntry {
pub(crate) fn map_name_to_kstring(&self) -> KString {
self
.get_map_name()
.pipe(to_kstr)
}
}
#[cfg(test)]
pub(crate) mod dbg_shared {
use crate::L10nResources;
pub(crate) const DIR: &str = "../../locales/";
pub(crate) fn new_resources() -> L10nResources {
L10nResources::new(DIR)
}
}
#[cfg(test)]
mod tests {
use std::{collections::BTreeMap, fs, io};
use testutils::simple_benchmark;
use super::*;
use crate::resources::dbg_shared::new_resources;
#[ignore]
#[test]
fn test_read_dir() -> io::Result<()> {
for (idx, path) in dbg_shared::DIR
.pipe(fs::read_dir)?
.filter_map(Result::ok)
.map(|x| x.path())
.filter(|e| e.is_dir())
.enumerate()
{
dbg!(path.file_name(), idx);
}
Ok(())
}
#[ignore]
#[test]
fn bench_init_res_data() {
simple_benchmark(|| {
dbg_shared::new_resources();
});
}
#[ignore]
#[test]
fn test_init_res_data() {
let res = dbg_shared::new_resources();
let map = res
.get_or_init_data()
.iter()
.collect::<BTreeMap<_, _>>();
dbg!(map);
}
#[ignore]
#[test]
fn test_only_includes_en() {
let res = new_resources()
.with_include_languages(["zh", "en"])
.with_exclude_map_names(["hi.tmpl", "test", "unread.tmpl"])
.with_exclude_languages(["zh"]);
let map = res.get_or_init_data();
dbg!(map);
}
#[ignore]
#[test]
fn test_only_includes_de_and_und() {
let res = new_resources()
.with_include_languages(["de", "und", "es"])
.with_exclude_languages(["es"]);
let map = res.get_or_init_data();
dbg!(map);
}
}