use std::{
fs,
io::{self, BufRead, Read, Write},
path::{Path, PathBuf},
time::Duration,
};
use leaky_bucket_lite::sync_threadsafe::LeakyBucket;
use rayon::iter::{IntoParallelIterator, IntoParallelRefIterator, ParallelIterator};
use sha2::Digest;
use unic_langid::LanguageIdentifier;
use crate::util;
pub fn translate(dir: &str, from: &str, to: &str, replace: bool, check: bool, verbose: bool) {
let dir_path = Path::new(dir);
if !dir_path.exists() {
fatal!("dir `{dir}` does not exist")
}
let from = if from.is_empty() {
let name = dir_path.file_name().unwrap().to_string_lossy();
let name = &*name;
if name == "template" {
if verbose {
println!("from language set to `en`, default language for `template` folder");
}
LanguageIdentifier::from_bytes(b"en").unwrap()
} else {
let name = name.strip_suffix("-machine").unwrap_or(name);
match LanguageIdentifier::from_bytes(name.as_bytes()) {
Ok(l) => {
if verbose {
println!("from language language set to `{l}`, derived from `{name}` folder");
}
l
}
Err(e) => fatal!("cannot define from language, `{name}` is not a language, {e}\nset --translate-from to resolve"),
}
}
} else {
match LanguageIdentifier::from_bytes(from.as_bytes()) {
Ok(l) => l,
Err(e) => fatal!("cannot translate from `{from}`, {e}"),
}
};
let to = {
let mut t = vec![];
for l_str in to.split(',') {
let l_str = l_str.trim();
if l_str.is_empty() {
continue;
}
let l_str = l_str.strip_suffix("-machine").unwrap_or(l_str);
match LanguageIdentifier::from_bytes(l_str.as_bytes()) {
Ok(l) => t.push(l),
Err(e) => fatal!("cannot translate to `{l_str}`, {e}"),
}
}
t
};
if to.is_empty() {
fatal!("--translate-to requires at least one language")
}
let pattern = dir_path.join("**/*.ftl");
let mut files = vec![];
for entry in glob::glob(&pattern.display().to_string()).unwrap_or_else(|e| fatal!("cannot read `{dir}`, {e}")) {
let entry = entry.unwrap_or_else(|e| fatal!("cannot read `{dir}` entry, {e}"));
let relative_entry = entry.strip_prefix(dir_path).unwrap();
let file = match std::fs::read_to_string(&entry) {
Ok(f) => f,
Err(e) => fatal!("cannot read `{}`, {}", entry.display(), e),
};
let mut hasher = sha2::Sha256::new();
hasher.update(file.as_bytes());
let hash = format!("{:x}", base16ct::HexDisplay(&hasher.finalize()));
files.push((relative_entry.to_owned(), file, hash));
}
let files = files;
if verbose {
println!("collected {} source files", files.len());
}
let dir_path = dunce::canonicalize(dir_path).unwrap();
let dir_parent = dir_path.parent().unwrap();
let translator = Translator::default();
if verbose {
println!("using `{}`, RPM: {}", translator.path.display(), translator.rpm);
}
to.into_par_iter().for_each(|to| {
if to == from {
return;
}
println!("translating to {to}-machine");
let non_machine_dir = dir_parent.join(to.to_string());
if non_machine_dir.exists() {
return;
}
let dir_to = dir_parent.join(format!("{to}-machine"));
files.par_iter().for_each(|(relative_path, file, hash)| {
let file_to = dir_to.join(relative_path);
if verbose {
println!(" {}", file_to.display());
}
let _ = util::check_or_create_dir_all(check, file_to.parent().unwrap());
const HEADER_PREFIX: &str = "### Machine translated by `cargo zng l10n`, ";
let mut stale = false;
match fs::File::open(&file_to) {
Ok(f) => {
if !replace {
let mut f = io::BufReader::new(f);
let mut header = String::new();
let _ = f.read_line(&mut header);
if let Some(h) = header.strip_prefix(HEADER_PREFIX) {
if hash == h.trim_end() {
return;
}
stale = true;
} else {
return;
}
}
}
Err(_) => {
if check {
fatal!("expected `{}`", file_to.display());
}
}
}
if !check && (replace || stale || !file_to.exists()) {
let r = translator.translate(&from, &to, file, verbose);
let write = || -> io::Result<()> {
let mut f = fs::File::create(&file_to)?;
f.write_all(HEADER_PREFIX.as_bytes())?;
f.write_all(hash.as_bytes())?;
f.write_all("\n\n".as_bytes())?;
f.write_all(r.as_bytes())?;
Ok(())
};
if let Err(e) = write() {
fatal!("cannot write `{}`, {}", file_to.display(), e);
}
}
});
});
}
struct Translator {
path: PathBuf,
rpm: u64,
limiter: LeakyBucket,
}
impl Default for Translator {
fn default() -> Self {
let install_dir = std::env::current_exe().unwrap();
let install_dir = install_dir.parent().unwrap();
let mut t = if let Ok(translator) = std::env::var("ZNG_L10N_TRANSLATOR") {
let p = if translator.contains('/') || translator.contains('\\') {
PathBuf::from(&translator)
} else {
install_dir.join(format!("zng-l10n-translator-{translator}{}", std::env::consts::EXE_SUFFIX))
};
if !p.exists() {
fatal!("cannot find translator `{translator}`");
}
Translator {
path: p,
rpm: 0,
limiter: LeakyBucket::builder().build(),
}
} else {
let translators_pattern = install_dir.join("zng-l10n-translator-*").display().to_string();
let mut options = vec![];
for opt in glob::glob(&translators_pattern).unwrap() {
options.push(opt.unwrap());
}
if options.is_empty() {
fatal!("no translator installed\n install a zng-l10n-translator-* crate\nor set ZNG_L10N_TRANSLATOR to a path")
} else if options.len() > 1 {
let mut names: Vec<_> = options
.iter()
.map(|p| {
p.file_name()
.unwrap()
.to_string_lossy()
.trim_start_matches("zng-l10n-translator-")
.trim_end_matches(std::env::consts::EXE_SUFFIX)
.to_owned()
})
.collect();
names.sort();
let names = names.join(", ");
fatal!("multiple translators installed\n set ZNG_L10N_TRANSLATOR to one of: {names}");
}
Translator {
path: options.remove(0),
rpm: 0,
limiter: LeakyBucket::builder().build(),
}
};
t.read_limits();
t
}
}
impl Translator {
fn read_limits(&mut self) {
#[derive(serde::Deserialize)]
struct Limits {
#[serde(rename = "requests-per-minute")]
rpm: u64,
}
let output = std::process::Command::new(&self.path).arg("--limits").output().unwrap();
if output.status.success() {
match serde_json::from_slice::<Limits>(&output.stdout) {
Ok(l) => {
self.rpm = l.rpm;
let t: u32 = self.rpm.try_into().unwrap();
self.limiter = LeakyBucket::builder()
.max(t)
.tokens(t)
.refill_amount(t)
.refill_interval(Duration::from_mins(1))
.build();
}
Err(e) => {
let limits = String::from_utf8_lossy(&output.stdout);
fatal!("invalid response to --limits, {e}\n{limits}")
}
}
} else {
let error = String::from_utf8_lossy(&output.stderr);
fatal!(
"{error}\n{} exited with code {}",
self.path.file_name().unwrap().to_string_lossy(),
output.status.code().unwrap_or(0)
);
}
}
fn translate(&self, from: &LanguageIdentifier, to: &LanguageIdentifier, file: &str, _verbose: bool) -> String {
self.limiter.acquire_one();
let mut cmd = std::process::Command::new(&self.path)
.arg("--from-lang")
.arg(from.to_string())
.arg("--to-lang")
.arg(to.to_string())
.stdin(std::process::Stdio::piped())
.stdout(std::process::Stdio::piped())
.stderr(std::process::Stdio::piped())
.spawn()
.unwrap();
let mut stdin = cmd.stdin.take().unwrap();
stdin.write_all(file.as_bytes()).unwrap();
stdin.flush().unwrap();
drop(stdin);
let s = cmd.wait().unwrap();
if !s.success() {
let mut error = String::new();
cmd.stderr.unwrap().read_to_string(&mut error).unwrap();
fatal!(
"{error}\n{} exited with code {}",
self.path.file_name().unwrap().to_string_lossy(),
s.code().unwrap_or(0)
);
}
let mut out = String::new();
cmd.stdout.unwrap().read_to_string(&mut out).unwrap();
if let Err(e) = fluent_syntax::parser::parse(out.as_str()) {
fatal!("translator output is not valid Fluent, {}", &e.1[0]);
}
out
}
}