use std::collections::HashMap;
use std::collections::HashSet;
use std::env;
use std::fs::File;
use std::io;
use std::io::Write;
use std::iter;
use std::path::Path;
use daachorse::{CharwiseDoubleArrayAhoCorasickBuilder, MatchKind};
#[cfg(any(feature = "_mediawiki-base", feature = "_opencc-base"))]
use hex_literal::hex;
use vergen::EmitBuilder;
#[cfg(feature = "_opencc-base")]
use self::opencc::load_opencc_to;
#[cfg(feature = "_mediawiki-base")]
const MEDIAWIKI_COMMIT: &str = "ecf4342132cf089ac0c42436827e9038a738bb6f";
#[cfg(feature = "_mediawiki-base")]
const MEDIAWIKI_SHA256: [u8; 32] =
hex!("5cb0019b32bb39ec5c6e662029f90bd166f7a844efb3bc877f9be41fdd511bf2");
#[cfg(feature = "_opencc-base")]
const OPENCC_COMMIT: &str = "26753884f1984add422f3b0249ccee8613deaff6";
#[cfg(feature = "_opencc-base")]
const OPENCC_SHA256: [(&str, [u8; 32]); 9] = [
(
"HKVariants.txt",
hex!("e5cd4345303224587102f2c9e4d2b67d2b7e349c6ce9152e4a118f4656cf7302"),
),
(
"HKVariantsRevPhrases.txt",
hex!("35352aef4833c2631b2144bc85623cc44d5a09221dda9c32178ea024300d34d3"),
),
(
"STCharacters.txt",
hex!("a0ca1601c70648cf48b33c3c6210ccbecc5c7eead4b4c3daf76587ba2c03582b"),
),
(
"STPhrases.txt",
hex!("f6eab5e5c6dd7640597878d3dfc6599ee1279d2bc91561eadd8e114194e2925a"),
),
(
"TSCharacters.txt",
hex!("737c21c66f55a419dd6956cb3089476cdefc5a36877452631617696df1e5d925"),
),
(
"TSPhrases.txt",
hex!("362fa1b9a7d6edd04b462a32e12c9fef3adae822ab1dee9c83561cc37c06cb1f"),
),
(
"TWPhrases.txt",
hex!("bcb435b744ee3e522beb9b18fcc5486a36ed4763c6aa642ce18112fb5d604e31"),
),
(
"TWVariants.txt",
hex!("e187278e119c427ca561180ac5da5b20e9f8681190458f35c327ce499e95a6a5"),
),
(
"TWVariantsRevPhrases.txt",
hex!("5ebfb4bdc938c2b14e01ace378988d5d3dc12462b3496ef1d424951ccd371256"),
),
];
const DELIMITER: &str = "|";
fn main() -> io::Result<()> {
#[cfg(all(
feature = "opencc-twp",
not(any(feature = "opencc-tw", feature = "opencc-cn"))
))]
panic!("opencc-twp should only be enabled together with opencc-tw or opencc-cn");
let mut diagnostics_file =
File::create(Path::new(&env::var_os("OUT_DIR").unwrap()).join("zhconv-diagnostics.txt"))?;
macro_rules! log_diag {
($fmt:expr $(, $arg:expr)*) => {
write!(diagnostics_file, $fmt $(, $arg)*)
};
}
log_diag!("=== BUILDING ===\n")?;
for (k, _v) in std::env::vars() {
if k.starts_with("CARGO_FEATURE_") {
log_diag!("feature: {}\n", k.strip_prefix("CARGO_FEATURE_").unwrap())?;
}
}
#[cfg(feature = "_mediawiki-base")]
log_diag!("MEDIAWIKI_COMMIT={}\n", MEDIAWIKI_COMMIT)?;
#[cfg(feature = "_opencc-base")]
log_diag!("OPENCC_COMMIT={}\n", OPENCC_COMMIT)?;
let start_time = std::time::Instant::now();
#[cfg(feature = "_mediawiki-base")]
let mut zhconvs = parse_mediawiki(&read_and_validate_file(
"data/ZhConversion.php",
&MEDIAWIKI_SHA256,
));
#[cfg(not(feature = "_mediawiki-base"))]
let mut zhconvs: HashMap<String, Vec<(String, String)>> = HashMap::new();
for name in [
"ZH_TO_HANT",
"ZH_TO_TW",
"ZH_TO_HK",
"ZH_TO_HANS",
"ZH_TO_CN",
] {
#[allow(unused_mut)]
let mut pairs = zhconvs.entry(name.to_owned()).or_default();
log_diag!("Processing {}: MediaWiki.len = {}", name, pairs.len())?;
#[cfg(feature = "_opencc-base")]
match name {
#[cfg(any(feature = "opencc-hans", feature = "opencc-cn"))]
"ZH_TO_HANS" => {
load_opencc_to!(&mut pairs, [TSCharacters, TSPhrases]);
}
#[cfg(any(feature = "opencc-hant", feature = "opencc-tw", feature = "opencc-hk"))]
"ZH_TO_HANT" => {
load_opencc_to!(&mut pairs, [STCharacters, STPhrases]);
}
#[cfg(feature = "opencc-tw")]
"ZH_TO_TW" => {
if cfg!(feature = "opencc-twp") {
load_opencc_to!(
&mut pairs,
[STPhrases, STCharacters],
[TWPhrases],
[TWVariants]
);
} else {
load_opencc_to!(&mut pairs, [STPhrases, STCharacters], [TWVariants]);
}
}
#[cfg(feature = "opencc-hk")]
"ZH_TO_HK" => {
load_opencc_to!(&mut pairs, [STPhrases, STCharacters], [HKVariants]);
}
#[cfg(feature = "opencc-cn")]
"ZH_TO_CN" => {
if cfg!(feature = "opencc-twp") {
load_opencc_to!(
&mut pairs,
[!TWPhrases, TWVariantsRevPhrases],
[TSPhrases, TSCharacters]
);
} else {
load_opencc_to!(
&mut pairs,
[TWVariantsRevPhrases],
[TSPhrases, TSCharacters]
);
}
load_opencc_to!(
&mut pairs,
[HKVariantsRevPhrases],
[TSPhrases, TSCharacters]
);
}
_ => (),
}
log_diag!(", withOpenCC.len = {}", pairs.len())?;
sort_and_dedup(pairs);
log_diag!(", sortedAndDeduped.len = {}\n", pairs.len())?;
}
let hans_pairs = zhconvs.remove("ZH_TO_HANS").unwrap();
if cfg!(any(
feature = "mediawiki-hans",
feature = "opencc-hans",
feature = "mediawiki-cn",
feature = "opencc-cn"
)) {
write_conv_file("ZH_TO_HANS", &hans_pairs)?;
write_daac_file("ZH_TO_HANS", &hans_pairs)?;
}
let hant_pairs = zhconvs.remove("ZH_TO_HANT").unwrap();
if cfg!(any(
feature = "mediawiki-hant",
feature = "opencc-hant",
feature = "mediawiki-tw",
feature = "opencc-tw",
feature = "mediawiki-hk",
feature = "opencc-hk"
)) {
write_conv_file("ZH_TO_HANT", &hant_pairs)?;
write_daac_file("ZH_TO_HANT", &hant_pairs)?;
}
let mut cn_pairs = zhconvs.remove("ZH_TO_CN").unwrap();
if cfg!(any(feature = "mediawiki-cn", feature = "opencc-cn")) {
let hans_map: HashMap<_, _> = hans_pairs.iter().cloned().collect();
cn_pairs.retain(|(from, to)| hans_map.get(from.as_str()) != Some(to));
write_conv_file("ZH_TO_CN", &cn_pairs)?;
let mut hans_cn_pairs = hans_pairs;
hans_cn_pairs.extend(cn_pairs);
write_daac_file("ZH_TO_HANS_CN", &hans_cn_pairs)?;
log_diag!("ZH_TO_HANS_CN: final.len = {}\n", hans_cn_pairs.len())?;
}
if cfg!(any(
feature = "mediawiki-tw",
feature = "opencc-tw",
feature = "mediawiki-hk",
feature = "opencc-hk"
)) {
let hant_map: HashMap<_, _> = hant_pairs.iter().cloned().collect();
let mut tw_pairs = zhconvs.remove("ZH_TO_TW").unwrap();
if cfg!(any(feature = "mediawiki-tw", feature = "opencc-tw")) {
tw_pairs.retain(|(from, to)| hant_map.get(from.as_str()) != Some(to));
write_conv_file("ZH_TO_TW", &tw_pairs)?;
let mut hant_tw_pairs = hant_pairs.clone();
hant_tw_pairs.extend(tw_pairs);
write_daac_file("ZH_TO_HANT_TW", &hant_tw_pairs)?;
log_diag!("ZH_TO_HANT_TW: final.len = {}\n", hant_tw_pairs.len())?;
}
let mut hk_pairs = zhconvs.remove("ZH_TO_HK").unwrap();
if cfg!(any(feature = "mediawiki-hk", feature = "opencc-hk")) {
hk_pairs.retain(|(from, to)| hant_map.get(from.as_str()) != Some(to));
write_conv_file("ZH_TO_HK", &hk_pairs)?;
let mut hant_hk_pairs = hant_pairs;
hant_hk_pairs.extend(hk_pairs);
write_daac_file("ZH_TO_HANT_HK", &hant_hk_pairs)?;
log_diag!("ZH_TO_HANT_HK: final.len = {}\n", hant_hk_pairs.len())?;
}
}
log_diag!("Built in: {:?}\n=== DONE ===\n", start_time.elapsed())?;
if std::env::var("DOCS_RS").is_err() {
if env::var("CARGO_CFG_TARGET_ARCH") == Ok("wasm32".to_owned()) {
EmitBuilder::builder()
.all_build()
.all_git()
.emit()
.unwrap_or_else(|e| println!("cargo:warning=vergen failed: {:?}", e));
}
}
#[cfg(feature = "_mediawiki-base")]
println!("cargo:rustc-env=MEDIAWIKI_COMMIT_HASH={}", MEDIAWIKI_COMMIT);
#[cfg(feature = "_opencc-base")]
println!("cargo:rustc-env=OPENCC_COMMIT_HASH={}", OPENCC_COMMIT);
println!("cargo:rerun-if-changed=build.rs");
#[cfg(feature = "_mediawiki-base")]
println!("cargo:rerun-if-changed=data/ZhConversion.php");
#[cfg(feature = "_opencc-base")]
for (opencc, _) in OPENCC_SHA256.iter() {
println!("cargo:rerun-if-changed=data/{}", opencc);
}
println!("cargo:rerun-if-changed=Cargo.toml");
Ok(())
}
#[cfg(feature = "_mediawiki-base")]
fn parse_mediawiki(text: &str) -> HashMap<String, Vec<(String, String)>> {
let patb = regex::Regex::new(r"public const (\w+) = \[([^]]+)\]?;").unwrap();
let patl = regex::Regex::new(r"'(.+?)' *=> *'(.+?)' *,?\n").unwrap();
let mut res = HashMap::new();
for block in patb.captures_iter(text) {
let name = block.get(1).unwrap().as_str();
let body = block.get(2).unwrap().as_str();
let mut pairs = vec![];
for line in patl.captures_iter(body) {
let from = line.get(1).unwrap().as_str();
let to = line.get(2).unwrap().as_str();
pairs.push((from.to_owned(), to.to_owned()));
}
assert!(res.insert(name.to_owned(), pairs).is_none());
}
for name in [
"ZH_TO_HANS",
"ZH_TO_HANT",
"ZH_TO_CN",
"ZH_TO_TW",
"ZH_TO_HK",
] {
assert!(res.contains_key(name));
}
res
}
fn write_conv_file(name: &str, pairs: &[(String, String)]) -> io::Result<()> {
let out_dir = env::var_os("OUT_DIR").unwrap();
let dest_path_from = Path::new(&out_dir).join(format!("{}.from.conv", name));
let dest_path_to = Path::new(&out_dir).join(format!("{}.to.conv", name));
let mut ffrom = File::create(dest_path_from)?;
let mut fto = File::create(dest_path_to)?;
let mut it = pairs.iter().peekable();
let mut last_from = "";
while let Some((from, to)) = it.next().map(|(f, t)| (f, t)) {
debug_assert!(
!from.contains(DELIMITER) && !to.contains(DELIMITER),
"Unexpected delimiter {} in pair {} -> {}",
DELIMITER,
from,
to
);
debug_assert!(
!from
.chars()
.any(|c| (SURROGATE_START..SURROGATE_END).contains(&c))
&& !to
.chars()
.any(|c| (SURROGATE_START..SURROGATE_END).contains(&c)),
"Unexpected surrogate char in pair {} -> {}",
from,
to
);
for c in pair_reduce(from.chars(), last_from.chars()) {
write!(ffrom, "{}", c)?;
}
for c in pair_reduce(to.chars(), from.chars()) {
write!(fto, "{}", c)?;
}
if it.peek().is_some() {
write!(ffrom, "{}", DELIMITER)?;
write!(fto, "{}", DELIMITER)?;
}
last_from = from;
}
Ok(())
}
fn write_daac_file(name: &str, pairs: &[(String, String)]) -> io::Result<()> {
let mut seen = HashSet::new();
let out_dir = env::var_os("OUT_DIR").unwrap();
let dest_path_daac = Path::new(&out_dir).join(format!("{}.daac", name));
let daac = CharwiseDoubleArrayAhoCorasickBuilder::new()
.match_kind(MatchKind::LeftmostLongest)
.build_with_values::<_, _, u32>(pairs.iter().enumerate().rev().filter_map(
|(i, (f, _t))| {
if seen.contains(f) {
None
} else {
seen.insert(f);
Some((f, i as u32))
}
},
))
.expect(name)
.serialize();
#[cfg(feature = "compress")]
let daac = zstd::bulk::Compressor::new(21)
.unwrap()
.compress(&daac)
.unwrap();
File::create(dest_path_daac)?.write_all(&daac)
}
const SURROGATE_START: char = '\x00';
const SURROGATE_END: char = '\x20';
fn pair_reduce<'s>(
mut s: impl Iterator<Item = char> + 's + Clone,
mut base: impl Iterator<Item = char> + 's + Clone,
) -> impl Iterator<Item = char> + 's + Clone {
let mut it = iter::from_fn(move || match (s.next(), base.next()) {
(Some(a), Some(b)) if a == b => Some(SURROGATE_START),
(Some(a), _) => Some(a),
(None, _) => None,
})
.peekable();
iter::from_fn(move || {
it.next().map(|curr| {
if curr == SURROGATE_START {
let mut count = 1;
while Some(&SURROGATE_START) == it.peek() {
if (SURROGATE_START as u32) + (count + 1) >= (SURROGATE_END as u32) {
break;
}
let _ = it.next();
count += 1;
}
char::from_u32(SURROGATE_START as u32 + count).unwrap()
} else {
curr
}
})
})
}
fn sort_and_dedup(pairs: &mut Vec<(String, String)>) {
pairs.sort_by(|a, b| b.0.len().cmp(&a.0.len()).then(a.0.cmp(&b.0)));
pairs.dedup_by(|a, b| a.0 == b.0);
}
#[cfg(feature = "_opencc-base")]
mod opencc {
use daachorse::{
CharwiseDoubleArrayAhoCorasick, CharwiseDoubleArrayAhoCorasickBuilder, MatchKind,
};
use std::collections::HashMap;
use std::sync::LazyLock;
use super::OPENCC_SHA256;
pub static OPENCC_SHA256_MAP: LazyLock<HashMap<String, [u8; 32]>> = LazyLock::new(|| {
OPENCC_SHA256
.into_iter()
.map(|(n, s)| (n.to_owned(), s))
.collect()
});
macro_rules! load_opencc_to {
( @read_to $out_conv: expr, $out_revconv: expr, $name: ident) => {
let s = read_and_validate_file(concat!("data/", stringify!($name), ".txt"), crate::opencc::OPENCC_SHA256_MAP.get(stringify!($name.txt)).expect(stringify!($name.txt not found)));
crate::opencc::parse_opencc_to($out_conv, $out_revconv, &s);
};
( @parse_to $out_conv: expr, $out_revconv: expr, $name: ident, $($remainings: tt)* ) => {
load_opencc_to!(@read_to $out_conv, $out_revconv, $name);
load_opencc_to!(@parse_to $out_conv, $out_revconv, $($remainings)*);
};
( @parse_to $out_conv: expr, $out_revconv: expr, $name: ident ) => {
load_opencc_to!(@read_to $out_conv, $out_revconv, $name);
};
( @parse_to $out_conv: expr, $out_revconv: expr, ! $name: ident, $($remainings: tt)* ) => {
load_opencc_to!(@read_to $out_revconv, $out_conv, $name);
load_opencc_to!(@parse_to $out_conv, $out_revconv, $($remainings)*);
};
( @parse_to $out_conv: expr, $out_revconv: expr, ! $name: ident ) => {
load_opencc_to!(@read_to $out_revconv, $out_conv, $name);
};
( @load_stage $out: expr, $prev_stage: ident, [ $($rule: tt)+ ] ) => {
let (mut prev_convs, prev_revconvs): (HashMap<String, String>, HashMap<String, String>) = $prev_stage.unwrap_or_else(|| (HashMap::new(), HashMap::new()));
let mut convs: HashMap<String, String> = HashMap::new();
let mut revconvs: HashMap<String, String> = HashMap::new();
load_opencc_to!(@parse_to &mut convs, &mut revconvs, $($rule)*);
let conver: crate::opencc::SimpleConverter = convs.clone().into();
let prev_revconver: crate::opencc::SimpleConverter = prev_revconvs.clone().into();
for (_f, t) in prev_convs.iter_mut() {
*t = conver.convert(t);
}
for (f, t) in convs.iter() {
prev_convs.insert(f.clone(), t.clone());
let ff = prev_revconver.convert(f);
if &ff != f && &ff != t {
prev_convs.insert(ff.to_owned(), t.to_owned());
}
}
for (_f, t) in revconvs.iter_mut() {
*t = prev_revconver.convert(t);
}
revconvs.extend(prev_revconvs.iter().map(|(f, t)| (conver.convert(f), t.to_owned())));
revconvs.extend(prev_revconvs.iter().map(|(f, t)| (f.to_owned(), t.to_owned())));
$prev_stage = Some((prev_convs, revconvs));
};
( $out: expr, $($stage: tt),+ ) => {
let mut prev_stage = None;
$(load_opencc_to!(@load_stage $out, prev_stage, $stage);)*
let (convs, _) = prev_stage.unwrap();
$out.extend(convs.into_iter());
};
}
pub(crate) use load_opencc_to;
pub fn parse_opencc_to(
out_conv: &mut HashMap<String, String>,
out_revconv: &mut HashMap<String, String>,
s: &str,
) {
let s = s.strip_prefix('\u{feff}').unwrap_or(s);
for line in s
.lines()
.map(|l| l.trim())
.filter(|l| !l.is_empty() && !l.starts_with('#'))
{
if let Some((f, ts)) = line.split_once(char::is_whitespace) {
if f.is_empty() || ts.is_empty() {
continue;
}
let ts: Vec<_> = ts.split_whitespace().collect();
if !(ts.len() > 1 && ts.contains(&f)) {
out_conv.insert(f.to_owned(), ts[0].to_owned());
}
for t in ts {
if !out_revconv.contains_key(t) {
out_revconv.insert(t.to_owned(), f.to_owned());
}
}
}
}
}
pub struct SimpleConverter {
automaton: Option<CharwiseDoubleArrayAhoCorasick<usize>>,
target_words: Vec<String>,
}
impl From<HashMap<String, String>> for SimpleConverter {
fn from(mapping: HashMap<String, String>) -> Self {
let mut target_words = Vec::with_capacity(mapping.len());
let automaton = if mapping.is_empty() {
None
} else {
Some(
CharwiseDoubleArrayAhoCorasickBuilder::new()
.match_kind(MatchKind::LeftmostLongest)
.build(mapping.into_iter().map(|(f, t)| {
target_words.push(t);
f
}))
.expect("Conversion table is valid"),
)
};
Self {
automaton,
target_words,
}
}
}
impl SimpleConverter {
#[allow(dead_code)]
pub fn build<'s>(pairs: impl Iterator<Item = (&'s str, &'s str)>) -> Self {
let mapping = HashMap::from_iter(pairs.map(|(a, b)| (a.to_owned(), b.to_owned())));
mapping.into()
}
pub fn convert(&self, text: &str) -> String {
match &self.automaton {
Some(automaton) => {
let mut output = String::new();
let mut last = 0;
for (s, e, ti) in automaton
.leftmost_find_iter(text)
.map(|m| (m.start(), m.end(), m.value()))
{
if s > last {
output.push_str(&text[last..s]);
}
output.push_str(&self.target_words[ti]);
last = e;
}
output.push_str(&text[last..]);
output
}
None => String::from(text),
}
}
}
}
#[cfg(any(feature = "_mediawiki-base", feature = "_opencc-base"))]
fn read_and_validate_file(path: &str, sha256sum: &[u8; 32]) -> String {
fn sha256(text: &str) -> [u8; 32] {
use sha2::{Digest, Sha256};
let mut hasher = Sha256::new();
hasher.update(text.as_bytes());
hasher.finalize().into()
}
let data_dir = env::var_os("CARGO_MANIFEST_DIR").unwrap();
let path = Path::new(&data_dir).join(path);
let content = String::from_utf8(
std::fs::read(&path).unwrap_or_else(|e| panic!("{} when reading {}", e, path.display())),
)
.unwrap_or_else(|e| panic!("{} is not in valid UTF-8 ({})", path.display(), e));
assert_eq!(
&sha256(&content),
sha256sum,
"Validating the checksum of {}",
path.display()
);
content
}