pub mod split;
pub mod text;
pub use madi_core::{Options, Tag, TagStyle, Token};
pub use madi_lm::precise::PreciseOpt;
pub use madi_news as news;
use anyhow::{bail, Context, Result};
use madi_core::{Analyzer, Dict, LangModel, Worker};
use madi_lm::tlm::TlmState;
use madi_lm::{KnLm, Tlm};
#[cfg(feature = "nlm")]
use madi_lm::{Mixed, Nlm};
use std::io::Read;
use std::path::{Path, PathBuf};
use text::SkipRules;
#[derive(Debug, Clone, PartialEq)]
pub enum Skip {
Builtin,
File(PathBuf),
Off,
}
#[derive(Debug, Clone)]
pub struct EngineConfig {
pub dict: PathBuf,
pub knlm: Option<PathBuf>,
pub nlm: Option<PathBuf>,
pub lm_weight: f32,
pub nlm_weight: f32,
pub unk_scale: f32,
pub options: Options,
pub skip: Skip,
pub min_len: usize,
pub hanja: Option<PathBuf>,
pub user: Vec<PathBuf>,
pub precise: Option<PathBuf>,
pub precise_opt: PreciseOpt,
pub teacher_only: bool,
}
impl EngineConfig {
pub fn new(dict: impl Into<PathBuf>) -> Self {
Self {
dict: dict.into(),
knlm: None,
nlm: None,
lm_weight: 1.0,
nlm_weight: 1.0,
unk_scale: 0.0,
options: Options::default(),
skip: Skip::Builtin,
min_len: 5,
hanja: None,
user: Vec::new(),
precise: None,
precise_opt: PreciseOpt::default(),
teacher_only: false,
}
}
}
enum Inner {
Conn(Analyzer<madi_core::ConnCost>),
Kn(Analyzer<KnLm>),
#[cfg(feature = "nlm")]
Nl(Analyzer<Nlm>),
#[cfg(feature = "nlm")]
KnNl(Analyzer<Mixed<KnLm, Nlm>>),
}
pub struct Engine {
inner: Inner,
skip: Option<SkipRules>,
min_len: usize,
options: Options,
version: String,
precise: Option<(Tlm, PreciseOpt)>,
teacher_only: bool,
user_h: Option<u64>,
}
enum WInner<'a> {
Conn(Worker<'a, madi_core::ConnCost>),
Kn(Worker<'a, KnLm>),
#[cfg(feature = "nlm")]
Nl(Worker<'a, Nlm>),
#[cfg(feature = "nlm")]
KnNl(Worker<'a, Mixed<KnLm, Nlm>>),
}
pub struct EngineWorker<'a> {
w: WInner<'a>,
e: &'a Engine,
nbest_opt: Option<Options>,
tst: Option<TlmState>,
teacher_score: bool,
}
#[derive(Debug, Clone)]
pub struct Sentence {
pub text: String,
pub tokens: Vec<Token>,
}
#[derive(Debug, Clone)]
pub struct Morph {
pub token: Token,
pub word: u32,
pub sent: u32,
}
#[derive(Debug, Clone)]
pub struct SentSpan {
pub start: usize,
pub end: usize,
pub text: String,
pub tokens: Vec<Morph>,
}
#[derive(Debug, Clone)]
pub struct Pretoken {
pub start: usize,
pub end: usize,
pub form: Option<String>,
pub tag: Option<String>,
}
pub fn lemma(t: &Token) -> String {
match t.tag {
Tag::VV | Tag::VA | Tag::VX | Tag::VCP | Tag::VCN => format!("{}다", t.form),
_ => t.form.clone(),
}
}
pub fn hash_bytes(h: u64, b: &[u8]) -> u64 {
let mut h = h;
let mut ch = b.chunks_exact(8);
for c in &mut ch {
h = (h ^ u64::from_le_bytes(c.try_into().unwrap())).wrapping_mul(0x100000001b3).rotate_left(23);
}
for &x in ch.remainder() {
h = (h ^ x as u64).wrapping_mul(0x100000001b3);
}
h
}
pub fn hash_text(t: &str) -> u64 {
let t = t.strip_prefix('\u{feff}').unwrap_or(t);
if t.contains('\r') {
hash_bytes(0xcbf29ce484222325, t.replace("\r\n", "\n").as_bytes())
} else {
hash_bytes(0xcbf29ce484222325, t.as_bytes())
}
}
pub fn hash_file(path: &Path) -> Result<u64> {
let mut f = std::fs::File::open(path).with_context(|| path.display().to_string())?;
let mut buf = vec![0u8; 1 << 20];
let mut h = 0xcbf29ce484222325u64;
let mut carry: Vec<u8> = Vec::new();
loop {
let n = f.read(&mut buf)?;
if n == 0 {
break;
}
carry.extend_from_slice(&buf[..n]);
let k = carry.len() / 8 * 8;
h = hash_bytes(h, &carry[..k]);
carry.drain(..k);
}
Ok(hash_bytes(h, &carry))
}
pub fn hash_words(words: &[u64], len: usize) -> u64 {
let full = len / 8;
let mut h = 0xcbf29ce484222325u64;
for &w in &words[..full] {
h = (h ^ u64::from_le(w)).wrapping_mul(0x100000001b3).rotate_left(23);
}
if len % 8 != 0 {
let b = words[full].to_le_bytes();
for &x in &b[..len % 8] {
h = (h ^ x as u64).wrapping_mul(0x100000001b3);
}
}
h
}
fn short(h: u64) -> String {
format!("{:04x}", (h ^ (h >> 16) ^ (h >> 32) ^ (h >> 48)) & 0xffff)
}
#[cfg(feature = "nlm")]
fn load_nlm(p: &Path, weight: f32, dict: &Dict) -> Result<Nlm> {
let mut n = madi_lm::nlm::load_checked(p).with_context(|| format!("load {}", p.display()))?;
n.weight = weight;
n.attach(dict);
Ok(n)
}
macro_rules! with_an {
($inner:expr, $an:ident => $body:expr) => {
match &mut $inner {
Inner::Conn($an) => $body,
Inner::Kn($an) => $body,
#[cfg(feature = "nlm")]
Inner::Nl($an) => $body,
#[cfg(feature = "nlm")]
Inner::KnNl($an) => $body,
}
};
}
fn user_hash(cfg: &EngineConfig) -> Result<Option<u64>> {
if cfg.hanja.is_none() && cfg.user.is_empty() {
return Ok(None);
}
let mut h = 0u64;
if let Some(d) = &cfg.hanja {
for f in ["hanja_char.csv", "dueum.csv", "hanja_word.csv"] {
let t = std::fs::read_to_string(d.join(f)).with_context(|| d.join(f).display().to_string())?;
h = h.rotate_left(5) ^ hash_text(&t);
}
}
for p in &cfg.user {
let t = std::fs::read_to_string(p).with_context(|| p.display().to_string())?;
h = h.rotate_left(5) ^ hash_text(&t);
}
Ok(Some(h))
}
fn prepare<L: LangModel>(mut an: Analyzer<L>, cfg: &EngineConfig) -> Result<Analyzer<L>> {
if let Some(dir) = &cfg.hanja {
an.load_hanja(dir).with_context(|| format!("load hanja from {}", dir.display()))?;
}
for p in &cfg.user {
let bad = an.load_user_dict(p).with_context(|| format!("read {}", p.display()))?;
if let Some((ln, line)) = bad.first() {
bail!("{}:{}: 사용자 사전 줄 형식 오류: {}", p.display(), ln, line);
}
}
Ok(an)
}
impl Engine {
pub fn load(cfg: &EngineConfig) -> Result<Self> {
let dict = Dict::open(&cfg.dict).with_context(|| format!("open {}", cfg.dict.display()))?;
Self::from_dict(dict, cfg)
}
pub fn from_dict(dict: Dict, cfg: &EngineConfig) -> Result<Self> {
#[cfg(not(feature = "nlm"))]
if cfg.nlm.is_some() {
bail!("this madi was built without feature \"nlm\"");
}
let knlm = match &cfg.knlm {
Some(p) => {
let mut lm = KnLm::load(p).with_context(|| format!("load {}", p.display()))?;
lm.weight = cfg.lm_weight;
lm.unk_scale = cfg.unk_scale;
lm.attach(&dict);
Some(lm)
}
None => None,
};
let inner = match (knlm, &cfg.nlm) {
(None, None) => Inner::Conn(prepare(Analyzer::with_lm(dict, madi_core::ConnCost), cfg)?),
(Some(lm), None) => Inner::Kn(prepare(Analyzer::with_lm(dict, lm), cfg)?),
#[cfg(feature = "nlm")]
(None, Some(np)) => {
let nlm = load_nlm(np, cfg.nlm_weight, &dict)?;
Inner::Nl(prepare(Analyzer::with_lm(dict, nlm), cfg)?)
}
#[cfg(feature = "nlm")]
(Some(lm), Some(np)) => {
let mut nlm = load_nlm(np, cfg.nlm_weight, &dict)?;
nlm.weight *= cfg.options.search.nlm_scale; Inner::KnNl(prepare(Analyzer::with_lm(dict, Mixed { a: lm, b: nlm }), cfg)?)
}
#[cfg(not(feature = "nlm"))]
(_, Some(_)) => unreachable!(),
};
let (skip, skip_text) = match &cfg.skip {
Skip::Builtin => (Some(SkipRules::builtin()), Some(text::BUILTIN_SKIP.to_string())),
Skip::File(p) => {
let t = std::fs::read_to_string(p).with_context(|| p.display().to_string())?;
(Some(SkipRules::parse(&t, &p.display().to_string())?), Some(t))
}
Skip::Off => (None, None),
};
let version = Self::make_version(cfg, skip_text.as_deref())?;
let precise = match &cfg.precise {
Some(p) => Some((Tlm::load(p).with_context(|| format!("load {} (정밀 모드 교사)", p.display()))?, cfg.precise_opt)),
None => None,
};
Ok(Self { inner, skip, min_len: cfg.min_len, options: cfg.options.clone(), version, precise, teacher_only: cfg.teacher_only, user_h: user_hash(cfg)? })
}
pub fn from_memory(dict_words: Vec<u64>, dict_len: usize, knlm: Option<(Vec<u64>, usize)>, cfg: &EngineConfig) -> Result<Self> {
if cfg.nlm.is_some() || cfg.precise.is_some() || !cfg.user.is_empty() || cfg.hanja.is_some() {
bail!("from_memory: NLM·정밀 모드·사용자 사전·한자 자료는 아직 지원하지 않는다");
}
let mut v = format!("{}+d{}", env!("CARGO_PKG_VERSION"), short(hash_words(&dict_words, dict_len)));
v += &rev_tag(cfg);
let kh = knlm.as_ref().map(|(w, n)| hash_words(w, *n));
let dict = Dict::from_words(dict_words, dict_len).context("사전")?;
let lm = match knlm {
Some((w, n)) => {
let mut lm = KnLm::from_raw(madi_core::dict::RawSections::from_words(w, n).context("KnLM")?)?;
lm.weight = cfg.lm_weight;
lm.unk_scale = cfg.unk_scale;
lm.attach(&dict);
Some(lm)
}
None => None,
};
match kh {
Some(h) => v += &format!(".k{}", short(h)),
None => v += ".k-",
}
v += ".n-";
let skip = match &cfg.skip {
Skip::Builtin => {
v += &format!(".s{}", short(hash_text(text::BUILTIN_SKIP)));
Some(SkipRules::builtin())
}
Skip::File(_) => bail!("from_memory: Skip::File 은 쓸 수 없다 (Builtin 또는 Off)"),
Skip::Off => {
v += ".s-";
None
}
};
v += &knob_tag(cfg);
let inner = match lm {
None => Inner::Conn(Analyzer::with_lm(dict, madi_core::ConnCost)),
Some(lm) => Inner::Kn(Analyzer::with_lm(dict, lm)),
};
Ok(Self { inner, skip, min_len: cfg.min_len, options: cfg.options.clone(), version: v, precise: None, teacher_only: false, user_h: None })
}
fn make_version(cfg: &EngineConfig, skip_text: Option<&str>) -> Result<String> {
let mut v = format!("{}+d{}", env!("CARGO_PKG_VERSION"), short(hash_file(&cfg.dict)?));
v += &rev_tag(cfg);
match &cfg.knlm {
Some(p) => v += &format!(".k{}", short(hash_file(p)?)),
None => v += ".k-",
}
match &cfg.nlm {
Some(p) => {
#[allow(unused_mut)]
let mut h = hash_file(p)?;
#[cfg(feature = "nlm")]
if !Nlm::is_packed(p) {
h ^= hash_file(&p.parent().unwrap_or(Path::new(".")).join("vocab.tsv"))?.rotate_left(7);
}
v += &format!(".n{}", short(h));
}
None => v += ".n-",
}
match skip_text {
Some(t) => v += &format!(".s{}", short(hash_text(t))),
None => v += ".s-",
}
if let Some(h) = user_hash(cfg)? {
v += &format!(".u{}", short(h));
}
v += &knob_tag(cfg);
if let Some(p) = &cfg.precise {
let mut h = hash_file(p)?;
let po = cfg.precise_opt.diff_default();
if !po.is_empty() {
h ^= hash_bytes(11, po.join(" ").as_bytes());
}
v += &format!(".p{}", short(h));
}
Ok(v)
}
pub fn version(&self) -> &str {
&self.version
}
pub fn add_user_word(&mut self, form: &str, tag: Tag, score: Option<f32>) {
with_an!(self.inner, an => an.add_user_word(form, tag, score));
self.bump_user(&format!("{form}\t{}\t{score:?}", tag.code()));
}
pub fn add_pre_analyzed_word(&mut self, surface: &str, morphs: &[(String, Tag)], score: Option<f32>) {
with_an!(self.inner, an => an.add_preanalysis(surface, morphs, score));
let ms: Vec<String> = morphs.iter().map(|(f, t)| format!("{f}/{}", t.code())).collect();
self.bump_user(&format!("{surface}\t=\t{}\t{score:?}", ms.join(" ")));
}
pub fn load_user_dictionary(&mut self, path: &Path) -> Result<()> {
let t = std::fs::read_to_string(path).with_context(|| path.display().to_string())?;
let bad = with_an!(self.inner, an => an.load_user_dict(path)).with_context(|| format!("read {}", path.display()))?;
if let Some((ln, line)) = bad.first() {
bail!("{}:{}: 사용자 사전 줄 형식 오류: {}", path.display(), ln, line);
}
self.bump_user(&t);
Ok(())
}
fn bump_user(&mut self, text: &str) {
let h = self.user_h.unwrap_or(0).rotate_left(5) ^ hash_text(text);
self.user_h = Some(h);
let (head, rest) = self.version.split_once('+').map(|(a, b)| (a.to_string(), b.to_string())).unwrap_or_default();
let mut segs: Vec<String> = rest.split('.').filter(|x| !x.starts_with('u')).map(String::from).collect();
let at = segs.iter().position(|x| x.starts_with('s')).map(|i| i + 1).unwrap_or(segs.len());
segs.insert(at, format!("u{}", short(h)));
self.version = format!("{head}+{}", segs.join("."));
}
pub fn options(&self) -> &Options {
&self.options
}
pub fn is_precise(&self) -> bool {
self.precise.is_some()
}
pub fn lm_name(&self) -> &'static str {
match &self.inner {
Inner::Conn(_) => "conn",
Inner::Kn(_) => "knlm",
#[cfg(feature = "nlm")]
Inner::Nl(_) => "nlm",
#[cfg(feature = "nlm")]
Inner::KnNl(_) => "knlm+nlm",
}
}
pub fn dict(&self) -> &Dict {
match &self.inner {
Inner::Conn(a) => &a.dict,
Inner::Kn(a) => &a.dict,
#[cfg(feature = "nlm")]
Inner::Nl(a) => &a.dict,
#[cfg(feature = "nlm")]
Inner::KnNl(a) => &a.dict,
}
}
pub fn n_skip_rules(&self) -> usize {
self.skip.as_ref().map(|r| r.len()).unwrap_or(0)
}
pub fn worker(&self) -> EngineWorker<'_> {
let w = match &self.inner {
Inner::Conn(a) => WInner::Conn(a.worker()),
Inner::Kn(a) => WInner::Kn(a.worker()),
#[cfg(feature = "nlm")]
Inner::Nl(a) => WInner::Nl(a.worker()),
#[cfg(feature = "nlm")]
Inner::KnNl(a) => WInner::KnNl(a.worker()),
};
let tst = self.precise.as_ref().map(|(t, _)| t.new_state());
EngineWorker { w, e: self, nbest_opt: None, tst, teacher_score: false }
}
}
pub struct DocResult {
pub sentences: Vec<Sentence>,
pub skipped: usize,
pub short: usize,
}
impl EngineWorker<'_> {
fn run(&mut self, text: &str) -> Vec<Token> {
let opt = &self.e.options;
if let (Some((tlm, po)), Some(st), false) = (&self.e.precise, &mut self.tst, self.e.teacher_only) {
let r = match &mut self.w {
WInner::Conn(w) => w.analyze_alts_local_gated(text, opt, po.cand_margin, po.max_alts, Some(po.tau)),
WInner::Kn(w) => w.analyze_alts_local_gated(text, opt, po.cand_margin, po.max_alts, Some(po.tau)),
#[cfg(feature = "nlm")]
WInner::Nl(w) => w.analyze_alts_local_gated(text, opt, po.cand_margin, po.max_alts, Some(po.tau)),
#[cfg(feature = "nlm")]
WInner::KnNl(w) => w.analyze_alts_local_gated(text, opt, po.cand_margin, po.max_alts, Some(po.tau)),
};
let mut stats = Default::default();
return madi_lm::precise::apply(tlm, st, &r, po, &mut stats);
}
match &mut self.w {
WInner::Conn(w) => w.analyze(text, opt),
WInner::Kn(w) => w.analyze(text, opt),
#[cfg(feature = "nlm")]
WInner::Nl(w) => w.analyze(text, opt),
#[cfg(feature = "nlm")]
WInner::KnNl(w) => w.analyze(text, opt),
}
}
pub fn analyze_doc(&mut self, text: &str) -> Vec<Sentence> {
self.analyze_doc_full(text).sentences
}
pub fn analyze_doc_full(&mut self, text: &str) -> DocResult {
let e = self.e;
let cleaned;
let text = match &e.skip {
Some(r) => {
cleaned = r.clean(text);
cleaned.as_str()
}
None => text,
};
let mut ss = text::split_sentences(text);
let mut skipped = 0;
if let Some(r) = &e.skip {
let before = ss.len();
ss.retain(|s| !r.skip(s));
skipped = before - ss.len();
}
let mut short = 0;
let mut sentences = Vec::with_capacity(ss.len());
for s in ss {
if s.chars().count() < e.min_len {
short += 1;
continue;
}
let tokens = self.run(&s);
sentences.push(Sentence { text: s, tokens });
}
DocResult { sentences, skipped, short }
}
pub fn split_sentences(&mut self, text: &str) -> Vec<String> {
let mut out = Vec::new();
for line in text.lines() {
if line.trim().is_empty() {
continue;
}
let toks = self.run(line);
let chars: Vec<char> = line.chars().collect();
for (s, e) in split::split_line(line, &toks) {
out.push(chars[s..e].iter().collect());
}
}
out
}
pub fn sentences(&mut self, text: &str) -> Vec<SentSpan> {
let mut out = Vec::new();
let mut off = 0usize; for raw in text.split('\n') {
let line = raw.strip_suffix('\r').unwrap_or(raw);
let n_line = raw.chars().count() + 1;
if !line.trim().is_empty() {
let toks = self.run(line);
let chars: Vec<char> = line.chars().collect();
for (s, e) in split::split_line(line, &toks) {
let sent = out.len() as u32;
let mut word = 0u32;
let mut ms = Vec::new();
for t in toks.iter().filter(|t| t.start >= s && t.end <= e) {
if !ms.is_empty() && t.space_before {
word += 1;
}
let mut t = t.clone();
t.start += off;
t.end += off;
ms.push(Morph { token: t, word, sent });
}
out.push(SentSpan { start: off + s, end: off + e, text: chars[s..e].iter().collect(), tokens: ms });
}
}
off += n_line;
}
out
}
pub fn tokenize(&mut self, text: &str) -> Vec<Morph> {
self.sentences(text).into_iter().flat_map(|s| s.tokens).collect()
}
pub fn analyze_nbest(&mut self, text: &str, n: usize) -> Vec<(f32, Vec<Token>)> {
let n = n.max(1);
let mut o2 = self.e.options.clone();
if n > 1 {
o2.search.nlm_gate = 0.0;
}
let opt = &o2;
let mut r = match &mut self.w {
WInner::Conn(w) => w.analyze_nbest(text, n, opt),
WInner::Kn(w) => w.analyze_nbest(text, n, opt),
#[cfg(feature = "nlm")]
WInner::Nl(w) => w.analyze_nbest(text, n, opt),
#[cfg(feature = "nlm")]
WInner::KnNl(w) => w.analyze_nbest(text, n, opt),
};
let best = self.run(text);
let same = |a: &[Token], b: &[Token]| a.len() == b.len() && a.iter().zip(b).all(|(x, y)| x.form == y.form && x.tag == y.tag && x.start == y.start && x.end == y.end);
if r.first().map_or(true, |(_, t)| !same(t, &best)) {
let sc = r.first().map_or(0.0, |x| x.0);
r.retain(|(_, t)| !same(t, &best));
r.insert(0, (sc, best));
r.truncate(n);
}
r
}
pub fn analyze_pretokenized(&mut self, text: &str, pre: &[Pretoken]) -> (Vec<Token>, Vec<bool>) {
let chars: Vec<char> = text.chars().collect();
let cons: Vec<madi_core::lattice::Boost> = pre
.iter()
.map(|p| {
let e = p.end.min(chars.len());
let s = p.start.min(e);
madi_core::lattice::Boost {
c0: s as u32,
c1: e as u32,
form: p.form.clone().unwrap_or_else(|| chars[s..e].iter().collect()),
tag: p.tag.clone().unwrap_or_default(),
bonus: 30.0,
}
})
.collect();
let opt = &self.e.options;
match &mut self.w {
WInner::Conn(w) => w.analyze_constrained(text, opt, &cons),
WInner::Kn(w) => w.analyze_constrained(text, opt, &cons),
#[cfg(feature = "nlm")]
WInner::Nl(w) => w.analyze_constrained(text, opt, &cons),
#[cfg(feature = "nlm")]
WInner::KnNl(w) => w.analyze_constrained(text, opt, &cons),
}
}
pub fn analyze_sentence(&mut self, text: &str) -> Vec<Token> {
self.run(text)
}
pub fn analyze_query(&mut self, text: &str) -> Vec<Token> {
self.run(text.trim())
}
pub fn set_nbest_options(&mut self, opt: Option<Options>) {
self.nbest_opt = opt;
}
pub fn analyze_query_nbest(&mut self, text: &str, n: usize) -> Vec<(f32, Vec<Token>)> {
let opt = self.nbest_opt.as_ref().unwrap_or(&self.e.options);
let text = text.trim();
let mut r = match &mut self.w {
WInner::Conn(w) => w.analyze_nbest(text, n, opt),
WInner::Kn(w) => w.analyze_nbest(text, n, opt),
#[cfg(feature = "nlm")]
WInner::Nl(w) => w.analyze_nbest(text, n, opt),
#[cfg(feature = "nlm")]
WInner::KnNl(w) => w.analyze_nbest(text, n, opt),
};
if self.teacher_score {
if let (Some((tlm, _)), Some(st)) = (&self.e.precise, &mut self.tst) {
for (sc, toks) in r.iter_mut() {
*sc = teacher_logprob(tlm, st, toks);
}
}
}
r
}
pub fn set_teacher_score(&mut self, on: bool) {
self.teacher_score = on;
}
}
fn teacher_logprob(tlm: &Tlm, st: &mut TlmState, toks: &[Token]) -> f32 {
use madi_lm::precise::{BOS, EOS};
let mut seq: Vec<u16> = toks.iter().map(|t| tlm.id_of(&t.form, t.tag)).collect();
seq.push(EOS);
let mut prefix: Vec<u16> = vec![BOS, BOS, BOS];
let mut total = 0.0;
for chunk in seq.chunks(64) {
total += tlm.score(st, &prefix, &[chunk])[0];
prefix.extend_from_slice(chunk);
}
total
}
pub fn format_tokens(toks: &[Token]) -> String {
text::format_tokens(toks)
}
fn rev_tag(cfg: &EngineConfig) -> String {
let s = &cfg.options.search;
let r = if s.is_rev3() && !s.is_rev3p() { format!(".r{}", madi_core::lattice::SCORE_REV) } else { format!(".r{}p", madi_core::lattice::SCORE_REV) };
let f = s.fb_cutoff;
let f = if f == f.trunc() { format!("{}", f as i64) } else { format!("{}", f) };
format!("{r}.f{f}")
}
pub const NLM_MIX_WEIGHTS: (f32, f32) = (0.7, 0.3);
fn knob_tag(cfg: &EngineConfig) -> String {
let d = knob_diff(cfg);
if d.is_empty() { String::new() } else { format!(".w{}", short(hash_bytes(7, d.join(" ").as_bytes()))) }
}
pub fn knob_diff(cfg: &EngineConfig) -> Vec<String> {
let def = EngineConfig::new("");
let mut d: Vec<String> = Vec::new();
if cfg.nlm.is_some() {
if cfg.lm_weight != NLM_MIX_WEIGHTS.0 { d.push(format!("lm_weight={}", cfg.lm_weight)); }
if cfg.nlm_weight != NLM_MIX_WEIGHTS.1 { d.push(format!("nlm_weight={}", cfg.nlm_weight)); }
} else if cfg.lm_weight != def.lm_weight {
d.push(format!("lm_weight={}", cfg.lm_weight));
}
if cfg.unk_scale != def.unk_scale { d.push(format!("unk_scale={}", cfg.unk_scale)); }
let min_def = if cfg.skip == Skip::Off { 0 } else { def.min_len };
if cfg.min_len != min_def { d.push(format!("min_len={}", cfg.min_len)); }
let s = &cfg.options.search;
let preset = s.is_rev3() || s.is_rev3p();
d.extend(s.diff_default(!preset, false));
let o = &cfg.options;
let od = &def.options;
madi_core::diff_fields!(d, o, od; split_compound, split_unknown_compound, fine_tags, merge_nouns, std_2025);
d
}