use std::collections::HashMap;
use std::path::Path;
use sha2::{Digest, Sha256};
use udpipe_rs::Model;
use crate::domain::Error;
use crate::domain::{Sentence, Token};
use super::NlpProvider;
const ENGLISH_MODEL_SHA256: &str =
"784bd0fa85e3d831fd02a55290d0acfd05c953159dc38cc33d52e1b28add9957";
const ENGLISH_MODEL_SIZE: u64 = 16_309_608;
const ENGLISH_MODEL_URL: &str = "https://lindat.mff.cuni.cz/repository/server/api/core/bitstreams/handle/11234/1-3131/english-ewt-ud-2.5-191206.udpipe?sequence=17&isAllowed=y";
pub struct Udpipe {
model: Model,
}
impl std::fmt::Debug for Udpipe {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.debug_struct("Udpipe").finish_non_exhaustive()
}
}
impl Udpipe {
pub fn from_path(path: impl AsRef<Path>) -> crate::domain::Result<Self> {
let path = path.as_ref();
if !path.exists() {
return Err(Error::ModelNotFound(path.to_path_buf()));
}
let model = Model::load(path).map_err(|e| Error::ModelInvalid(e.to_string()))?;
Ok(Self { model })
}
pub fn from_bytes(data: &[u8]) -> crate::domain::Result<Self> {
let model =
Model::load_from_memory(data).map_err(|e| Error::ModelInvalid(e.to_string()))?;
Ok(Self { model })
}
pub fn english(model_dir: impl AsRef<Path>) -> crate::domain::Result<Self> {
let dir = model_dir.as_ref();
std::fs::create_dir_all(dir)?;
let path = dir.join(ENGLISH_MODEL_FILENAME);
if !path.exists() {
download_english(dir, &path)?;
}
if let Some(bytes) = read_and_verify(&path, ENGLISH_MODEL_SIZE, ENGLISH_MODEL_SHA256)? {
return Self::from_bytes(&bytes);
}
std::fs::remove_file(&path)?;
download_english(dir, &path)?;
match read_and_verify(&path, ENGLISH_MODEL_SIZE, ENGLISH_MODEL_SHA256)? {
Some(bytes) => Self::from_bytes(&bytes),
None => Err(Error::ModelInvalid(format!(
"SHA-256 mismatch after re-download: {}",
path.display()
))),
}
}
}
const ENGLISH_MODEL_FILENAME: &str = "english-ewt-ud-2.5-191206.udpipe";
fn with_temp_subdir<F, T>(parent: &Path, f: F) -> crate::domain::Result<T>
where
F: FnOnce(&Path) -> crate::domain::Result<T>,
{
struct Cleanup<'a>(&'a Path);
impl Drop for Cleanup<'_> {
fn drop(&mut self) {
let _ = std::fs::remove_dir_all(self.0);
}
}
let tmp_name = format!(".tmp.download.{}", std::process::id());
let tmp_dir = parent.join(&tmp_name);
let _ = std::fs::remove_dir_all(&tmp_dir);
std::fs::create_dir_all(&tmp_dir)?;
let _cleanup = Cleanup(&tmp_dir);
f(&tmp_dir)
}
fn download_english(dir: &Path, final_path: &Path) -> crate::domain::Result<()> {
with_temp_subdir(dir, |tmp_dir| {
let tmp_file = tmp_dir.join(ENGLISH_MODEL_FILENAME);
udpipe_rs::download_model_from_url(ENGLISH_MODEL_URL, &tmp_file)
.map_err(|e| Error::ModelInvalid(e.to_string()))?;
std::fs::rename(&tmp_file, final_path)?;
Ok(())
})
}
fn read_and_verify(
path: &Path,
expected_size: u64,
expected_hash: &str,
) -> crate::domain::Result<Option<Vec<u8>>> {
let meta = std::fs::metadata(path)?;
if meta.len() != expected_size {
return Ok(None);
}
let bytes = std::fs::read(path)?;
let mut hasher = Sha256::new();
hasher.update(&bytes);
let got = hex_encode(&hasher.finalize());
if got.eq_ignore_ascii_case(expected_hash) {
Ok(Some(bytes))
} else {
Ok(None)
}
}
fn hex_encode(bytes: &[u8]) -> String {
let mut s = String::with_capacity(bytes.len() * 2);
for b in bytes {
s.push_str(&format!("{b:02x}"));
}
s
}
fn catch_parse_panic<F, T>(f: F) -> crate::domain::Result<T>
where
F: FnOnce() -> crate::domain::Result<T>,
{
use std::panic::{AssertUnwindSafe, catch_unwind};
match catch_unwind(AssertUnwindSafe(f)) {
Ok(result) => result,
Err(payload) => {
let message = payload
.downcast_ref::<&'static str>()
.map(|s| (*s).to_string())
.or_else(|| payload.downcast_ref::<String>().cloned())
.unwrap_or_else(|| "udpipe panic (no message captured)".to_string());
Err(Error::ParseFailed(format!("udpipe panicked: {message}")))
}
}
}
impl NlpProvider for Udpipe {
fn parse(&self, text: &str) -> crate::domain::Result<Vec<Sentence>> {
let words = catch_parse_panic(|| {
self.model
.parse(text)
.map_err(|e| Error::ParseFailed(e.to_string()))
})?;
let mut by_sentence: HashMap<i32, Vec<&udpipe_rs::Word>> = HashMap::new();
for word in &words {
by_sentence.entry(word.sentence_id).or_default().push(word);
}
let mut ids: Vec<i32> = by_sentence.keys().copied().collect();
ids.sort();
ids.into_iter()
.map(|id| {
let sent_words = &by_sentence[&id];
let tokens: Vec<Token> = sent_words
.iter()
.map(|w| {
let id = usize::try_from(w.id).map_err(|_| {
Error::ParseFailed(format!(
"invalid token id {} in sentence {}",
w.id, w.sentence_id
))
})?;
let head = usize::try_from(w.head).map_err(|_| {
Error::ParseFailed(format!(
"invalid head {} for token {} in sentence {}",
w.head, w.id, w.sentence_id
))
})?;
Ok(Token {
id,
text: w.form.clone(),
lemma: w.lemma.clone(),
pos: w.upostag.clone(),
xpos: w.xpostag.clone(),
feats: w.feats.clone(),
dep: w.deprel.clone(),
head,
deps: String::from("_"),
misc: w.misc.clone(),
is_punct: w.is_punct(),
})
})
.collect::<crate::domain::Result<Vec<Token>>>()?;
let text = {
let mut buf = String::new();
for (i, tok) in tokens.iter().enumerate() {
buf.push_str(&tok.text);
if i + 1 < tokens.len() && !tok.misc.contains("SpaceAfter=No") {
buf.push(' ');
}
}
buf
};
Ok(Sentence::new(text, tokens))
})
.collect()
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn catch_parse_panic_converts_str_panic_to_parse_failed() {
let result: crate::domain::Result<()> = catch_parse_panic(|| {
panic!("simulated udpipe-rs panic");
});
match result {
Err(Error::ParseFailed(msg)) => {
assert!(
msg.contains("udpipe panicked"),
"expected wrapper prefix in: {msg}"
);
assert!(
msg.contains("simulated udpipe-rs panic"),
"expected payload in: {msg}"
);
}
other => panic!("expected ParseFailed, got {other:?}"),
}
}
#[test]
fn catch_parse_panic_converts_string_panic_to_parse_failed() {
let result: crate::domain::Result<()> =
catch_parse_panic(|| panic!("{}", "owned string panic"));
match result {
Err(Error::ParseFailed(msg)) => {
assert!(
msg.contains("owned string panic"),
"expected payload in: {msg}"
);
}
other => panic!("expected ParseFailed, got {other:?}"),
}
}
#[test]
fn catch_parse_panic_passes_through_non_panic_results() {
let ok: crate::domain::Result<i32> = catch_parse_panic(|| Ok(42));
assert_eq!(ok.unwrap(), 42);
let err: crate::domain::Result<i32> =
catch_parse_panic(|| Err(Error::ParseFailed("boring failure".into())));
match err {
Err(Error::ParseFailed(msg)) => assert_eq!(msg, "boring failure"),
other => panic!("expected pass-through ParseFailed, got {other:?}"),
}
}
const HELLO_HASH: &str = "2cf24dba5fb0a30e26e83b2ac5b9e29e1b161e5c1fa7425e73043362938b9824";
#[test]
fn read_and_verify_returns_bytes_on_match() {
let dir = tempfile::tempdir().unwrap();
let path = dir.path().join("hello.bin");
std::fs::write(&path, b"hello").unwrap();
let result = read_and_verify(&path, 5, HELLO_HASH).unwrap();
assert_eq!(result.as_deref(), Some(b"hello".as_slice()));
}
#[test]
fn read_and_verify_returns_none_on_size_mismatch() {
let dir = tempfile::tempdir().unwrap();
let path = dir.path().join("hello.bin");
std::fs::write(&path, b"hello").unwrap();
let result = read_and_verify(&path, 6, HELLO_HASH).unwrap();
assert!(result.is_none());
}
#[test]
fn read_and_verify_returns_none_on_hash_mismatch() {
let dir = tempfile::tempdir().unwrap();
let path = dir.path().join("five.bin");
std::fs::write(&path, b"world").unwrap();
let result = read_and_verify(&path, 5, HELLO_HASH).unwrap();
assert!(result.is_none());
}
#[test]
fn with_temp_subdir_creates_and_cleans_up_on_success() {
let parent = tempfile::tempdir().unwrap();
let mut captured: Option<std::path::PathBuf> = None;
let _ = with_temp_subdir(parent.path(), |tmp| {
assert!(tmp.exists(), "temp subdir should exist inside the closure");
assert!(
tmp.starts_with(parent.path()),
"temp subdir should be inside parent"
);
captured = Some(tmp.to_path_buf());
Ok(())
});
let tmp_path = captured.unwrap();
assert!(
!tmp_path.exists(),
"temp subdir should be removed after the closure returns"
);
}
#[test]
fn with_temp_subdir_cleans_up_on_error() {
let parent = tempfile::tempdir().unwrap();
let mut captured: Option<std::path::PathBuf> = None;
let _ = with_temp_subdir(parent.path(), |tmp| {
captured = Some(tmp.to_path_buf());
Err::<(), Error>(Error::ModelInvalid("synthetic".into()))
});
let tmp_path = captured.unwrap();
assert!(
!tmp_path.exists(),
"temp subdir should be removed even when the closure returns Err"
);
}
#[test]
fn read_and_verify_returned_bytes_are_what_was_hashed() {
let dir = tempfile::tempdir().unwrap();
let path = dir.path().join("hello.bin");
std::fs::write(&path, b"hello").unwrap();
let bytes = read_and_verify(&path, 5, HELLO_HASH).unwrap().unwrap();
std::fs::write(&path, b"WORLD").unwrap();
assert_eq!(&bytes, b"hello");
}
}