use std::collections::HashMap;
use std::path::Path;
use std::sync::atomic::{AtomicU64, Ordering};
use std::time::{Duration, SystemTime};
use udpipe_rs::Model;
use crate::config::Config;
use crate::digest::sha256_hex;
use crate::domain::Error;
use crate::domain::{Identity, ProvisionNotice, Span, Token};
use super::NlpProvider;
const ENGLISH_MODEL_SHA256: &str =
"784bd0fa85e3d831fd02a55290d0acfd05c953159dc38cc33d52e1b28add9957";
const ENGLISH_MODEL_NAME: &str = "english-ewt-ud-2.5-191206";
const ENGLISH_MODEL_SIZE: u64 = 16_309_608;
const ENGLISH_MODEL_URL: &str = "https://lindat.mff.cuni.cz/repository/server/api/core/bitstreams/handle/11234/1-3131/english-ewt-ud-2.5-191206.udpipe?sequence=17&isAllowed=y";
const ENGLISH_MODEL_LICENSE: &str = "CC BY-NC-SA 4.0 (non-commercial)";
const ENGLISH_MODEL_LICENSE_URL: &str = "https://creativecommons.org/licenses/by-nc-sa/4.0/";
const ENGLISH_MODEL: Pin<'static> = Pin {
filename: ENGLISH_MODEL_FILENAME,
size: ENGLISH_MODEL_SIZE,
sha256: ENGLISH_MODEL_SHA256,
url: ENGLISH_MODEL_URL,
license: ENGLISH_MODEL_LICENSE,
license_url: ENGLISH_MODEL_LICENSE_URL,
};
#[derive(Debug, Clone, Copy)]
struct Pin<'a> {
filename: &'a str,
size: u64,
sha256: &'a str,
url: &'a str,
license: &'a str,
license_url: &'a str,
}
use crate::domain::limits::MAX_DOWNLOAD_BYTES as MAX_MODEL_BYTES;
const FETCH_TIMEOUT: Duration = Duration::from_secs(300);
const CONNECT_TIMEOUT: Duration = Duration::from_secs(30);
const STALE_TEMP_AGE: Duration = Duration::from_secs(600);
const TEMP_DIR_PREFIX: &str = ".tmp.download.";
const PARSER_NAME: &str = "udpipe";
pub(crate) struct Udpipe {
model: Model,
identity: Identity,
}
impl std::fmt::Debug for Udpipe {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.debug_struct("Udpipe").finish_non_exhaustive()
}
}
impl Udpipe {
pub(crate) fn from_bytes(data: &[u8]) -> crate::domain::Result<Self> {
let model =
Model::load_from_memory(data).map_err(|e| Error::ModelInvalid(e.to_string()))?;
Ok(Self {
model,
identity: identity_of(data),
})
}
pub(crate) fn english(model_dir: impl AsRef<Path>) -> crate::domain::Result<Self> {
Self::english_with_notice(model_dir, |_| {})
}
pub(crate) fn english_with_notice(
model_dir: impl AsRef<Path>,
mut notice: impl FnMut(&ProvisionNotice),
) -> crate::domain::Result<Self> {
let bytes = provision(
model_dir.as_ref(),
ENGLISH_MODEL,
&mut notice,
&fetch_capped,
)?;
Self::from_bytes(&bytes)
}
pub(crate) fn from_config(cfg: &Config) -> crate::domain::Result<Self> {
Self::english(cfg.model_dir())
}
pub(crate) fn from_config_with_notice(
cfg: &Config,
notice: impl FnMut(&ProvisionNotice),
) -> crate::domain::Result<Self> {
Self::english_with_notice(cfg.model_dir(), notice)
}
}
const ENGLISH_MODEL_FILENAME: &str = "english-ewt-ud-2.5-191206.udpipe";
fn provision(
dir: &Path,
pin: Pin<'_>,
notice: &mut dyn FnMut(&ProvisionNotice),
fetch: &dyn Fn(&str) -> crate::domain::Result<Vec<u8>>,
) -> crate::domain::Result<Vec<u8>> {
create_model_dir(dir)?;
let path = dir.join(pin.filename);
if path.exists() {
if let Some(bytes) = read_and_verify(&path, pin.size, pin.sha256)? {
return Ok(bytes);
}
}
let bytes = fetch_verified(dir, pin, notice, fetch)?;
install(dir, pin.filename, &bytes)?;
Ok(bytes)
}
fn fetch_verified(
dir: &Path,
pin: Pin<'_>,
notice: &mut dyn FnMut(&ProvisionNotice),
fetch: &dyn Fn(&str) -> crate::domain::Result<Vec<u8>>,
) -> crate::domain::Result<Vec<u8>> {
for _ in 0..2 {
notice(&ProvisionNotice {
artifact: pin.filename.to_string(),
bytes: pin.size,
destination: dir.to_path_buf(),
license: pin.license.to_string(),
license_url: pin.license_url.to_string(),
});
let bytes = fetch(pin.url)?;
if bytes.len() > MAX_MODEL_BYTES {
return Err(Error::InputTooLarge {
limit: MAX_MODEL_BYTES,
actual: bytes.len(),
what: "parser_model_download",
});
}
if verified(&bytes, pin.size, pin.sha256) {
return Ok(bytes);
}
}
Err(Error::ModelInvalid(format!(
"SHA-256 mismatch after re-download from {}",
pin.url
)))
}
fn install(dir: &Path, filename: &str, bytes: &[u8]) -> crate::domain::Result<()> {
use std::io::Write;
with_temp_subdir(dir, |tmp_dir| {
let tmp_file = tmp_dir.join(filename);
let mut file = std::fs::OpenOptions::new()
.write(true)
.create_new(true)
.open(&tmp_file)
.map_err(|e| io_at("create the temporary file", &tmp_file, &e))?;
file.write_all(bytes)
.map_err(|e| io_at("write the model to", &tmp_file, &e))?;
drop(file);
let final_path = dir.join(filename);
std::fs::rename(&tmp_file, &final_path)
.map_err(|e| io_at("move the model into place at", &final_path, &e))
})
}
fn fetch_capped(url: &str) -> crate::domain::Result<Vec<u8>> {
use std::io::Read;
let response = download_agent()
.get(url)
.call()
.map_err(|e| transport_failure(url, &e))?;
let mut bytes = Vec::new();
response
.into_body()
.into_reader()
.take(MAX_MODEL_BYTES as u64 + 1)
.read_to_end(&mut bytes)
.map_err(|e| body_failure(url, e))?;
Ok(bytes)
}
fn download_agent() -> ureq::Agent {
ureq::Agent::config_builder()
.timeout_global(Some(FETCH_TIMEOUT))
.timeout_connect(Some(CONNECT_TIMEOUT))
.https_only(true)
.build()
.into()
}
fn body_failure(url: &str, error: std::io::Error) -> Error {
transport_failure(url, &ureq::Error::from(error))
}
fn transport_failure(url: &str, error: &ureq::Error) -> Error {
use std::io::ErrorKind;
let kind = match error {
ureq::Error::Io(e) => e.kind(),
ureq::Error::Timeout(_) => ErrorKind::TimedOut,
ureq::Error::ConnectionFailed | ureq::Error::HostNotFound => ErrorKind::NotConnected,
_ => ErrorKind::Other,
};
let retryable = !is_certificate_rejection(&error.to_string(), error);
Error::Io(crate::domain::IoContext::download(
kind,
url,
download_message(url, error),
retryable,
))
}
fn download_message(url: &str, error: &ureq::Error) -> String {
let detail = error.to_string();
if !is_certificate_rejection(&detail, error) {
return format!("download {url}: {detail}");
}
format!(
"download {url}: the TLS certificate offered for {} was rejected. matra verifies \
TLS against root certificates compiled into it and never reads the system trust \
store, so a proxy that re-signs TLS cannot be trusted by installing its CA. \
Fetch {} by hand and put it in the model directory instead. Underlying failure: {detail}",
host_of(url),
ENGLISH_MODEL_FILENAME,
)
}
fn is_certificate_rejection(detail: &str, error: &ureq::Error) -> bool {
if matches!(error, ureq::Error::Tls(_)) {
return true;
}
let lowered = detail.to_ascii_lowercase();
lowered.contains("certificate") || lowered.contains("rustls")
}
fn host_of(url: &str) -> &str {
let after_scheme = url.split_once("://").map_or(url, |(_, rest)| rest);
let authority = after_scheme
.split(['/', '?', '#'])
.next()
.unwrap_or(after_scheme);
let host = authority.rsplit_once('@').map_or(authority, |(_, h)| h);
host.split(':').next().unwrap_or(host)
}
fn io_at(operation: &str, path: &Path, error: &std::io::Error) -> Error {
Error::Io(crate::domain::IoContext::cannot(operation, path, error))
}
fn create_model_dir(dir: &Path) -> crate::domain::Result<()> {
std::fs::create_dir_all(dir).map_err(|e| io_at("create the model directory", dir, &e))
}
fn verified(bytes: &[u8], expected_size: u64, expected_hash: &str) -> bool {
if bytes.len() as u64 != expected_size {
return false;
}
sha256_hex(bytes).eq_ignore_ascii_case(expected_hash)
}
fn with_temp_subdir<F, T>(parent: &Path, f: F) -> crate::domain::Result<T>
where
F: FnOnce(&Path) -> crate::domain::Result<T>,
{
struct Cleanup<'a>(&'a Path);
impl Drop for Cleanup<'_> {
fn drop(&mut self) {
let _ = std::fs::remove_dir_all(self.0);
}
}
reclaim_stale_temp_dirs(parent, SystemTime::now());
let tmp_dir = parent.join(format!("{TEMP_DIR_PREFIX}{}", temp_stamp()));
std::fs::create_dir(&tmp_dir)
.map_err(|e| io_at("create the temporary directory", &tmp_dir, &e))?;
let _cleanup = Cleanup(&tmp_dir);
f(&tmp_dir)
}
fn temp_stamp() -> String {
static COUNTER: AtomicU64 = AtomicU64::new(0);
let nanos = SystemTime::now()
.duration_since(SystemTime::UNIX_EPOCH)
.map_or(0, |d| d.as_nanos() as u64);
format!(
"{}.{nanos:x}.{:x}",
std::process::id(),
COUNTER.fetch_add(1, Ordering::Relaxed)
)
}
fn reclaim_stale_temp_dirs(parent: &Path, now: SystemTime) {
let Ok(entries) = std::fs::read_dir(parent) else {
return;
};
for entry in entries.flatten() {
let name = entry.file_name();
let Some(name) = name.to_str() else { continue };
if !name.starts_with(TEMP_DIR_PREFIX) {
continue;
}
let stale = entry
.metadata()
.and_then(|m| m.modified())
.ok()
.and_then(|modified| now.duration_since(modified).ok())
.is_some_and(|age| age >= STALE_TEMP_AGE);
if stale {
let _ = std::fs::remove_dir_all(entry.path());
}
}
}
fn read_and_verify(
path: &Path,
expected_size: u64,
expected_hash: &str,
) -> crate::domain::Result<Option<Vec<u8>>> {
let meta = std::fs::metadata(path).map_err(|e| io_at("read", path, &e))?;
if meta.len() != expected_size {
return Ok(None);
}
let bytes = std::fs::read(path).map_err(|e| io_at("read", path, &e))?;
if verified(&bytes, expected_size, expected_hash) {
Ok(Some(bytes))
} else {
Ok(None)
}
}
fn identity_of(bytes: &[u8]) -> Identity {
let sha256 = sha256_hex(bytes);
let pinned = sha256.eq_ignore_ascii_case(ENGLISH_MODEL.sha256);
Identity::new(
PARSER_NAME,
pinned.then(|| ENGLISH_MODEL_NAME.to_string()),
Some(sha256),
pinned.then(|| ENGLISH_MODEL.license.to_string()),
)
}
fn catch_parse_panic<F, T>(f: F) -> crate::domain::Result<T>
where
F: FnOnce() -> crate::domain::Result<T>,
{
use std::panic::{AssertUnwindSafe, catch_unwind};
match catch_unwind(AssertUnwindSafe(f)) {
Ok(result) => result,
Err(payload) => {
let message = payload
.downcast_ref::<&'static str>()
.map(|s| (*s).to_string())
.or_else(|| payload.downcast_ref::<String>().cloned())
.unwrap_or_else(|| "udpipe panic (no message captured)".to_string());
Err(Error::ParseFailed {
reason: None,
cause: format!("udpipe panicked: {message}"),
})
}
}
}
impl NlpProvider for Udpipe {
fn parse(&self, text: &str) -> crate::domain::Result<Vec<Vec<Token>>> {
let words = catch_parse_panic(|| {
self.model.parse(text).map_err(|e| Error::ParseFailed {
reason: None,
cause: e.to_string(),
})
})?;
let mut by_sentence: HashMap<i32, Vec<&udpipe_rs::Word>> = HashMap::new();
for word in &words {
by_sentence.entry(word.sentence_id).or_default().push(word);
}
let mut ids: Vec<i32> = by_sentence.keys().copied().collect();
ids.sort();
let mut cursor = 0;
ids.into_iter()
.map(|id| {
let mut tokens = by_sentence[&id]
.iter()
.map(|w| token_of(w))
.collect::<crate::domain::Result<Vec<Token>>>()?;
for token in &mut tokens {
let (start, end) = locate(text, cursor, &token.text)
.ok_or_else(|| unplaced(token, id + 1, cursor))?;
token.span = Span::new(start, end);
cursor = end;
}
Ok(tokens)
})
.collect()
}
fn identity(&self) -> Identity {
self.identity.clone()
}
}
fn locate(text: &str, cursor: usize, form: &str) -> Option<(usize, usize)> {
let rest = text.get(cursor..)?;
let start = cursor + (rest.len() - rest.trim_start().len());
let end = crate::domain::form_end(text, start, form)?;
Some((start, end))
}
fn token_of(word: &udpipe_rs::Word) -> crate::domain::Result<Token> {
let id = usize::try_from(word.id).map_err(|_| Error::ParseFailed {
reason: None,
cause: format!(
"invalid token id {} in sentence {}",
word.id, word.sentence_id
),
})?;
let head = usize::try_from(word.head).map_err(|_| Error::ParseFailed {
reason: None,
cause: format!(
"invalid head {} for token {} in sentence {}",
word.head, word.id, word.sentence_id
),
})?;
Ok(Token {
id,
span: Span::default(),
text: word.form.clone(),
lemma: word.lemma.clone(),
pos: word.upostag.clone(),
xpos: word.xpostag.clone(),
features: crate::domain::features_from_conllu(&word.feats),
dep: word.deprel.clone(),
head,
deps: String::from("_"),
misc: word.misc.clone(),
is_punct: word.is_punct(),
})
}
fn unplaced(token: &Token, sentence: i32, cursor: usize) -> Error {
Error::ParseFailed {
reason: None,
cause: format!(
"token {} ({:?}) of sentence {sentence} is not in the parsed text at or after byte {cursor}, so it has no span",
token.id, token.text
),
}
}
#[cfg(test)]
mod tests {
use super::*;
use crate::domain::DetailValue;
#[test]
fn locate_skips_leading_whitespace_and_matches_exactly() {
let text = " Hello, wörld.";
assert_eq!(locate(text, 0, "Hello"), Some((2, 7)));
assert_eq!(locate(text, 7, ","), Some((7, 8)));
assert_eq!(locate(text, 8, "wörld"), Some((9, 15)));
assert_eq!(locate(text, 8, "world"), None);
}
#[test]
fn locate_lets_whitespace_in_a_form_match_any_whitespace() {
let text = "e.\r\n g. done";
assert_eq!(locate(text, 0, "e. g."), Some((0, 7)));
}
#[test]
fn locate_is_none_past_the_end() {
assert_eq!(locate("abc", 3, "d"), None);
assert_eq!(locate("abc", 9, "a"), None);
}
#[test]
fn an_unpinned_model_reports_its_digest_and_no_name() {
let id = identity_of(b"hello");
assert_eq!(id.name, "udpipe");
assert_eq!(id.sha256.as_deref(), Some(HELLO_HASH));
assert_eq!(id.model, None);
assert_eq!(id.license, None);
}
#[test]
fn catch_parse_panic_converts_str_panic_to_parse_failed() {
let result: crate::domain::Result<()> = catch_parse_panic(|| {
panic!("simulated udpipe-rs panic");
});
match result {
Err(Error::ParseFailed { cause: msg, .. }) => {
assert!(
msg.contains("udpipe panicked"),
"expected wrapper prefix in: {msg}"
);
assert!(
msg.contains("simulated udpipe-rs panic"),
"expected payload in: {msg}"
);
}
other => panic!("expected ParseFailed, got {other:?}"),
}
}
#[test]
fn catch_parse_panic_converts_string_panic_to_parse_failed() {
let result: crate::domain::Result<()> =
catch_parse_panic(|| panic!("{}", "owned string panic"));
match result {
Err(Error::ParseFailed { cause: msg, .. }) => {
assert!(
msg.contains("owned string panic"),
"expected payload in: {msg}"
);
}
other => panic!("expected ParseFailed, got {other:?}"),
}
}
#[test]
fn catch_parse_panic_passes_through_non_panic_results() {
let ok: crate::domain::Result<i32> = catch_parse_panic(|| Ok(42));
assert_eq!(ok.unwrap(), 42);
let err: crate::domain::Result<i32> = catch_parse_panic(|| {
Err(Error::ParseFailed {
reason: None,
cause: "boring failure".into(),
})
});
match err {
Err(Error::ParseFailed { cause: msg, .. }) => assert_eq!(msg, "boring failure"),
other => panic!("expected pass-through ParseFailed, got {other:?}"),
}
}
const HELLO_HASH: &str = "2cf24dba5fb0a30e26e83b2ac5b9e29e1b161e5c1fa7425e73043362938b9824";
#[test]
fn read_and_verify_returns_bytes_on_match() {
let dir = tempfile::tempdir().unwrap();
let path = dir.path().join("hello.bin");
std::fs::write(&path, b"hello").unwrap();
let result = read_and_verify(&path, 5, HELLO_HASH).unwrap();
assert_eq!(result.as_deref(), Some(b"hello".as_slice()));
}
#[test]
fn read_and_verify_returns_none_on_size_mismatch() {
let dir = tempfile::tempdir().unwrap();
let path = dir.path().join("hello.bin");
std::fs::write(&path, b"hello").unwrap();
let result = read_and_verify(&path, 6, HELLO_HASH).unwrap();
assert!(result.is_none());
}
#[test]
fn read_and_verify_returns_none_on_hash_mismatch() {
let dir = tempfile::tempdir().unwrap();
let path = dir.path().join("five.bin");
std::fs::write(&path, b"world").unwrap();
let result = read_and_verify(&path, 5, HELLO_HASH).unwrap();
assert!(result.is_none());
}
#[test]
fn with_temp_subdir_creates_and_cleans_up_on_success() {
let parent = tempfile::tempdir().unwrap();
let mut captured: Option<std::path::PathBuf> = None;
let _ = with_temp_subdir(parent.path(), |tmp| {
assert!(tmp.exists(), "temp subdir should exist inside the closure");
assert!(
tmp.starts_with(parent.path()),
"temp subdir should be inside parent"
);
captured = Some(tmp.to_path_buf());
Ok(())
});
let tmp_path = captured.unwrap();
assert!(
!tmp_path.exists(),
"temp subdir should be removed after the closure returns"
);
}
#[test]
fn with_temp_subdir_cleans_up_on_error() {
let parent = tempfile::tempdir().unwrap();
let mut captured: Option<std::path::PathBuf> = None;
let _ = with_temp_subdir(parent.path(), |tmp| {
captured = Some(tmp.to_path_buf());
Err::<(), Error>(Error::ModelInvalid("synthetic".into()))
});
let tmp_path = captured.unwrap();
assert!(
!tmp_path.exists(),
"temp subdir should be removed even when the closure returns Err"
);
}
const FIXTURE: &[u8] = b"hello";
const FIXTURE_NAME: &str = "fixture.udpipe";
const FIXTURE_URL: &str = "https://models.example/fixture.udpipe";
const FIXTURE_LICENSE: &str = "Fixture License 1.0";
const FIXTURE_LICENSE_URL: &str = "https://licenses.example/fixture";
struct Recorder {
notices: Vec<ProvisionNotice>,
fetches: usize,
}
fn provision_fixture(
dir: &Path,
recorder: &mut Recorder,
body: &dyn Fn() -> crate::domain::Result<Vec<u8>>,
) -> crate::domain::Result<Vec<u8>> {
let fetches = std::cell::Cell::new(0usize);
let fetch = |_: &str| {
fetches.set(fetches.get() + 1);
body()
};
let mut notice = |n: &ProvisionNotice| recorder.notices.push(n.clone());
let pin = Pin {
filename: FIXTURE_NAME,
size: FIXTURE.len() as u64,
sha256: HELLO_HASH,
url: FIXTURE_URL,
license: FIXTURE_LICENSE,
license_url: FIXTURE_LICENSE_URL,
};
let result = provision(dir, pin, &mut notice, &fetch);
recorder.fetches += fetches.get();
result
}
fn recorder() -> Recorder {
Recorder {
notices: Vec::new(),
fetches: 0,
}
}
#[test]
fn a_cold_run_announces_the_download_once() {
let dir = tempfile::tempdir().unwrap();
let mut rec = recorder();
let bytes = provision_fixture(dir.path(), &mut rec, &|| Ok(FIXTURE.to_vec())).unwrap();
assert_eq!(bytes, FIXTURE);
assert_eq!(rec.notices.len(), 1, "one notice per fetch, one fetch");
assert_eq!(rec.notices[0].artifact, FIXTURE_NAME);
assert_eq!(rec.notices[0].bytes, FIXTURE.len() as u64);
assert_eq!(rec.notices[0].destination, dir.path());
assert_eq!(rec.notices[0].license, FIXTURE_LICENSE);
assert_eq!(rec.notices[0].license_url, FIXTURE_LICENSE_URL);
assert_eq!(
std::fs::read(dir.path().join(FIXTURE_NAME)).unwrap(),
FIXTURE
);
}
#[test]
fn a_warm_run_announces_nothing_and_fetches_nothing() {
let dir = tempfile::tempdir().unwrap();
std::fs::write(dir.path().join(FIXTURE_NAME), FIXTURE).unwrap();
let mut rec = recorder();
let bytes = provision_fixture(dir.path(), &mut rec, &|| {
panic!("a cached, verified model must not be refetched")
})
.unwrap();
assert_eq!(bytes, FIXTURE);
assert!(rec.notices.is_empty(), "nothing to announce");
assert_eq!(rec.fetches, 0);
}
#[test]
fn bytes_that_fail_the_digest_are_refetched_once_then_refused() {
let dir = tempfile::tempdir().unwrap();
let mut rec = recorder();
let err = provision_fixture(dir.path(), &mut rec, &|| Ok(b"WORLD".to_vec()))
.expect_err("wrong bytes are refused");
match err {
Error::ModelInvalid(msg) => {
assert!(msg.contains("SHA-256 mismatch after re-download"), "{msg}");
assert!(
msg.contains(FIXTURE_URL),
"the message names the URL: {msg}"
);
}
other => panic!("expected ModelInvalid, got {other:?}"),
}
assert_eq!(rec.fetches, 2, "one retry, not more");
assert!(
!dir.path().join(FIXTURE_NAME).exists(),
"nothing that failed the digest is written"
);
assert!(leftovers(dir.path()).is_empty(), "no temporary directory");
}
#[test]
fn a_retry_that_succeeds_lands_the_model() {
let dir = tempfile::tempdir().unwrap();
let attempts = std::cell::Cell::new(0usize);
let mut rec = recorder();
let bytes = provision_fixture(dir.path(), &mut rec, &|| {
attempts.set(attempts.get() + 1);
if attempts.get() == 1 {
Ok(b"WORLD".to_vec())
} else {
Ok(FIXTURE.to_vec())
}
})
.unwrap();
assert_eq!(bytes, FIXTURE);
assert_eq!(rec.notices.len(), 2, "a notice per fetch");
assert_eq!(
std::fs::read(dir.path().join(FIXTURE_NAME)).unwrap(),
FIXTURE
);
}
#[test]
fn a_corrupt_cache_is_replaced() {
let dir = tempfile::tempdir().unwrap();
std::fs::write(dir.path().join(FIXTURE_NAME), b"junk").unwrap();
let mut rec = recorder();
let bytes = provision_fixture(dir.path(), &mut rec, &|| Ok(FIXTURE.to_vec())).unwrap();
assert_eq!(bytes, FIXTURE);
assert_eq!(rec.fetches, 1);
assert_eq!(
std::fs::read(dir.path().join(FIXTURE_NAME)).unwrap(),
FIXTURE
);
}
#[test]
fn a_corrupt_cache_outlives_a_failed_refetch() {
let dir = tempfile::tempdir().unwrap();
let path = dir.path().join(FIXTURE_NAME);
std::fs::write(&path, b"junk").unwrap();
let mut rec = recorder();
let err = provision_fixture(dir.path(), &mut rec, &|| {
Err(Error::Io(std::io::Error::new(
std::io::ErrorKind::NotConnected,
"download https://models.example/fixture.udpipe: host not found",
)))
})
.unwrap_err();
assert!(matches!(err, Error::Io(_)), "got {err:?}");
assert_eq!(std::fs::read(&path).unwrap(), b"junk");
}
#[test]
fn a_transport_failure_is_not_an_invalid_model() {
let dir = tempfile::tempdir().unwrap();
let mut rec = recorder();
let err = provision_fixture(dir.path(), &mut rec, &|| {
Err(Error::Io(std::io::Error::new(
std::io::ErrorKind::NotConnected,
"download https://models.example/fixture.udpipe: host not found",
)))
})
.expect_err("an unreachable host fails");
assert_eq!(err.kind(), "io", "{err}");
assert!(err.to_string().contains("models.example"), "{err}");
assert!(leftovers(dir.path()).is_empty());
}
#[test]
fn a_response_over_the_cap_is_refused_by_size() {
let dir = tempfile::tempdir().unwrap();
let mut rec = recorder();
let err = provision_fixture(dir.path(), &mut rec, &|| Ok(vec![0u8; MAX_MODEL_BYTES + 1]))
.expect_err("over the cap");
match err {
Error::InputTooLarge {
limit,
actual,
what,
} => {
assert_eq!(limit, MAX_MODEL_BYTES);
assert_eq!(actual, MAX_MODEL_BYTES + 1);
assert_eq!(what, "parser_model_download");
}
other => panic!("expected InputTooLarge, got {other:?}"),
}
assert_eq!(rec.fetches, 1, "the cap fails the call, it does not retry");
}
#[test]
fn a_filesystem_failure_names_the_operation_and_the_path() {
let parent = tempfile::tempdir().unwrap();
let blocked = parent.path().join("not-a-directory");
std::fs::write(&blocked, b"x").unwrap();
let dir = blocked.join("models");
let mut rec = recorder();
let err = provision_fixture(&dir, &mut rec, &|| {
panic!("the directory must fail before any fetch")
})
.expect_err("cannot create the model directory");
assert_eq!(err.kind(), "io");
let message = err.to_string();
assert!(
message.contains("create the model directory"),
"names the operation: {message}"
);
assert!(
message.contains(&dir.display().to_string()),
"names the path: {message}"
);
assert_eq!(
err.details().get("path"),
Some(&DetailValue::Text(dir.display().to_string())),
"the path is data, not only words in the message"
);
}
#[test]
fn an_aged_orphan_temp_directory_is_reclaimed() {
let dir = tempfile::tempdir().unwrap();
let orphan = dir.path().join(format!("{TEMP_DIR_PREFIX}424242"));
std::fs::create_dir_all(&orphan).unwrap();
std::fs::write(orphan.join("partial.udpipe"), b"half a model").unwrap();
reclaim_stale_temp_dirs(
dir.path(),
SystemTime::now() + STALE_TEMP_AGE + Duration::from_secs(1),
);
assert!(!orphan.exists(), "an aged leftover is reclaimed");
}
#[test]
fn a_live_concurrent_temp_directory_is_left_alone() {
let dir = tempfile::tempdir().unwrap();
let peer = dir.path().join(format!("{TEMP_DIR_PREFIX}424242"));
std::fs::create_dir_all(&peer).unwrap();
std::fs::write(peer.join("partial.udpipe"), b"in flight").unwrap();
reclaim_stale_temp_dirs(dir.path(), SystemTime::now());
assert!(peer.exists(), "a fresh peer is not this call's to remove");
let mut rec = recorder();
provision_fixture(dir.path(), &mut rec, &|| Ok(FIXTURE.to_vec())).unwrap();
assert!(peer.exists(), "a download does not evict a live peer");
}
#[test]
fn a_temp_directory_from_the_future_is_left_alone() {
let dir = tempfile::tempdir().unwrap();
let odd = dir.path().join(format!("{TEMP_DIR_PREFIX}7"));
std::fs::create_dir_all(&odd).unwrap();
reclaim_stale_temp_dirs(dir.path(), SystemTime::now() - Duration::from_secs(3600));
assert!(odd.exists());
}
#[test]
fn the_sweep_only_matches_its_own_temporary_names() {
let dir = tempfile::tempdir().unwrap();
let model = dir.path().join("english-ewt-ud-2.5-191206.udpipe");
std::fs::write(&model, b"a hand-placed model").unwrap();
let other = dir.path().join("potion-base-8M");
std::fs::create_dir_all(&other).unwrap();
reclaim_stale_temp_dirs(dir.path(), SystemTime::now() + Duration::from_secs(86_400));
assert!(model.exists(), "a model is not a temporary directory");
assert!(other.exists(), "another model's directory is not either");
}
fn leftovers(dir: &Path) -> Vec<String> {
std::fs::read_dir(dir)
.expect("list")
.filter_map(Result::ok)
.map(|e| e.file_name().to_string_lossy().into_owned())
.filter(|name| name.starts_with(TEMP_DIR_PREFIX))
.collect()
}
#[test]
fn a_url_yields_the_host_a_message_should_name() {
assert_eq!(host_of(ENGLISH_MODEL_URL), "lindat.mff.cuni.cz");
assert_eq!(host_of("https://example.org:8443/a/b?c=d"), "example.org");
assert_eq!(host_of("https://user@example.org/a"), "example.org");
assert_eq!(host_of("not a url"), "not a url");
}
#[test]
fn a_rejected_certificate_reads_as_a_sentence() {
let error = ureq::Error::Io(std::io::Error::other(
"invalid peer certificate: Other(OtherError(CaUsedAsEndEntity))",
));
let message = download_message(ENGLISH_MODEL_URL, &error);
assert!(message.contains("lindat.mff.cuni.cz"), "{message}");
assert!(message.contains("system trust store"), "{message}");
assert!(message.contains(ENGLISH_MODEL_FILENAME), "{message}");
assert!(
message.contains("CaUsedAsEndEntity"),
"the underlying failure survives: {message}"
);
let mapped = transport_failure(ENGLISH_MODEL_URL, &error);
assert_eq!(mapped.kind(), "io");
assert_eq!(
mapped.details().get("retryable"),
Some(&DetailValue::Bool(false)),
"the message says to fetch by hand, so a retry is not offered"
);
}
#[test]
fn a_transport_failure_is_retryable_and_names_its_url() {
for error in [
ureq::Error::HostNotFound,
ureq::Error::ConnectionFailed,
ureq::Error::Timeout(ureq::Timeout::Global),
ureq::Error::StatusCode(503),
] {
let details = transport_failure(ENGLISH_MODEL_URL, &error).details();
assert_eq!(
details.get("url"),
Some(&DetailValue::Text(ENGLISH_MODEL_URL.to_string())),
"{error}"
);
assert_eq!(
details.get("retryable"),
Some(&DetailValue::Bool(true)),
"{error}"
);
}
}
#[test]
fn an_ordinary_transport_failure_is_reported_plainly() {
let error = ureq::Error::HostNotFound;
let message = download_message(ENGLISH_MODEL_URL, &error);
assert!(message.starts_with("download https://lindat"), "{message}");
assert!(!message.contains("trust store"), "{message}");
let mapped = transport_failure(ENGLISH_MODEL_URL, &error);
match mapped {
Error::Io(e) => assert_eq!(e.kind(), std::io::ErrorKind::NotConnected),
other => panic!("expected Io, got {other:?}"),
}
}
#[test]
fn a_timeout_while_reading_the_body_is_reported_as_a_timeout() {
let from_ureq = std::io::Error::other(ureq::Error::Timeout(ureq::Timeout::Global));
assert_eq!(
from_ureq.kind(),
std::io::ErrorKind::Other,
"the wrapping this test exists to undo"
);
match body_failure(ENGLISH_MODEL_URL, from_ureq) {
Error::Io(e) => {
assert_eq!(e.kind(), std::io::ErrorKind::TimedOut);
assert!(e.to_string().contains(ENGLISH_MODEL_URL), "{e}");
assert!(e.to_string().contains("timeout"), "{e}");
}
other => panic!("expected Io(TimedOut), got {other:?}"),
}
}
#[test]
fn a_plain_body_read_failure_keeps_its_own_kind() {
let raw = std::io::Error::new(std::io::ErrorKind::UnexpectedEof, "peer disconnected");
match body_failure(ENGLISH_MODEL_URL, raw) {
Error::Io(e) => {
assert_eq!(e.kind(), std::io::ErrorKind::UnexpectedEof);
assert!(e.to_string().contains(ENGLISH_MODEL_URL), "{e}");
}
other => panic!("expected Io(UnexpectedEof), got {other:?}"),
}
}
#[test]
fn a_certificate_rejected_mid_stream_still_reads_as_a_sentence() {
let from_ureq = std::io::Error::other(ureq::Error::Io(std::io::Error::other(
"invalid peer certificate: Other(OtherError(CaUsedAsEndEntity))",
)));
match body_failure(ENGLISH_MODEL_URL, from_ureq) {
Error::Io(e) => {
let message = e.to_string();
assert!(message.contains("lindat.mff.cuni.cz"), "{message}");
assert!(message.contains("system trust store"), "{message}");
}
other => panic!("expected Io, got {other:?}"),
}
}
#[test]
fn the_download_agent_refuses_to_leave_https() {
let agent = download_agent();
assert!(agent.config().https_only());
assert!(
agent.config().max_redirects() > 0,
"the redirect following that makes https_only load-bearing"
);
}
#[test]
fn a_peer_temp_directory_bearing_this_pid_is_left_alone() {
let dir = tempfile::tempdir().unwrap();
let peer = dir
.path()
.join(format!("{TEMP_DIR_PREFIX}{}", std::process::id()));
std::fs::create_dir_all(&peer).unwrap();
std::fs::write(peer.join("partial.udpipe"), b"in flight").unwrap();
let mut rec = recorder();
provision_fixture(dir.path(), &mut rec, &|| Ok(FIXTURE.to_vec())).unwrap();
assert!(
peer.exists(),
"a fresh directory under this pid is a peer's, not this call's"
);
assert_eq!(
std::fs::read(peer.join("partial.udpipe")).unwrap(),
b"in flight"
);
}
#[test]
fn two_temporary_names_in_one_process_differ() {
assert_ne!(temp_stamp(), temp_stamp());
}
#[test]
fn read_and_verify_returned_bytes_are_what_was_hashed() {
let dir = tempfile::tempdir().unwrap();
let path = dir.path().join("hello.bin");
std::fs::write(&path, b"hello").unwrap();
let bytes = read_and_verify(&path, 5, HELLO_HASH).unwrap().unwrap();
std::fs::write(&path, b"WORLD").unwrap();
assert_eq!(&bytes, b"hello");
}
}