#![cfg(not(target_arch = "wasm32"))]
use std::path::PathBuf;
use std::sync::atomic::{AtomicU32, Ordering};
use fxtranslate::cache::{ensure_model, sha256_hex, zstd_decode, Cache};
use fxtranslate::fetch::{status_is_retryable, Fetch, RetryPolicy};
use fxtranslate::remote::{
fetch_records, language_matches, pairs, parse_records, records_url, Record,
};
mod common;
use common::MockFetch;
fn fixture(name: &str) -> Vec<u8> {
let path = PathBuf::from(env!("CARGO_MANIFEST_DIR"))
.join("tests/fixtures")
.join(name);
std::fs::read(&path).unwrap_or_else(|e| panic!("read fixture {}: {e}", path.display()))
}
fn tmp_cache() -> Cache {
static N: AtomicU32 = AtomicU32::new(0);
let dir = std::env::temp_dir().join(format!(
"fxtranslate-test-{}-{}",
std::process::id(),
N.fetch_add(1, Ordering::Relaxed)
));
Cache::with_root(dir)
}
const TINY_HASH: &str = "5a9aaf6b319b6cdb5f3ef4ff520599018f2df654ddc4d9bb73dcb687092c77b8";
const TINY_PLAIN: &[u8] = b"fxtranslate tiny model fixture\n";
fn tiny_record(name: &str, file_type: &str, src: &str, trg: &str) -> Record {
Record {
name: name.into(),
file_type: file_type.into(),
src: src.into(),
trg: trg.into(),
version: "3.0".into(),
architecture: Some("base".into()),
decompressed_hash: Some(TINY_HASH.into()),
location: format!("cdn/{name}.zst"),
}
}
mod discovery {
use super::*;
#[test]
fn parses_fixture_records() {
let recs =
parse_records(std::str::from_utf8(&fixture("rs-models-v2.json")).unwrap()).unwrap();
assert_eq!(recs.len(), 7, "fixture has 7 records");
let ps = pairs(&recs);
assert!(ps.contains(&("en".into(), "es".into())), "en-es present");
assert!(ps.contains(&("en".into(), "ja".into())), "en-ja present");
assert!(fxtranslate::remote::pick(&recs, "model", "en", "es").is_some());
assert!(fxtranslate::remote::pick(&recs, "vocab", "en", "es").is_some());
assert!(fxtranslate::remote::pick(&recs, "srcvocab", "en", "es").is_none());
assert!(fxtranslate::remote::pick(&recs, "vocab", "en", "ja").is_none());
assert!(fxtranslate::remote::pick(&recs, "srcvocab", "en", "ja").is_some());
assert!(fxtranslate::remote::pick(&recs, "trgvocab", "en", "ja").is_some());
}
#[test]
fn fetch_records_goes_through_http() {
let mock = MockFetch::new().route(&records_url(), fixture("rs-models-v2.json"));
let recs = fetch_records(&mock).unwrap();
assert_eq!(recs.len(), 7);
assert_eq!(mock.hit_count(), 1);
}
#[test]
fn pick_selects_latest_supported_minor_ignoring_higher_major() {
let mk = |v: &str| Record {
version: v.into(),
..tiny_record("model.enes", "model", "en", "es")
};
let recs = vec![mk("3.2"), mk("3.10"), mk("4.0"), mk("100.0")];
assert_eq!(
fxtranslate::remote::pick(&recs, "model", "en", "es")
.unwrap()
.version,
"3.10"
);
}
#[test]
fn pick_tolerates_prerelease_version_suffix() {
let mk = |v: &str| Record {
version: v.into(),
..tiny_record("model.enes", "model", "en", "es")
};
assert_eq!(
fxtranslate::remote::pick(&[mk("3.0a1")], "model", "en", "es")
.unwrap()
.version,
"3.0a1"
);
let recs = vec![mk("3.0a1"), mk("3.0"), mk("3.1")];
assert_eq!(
fxtranslate::remote::pick(&recs, "model", "en", "es")
.unwrap()
.version,
"3.1"
);
assert!(fxtranslate::remote::pick(&[mk("4.0a1")], "model", "en", "es").is_none());
}
#[test]
fn pick_rejects_unsupported_major_only_pair() {
let mk = |v: &str| Record {
version: v.into(),
..tiny_record("model.enes", "model", "en", "es")
};
let recs = vec![mk("100.0"), mk("100.5")];
assert!(fxtranslate::remote::pick(&recs, "model", "en", "es").is_none());
}
#[test]
fn list_query_matches_both_directions() {
assert!(language_matches("es", "en", "es"), "es → en matches `es`");
assert!(
language_matches("en", "es", "es"),
"en → es also matches `es`"
);
assert!(language_matches("zh-Hans", "en", "zh"));
assert!(language_matches("en", "zh-Hant", "zh"));
assert!(language_matches("en", "es", "en-es"));
assert!(!language_matches("es", "en", "en-es"));
assert!(language_matches("zh-Hans", "en", "zh-en"));
assert!(language_matches("zh-Hant", "en", "zh-en"));
assert!(!language_matches("en", "zh-Hans", "zh-en"));
assert!(!language_matches("en", "es", "fr"));
}
}
mod decompression {
use super::*;
#[test]
fn decompress_and_hash_roundtrip() {
let plain = zstd_decode(&fixture("tiny.bin.zst")).unwrap();
assert_eq!(plain, TINY_PLAIN);
assert_eq!(sha256_hex(&plain), TINY_HASH);
}
}
mod cache_behavior {
use super::*;
#[test]
fn cache_downloads_then_hits() {
let cache = tmp_cache();
let rec = tiny_record("model.enes.bin", "model", "en", "es");
let mock = MockFetch::new().route(&rec.cdn_url(), fixture("tiny.bin.zst"));
let path = cache.ensure(&mock, &rec).unwrap();
assert!(path.is_file());
assert_eq!(std::fs::read(&path).unwrap(), TINY_PLAIN);
assert_eq!(mock.hit_count(), 1, "first ensure downloads");
let path2 = cache.ensure(&mock, &rec).unwrap();
assert_eq!(path, path2);
assert_eq!(mock.hit_count(), 1, "second ensure is a cache hit");
}
#[test]
fn cache_refetches_corrupt_file() {
let cache = tmp_cache();
let rec = tiny_record("model.enes.bin", "model", "en", "es");
let mock = MockFetch::new().route(&rec.cdn_url(), fixture("tiny.bin.zst"));
let path = cache.ensure(&mock, &rec).unwrap();
assert_eq!(mock.hit_count(), 1);
std::fs::write(&path, b"corrupted!!!").unwrap();
let path2 = cache.ensure(&mock, &rec).unwrap();
assert_eq!(path, path2);
assert_eq!(
std::fs::read(&path2).unwrap(),
TINY_PLAIN,
"restored good bytes"
);
assert_eq!(mock.hit_count(), 2, "corrupt file triggered a re-fetch");
}
#[test]
fn cache_rejects_download_hash_mismatch() {
let cache = tmp_cache();
let mut rec = tiny_record("model.enes.bin", "model", "en", "es");
rec.decompressed_hash = Some("0".repeat(64));
let mock = MockFetch::new().route(&rec.cdn_url(), fixture("tiny.bin.zst"));
let err = cache.ensure(&mock, &rec).unwrap_err();
assert!(err.contains("hash mismatch"), "got: {err}");
assert!(
!cache.pair_dir("en", "es").join(&rec.name).is_file(),
"no bad file left"
);
}
}
mod resilience {
use super::*;
use std::io::Cursor;
fn no_delay_cache() -> Cache {
tmp_cache().with_retry(RetryPolicy::no_delay())
}
#[test]
fn retryable_status_classification() {
assert!(status_is_retryable(429));
assert!(status_is_retryable(500));
assert!(status_is_retryable(503));
assert!(!status_is_retryable(200));
assert!(!status_is_retryable(400));
assert!(!status_is_retryable(404));
assert!(!status_is_retryable(403));
}
#[test]
fn retries_transient_failures_then_succeeds() {
let cache = no_delay_cache();
let rec = tiny_record("model.enes.bin", "model", "en", "es");
let mock = MockFetch::new()
.route(&rec.cdn_url(), fixture("tiny.bin.zst"))
.fail_times(2, true);
let path = cache.ensure(&mock, &rec).unwrap();
assert_eq!(
std::fs::read(&path).unwrap(),
TINY_PLAIN,
"good bytes cached"
);
assert_eq!(
mock.hit_count(),
3,
"two failed attempts + one success = 3 get_to attempts"
);
}
#[test]
fn gives_up_after_max_transient_failures_leaving_nothing() {
let cache = no_delay_cache();
let rec = tiny_record("model.enes.bin", "model", "en", "es");
let mock = MockFetch::new()
.route(&rec.cdn_url(), fixture("tiny.bin.zst"))
.fail_times(10, true);
let err = cache.ensure(&mock, &rec).unwrap_err();
assert!(err.contains("scripted failure"), "got: {err}");
assert_eq!(
mock.hit_count(),
RetryPolicy::no_delay().max_attempts as usize,
"stops at the attempt cap"
);
assert!(
!cache.pair_dir("en", "es").join(&rec.name).is_file(),
"a give-up leaves no file in the cache"
);
}
#[test]
fn does_not_retry_non_retryable_failure() {
let cache = no_delay_cache();
let rec = tiny_record("model.enes.bin", "model", "en", "es");
let mock = MockFetch::new()
.route(&rec.cdn_url(), fixture("tiny.bin.zst"))
.fail_times(1, false);
cache.ensure(&mock, &rec).unwrap_err();
assert_eq!(mock.hit_count(), 1, "no retry on a permanent failure");
}
#[test]
fn streams_progress_monotonically_to_total() {
let body = fixture("tiny.bin.zst");
let url = "https://example.test/attachment.zst";
let mock = MockFetch::new().route(url, body.clone()).chunk_size(8);
let mut sink = Cursor::new(Vec::new());
let mut samples: Vec<(u64, Option<u64>)> = Vec::new();
let mut on_progress = |done, total| samples.push((done, total));
let outcome = mock.get_to(url, 0, &mut sink, &mut on_progress).unwrap();
assert!(
!outcome.resumed,
"a range_from=0 request is a full body, not a resume"
);
assert_eq!(sink.into_inner(), body, "sink received the full body");
assert!(
samples.len() > 2,
"chunked download reports progress more than once (got {})",
samples.len()
);
for w in samples.windows(2) {
assert!(w[1].0 >= w[0].0, "progress is monotonic: {samples:?}");
}
let total = Some(body.len() as u64);
assert!(
samples.iter().all(|&(_, t)| t == total),
"total is constant"
);
assert_eq!(
samples.last().unwrap().0,
body.len() as u64,
"final report equals the byte length"
);
}
#[test]
fn resumes_from_partial_after_midstream_drop() {
let cache = no_delay_cache();
let rec = tiny_record("model.enes.bin", "model", "en", "es");
let mock = MockFetch::new()
.route(&rec.cdn_url(), fixture("tiny.bin.zst"))
.chunk_size(4)
.fail_after(8, true);
let path = cache.ensure(&mock, &rec).unwrap();
assert_eq!(
std::fs::read(&path).unwrap(),
TINY_PLAIN,
"resumed body verifies"
);
let ranges = mock.get_to_ranges();
assert_eq!(ranges.len(), 2, "one drop + one resume");
assert_eq!(ranges[0], 0, "first attempt starts at 0");
assert!(
ranges[1] >= 8,
"resume requests the tail from where it dropped: {ranges:?}"
);
}
#[test]
fn restarts_when_server_ignores_range() {
let cache = no_delay_cache();
let rec = tiny_record("model.enes.bin", "model", "en", "es");
let mock = MockFetch::new()
.route(&rec.cdn_url(), fixture("tiny.bin.zst"))
.chunk_size(4)
.fail_after(8, true)
.ignore_range();
let path = cache.ensure(&mock, &rec).unwrap();
assert_eq!(
std::fs::read(&path).unwrap(),
TINY_PLAIN,
"full restart still yields the correct body"
);
let ranges = mock.get_to_ranges();
assert_eq!(ranges.len(), 2, "one drop + one restart");
assert!(
ranges[1] > 0,
"the client still *asked* to resume: {ranges:?}"
);
}
#[test]
fn genuine_hash_mismatch_fails_without_wasteful_retry() {
let cache = no_delay_cache();
let mut rec = tiny_record("model.enes.bin", "model", "en", "es");
rec.decompressed_hash = Some("0".repeat(64));
let mock = MockFetch::new().route(&rec.cdn_url(), fixture("tiny.bin.zst"));
let err = cache.ensure(&mock, &rec).unwrap_err();
assert!(err.contains("hash mismatch"), "got: {err}");
assert_eq!(
mock.hit_count(),
1,
"one clean attempt, no pointless re-download"
);
assert!(
!cache.pair_dir("en", "es").join(&rec.name).is_file(),
"nothing left in the cache"
);
}
}
mod ensure_model_wiring {
use super::*;
#[test]
fn ensure_model_shared_vocab() {
let cache = tmp_cache();
let recs = vec![
tiny_record("model.enes.bin", "model", "en", "es"),
tiny_record("vocab.enes.spm", "vocab", "en", "es"),
tiny_record("lex.enes.bin", "lex", "en", "es"),
];
let mut mock = MockFetch::new();
for r in &recs {
mock = mock.route(&r.cdn_url(), fixture("tiny.bin.zst"));
}
let files = ensure_model(&mock, &cache, &recs, "en", "es").unwrap();
assert_eq!(files.src_vocab, files.trg_vocab, "shared vocab reused");
assert!(files.lex.is_some(), "lex present");
assert!(files.model.is_file());
}
#[test]
fn ensure_model_split_vocab() {
let cache = tmp_cache();
let recs = vec![
tiny_record("model.enja.bin", "model", "en", "ja"),
tiny_record("srcvocab.enja.spm", "srcvocab", "en", "ja"),
tiny_record("trgvocab.enja.spm", "trgvocab", "en", "ja"),
];
let mut mock = MockFetch::new();
for r in &recs {
mock = mock.route(&r.cdn_url(), fixture("tiny.bin.zst"));
}
let files = ensure_model(&mock, &cache, &recs, "en", "ja").unwrap();
assert_ne!(files.src_vocab, files.trg_vocab, "split vocab differs");
assert!(files.lex.is_none(), "no shortlist for CJK split pair");
}
}
mod pivot_wiring {
use super::*;
use fxtranslate::route::{resolve_route, Route};
#[test]
fn pivot_resolves_and_downloads_both_legs() {
let cache = tmp_cache();
let recs = vec![
tiny_record("model.esen.bin", "model", "es", "en"),
tiny_record("vocab.esen.spm", "vocab", "es", "en"),
tiny_record("model.enfr.bin", "model", "en", "fr"),
tiny_record("vocab.enfr.spm", "vocab", "en", "fr"),
];
let mut mock = MockFetch::new();
for r in &recs {
mock = mock.route(&r.cdn_url(), fixture("tiny.bin.zst"));
}
let route = resolve_route(&recs, "es", "fr").unwrap();
let (src, pivot, trg) = match route {
Route::Pivot { src, pivot, trg } => (src, pivot, trg),
Route::Direct { .. } => panic!("es→fr must pivot, not resolve directly"),
};
assert_eq!(
(src.as_str(), pivot.as_str(), trg.as_str()),
("es", "en", "fr")
);
let leg1 = ensure_model(&mock, &cache, &recs, &src, &pivot).unwrap();
let leg2 = ensure_model(&mock, &cache, &recs, &pivot, &trg).unwrap();
assert!(leg1.model.is_file(), "es→en model cached");
assert!(leg2.model.is_file(), "en→fr model cached");
}
}
mod local_cache {
use super::*;
use fxtranslate::cache::dir_size;
use std::fs;
fn write_pair_file(cache: &Cache, pair: &str, file: &str, n: usize) {
let dir = cache.root().join(pair);
fs::create_dir_all(&dir).unwrap();
fs::write(dir.join(file), vec![0u8; n]).unwrap();
}
#[test]
fn list_cached_empty_when_root_absent() {
let cache = tmp_cache();
assert!(cache.list_cached().unwrap().is_empty());
assert_eq!(
dir_size(&cache.root().join("nope")).unwrap(),
0,
"missing dir is 0 bytes"
);
}
#[test]
fn list_cached_sums_sizes_sorted_and_skips_temps() {
let cache = tmp_cache();
write_pair_file(&cache, "en-es", "model.enes.bin", 100);
write_pair_file(&cache, "en-es", "vocab.enes.spm", 20);
write_pair_file(&cache, "en-es", ".model.enes.bin.download", 999);
write_pair_file(&cache, "es-en", "model.esen.bin", 50);
let cached = cache.list_cached().unwrap();
assert_eq!(
cached.iter().map(|c| c.name.as_str()).collect::<Vec<_>>(),
["en-es", "es-en"],
"pairs listed sorted by name"
);
assert_eq!(cached[0].bytes, 120, "temp file excluded from size");
assert_eq!(cached[1].bytes, 50);
}
#[test]
fn pair_files_lists_sorted_skipping_temps() {
let cache = tmp_cache();
write_pair_file(&cache, "en-es", "vocab.enes.spm", 20);
write_pair_file(&cache, "en-es", "model.enes.bin", 100);
write_pair_file(&cache, "en-es", ".model.enes.bin.partial", 5);
let files = cache.pair_files("en-es").unwrap();
let names: Vec<&str> = files.iter().map(|(n, ..)| n.as_str()).collect();
assert_eq!(
names,
["model.enes.bin", "vocab.enes.spm"],
"sorted, no temps"
);
assert_eq!(files[0].1, 100, "size reported per file");
assert!(
cache.pair_files("nope").unwrap().is_empty(),
"absent pair is empty"
);
}
#[test]
fn remove_pair_is_idempotent_and_isolated() {
let cache = tmp_cache();
write_pair_file(&cache, "en-es", "model.enes.bin", 10);
write_pair_file(&cache, "es-en", "model.esen.bin", 10);
assert!(cache.remove_pair("en-es").unwrap(), "present pair removed");
assert!(!cache.root().join("en-es").exists(), "dir gone");
assert!(cache.root().join("es-en").exists(), "sibling untouched");
assert!(
!cache.remove_pair("en-es").unwrap(),
"second remove is a no-op"
);
}
#[test]
fn pair_ops_reject_paths_escaping_the_cache() {
let cache = tmp_cache();
assert!(cache.remove_pair("../evil").is_err());
assert!(cache.pair_files("a/b").is_err());
}
#[test]
fn cached_model_reads_shared_vocab_pair_offline() {
let cache = tmp_cache();
write_pair_file(&cache, "en-es", "model.enes.intgemm.alphas.bin", 10);
write_pair_file(&cache, "en-es", "vocab.enes.spm", 10);
write_pair_file(&cache, "en-es", "lex.50.50.enes.s2t.bin", 10);
write_pair_file(&cache, "en-es", ".model.enes.bin.download", 999);
let files = cache
.cached_model("en", "es")
.expect("assembled from cache");
assert!(files.model.ends_with("model.enes.intgemm.alphas.bin"));
assert_eq!(files.src_vocab, files.trg_vocab, "shared vocab serves both");
assert!(files.src_vocab.ends_with("vocab.enes.spm"));
assert!(files.lex.is_some(), "lex picked up");
}
#[test]
fn cached_model_reads_split_vocab_pair_offline() {
let cache = tmp_cache();
write_pair_file(&cache, "en-ja", "model.enja.intgemm.alphas.bin", 10);
write_pair_file(&cache, "en-ja", "srcvocab.enja.spm", 10);
write_pair_file(&cache, "en-ja", "trgvocab.enja.spm", 10);
let files = cache
.cached_model("en", "ja")
.expect("assembled from cache");
assert_ne!(
files.src_vocab, files.trg_vocab,
"split vocab halves differ"
);
assert!(files.lex.is_none(), "no shortlist for this pair");
}
#[test]
fn cached_model_none_when_incomplete() {
let cache = tmp_cache();
assert!(cache.cached_model("en", "es").is_none(), "absent pair");
write_pair_file(&cache, "en-es", "model.enes.bin", 10);
assert!(
cache.cached_model("en", "es").is_none(),
"model but no vocab"
);
}
}
mod offline_fallback {
use super::*;
use fxtranslate::loader::load_translation;
#[test]
fn reports_missing_pair_and_unreachable_discovery() {
let cache = tmp_cache();
let err = match load_translation(&MockFetch::new(), &cache, "en", "es") {
Err(e) => e,
Ok(_) => panic!("expected an error with no network and an empty cache"),
};
assert!(err.contains("no cached model for en→es"), "got: {err}");
assert!(
err.contains("unreachable"),
"names the discovery failure: {err}"
);
}
}
mod route_download {
use super::*;
use fxtranslate::loader::ensure_route_files;
use fxtranslate::route::Route;
fn records_body() -> String {
let rec = |name: &str, ft: &str, src: &str, trg: &str, loc: &str| {
format!(
r#"{{"name":"{name}","version":"3.0","fileType":"{ft}","sourceLanguage":"{src}","targetLanguage":"{trg}","attachment":{{"location":"{loc}"}}}}"#
)
};
let recs = [
rec("model.enes", "model", "en", "es", "cdn/enes-model.zst"),
rec("vocab.enes", "vocab", "en", "es", "cdn/enes-vocab.zst"),
rec("model.esen", "model", "es", "en", "cdn/esen-model.zst"),
rec("vocab.esen", "vocab", "es", "en", "cdn/esen-vocab.zst"),
rec("model.enfr", "model", "en", "fr", "cdn/enfr-model.zst"),
rec("vocab.enfr", "vocab", "en", "fr", "cdn/enfr-vocab.zst"),
];
format!(r#"{{"data":[{}]}}"#, recs.join(","))
}
fn mock() -> MockFetch {
let body = records_body();
let recs = parse_records(&body).unwrap();
let mut m = MockFetch::new().route(&records_url(), body.into_bytes());
for r in &recs {
m = m.route(&r.cdn_url(), fixture("tiny.bin.zst"));
}
m
}
#[test]
fn direct_pair_caches_one_leg() {
let cache = tmp_cache();
let (route, files) = ensure_route_files(&mock(), &cache, "en", "es").unwrap();
assert!(matches!(route, Route::Direct { .. }), "en→es is direct");
assert_eq!(files.len(), 1, "one leg for a direct pair");
assert!(files[0].model.is_file());
}
#[test]
fn pivot_pair_caches_both_legs() {
let cache = tmp_cache();
let (route, files) = ensure_route_files(&mock(), &cache, "es", "fr").unwrap();
assert_eq!(
route,
Route::Pivot {
src: "es".into(),
pivot: "en".into(),
trg: "fr".into()
}
);
assert_eq!(files.len(), 2, "both pivot legs downloaded");
assert!(cache.root().join("es-en").is_dir(), "es→en leg cached");
assert!(cache.root().join("en-fr").is_dir(), "en→fr leg cached");
}
}