use std::io::Write;
use std::path::{Path, PathBuf};
use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH};
use super::traits::PoolingStrategy;
use crate::error::{MemdError, Result};
#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
pub enum EmbeddingModel {
#[default]
AllMiniLmL6V2,
Qwen3Embedding0_6B,
}
impl EmbeddingModel {
pub fn dimension(&self) -> usize {
match self {
Self::AllMiniLmL6V2 => 384,
Self::Qwen3Embedding0_6B => 1024,
}
}
pub fn pooling_strategy(&self) -> PoolingStrategy {
match self {
Self::AllMiniLmL6V2 => PoolingStrategy::Mean,
Self::Qwen3Embedding0_6B => PoolingStrategy::LastToken,
}
}
pub fn model_url(&self) -> &'static str {
match self {
Self::AllMiniLmL6V2 => {
"https://huggingface.co/Xenova/all-MiniLM-L6-v2/resolve/main/onnx/model_quantized.onnx"
}
Self::Qwen3Embedding0_6B => {
"https://huggingface.co/onnx-community/Qwen3-Embedding-0.6B-ONNX/resolve/main/onnx/model_int8.onnx"
}
}
}
pub fn tokenizer_url(&self) -> &'static str {
match self {
Self::AllMiniLmL6V2 => {
"https://huggingface.co/Xenova/all-MiniLM-L6-v2/resolve/main/tokenizer.json"
}
Self::Qwen3Embedding0_6B => {
"https://huggingface.co/onnx-community/Qwen3-Embedding-0.6B-ONNX/resolve/main/tokenizer.json"
}
}
}
pub fn model_filename(&self) -> &'static str {
match self {
Self::AllMiniLmL6V2 => "all-MiniLM-L6-v2-quantized.onnx",
Self::Qwen3Embedding0_6B => "qwen3-embedding-0.6b-q8.onnx",
}
}
pub fn tokenizer_filename(&self) -> &'static str {
match self {
Self::AllMiniLmL6V2 => "all-minilm-l6-v2-tokenizer.json",
Self::Qwen3Embedding0_6B => "qwen3-embedding-0.6b-tokenizer.json",
}
}
pub fn min_model_size(&self) -> u64 {
match self {
Self::AllMiniLmL6V2 => 20_000_000, Self::Qwen3Embedding0_6B => 500_000_000, }
}
pub fn min_tokenizer_size(&self) -> u64 {
match self {
Self::AllMiniLmL6V2 => 500_000, Self::Qwen3Embedding0_6B => 10_000, }
}
pub fn uses_instruction_format(&self) -> bool {
match self {
Self::AllMiniLmL6V2 => false,
Self::Qwen3Embedding0_6B => true,
}
}
pub fn requires_position_ids(&self) -> bool {
match self {
Self::AllMiniLmL6V2 => false,
Self::Qwen3Embedding0_6B => true,
}
}
pub fn kv_cache_config(&self) -> Option<KvCacheConfig> {
match self {
Self::AllMiniLmL6V2 => None,
Self::Qwen3Embedding0_6B => Some(KvCacheConfig {
num_layers: 28,
num_kv_heads: 8,
head_dim: 128,
}),
}
}
}
#[derive(Debug, Clone, Copy)]
pub struct KvCacheConfig {
pub num_layers: usize,
pub num_kv_heads: usize,
pub head_dim: usize,
}
const MODEL_URL: &str =
"https://huggingface.co/Xenova/all-MiniLM-L6-v2/resolve/main/onnx/model_quantized.onnx";
const MODEL_FILENAME: &str = "all-MiniLM-L6-v2-quantized.onnx";
const TOKENIZER_URL: &str =
"https://huggingface.co/Xenova/all-MiniLM-L6-v2/resolve/main/tokenizer.json";
const TOKENIZER_FILENAME: &str = "tokenizer.json";
const MIN_MODEL_SIZE: u64 = 20_000_000;
const MIN_TOKENIZER_SIZE: u64 = 500_000;
const CANDLE_BERT_CONFIG_URL: &str =
"https://huggingface.co/sentence-transformers/all-MiniLM-L6-v2/resolve/main/config.json";
const CANDLE_BERT_CONFIG_FILENAME: &str = "sentence-transformers-all-MiniLM-L6-v2-config.json";
const CANDLE_BERT_TOKENIZER_URL: &str =
"https://huggingface.co/sentence-transformers/all-MiniLM-L6-v2/resolve/main/tokenizer.json";
const CANDLE_BERT_TOKENIZER_FILENAME: &str =
"sentence-transformers-all-MiniLM-L6-v2-tokenizer.json";
const CANDLE_BERT_WEIGHTS_URL: &str =
"https://huggingface.co/sentence-transformers/all-MiniLM-L6-v2/resolve/main/model.safetensors";
const CANDLE_BERT_WEIGHTS_FILENAME: &str = "sentence-transformers-all-MiniLM-L6-v2.safetensors";
const MIN_CANDLE_BERT_CONFIG_SIZE: u64 = 100; const MIN_CANDLE_BERT_TOKENIZER_SIZE: u64 = 100_000; const MIN_CANDLE_BERT_WEIGHTS_SIZE: u64 = 80_000_000;
pub fn get_cache_dir() -> Result<PathBuf> {
let cache_dir = dirs::cache_dir()
.ok_or_else(|| MemdError::StorageError("cannot determine cache directory".into()))?
.join("memd")
.join("models");
Ok(cache_dir)
}
pub fn get_model_path() -> Result<PathBuf> {
let cache_dir = get_cache_dir()?;
let model_path = cache_dir.join(MODEL_FILENAME);
if !model_path.exists() {
download_model(&cache_dir)?;
}
verify_model_exists(&model_path)?;
Ok(model_path)
}
pub fn get_tokenizer_path() -> Result<PathBuf> {
let cache_dir = get_cache_dir()?;
let tokenizer_path = cache_dir.join(TOKENIZER_FILENAME);
if !tokenizer_path.exists() {
download_tokenizer(&cache_dir)?;
}
verify_tokenizer_exists(&tokenizer_path)?;
Ok(tokenizer_path)
}
pub fn verify_model_exists(path: &PathBuf) -> Result<()> {
if !path.exists() {
return Err(MemdError::StorageError(format!(
"model file not found at {:?}",
path
)));
}
let metadata = std::fs::metadata(path)?;
if metadata.len() < MIN_MODEL_SIZE {
return Err(MemdError::StorageError(format!(
"model file too small ({} bytes), expected >= {} bytes. File may be corrupted, delete and retry.",
metadata.len(),
MIN_MODEL_SIZE
)));
}
Ok(())
}
fn verify_tokenizer_exists(path: &PathBuf) -> Result<()> {
if !path.exists() {
return Err(MemdError::StorageError(format!(
"tokenizer file not found at {:?}",
path
)));
}
let metadata = std::fs::metadata(path)?;
if metadata.len() < MIN_TOKENIZER_SIZE {
return Err(MemdError::StorageError(format!(
"tokenizer file too small ({} bytes), expected >= {} bytes. File may be corrupted, delete and retry.",
metadata.len(),
MIN_TOKENIZER_SIZE
)));
}
Ok(())
}
pub fn download_model(cache_dir: &PathBuf) -> Result<()> {
let model_path = cache_dir.join(MODEL_FILENAME);
download_file(MODEL_URL, &model_path, "embedding model")
}
fn download_tokenizer(cache_dir: &PathBuf) -> Result<()> {
let tokenizer_path = cache_dir.join(TOKENIZER_FILENAME);
download_file(TOKENIZER_URL, &tokenizer_path, "tokenizer")
}
pub fn get_model_path_for(model: EmbeddingModel) -> Result<PathBuf> {
let cache_dir = get_cache_dir()?;
let model_path = cache_dir.join(model.model_filename());
if !model_path.exists() {
download_file(model.model_url(), &model_path, model.model_filename())?;
}
verify_file_size(&model_path, model.min_model_size(), "model")?;
Ok(model_path)
}
pub fn get_tokenizer_path_for(model: EmbeddingModel) -> Result<PathBuf> {
let cache_dir = get_cache_dir()?;
let tokenizer_path = cache_dir.join(model.tokenizer_filename());
if !tokenizer_path.exists() {
download_file(
model.tokenizer_url(),
&tokenizer_path,
model.tokenizer_filename(),
)?;
}
verify_file_size(&tokenizer_path, model.min_tokenizer_size(), "tokenizer")?;
Ok(tokenizer_path)
}
fn download_file(url: &str, path: &PathBuf, name: &str) -> Result<()> {
use std::sync::atomic::{AtomicU64, Ordering};
static COUNTER: AtomicU64 = AtomicU64::new(0);
let cache_dir = path.parent().unwrap();
std::fs::create_dir_all(cache_dir)?;
let lock_path = advisory_lock_path(path);
let mut _lock_guard: Option<LockGuard> = None;
match try_acquire_advisory_lock(&lock_path) {
AcquireLockOutcome::Acquired(guard) => {
_lock_guard = Some(guard);
}
AcquireLockOutcome::Contended => {
match wait_for_publish_or_release(path, &lock_path, advisory_lock_wait_timeout()) {
WaitOutcome::Published => {
tracing::info!("{} published by concurrent writer; reusing", name);
return Ok(());
}
WaitOutcome::LockReleased | WaitOutcome::Timeout => {
tracing::debug!(
"advisory lock wait fell through for {}; proceeding with race-safe download",
name
);
}
}
}
AcquireLockOutcome::Skipped => {
tracing::debug!(
"advisory lock unavailable for {} (non-fatal); proceeding with race-safe download",
name
);
}
}
tracing::info!("Downloading {} to {:?}", name, path);
let response = ureq::get(url)
.call()
.map_err(|e| MemdError::StorageError(format!("failed to download {}: {}", name, e)))?;
let tmp_path = path.with_extension(format!(
"partial.{}.{:?}.{}",
std::process::id(),
std::thread::current().id(),
COUNTER.fetch_add(1, Ordering::Relaxed)
));
let mut file = std::fs::File::create(&tmp_path)?;
let copy_result =
std::io::copy(&mut response.into_reader(), &mut file).and_then(|_| file.sync_all());
if let Err(e) = copy_result {
let _ = std::fs::remove_file(&tmp_path);
return Err(MemdError::StorageError(format!(
"failed to stream {} to {:?}: {}",
name, tmp_path, e
)));
}
drop(file);
match std::fs::hard_link(&tmp_path, path) {
Ok(()) => {
let _ = std::fs::remove_file(&tmp_path);
fsync_parent_dir(path);
}
Err(e) if e.kind() == std::io::ErrorKind::AlreadyExists => {
let _ = std::fs::remove_file(&tmp_path);
}
Err(e) => {
let _ = std::fs::remove_file(&tmp_path);
return Err(MemdError::StorageError(format!(
"failed to publish {} to {:?}: {}",
name, path, e
)));
}
}
tracing::info!("{} downloaded successfully", name);
Ok(())
}
fn fsync_parent_dir(path: &std::path::Path) {
let Some(parent) = path.parent() else {
return;
};
match std::fs::File::open(parent) {
Ok(dir) => {
if let Err(e) = dir.sync_all() {
tracing::debug!(
"fsync on parent dir {:?} failed (best-effort): {}",
parent,
e
);
}
}
Err(e) => {
tracing::debug!(
"opening parent dir {:?} for fsync failed (best-effort): {}",
parent,
e
);
}
}
}
const ADVISORY_LOCK_WAIT: Duration = Duration::from_secs(900);
const STALE_LOCK_TIMEOUT: Duration = Duration::from_secs(3600);
const WAIT_POLL_INITIAL: Duration = Duration::from_millis(50);
const WAIT_POLL_MAX: Duration = Duration::from_secs(2);
#[cfg(test)]
static TEST_WAIT_OVERRIDE_MS: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(0);
#[cfg(test)]
fn advisory_lock_wait_timeout() -> Duration {
let ms = TEST_WAIT_OVERRIDE_MS.load(std::sync::atomic::Ordering::Relaxed);
if ms == 0 {
ADVISORY_LOCK_WAIT
} else {
Duration::from_millis(ms)
}
}
#[cfg(not(test))]
fn advisory_lock_wait_timeout() -> Duration {
ADVISORY_LOCK_WAIT
}
fn advisory_lock_path(path: &Path) -> PathBuf {
match path.file_name() {
Some(name) => {
let mut new_name = name.to_os_string();
new_name.push(".lock");
path.with_file_name(new_name)
}
None => path.with_extension("lock"),
}
}
enum AcquireLockOutcome {
Acquired(LockGuard),
Contended,
Skipped,
}
enum WaitOutcome {
Published,
LockReleased,
Timeout,
}
fn try_acquire_advisory_lock(lock_path: &Path) -> AcquireLockOutcome {
loop {
match std::fs::OpenOptions::new()
.write(true)
.create_new(true)
.open(lock_path)
{
Ok(mut f) => {
let now_ms = SystemTime::now()
.duration_since(UNIX_EPOCH)
.map(|d| d.as_millis())
.unwrap_or(0);
let _ = writeln!(f, "pid={} created_ms={}", std::process::id(), now_ms);
let _ = f.sync_all();
drop(f);
return AcquireLockOutcome::Acquired(LockGuard {
path: lock_path.to_path_buf(),
});
}
Err(e) if e.kind() == std::io::ErrorKind::AlreadyExists => {
if lock_is_stale(lock_path) {
let _ = std::fs::remove_file(lock_path);
continue;
}
return AcquireLockOutcome::Contended;
}
Err(e) => {
tracing::debug!(
"advisory lock create at {:?} failed non-fatally: {}",
lock_path,
e
);
return AcquireLockOutcome::Skipped;
}
}
}
}
fn lock_is_stale(lock_path: &Path) -> bool {
let Ok(meta) = std::fs::metadata(lock_path) else {
return false;
};
let Ok(mtime) = meta.modified() else {
return false;
};
match SystemTime::now().duration_since(mtime) {
Ok(age) => age > STALE_LOCK_TIMEOUT,
Err(_) => false,
}
}
fn wait_for_publish_or_release(target: &Path, lock_path: &Path, timeout: Duration) -> WaitOutcome {
let start = Instant::now();
let mut delay = WAIT_POLL_INITIAL;
loop {
if target.exists() {
return WaitOutcome::Published;
}
if !lock_path.exists() {
if target.exists() {
return WaitOutcome::Published;
}
return WaitOutcome::LockReleased;
}
if start.elapsed() >= timeout {
return WaitOutcome::Timeout;
}
std::thread::sleep(delay);
delay = (delay * 2).min(WAIT_POLL_MAX);
}
}
struct LockGuard {
path: PathBuf,
}
impl Drop for LockGuard {
fn drop(&mut self) {
let _ = std::fs::remove_file(&self.path);
}
}
pub fn get_candle_bert_paths() -> Result<(PathBuf, PathBuf, PathBuf)> {
let cache_dir = get_cache_dir()?;
std::fs::create_dir_all(&cache_dir)?;
let config_path = cache_dir.join(CANDLE_BERT_CONFIG_FILENAME);
if !config_path.exists() {
download_file(CANDLE_BERT_CONFIG_URL, &config_path, "BERT config")?;
}
verify_file_size(&config_path, MIN_CANDLE_BERT_CONFIG_SIZE, "BERT config")?;
let tokenizer_path = cache_dir.join(CANDLE_BERT_TOKENIZER_FILENAME);
if !tokenizer_path.exists() {
download_file(CANDLE_BERT_TOKENIZER_URL, &tokenizer_path, "BERT tokenizer")?;
}
verify_file_size(
&tokenizer_path,
MIN_CANDLE_BERT_TOKENIZER_SIZE,
"BERT tokenizer",
)?;
let weights_path = cache_dir.join(CANDLE_BERT_WEIGHTS_FILENAME);
if !weights_path.exists() {
download_file(CANDLE_BERT_WEIGHTS_URL, &weights_path, "BERT weights")?;
}
verify_file_size(&weights_path, MIN_CANDLE_BERT_WEIGHTS_SIZE, "BERT weights")?;
Ok((config_path, tokenizer_path, weights_path))
}
fn verify_file_size(path: &PathBuf, min_size: u64, file_type: &str) -> Result<()> {
if !path.exists() {
return Err(MemdError::StorageError(format!(
"{} file not found at {:?}",
file_type, path
)));
}
let metadata = std::fs::metadata(path)?;
if metadata.len() < min_size {
return Err(MemdError::StorageError(format!(
"{} file too small ({} bytes), expected >= {} bytes. File may be corrupted, delete and retry.",
file_type,
metadata.len(),
min_size
)));
}
Ok(())
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_cache_dir() {
let dir = get_cache_dir().expect("should get cache dir");
assert!(dir.to_string_lossy().contains("memd"));
assert!(dir.to_string_lossy().contains("models"));
}
#[test]
fn test_embedding_model_defaults() {
let model = EmbeddingModel::default();
assert_eq!(model, EmbeddingModel::AllMiniLmL6V2);
assert_eq!(model.dimension(), 384);
assert_eq!(model.pooling_strategy(), PoolingStrategy::Mean);
assert!(!model.uses_instruction_format());
}
#[test]
fn test_qwen3_model_config() {
let model = EmbeddingModel::Qwen3Embedding0_6B;
assert_eq!(model.dimension(), 1024);
assert_eq!(model.pooling_strategy(), PoolingStrategy::LastToken);
assert!(model.uses_instruction_format());
}
#[test]
fn test_model_filenames() {
assert_eq!(
EmbeddingModel::AllMiniLmL6V2.model_filename(),
"all-MiniLM-L6-v2-quantized.onnx"
);
assert_eq!(
EmbeddingModel::Qwen3Embedding0_6B.model_filename(),
"qwen3-embedding-0.6b-q8.onnx"
);
}
#[test]
fn test_model_urls() {
assert!(EmbeddingModel::AllMiniLmL6V2
.model_url()
.contains("all-MiniLM-L6-v2"));
assert!(EmbeddingModel::Qwen3Embedding0_6B
.model_url()
.contains("Qwen3-Embedding"));
}
#[test]
fn test_candle_bert_constants() {
assert_eq!(
CANDLE_BERT_CONFIG_URL,
"https://huggingface.co/sentence-transformers/all-MiniLM-L6-v2/resolve/main/config.json"
);
assert_eq!(
CANDLE_BERT_TOKENIZER_URL,
"https://huggingface.co/sentence-transformers/all-MiniLM-L6-v2/resolve/main/tokenizer.json"
);
assert_eq!(
CANDLE_BERT_WEIGHTS_URL,
"https://huggingface.co/sentence-transformers/all-MiniLM-L6-v2/resolve/main/model.safetensors"
);
assert!(CANDLE_BERT_CONFIG_URL.starts_with("https://"));
assert!(CANDLE_BERT_TOKENIZER_URL.starts_with("https://"));
assert!(CANDLE_BERT_WEIGHTS_URL.starts_with("https://"));
}
#[test]
fn test_candle_bert_cache_filenames_are_distinct() {
assert_ne!(CANDLE_BERT_CONFIG_FILENAME, CANDLE_BERT_TOKENIZER_FILENAME);
assert_ne!(CANDLE_BERT_TOKENIZER_FILENAME, TOKENIZER_FILENAME);
assert!(CANDLE_BERT_TOKENIZER_FILENAME.contains("sentence-transformers"));
assert!(CANDLE_BERT_WEIGHTS_FILENAME.ends_with(".safetensors"));
}
#[test]
fn test_download_file_fails_cleanly_on_request_error() {
let tmp_dir = std::env::temp_dir().join(format!(
"memd-download-req-err-test-{}-{}",
std::process::id(),
std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.unwrap()
.as_nanos()
));
std::fs::create_dir_all(&tmp_dir).unwrap();
let target = tmp_dir.join("atomic-test.bin");
let result = download_file("http://127.0.0.1:1/never", &target, "test file");
assert!(result.is_err(), "expected download to error");
assert!(
!target.exists(),
"target should not exist after failed download: {:?}",
target
);
let leftovers: Vec<_> = std::fs::read_dir(&tmp_dir)
.unwrap()
.filter_map(|e| e.ok())
.map(|e| e.file_name())
.collect();
assert!(
leftovers.is_empty(),
"no temp/partial files should remain, found: {:?}",
leftovers
);
std::fs::remove_dir_all(&tmp_dir).ok();
}
#[test]
fn test_fsync_parent_dir_is_ok_on_real_dir() {
let tmp_dir = std::env::temp_dir().join(format!(
"memd-fsync-test-{}-{}",
std::process::id(),
std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.unwrap()
.as_nanos()
));
std::fs::create_dir_all(&tmp_dir).unwrap();
let f = tmp_dir.join("marker.bin");
std::fs::write(&f, b"x").unwrap();
fsync_parent_dir(&f);
fsync_parent_dir(std::path::Path::new("foo"));
std::fs::remove_dir_all(&tmp_dir).ok();
}
#[test]
fn test_download_file_fsyncs_parent_on_publish() {
use std::io::{Read, Write};
use std::net::TcpListener;
let body = b"candle-fsync-payload".to_vec();
let listener = TcpListener::bind("127.0.0.1:0").unwrap();
let addr = listener.local_addr().unwrap();
let body_for_server = body.clone();
let server_thread = std::thread::spawn(move || {
if let Ok((mut stream, _)) = listener.accept() {
let mut buf = [0u8; 1024];
let _ = stream.read(&mut buf);
let response = format!(
"HTTP/1.1 200 OK\r\nContent-Length: {}\r\nConnection: close\r\n\r\n",
body_for_server.len()
);
let _ = stream.write_all(response.as_bytes());
let _ = stream.write_all(&body_for_server);
}
});
let tmp_dir = std::env::temp_dir().join(format!(
"memd-download-fsync-test-{}-{}",
std::process::id(),
std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.unwrap()
.as_nanos()
));
std::fs::create_dir_all(&tmp_dir).unwrap();
let target = tmp_dir.join("candle.bin");
let url = format!("http://{}/x", addr);
let result = download_file(&url, &target, "fsync test");
server_thread.join().ok();
assert!(result.is_ok(), "download_file should succeed: {:?}", result);
let observed = std::fs::read(&target).unwrap();
assert_eq!(observed, body, "published bytes must match");
let leftovers: Vec<_> = std::fs::read_dir(&tmp_dir)
.unwrap()
.filter_map(|e| e.ok())
.map(|e| e.file_name().to_string_lossy().into_owned())
.filter(|n| n.contains("partial"))
.collect();
assert!(
leftovers.is_empty(),
"no partial tmp files should remain, found: {:?}",
leftovers
);
std::fs::remove_dir_all(&tmp_dir).ok();
}
#[test]
fn test_download_file_hf_config_integration() {
if std::env::var("MEMD_NETWORK_TESTS").ok().as_deref() != Some("1") {
eprintln!("skipping HF network test; set MEMD_NETWORK_TESTS=1 to run");
return;
}
let tmp_dir = std::env::temp_dir().join(format!(
"memd-hf-config-net-test-{}-{}",
std::process::id(),
std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.unwrap()
.as_nanos()
));
std::fs::create_dir_all(&tmp_dir).unwrap();
let target = tmp_dir.join(CANDLE_BERT_CONFIG_FILENAME);
let result = download_file(CANDLE_BERT_CONFIG_URL, &target, "BERT config");
assert!(
result.is_ok(),
"HF config download should succeed (network required): {:?}",
result
);
assert!(target.exists(), "config file should exist at target");
let meta = std::fs::metadata(&target).expect("metadata");
assert!(
meta.len() >= MIN_CANDLE_BERT_CONFIG_SIZE,
"downloaded config is too small ({} bytes, expected >= {})",
meta.len(),
MIN_CANDLE_BERT_CONFIG_SIZE
);
let leftovers: Vec<_> = std::fs::read_dir(&tmp_dir)
.unwrap()
.filter_map(|e| e.ok())
.map(|e| e.file_name().to_string_lossy().into_owned())
.filter(|n| n.contains("partial"))
.collect();
assert!(
leftovers.is_empty(),
"no partial tmp files should remain after a clean HF download, found: {:?}",
leftovers
);
std::fs::remove_dir_all(&tmp_dir).ok();
}
#[test]
fn test_download_file_is_first_writer_wins() {
use std::io::{Read, Write};
use std::net::TcpListener;
let body = b"fresh-download-payload".to_vec();
let listener = TcpListener::bind("127.0.0.1:0").unwrap();
let addr = listener.local_addr().unwrap();
let body_for_server = body.clone();
let server_thread = std::thread::spawn(move || {
if let Ok((mut stream, _)) = listener.accept() {
let mut buf = [0u8; 1024];
let _ = stream.read(&mut buf);
let response = format!(
"HTTP/1.1 200 OK\r\nContent-Length: {}\r\nConnection: close\r\n\r\n",
body_for_server.len()
);
let _ = stream.write_all(response.as_bytes());
let _ = stream.write_all(&body_for_server);
}
});
let tmp_dir = std::env::temp_dir().join(format!(
"memd-download-winner-test-{}-{}",
std::process::id(),
std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.unwrap()
.as_nanos()
));
std::fs::create_dir_all(&tmp_dir).unwrap();
let target = tmp_dir.join("winner.bin");
let sentinel = b"WINNER-BYTES".to_vec();
std::fs::write(&target, &sentinel).unwrap();
let url = format!("http://{}/x", addr);
let result = download_file(&url, &target, "race test");
server_thread.join().ok();
assert!(result.is_ok(), "download_file should succeed: {:?}", result);
let observed = std::fs::read(&target).unwrap();
assert_eq!(
observed, sentinel,
"pre-existing winner bytes must not be clobbered"
);
let leftovers: Vec<_> = std::fs::read_dir(&tmp_dir)
.unwrap()
.filter_map(|e| e.ok())
.map(|e| e.file_name().to_string_lossy().into_owned())
.filter(|n| n.contains("partial"))
.collect();
assert!(
leftovers.is_empty(),
"loser's partial tmp must be cleaned up, found: {:?}",
leftovers
);
std::fs::remove_dir_all(&tmp_dir).ok();
}
fn unique_tmp_dir(tag: &str) -> PathBuf {
std::env::temp_dir().join(format!(
"memd-{}-{}-{}",
tag,
std::process::id(),
SystemTime::now()
.duration_since(UNIX_EPOCH)
.unwrap()
.as_nanos()
))
}
#[test]
fn test_advisory_lock_path_preserves_full_filename() {
let p = PathBuf::from("/tmp/cache/model.onnx");
let lock = advisory_lock_path(&p);
assert_eq!(lock, PathBuf::from("/tmp/cache/model.onnx.lock"));
let p2 = PathBuf::from("/tmp/cache/tokenizer.json");
assert_eq!(
advisory_lock_path(&p2),
PathBuf::from("/tmp/cache/tokenizer.json.lock")
);
let p3 = PathBuf::from("/tmp/cache/blob");
assert_eq!(
advisory_lock_path(&p3),
PathBuf::from("/tmp/cache/blob.lock")
);
}
#[test]
fn test_advisory_lock_acquire_and_drop_cleans_up() {
let tmp_dir = unique_tmp_dir("lock-acquire");
std::fs::create_dir_all(&tmp_dir).unwrap();
let target = tmp_dir.join("target.bin");
let lock = advisory_lock_path(&target);
{
let outcome = try_acquire_advisory_lock(&lock);
match outcome {
AcquireLockOutcome::Acquired(_guard) => {
assert!(lock.exists(), "lock file should exist while guard is held");
}
_ => panic!("expected Acquired on first attempt"),
}
}
assert!(
!lock.exists(),
"lock file should be removed after guard drop"
);
std::fs::remove_dir_all(&tmp_dir).ok();
}
#[test]
fn test_advisory_lock_second_caller_is_contended() {
let tmp_dir = unique_tmp_dir("lock-contention");
std::fs::create_dir_all(&tmp_dir).unwrap();
let target = tmp_dir.join("target.bin");
let lock = advisory_lock_path(&target);
let first = try_acquire_advisory_lock(&lock);
let second = try_acquire_advisory_lock(&lock);
match (first, second) {
(AcquireLockOutcome::Acquired(_g1), AcquireLockOutcome::Contended) => {}
_ => panic!("expected Acquired then Contended on two racers"),
}
std::fs::remove_dir_all(&tmp_dir).ok();
}
#[test]
fn test_advisory_lock_stale_reclaim() {
let tmp_dir = unique_tmp_dir("lock-stale");
std::fs::create_dir_all(&tmp_dir).unwrap();
let target = tmp_dir.join("target.bin");
let lock = advisory_lock_path(&target);
std::fs::write(&lock, b"pid=99999 created_ms=0\n").unwrap();
assert!(lock.exists());
std::fs::remove_file(&lock).unwrap();
let outcome = try_acquire_advisory_lock(&lock);
assert!(matches!(outcome, AcquireLockOutcome::Acquired(_)));
std::fs::remove_dir_all(&tmp_dir).ok();
}
#[test]
fn test_lock_is_stale_by_mtime() {
let tmp_dir = unique_tmp_dir("lock-staleness");
std::fs::create_dir_all(&tmp_dir).unwrap();
let lock = tmp_dir.join("fresh.lock");
std::fs::write(&lock, b"x").unwrap();
assert!(
!lock_is_stale(&lock),
"fresh lock should not be classified stale"
);
let missing = tmp_dir.join("nope.lock");
assert!(!lock_is_stale(&missing));
std::fs::remove_dir_all(&tmp_dir).ok();
}
#[test]
fn test_wait_returns_published_when_target_appears() {
let tmp_dir = unique_tmp_dir("lock-wait-pub");
std::fs::create_dir_all(&tmp_dir).unwrap();
let target = tmp_dir.join("target.bin");
let lock = advisory_lock_path(&target);
std::fs::write(&lock, b"pid=1 created_ms=0").unwrap();
let target_clone = target.clone();
let lock_clone = lock.clone();
let t = std::thread::spawn(move || {
std::thread::sleep(Duration::from_millis(100));
std::fs::write(&target_clone, b"published").unwrap();
std::fs::remove_file(&lock_clone).ok();
});
let outcome = wait_for_publish_or_release(&target, &lock, Duration::from_secs(5));
t.join().unwrap();
assert!(matches!(outcome, WaitOutcome::Published));
std::fs::remove_dir_all(&tmp_dir).ok();
}
#[test]
fn test_wait_prefers_published_over_released_when_both_are_true() {
let tmp_dir = unique_tmp_dir("lock-wait-toctou");
std::fs::create_dir_all(&tmp_dir).unwrap();
let target = tmp_dir.join("target.bin");
let lock = advisory_lock_path(&target);
std::fs::write(&target, b"already-published").unwrap();
let outcome = wait_for_publish_or_release(&target, &lock, Duration::from_millis(200));
assert!(
matches!(outcome, WaitOutcome::Published),
"target present + lock absent must yield Published, got non-Published"
);
std::fs::remove_dir_all(&tmp_dir).ok();
}
#[test]
fn test_wait_returns_released_when_lock_disappears_without_target() {
let tmp_dir = unique_tmp_dir("lock-wait-rel");
std::fs::create_dir_all(&tmp_dir).unwrap();
let target = tmp_dir.join("target.bin");
let lock = advisory_lock_path(&target);
std::fs::write(&lock, b"pid=1 created_ms=0").unwrap();
let lock_clone = lock.clone();
let t = std::thread::spawn(move || {
std::thread::sleep(Duration::from_millis(100));
std::fs::remove_file(&lock_clone).ok();
});
let outcome = wait_for_publish_or_release(&target, &lock, Duration::from_secs(5));
t.join().unwrap();
assert!(
matches!(outcome, WaitOutcome::LockReleased),
"writer crash (lock gone, target absent) must yield LockReleased"
);
std::fs::remove_dir_all(&tmp_dir).ok();
}
#[test]
fn test_wait_returns_timeout_when_nothing_changes() {
let tmp_dir = unique_tmp_dir("lock-wait-to");
std::fs::create_dir_all(&tmp_dir).unwrap();
let target = tmp_dir.join("target.bin");
let lock = advisory_lock_path(&target);
std::fs::write(&lock, b"pid=1 created_ms=0").unwrap();
let outcome = wait_for_publish_or_release(&target, &lock, Duration::from_millis(200));
assert!(matches!(outcome, WaitOutcome::Timeout));
std::fs::remove_file(&lock).ok();
std::fs::remove_dir_all(&tmp_dir).ok();
}
#[test]
fn test_download_file_waiter_reuses_publication() {
use std::io::{Read, Write};
use std::net::TcpListener;
let body = b"one-download-shared".to_vec();
let listener = TcpListener::bind("127.0.0.1:0").unwrap();
let addr = listener.local_addr().unwrap();
let body_for_server = body.clone();
let server_thread = std::thread::spawn(move || {
if let Ok((mut stream, _)) = listener.accept() {
let mut buf = [0u8; 1024];
let _ = stream.read(&mut buf);
std::thread::sleep(Duration::from_millis(150));
let response = format!(
"HTTP/1.1 200 OK\r\nContent-Length: {}\r\nConnection: close\r\n\r\n",
body_for_server.len()
);
let _ = stream.write_all(response.as_bytes());
let _ = stream.write_all(&body_for_server);
}
});
let tmp_dir = unique_tmp_dir("download-coop");
std::fs::create_dir_all(&tmp_dir).unwrap();
let target = tmp_dir.join("shared.bin");
let target_clone = target.clone();
let url = format!("http://{}/x", addr);
let url_clone = url.clone();
let target_for_writer = target.clone();
let writer = std::thread::spawn(move || {
download_file(&url, &target_for_writer, "cooperative writer")
});
std::thread::sleep(Duration::from_millis(20));
let waiter = std::thread::spawn(move || {
download_file(&url_clone, &target_clone, "cooperative waiter")
});
let writer_res = writer.join().unwrap();
let waiter_res = waiter.join().unwrap();
server_thread.join().ok();
assert!(writer_res.is_ok(), "writer must succeed: {:?}", writer_res);
assert!(waiter_res.is_ok(), "waiter must succeed: {:?}", waiter_res);
let observed = std::fs::read(&target).unwrap();
assert_eq!(observed, body);
let leftovers: Vec<_> = std::fs::read_dir(&tmp_dir)
.unwrap()
.filter_map(|e| e.ok())
.map(|e| e.file_name().to_string_lossy().into_owned())
.filter(|n| n.contains("partial") || n.ends_with(".lock"))
.collect();
assert!(
leftovers.is_empty(),
"no partial or lock files should remain after coop download: {:?}",
leftovers
);
std::fs::remove_dir_all(&tmp_dir).ok();
}
}