use std::io::{Read, Write};
use std::path::{Path, PathBuf};
use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH};
use sha2::{Digest, Sha256};
use super::traits::PoolingStrategy;
use crate::error::{MemdError, Result};
#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
pub enum EmbeddingModel {
#[default]
AllMiniLmL6V2,
Qwen3Embedding0_6B,
}
impl EmbeddingModel {
pub fn dimension(&self) -> usize {
match self {
Self::AllMiniLmL6V2 => 384,
Self::Qwen3Embedding0_6B => 1024,
}
}
pub fn pooling_strategy(&self) -> PoolingStrategy {
match self {
Self::AllMiniLmL6V2 => PoolingStrategy::Mean,
Self::Qwen3Embedding0_6B => PoolingStrategy::LastToken,
}
}
pub fn model_url(&self) -> &'static str {
match self {
Self::AllMiniLmL6V2 => ALL_MINILM_MODEL_URL,
Self::Qwen3Embedding0_6B => QWEN3_MODEL_URL,
}
}
pub fn tokenizer_url(&self) -> &'static str {
match self {
Self::AllMiniLmL6V2 => ALL_MINILM_TOKENIZER_URL,
Self::Qwen3Embedding0_6B => QWEN3_TOKENIZER_URL,
}
}
pub fn model_sha256(&self) -> &'static str {
match self {
Self::AllMiniLmL6V2 => ALL_MINILM_MODEL_SHA256,
Self::Qwen3Embedding0_6B => QWEN3_MODEL_SHA256,
}
}
pub fn tokenizer_sha256(&self) -> &'static str {
match self {
Self::AllMiniLmL6V2 => ALL_MINILM_TOKENIZER_SHA256,
Self::Qwen3Embedding0_6B => QWEN3_TOKENIZER_SHA256,
}
}
pub fn model_filename(&self) -> &'static str {
match self {
Self::AllMiniLmL6V2 => "all-MiniLM-L6-v2-quantized.onnx",
Self::Qwen3Embedding0_6B => "qwen3-embedding-0.6b-q8.onnx",
}
}
pub fn tokenizer_filename(&self) -> &'static str {
match self {
Self::AllMiniLmL6V2 => "all-minilm-l6-v2-tokenizer.json",
Self::Qwen3Embedding0_6B => "qwen3-embedding-0.6b-tokenizer.json",
}
}
pub fn min_model_size(&self) -> u64 {
match self {
Self::AllMiniLmL6V2 => 20_000_000, Self::Qwen3Embedding0_6B => 500_000_000, }
}
pub fn min_tokenizer_size(&self) -> u64 {
match self {
Self::AllMiniLmL6V2 => 500_000, Self::Qwen3Embedding0_6B => 10_000, }
}
pub fn uses_instruction_format(&self) -> bool {
match self {
Self::AllMiniLmL6V2 => false,
Self::Qwen3Embedding0_6B => true,
}
}
pub fn requires_position_ids(&self) -> bool {
match self {
Self::AllMiniLmL6V2 => false,
Self::Qwen3Embedding0_6B => true,
}
}
pub fn kv_cache_config(&self) -> Option<KvCacheConfig> {
match self {
Self::AllMiniLmL6V2 => None,
Self::Qwen3Embedding0_6B => Some(KvCacheConfig {
num_layers: 28,
num_kv_heads: 8,
head_dim: 128,
}),
}
}
}
#[derive(Debug, Clone, Copy)]
pub struct KvCacheConfig {
pub num_layers: usize,
pub num_kv_heads: usize,
pub head_dim: usize,
}
const ALL_MINILM_MODEL_URL: &str =
"https://huggingface.co/Xenova/all-MiniLM-L6-v2/resolve/751bff37182d3f1213fa05d7196b954e230abad9/onnx/model_quantized.onnx";
const ALL_MINILM_MODEL_SHA256: &str =
"afdb6f1a0e45b715d0bb9b11772f032c399babd23bfc31fed1c170afc848bdb1";
const ALL_MINILM_TOKENIZER_URL: &str =
"https://huggingface.co/Xenova/all-MiniLM-L6-v2/resolve/751bff37182d3f1213fa05d7196b954e230abad9/tokenizer.json";
const ALL_MINILM_TOKENIZER_SHA256: &str =
"da0e79933b9ed51798a3ae27893d3c5fa4a201126cef75586296df9b4d2c62a0";
const QWEN3_MODEL_URL: &str =
"https://huggingface.co/onnx-community/Qwen3-Embedding-0.6B-ONNX/resolve/c25a394dd583836952667c12f008335071b3f43d/onnx/model_int8.onnx";
const QWEN3_MODEL_SHA256: &str = "6d0ea863f78b4a84afa3c7fcba1ec341572b5e28121aef77b7092b1dfdf679c7";
const QWEN3_TOKENIZER_URL: &str =
"https://huggingface.co/onnx-community/Qwen3-Embedding-0.6B-ONNX/resolve/c25a394dd583836952667c12f008335071b3f43d/tokenizer.json";
const QWEN3_TOKENIZER_SHA256: &str =
"def76fb086971c7867b829c23a26261e38d9d74e02139253b38aeb9df8b4b50a";
const MODEL_URL: &str = ALL_MINILM_MODEL_URL;
const MODEL_SHA256: &str = ALL_MINILM_MODEL_SHA256;
const MODEL_FILENAME: &str = "all-MiniLM-L6-v2-quantized.onnx";
const TOKENIZER_URL: &str = ALL_MINILM_TOKENIZER_URL;
const TOKENIZER_SHA256: &str = ALL_MINILM_TOKENIZER_SHA256;
const TOKENIZER_FILENAME: &str = "tokenizer.json";
const MIN_MODEL_SIZE: u64 = 20_000_000;
const MIN_TOKENIZER_SIZE: u64 = 500_000;
const CANDLE_BERT_CONFIG_URL: &str =
"https://huggingface.co/sentence-transformers/all-MiniLM-L6-v2/resolve/1110a243fdf4706b3f48f1d95db1a4f5529b4d41/config.json";
const CANDLE_BERT_CONFIG_SHA256: &str =
"953f9c0d463486b10a6871cc2fd59f223b2c70184f49815e7efbcab5d8908b41";
const CANDLE_BERT_CONFIG_FILENAME: &str = "sentence-transformers-all-MiniLM-L6-v2-config.json";
const CANDLE_BERT_TOKENIZER_URL: &str =
"https://huggingface.co/sentence-transformers/all-MiniLM-L6-v2/resolve/1110a243fdf4706b3f48f1d95db1a4f5529b4d41/tokenizer.json";
const CANDLE_BERT_TOKENIZER_SHA256: &str =
"be50c3628f2bf5bb5e3a7f17b1f74611b2561a3a27eeab05e5aa30f411572037";
const CANDLE_BERT_TOKENIZER_FILENAME: &str =
"sentence-transformers-all-MiniLM-L6-v2-tokenizer.json";
const CANDLE_BERT_WEIGHTS_URL: &str =
"https://huggingface.co/sentence-transformers/all-MiniLM-L6-v2/resolve/1110a243fdf4706b3f48f1d95db1a4f5529b4d41/model.safetensors";
const CANDLE_BERT_WEIGHTS_SHA256: &str =
"53aa51172d142c89d9012cce15ae4d6cc0ca6895895114379cacb4fab128d9db";
const CANDLE_BERT_WEIGHTS_FILENAME: &str = "sentence-transformers-all-MiniLM-L6-v2.safetensors";
const MIN_CANDLE_BERT_CONFIG_SIZE: u64 = 100; const MIN_CANDLE_BERT_TOKENIZER_SIZE: u64 = 100_000; const MIN_CANDLE_BERT_WEIGHTS_SIZE: u64 = 80_000_000;
const CANDLE_BGE_CONFIG_URL: &str =
"https://huggingface.co/BAAI/bge-base-en-v1.5/resolve/a5beb1e3e68b9ab74eb54cfd186867f64f240e1a/config.json";
const CANDLE_BGE_CONFIG_SHA256: &str =
"bc00af31a4a31b74040d73370aa83b62da34c90b75eb77bfa7db039d90abd591";
const CANDLE_BGE_CONFIG_FILENAME: &str = "bge-base-en-v1.5-config.json";
const CANDLE_BGE_TOKENIZER_URL: &str =
"https://huggingface.co/BAAI/bge-base-en-v1.5/resolve/a5beb1e3e68b9ab74eb54cfd186867f64f240e1a/tokenizer.json";
const CANDLE_BGE_TOKENIZER_SHA256: &str =
"d241a60d5e8f04cc1b2b3e9ef7a4921b27bf526d9f6050ab90f9267a1f9e5c66";
const CANDLE_BGE_TOKENIZER_FILENAME: &str = "bge-base-en-v1.5-tokenizer.json";
const CANDLE_BGE_WEIGHTS_URL: &str =
"https://huggingface.co/BAAI/bge-base-en-v1.5/resolve/a5beb1e3e68b9ab74eb54cfd186867f64f240e1a/model.safetensors";
const CANDLE_BGE_WEIGHTS_SHA256: &str =
"c7c1988aae201f80cf91a5dbbd5866409503b89dcaba877ca6dba7dd0a5167d7";
const CANDLE_BGE_WEIGHTS_FILENAME: &str = "bge-base-en-v1.5.safetensors";
const MIN_CANDLE_BGE_CONFIG_SIZE: u64 = 100; const MIN_CANDLE_BGE_TOKENIZER_SIZE: u64 = 100_000; const MIN_CANDLE_BGE_WEIGHTS_SIZE: u64 = 400_000_000;
#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
pub enum CandleModel {
#[default]
MiniLm,
BgeBase,
}
impl CandleModel {
pub fn pooling_strategy(&self) -> PoolingStrategy {
match self {
Self::MiniLm => PoolingStrategy::Mean,
Self::BgeBase => PoolingStrategy::Cls,
}
}
pub fn query_prefix(&self) -> Option<&'static str> {
match self {
Self::MiniLm => None,
Self::BgeBase => Some("Represent this sentence for searching relevant passages: "),
}
}
pub fn hf_id(&self) -> &'static str {
match self {
Self::MiniLm => "sentence-transformers/all-MiniLM-L6-v2",
Self::BgeBase => "BAAI/bge-base-en-v1.5",
}
}
}
pub fn get_cache_dir() -> Result<PathBuf> {
let cache_dir = dirs::cache_dir()
.ok_or_else(|| MemdError::StorageError("cannot determine cache directory".into()))?
.join("memd")
.join("models");
Ok(cache_dir)
}
pub fn get_model_path() -> Result<PathBuf> {
let cache_dir = get_cache_dir()?;
let model_path = cache_dir.join(MODEL_FILENAME);
if !model_path.exists() {
download_model(&cache_dir)?;
}
verify_model_exists(&model_path)?;
Ok(model_path)
}
pub fn get_tokenizer_path() -> Result<PathBuf> {
let cache_dir = get_cache_dir()?;
let tokenizer_path = cache_dir.join(TOKENIZER_FILENAME);
if !tokenizer_path.exists() {
download_tokenizer(&cache_dir)?;
}
verify_tokenizer_exists(&tokenizer_path)?;
Ok(tokenizer_path)
}
pub fn verify_model_exists(path: &PathBuf) -> Result<()> {
if !path.exists() {
return Err(MemdError::StorageError(format!(
"model file not found at {:?}",
path
)));
}
let metadata = std::fs::metadata(path)?;
if metadata.len() < MIN_MODEL_SIZE {
return Err(MemdError::StorageError(format!(
"model file too small ({} bytes), expected >= {} bytes. File may be corrupted, delete and retry.",
metadata.len(),
MIN_MODEL_SIZE
)));
}
Ok(())
}
fn verify_tokenizer_exists(path: &PathBuf) -> Result<()> {
if !path.exists() {
return Err(MemdError::StorageError(format!(
"tokenizer file not found at {:?}",
path
)));
}
let metadata = std::fs::metadata(path)?;
if metadata.len() < MIN_TOKENIZER_SIZE {
return Err(MemdError::StorageError(format!(
"tokenizer file too small ({} bytes), expected >= {} bytes. File may be corrupted, delete and retry.",
metadata.len(),
MIN_TOKENIZER_SIZE
)));
}
Ok(())
}
pub fn download_model(cache_dir: &Path) -> Result<()> {
let model_path = cache_dir.join(MODEL_FILENAME);
download_file(
MODEL_URL,
&model_path,
"embedding model",
Some(MODEL_SHA256),
)
}
fn download_tokenizer(cache_dir: &Path) -> Result<()> {
let tokenizer_path = cache_dir.join(TOKENIZER_FILENAME);
download_file(
TOKENIZER_URL,
&tokenizer_path,
"tokenizer",
Some(TOKENIZER_SHA256),
)
}
pub fn get_model_path_for(model: EmbeddingModel) -> Result<PathBuf> {
let cache_dir = get_cache_dir()?;
let model_path = cache_dir.join(model.model_filename());
if !model_path.exists() {
download_file(
model.model_url(),
&model_path,
model.model_filename(),
Some(model.model_sha256()),
)?;
}
verify_file_size(&model_path, model.min_model_size(), "model")?;
Ok(model_path)
}
pub fn get_tokenizer_path_for(model: EmbeddingModel) -> Result<PathBuf> {
let cache_dir = get_cache_dir()?;
let tokenizer_path = cache_dir.join(model.tokenizer_filename());
if !tokenizer_path.exists() {
download_file(
model.tokenizer_url(),
&tokenizer_path,
model.tokenizer_filename(),
Some(model.tokenizer_sha256()),
)?;
}
verify_file_size(&tokenizer_path, model.min_tokenizer_size(), "tokenizer")?;
Ok(tokenizer_path)
}
fn download_file(
url: &str,
path: &PathBuf,
name: &str,
expected_sha256: Option<&str>,
) -> Result<()> {
use std::sync::atomic::{AtomicU64, Ordering};
static COUNTER: AtomicU64 = AtomicU64::new(0);
let cache_dir = path.parent().unwrap();
std::fs::create_dir_all(cache_dir)?;
let lock_path = advisory_lock_path(path);
let mut _lock_guard: Option<LockGuard> = None;
match try_acquire_advisory_lock(&lock_path) {
AcquireLockOutcome::Acquired(guard) => {
_lock_guard = Some(guard);
}
AcquireLockOutcome::Contended => {
match wait_for_publish_or_release(path, &lock_path, advisory_lock_wait_timeout()) {
WaitOutcome::Published => {
tracing::info!("{} published by concurrent writer; reusing", name);
return Ok(());
}
WaitOutcome::LockReleased | WaitOutcome::Timeout => {
tracing::debug!(
"advisory lock wait fell through for {}; proceeding with race-safe download",
name
);
}
}
}
AcquireLockOutcome::Skipped => {
tracing::debug!(
"advisory lock unavailable for {} (non-fatal); proceeding with race-safe download",
name
);
}
}
tracing::info!("Downloading {} to {:?}", name, path);
let response = ureq::get(url)
.call()
.map_err(|e| MemdError::StorageError(format!("failed to download {}: {}", name, e)))?;
let tmp_path = path.with_extension(format!(
"partial.{}.{:?}.{}",
std::process::id(),
std::thread::current().id(),
COUNTER.fetch_add(1, Ordering::Relaxed)
));
let mut file = std::fs::File::create(&tmp_path)?;
let mut reader = response.into_reader();
let mut hasher = Sha256::new();
let mut buffer = [0u8; 16 * 1024];
let copy_result = (|| -> std::io::Result<()> {
loop {
let read = reader.read(&mut buffer)?;
if read == 0 {
break;
}
hasher.update(&buffer[..read]);
file.write_all(&buffer[..read])?;
}
file.sync_all()
})();
if let Err(e) = copy_result {
let _ = std::fs::remove_file(&tmp_path);
return Err(MemdError::StorageError(format!(
"failed to stream {} to {:?}: {}",
name, tmp_path, e
)));
}
let actual_sha256 = format!("{:x}", hasher.finalize());
drop(file);
if let Some(expected) = expected_sha256 {
if let Err(error) = enforce_sha256(&actual_sha256, expected, name) {
let _ = std::fs::remove_file(&tmp_path);
return Err(error);
}
}
match std::fs::hard_link(&tmp_path, path) {
Ok(()) => {
let _ = std::fs::remove_file(&tmp_path);
fsync_parent_dir(path);
}
Err(e) if e.kind() == std::io::ErrorKind::AlreadyExists => {
let _ = std::fs::remove_file(&tmp_path);
}
Err(e) => {
let _ = std::fs::remove_file(&tmp_path);
return Err(MemdError::StorageError(format!(
"failed to publish {} to {:?}: {}",
name, path, e
)));
}
}
tracing::info!("{} downloaded successfully", name);
Ok(())
}
#[cfg(test)]
fn sha256_hex_of_file(path: &Path) -> Result<String> {
let mut file = std::fs::File::open(path)?;
let mut hasher = Sha256::new();
let mut buffer = [0u8; 16 * 1024];
loop {
let read = file.read(&mut buffer)?;
if read == 0 {
break;
}
hasher.update(&buffer[..read]);
}
Ok(format!("{:x}", hasher.finalize()))
}
fn enforce_sha256(actual: &str, expected: &str, name: &str) -> Result<()> {
if actual.eq_ignore_ascii_case(expected) {
return Ok(());
}
Err(MemdError::StorageError(format!(
"{} sha256 mismatch: expected {}, actual {}; corrupt partial download was deleted",
name, expected, actual
)))
}
fn fsync_parent_dir(path: &std::path::Path) {
let Some(parent) = path.parent() else {
return;
};
match std::fs::File::open(parent) {
Ok(dir) => {
if let Err(e) = dir.sync_all() {
tracing::debug!(
"fsync on parent dir {:?} failed (best-effort): {}",
parent,
e
);
}
}
Err(e) => {
tracing::debug!(
"opening parent dir {:?} for fsync failed (best-effort): {}",
parent,
e
);
}
}
}
const ADVISORY_LOCK_WAIT: Duration = Duration::from_secs(900);
const STALE_LOCK_TIMEOUT: Duration = Duration::from_secs(3600);
const WAIT_POLL_INITIAL: Duration = Duration::from_millis(50);
const WAIT_POLL_MAX: Duration = Duration::from_secs(2);
#[cfg(test)]
static TEST_WAIT_OVERRIDE_MS: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(0);
#[cfg(test)]
fn advisory_lock_wait_timeout() -> Duration {
let ms = TEST_WAIT_OVERRIDE_MS.load(std::sync::atomic::Ordering::Relaxed);
if ms == 0 {
ADVISORY_LOCK_WAIT
} else {
Duration::from_millis(ms)
}
}
#[cfg(not(test))]
fn advisory_lock_wait_timeout() -> Duration {
ADVISORY_LOCK_WAIT
}
fn advisory_lock_path(path: &Path) -> PathBuf {
match path.file_name() {
Some(name) => {
let mut new_name = name.to_os_string();
new_name.push(".lock");
path.with_file_name(new_name)
}
None => path.with_extension("lock"),
}
}
enum AcquireLockOutcome {
Acquired(LockGuard),
Contended,
Skipped,
}
enum WaitOutcome {
Published,
LockReleased,
Timeout,
}
fn try_acquire_advisory_lock(lock_path: &Path) -> AcquireLockOutcome {
loop {
match std::fs::OpenOptions::new()
.write(true)
.create_new(true)
.open(lock_path)
{
Ok(mut f) => {
let now_ms = SystemTime::now()
.duration_since(UNIX_EPOCH)
.map(|d| d.as_millis())
.unwrap_or(0);
let _ = writeln!(f, "pid={} created_ms={}", std::process::id(), now_ms);
let _ = f.sync_all();
drop(f);
return AcquireLockOutcome::Acquired(LockGuard {
path: lock_path.to_path_buf(),
});
}
Err(e) if e.kind() == std::io::ErrorKind::AlreadyExists => {
if lock_is_stale(lock_path) {
let _ = std::fs::remove_file(lock_path);
continue;
}
return AcquireLockOutcome::Contended;
}
Err(e) => {
tracing::debug!(
"advisory lock create at {:?} failed non-fatally: {}",
lock_path,
e
);
return AcquireLockOutcome::Skipped;
}
}
}
}
fn lock_is_stale(lock_path: &Path) -> bool {
let Ok(meta) = std::fs::metadata(lock_path) else {
return false;
};
let Ok(mtime) = meta.modified() else {
return false;
};
match SystemTime::now().duration_since(mtime) {
Ok(age) => age > STALE_LOCK_TIMEOUT,
Err(_) => false,
}
}
fn wait_for_publish_or_release(target: &Path, lock_path: &Path, timeout: Duration) -> WaitOutcome {
let start = Instant::now();
let mut delay = WAIT_POLL_INITIAL;
loop {
if target.exists() {
return WaitOutcome::Published;
}
if !lock_path.exists() {
if target.exists() {
return WaitOutcome::Published;
}
return WaitOutcome::LockReleased;
}
if start.elapsed() >= timeout {
return WaitOutcome::Timeout;
}
std::thread::sleep(delay);
delay = (delay * 2).min(WAIT_POLL_MAX);
}
}
struct LockGuard {
path: PathBuf,
}
impl Drop for LockGuard {
fn drop(&mut self) {
let _ = std::fs::remove_file(&self.path);
}
}
pub fn get_candle_bert_paths() -> Result<(PathBuf, PathBuf, PathBuf)> {
get_candle_model_paths(CandleModel::MiniLm)
}
pub fn get_candle_model_paths(model: CandleModel) -> Result<(PathBuf, PathBuf, PathBuf)> {
struct Spec {
config: (&'static str, &'static str, &'static str, u64),
tokenizer: (&'static str, &'static str, &'static str, u64),
weights: (&'static str, &'static str, &'static str, u64),
}
let spec = match model {
CandleModel::MiniLm => Spec {
config: (
CANDLE_BERT_CONFIG_URL,
CANDLE_BERT_CONFIG_FILENAME,
CANDLE_BERT_CONFIG_SHA256,
MIN_CANDLE_BERT_CONFIG_SIZE,
),
tokenizer: (
CANDLE_BERT_TOKENIZER_URL,
CANDLE_BERT_TOKENIZER_FILENAME,
CANDLE_BERT_TOKENIZER_SHA256,
MIN_CANDLE_BERT_TOKENIZER_SIZE,
),
weights: (
CANDLE_BERT_WEIGHTS_URL,
CANDLE_BERT_WEIGHTS_FILENAME,
CANDLE_BERT_WEIGHTS_SHA256,
MIN_CANDLE_BERT_WEIGHTS_SIZE,
),
},
CandleModel::BgeBase => Spec {
config: (
CANDLE_BGE_CONFIG_URL,
CANDLE_BGE_CONFIG_FILENAME,
CANDLE_BGE_CONFIG_SHA256,
MIN_CANDLE_BGE_CONFIG_SIZE,
),
tokenizer: (
CANDLE_BGE_TOKENIZER_URL,
CANDLE_BGE_TOKENIZER_FILENAME,
CANDLE_BGE_TOKENIZER_SHA256,
MIN_CANDLE_BGE_TOKENIZER_SIZE,
),
weights: (
CANDLE_BGE_WEIGHTS_URL,
CANDLE_BGE_WEIGHTS_FILENAME,
CANDLE_BGE_WEIGHTS_SHA256,
MIN_CANDLE_BGE_WEIGHTS_SIZE,
),
},
};
let cache_dir = get_cache_dir()?;
std::fs::create_dir_all(&cache_dir)?;
let fetch =
|(url, filename, sha, min_size): (&str, &str, &str, u64), kind: &str| -> Result<PathBuf> {
let path = cache_dir.join(filename);
if !path.exists() {
download_file(url, &path, kind, Some(sha))?;
}
verify_file_size(&path, min_size, kind)?;
Ok(path)
};
let config_path = fetch(spec.config, "BERT config")?;
let tokenizer_path = fetch(spec.tokenizer, "BERT tokenizer")?;
let weights_path = fetch(spec.weights, "BERT weights")?;
Ok((config_path, tokenizer_path, weights_path))
}
fn verify_file_size(path: &PathBuf, min_size: u64, file_type: &str) -> Result<()> {
if !path.exists() {
return Err(MemdError::StorageError(format!(
"{} file not found at {:?}",
file_type, path
)));
}
let metadata = std::fs::metadata(path)?;
if metadata.len() < min_size {
return Err(MemdError::StorageError(format!(
"{} file too small ({} bytes), expected >= {} bytes. File may be corrupted, delete and retry.",
file_type,
metadata.len(),
min_size
)));
}
Ok(())
}
#[cfg(test)]
mod tests {
use super::*;
fn loopback_listener_or_skip(test_name: &str) -> Option<std::net::TcpListener> {
match std::net::TcpListener::bind("127.0.0.1:0") {
Ok(listener) => Some(listener),
Err(err) if err.kind() == std::io::ErrorKind::PermissionDenied => {
eprintln!(
"skipping {test_name}: loopback bind is not permitted in this environment"
);
None
}
Err(err) => panic!("bind loopback listener for {test_name}: {err}"),
}
}
#[test]
fn test_cache_dir() {
let dir = get_cache_dir().expect("should get cache dir");
assert!(dir.to_string_lossy().contains("memd"));
assert!(dir.to_string_lossy().contains("models"));
}
#[test]
fn test_embedding_model_defaults() {
let model = EmbeddingModel::default();
assert_eq!(model, EmbeddingModel::AllMiniLmL6V2);
assert_eq!(model.dimension(), 384);
assert_eq!(model.pooling_strategy(), PoolingStrategy::Mean);
assert!(!model.uses_instruction_format());
}
#[test]
fn test_qwen3_model_config() {
let model = EmbeddingModel::Qwen3Embedding0_6B;
assert_eq!(model.dimension(), 1024);
assert_eq!(model.pooling_strategy(), PoolingStrategy::LastToken);
assert!(model.uses_instruction_format());
}
#[test]
fn test_model_filenames() {
assert_eq!(
EmbeddingModel::AllMiniLmL6V2.model_filename(),
"all-MiniLM-L6-v2-quantized.onnx"
);
assert_eq!(
EmbeddingModel::Qwen3Embedding0_6B.model_filename(),
"qwen3-embedding-0.6b-q8.onnx"
);
}
#[test]
fn test_model_urls() {
assert!(EmbeddingModel::AllMiniLmL6V2
.model_url()
.contains("all-MiniLM-L6-v2"));
assert!(EmbeddingModel::Qwen3Embedding0_6B
.model_url()
.contains("Qwen3-Embedding"));
}
fn assert_pinned_hf_url(url: &str) {
let marker = "/resolve/";
let start = url
.find(marker)
.expect("url should contain revision marker")
+ marker.len();
let rest = &url[start..];
let revision = rest.split('/').next().expect("revision segment");
assert_eq!(
revision.len(),
40,
"revision should be a commit hash: {url}"
);
assert!(
revision.bytes().all(|byte| byte.is_ascii_hexdigit()),
"revision should be ascii hex: {url}"
);
let mutable_ref = ["resolve", "main"].join("/");
assert!(
!url.contains(&mutable_ref),
"url should not use a mutable ref: {url}"
);
}
fn assert_sha256_constant(hash: &str) {
assert_eq!(hash.len(), 64, "sha256 should be 64 hex chars: {hash}");
assert!(
hash.bytes()
.all(|byte| byte.is_ascii_hexdigit() && !byte.is_ascii_uppercase()),
"sha256 should be lowercase hex: {hash}"
);
}
fn sha256_hex_of_bytes(bytes: &[u8]) -> String {
let mut hasher = Sha256::new();
hasher.update(bytes);
format!("{:x}", hasher.finalize())
}
#[test]
fn test_all_download_urls_are_revision_pinned() {
let urls = [
ALL_MINILM_MODEL_URL,
ALL_MINILM_TOKENIZER_URL,
QWEN3_MODEL_URL,
QWEN3_TOKENIZER_URL,
MODEL_URL,
TOKENIZER_URL,
CANDLE_BERT_CONFIG_URL,
CANDLE_BERT_TOKENIZER_URL,
CANDLE_BERT_WEIGHTS_URL,
CANDLE_BGE_CONFIG_URL,
CANDLE_BGE_TOKENIZER_URL,
CANDLE_BGE_WEIGHTS_URL,
EmbeddingModel::AllMiniLmL6V2.model_url(),
EmbeddingModel::AllMiniLmL6V2.tokenizer_url(),
EmbeddingModel::Qwen3Embedding0_6B.model_url(),
EmbeddingModel::Qwen3Embedding0_6B.tokenizer_url(),
];
for url in urls {
assert_pinned_hf_url(url);
}
}
#[test]
fn test_sha256_constants_are_lowercase_hex() {
let hashes = [
ALL_MINILM_MODEL_SHA256,
ALL_MINILM_TOKENIZER_SHA256,
QWEN3_MODEL_SHA256,
QWEN3_TOKENIZER_SHA256,
MODEL_SHA256,
TOKENIZER_SHA256,
CANDLE_BERT_CONFIG_SHA256,
CANDLE_BERT_TOKENIZER_SHA256,
CANDLE_BERT_WEIGHTS_SHA256,
CANDLE_BGE_CONFIG_SHA256,
CANDLE_BGE_TOKENIZER_SHA256,
CANDLE_BGE_WEIGHTS_SHA256,
];
for hash in hashes {
assert_sha256_constant(hash);
}
}
#[test]
fn test_candle_model_selection_and_properties() {
assert_eq!(CandleModel::default(), CandleModel::MiniLm);
assert_eq!(
CandleModel::MiniLm.pooling_strategy(),
PoolingStrategy::Mean
);
assert!(CandleModel::MiniLm.query_prefix().is_none());
assert_eq!(
CandleModel::BgeBase.pooling_strategy(),
PoolingStrategy::Cls
);
let prefix = CandleModel::BgeBase.query_prefix().unwrap();
assert!(prefix.starts_with("Represent this sentence"));
assert!(CandleModel::BgeBase.hf_id().contains("bge-base-en-v1.5"));
}
#[test]
fn test_candle_bert_constants() {
assert_eq!(
CANDLE_BERT_CONFIG_URL,
"https://huggingface.co/sentence-transformers/all-MiniLM-L6-v2/resolve/1110a243fdf4706b3f48f1d95db1a4f5529b4d41/config.json"
);
assert_eq!(
CANDLE_BERT_TOKENIZER_URL,
"https://huggingface.co/sentence-transformers/all-MiniLM-L6-v2/resolve/1110a243fdf4706b3f48f1d95db1a4f5529b4d41/tokenizer.json"
);
assert_eq!(
CANDLE_BERT_WEIGHTS_URL,
"https://huggingface.co/sentence-transformers/all-MiniLM-L6-v2/resolve/1110a243fdf4706b3f48f1d95db1a4f5529b4d41/model.safetensors"
);
assert!(CANDLE_BERT_CONFIG_URL.starts_with("https://"));
assert!(CANDLE_BERT_TOKENIZER_URL.starts_with("https://"));
assert!(CANDLE_BERT_WEIGHTS_URL.starts_with("https://"));
}
#[test]
fn test_sha256_verification_reports_expected_and_actual() {
let tmp_dir = unique_tmp_dir("hash-verify");
std::fs::create_dir_all(&tmp_dir).unwrap();
let file = tmp_dir.join("wrong.bin");
std::fs::write(&file, b"wrong content").unwrap();
let actual = sha256_hex_of_file(&file).unwrap();
let expected = sha256_hex_of_bytes(b"expected content");
let err = enforce_sha256(&actual, &expected, "wrong test file").unwrap_err();
let message = err.to_string();
assert!(message.contains(&expected));
assert!(message.contains(&actual));
std::fs::remove_dir_all(&tmp_dir).ok();
}
#[test]
fn test_download_file_rejects_corrupt_hash_without_publish() {
use std::io::{Read, Write};
let body = b"wrong loopback bytes".to_vec();
let Some(listener) =
loopback_listener_or_skip("test_download_file_rejects_corrupt_hash_without_publish")
else {
return;
};
let addr = listener.local_addr().unwrap();
let body_for_server = body.clone();
let server_thread = std::thread::spawn(move || {
if let Ok((mut stream, _)) = listener.accept() {
let mut buf = [0u8; 1024];
let _ = stream.read(&mut buf);
let response = format!(
"HTTP/1.1 200 OK\r\nContent-Length: {}\r\nConnection: close\r\n\r\n",
body_for_server.len()
);
let _ = stream.write_all(response.as_bytes());
let _ = stream.write_all(&body_for_server);
}
});
let tmp_dir = unique_tmp_dir("download-hash-reject");
std::fs::create_dir_all(&tmp_dir).unwrap();
let target = tmp_dir.join("corrupt.bin");
let expected = sha256_hex_of_bytes(b"different expected bytes");
let url = format!("http://{}/x", addr);
let result = download_file(&url, &target, "corrupt test", Some(&expected));
server_thread.join().ok();
assert!(result.is_err(), "hash mismatch should fail");
assert!(!target.exists(), "mismatched download must not publish");
let leftovers: Vec<_> = std::fs::read_dir(&tmp_dir)
.unwrap()
.filter_map(|entry| entry.ok())
.map(|entry| entry.file_name().to_string_lossy().into_owned())
.filter(|name| name.contains("partial"))
.collect();
assert!(
leftovers.is_empty(),
"no partial tmp files should remain, found: {:?}",
leftovers
);
std::fs::remove_dir_all(&tmp_dir).ok();
}
#[test]
fn test_download_file_accepts_matching_hash_and_publishes() {
use std::io::{Read, Write};
let body = b"known loopback bytes".to_vec();
let Some(listener) =
loopback_listener_or_skip("test_download_file_accepts_matching_hash_and_publishes")
else {
return;
};
let addr = listener.local_addr().unwrap();
let body_for_server = body.clone();
let server_thread = std::thread::spawn(move || {
if let Ok((mut stream, _)) = listener.accept() {
let mut buf = [0u8; 1024];
let _ = stream.read(&mut buf);
let response = format!(
"HTTP/1.1 200 OK\r\nContent-Length: {}\r\nConnection: close\r\n\r\n",
body_for_server.len()
);
let _ = stream.write_all(response.as_bytes());
let _ = stream.write_all(&body_for_server);
}
});
let tmp_dir = unique_tmp_dir("download-hash-ok");
std::fs::create_dir_all(&tmp_dir).unwrap();
let target = tmp_dir.join("pinned.bin");
let expected = sha256_hex_of_bytes(&body);
let url = format!("http://{}/x", addr);
let result = download_file(&url, &target, "pinned test", Some(&expected));
server_thread.join().ok();
assert!(result.is_ok(), "matching hash should succeed: {:?}", result);
assert_eq!(std::fs::read(&target).unwrap(), body);
std::fs::remove_dir_all(&tmp_dir).ok();
}
#[test]
fn test_candle_bert_cache_filenames_are_distinct() {
assert_ne!(CANDLE_BERT_CONFIG_FILENAME, CANDLE_BERT_TOKENIZER_FILENAME);
assert_ne!(CANDLE_BERT_TOKENIZER_FILENAME, TOKENIZER_FILENAME);
assert!(CANDLE_BERT_TOKENIZER_FILENAME.contains("sentence-transformers"));
assert!(CANDLE_BERT_WEIGHTS_FILENAME.ends_with(".safetensors"));
}
#[test]
fn test_download_file_fails_cleanly_on_request_error() {
let tmp_dir = std::env::temp_dir().join(format!(
"memd-download-req-err-test-{}-{}",
std::process::id(),
std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.unwrap()
.as_nanos()
));
std::fs::create_dir_all(&tmp_dir).unwrap();
let target = tmp_dir.join("atomic-test.bin");
let result = download_file("http://127.0.0.1:1/never", &target, "test file", None);
assert!(result.is_err(), "expected download to error");
assert!(
!target.exists(),
"target should not exist after failed download: {:?}",
target
);
let leftovers: Vec<_> = std::fs::read_dir(&tmp_dir)
.unwrap()
.filter_map(|e| e.ok())
.map(|e| e.file_name())
.collect();
assert!(
leftovers.is_empty(),
"no temp/partial files should remain, found: {:?}",
leftovers
);
std::fs::remove_dir_all(&tmp_dir).ok();
}
#[test]
fn test_fsync_parent_dir_is_ok_on_real_dir() {
let tmp_dir = std::env::temp_dir().join(format!(
"memd-fsync-test-{}-{}",
std::process::id(),
std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.unwrap()
.as_nanos()
));
std::fs::create_dir_all(&tmp_dir).unwrap();
let f = tmp_dir.join("marker.bin");
std::fs::write(&f, b"x").unwrap();
fsync_parent_dir(&f);
fsync_parent_dir(std::path::Path::new("foo"));
std::fs::remove_dir_all(&tmp_dir).ok();
}
#[test]
fn test_download_file_fsyncs_parent_on_publish() {
use std::io::{Read, Write};
let body = b"candle-fsync-payload".to_vec();
let Some(listener) =
loopback_listener_or_skip("test_download_file_fsyncs_parent_on_publish")
else {
return;
};
let addr = listener.local_addr().unwrap();
let body_for_server = body.clone();
let server_thread = std::thread::spawn(move || {
if let Ok((mut stream, _)) = listener.accept() {
let mut buf = [0u8; 1024];
let _ = stream.read(&mut buf);
let response = format!(
"HTTP/1.1 200 OK\r\nContent-Length: {}\r\nConnection: close\r\n\r\n",
body_for_server.len()
);
let _ = stream.write_all(response.as_bytes());
let _ = stream.write_all(&body_for_server);
}
});
let tmp_dir = std::env::temp_dir().join(format!(
"memd-download-fsync-test-{}-{}",
std::process::id(),
std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.unwrap()
.as_nanos()
));
std::fs::create_dir_all(&tmp_dir).unwrap();
let target = tmp_dir.join("candle.bin");
let url = format!("http://{}/x", addr);
let result = download_file(&url, &target, "fsync test", None);
server_thread.join().ok();
assert!(result.is_ok(), "download_file should succeed: {:?}", result);
let observed = std::fs::read(&target).unwrap();
assert_eq!(observed, body, "published bytes must match");
let leftovers: Vec<_> = std::fs::read_dir(&tmp_dir)
.unwrap()
.filter_map(|e| e.ok())
.map(|e| e.file_name().to_string_lossy().into_owned())
.filter(|n| n.contains("partial"))
.collect();
assert!(
leftovers.is_empty(),
"no partial tmp files should remain, found: {:?}",
leftovers
);
std::fs::remove_dir_all(&tmp_dir).ok();
}
#[test]
fn test_download_file_hf_config_integration() {
if std::env::var("MEMD_NETWORK_TESTS").ok().as_deref() != Some("1") {
eprintln!("skipping HF network test; set MEMD_NETWORK_TESTS=1 to run");
return;
}
let tmp_dir = std::env::temp_dir().join(format!(
"memd-hf-config-net-test-{}-{}",
std::process::id(),
std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.unwrap()
.as_nanos()
));
std::fs::create_dir_all(&tmp_dir).unwrap();
let target = tmp_dir.join(CANDLE_BERT_CONFIG_FILENAME);
let result = download_file(
CANDLE_BERT_CONFIG_URL,
&target,
"BERT config",
Some(CANDLE_BERT_CONFIG_SHA256),
);
assert!(
result.is_ok(),
"HF config download should succeed (network required): {:?}",
result
);
assert!(target.exists(), "config file should exist at target");
let meta = std::fs::metadata(&target).expect("metadata");
assert!(
meta.len() >= MIN_CANDLE_BERT_CONFIG_SIZE,
"downloaded config is too small ({} bytes, expected >= {})",
meta.len(),
MIN_CANDLE_BERT_CONFIG_SIZE
);
let leftovers: Vec<_> = std::fs::read_dir(&tmp_dir)
.unwrap()
.filter_map(|e| e.ok())
.map(|e| e.file_name().to_string_lossy().into_owned())
.filter(|n| n.contains("partial"))
.collect();
assert!(
leftovers.is_empty(),
"no partial tmp files should remain after a clean HF download, found: {:?}",
leftovers
);
std::fs::remove_dir_all(&tmp_dir).ok();
}
#[test]
fn test_download_file_is_first_writer_wins() {
use std::io::{Read, Write};
let body = b"fresh-download-payload".to_vec();
let Some(listener) = loopback_listener_or_skip("test_download_file_is_first_writer_wins")
else {
return;
};
let addr = listener.local_addr().unwrap();
let body_for_server = body.clone();
let server_thread = std::thread::spawn(move || {
if let Ok((mut stream, _)) = listener.accept() {
let mut buf = [0u8; 1024];
let _ = stream.read(&mut buf);
let response = format!(
"HTTP/1.1 200 OK\r\nContent-Length: {}\r\nConnection: close\r\n\r\n",
body_for_server.len()
);
let _ = stream.write_all(response.as_bytes());
let _ = stream.write_all(&body_for_server);
}
});
let tmp_dir = std::env::temp_dir().join(format!(
"memd-download-winner-test-{}-{}",
std::process::id(),
std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.unwrap()
.as_nanos()
));
std::fs::create_dir_all(&tmp_dir).unwrap();
let target = tmp_dir.join("winner.bin");
let sentinel = b"WINNER-BYTES".to_vec();
std::fs::write(&target, &sentinel).unwrap();
let url = format!("http://{}/x", addr);
let result = download_file(&url, &target, "race test", None);
server_thread.join().ok();
assert!(result.is_ok(), "download_file should succeed: {:?}", result);
let observed = std::fs::read(&target).unwrap();
assert_eq!(
observed, sentinel,
"pre-existing winner bytes must not be clobbered"
);
let leftovers: Vec<_> = std::fs::read_dir(&tmp_dir)
.unwrap()
.filter_map(|e| e.ok())
.map(|e| e.file_name().to_string_lossy().into_owned())
.filter(|n| n.contains("partial"))
.collect();
assert!(
leftovers.is_empty(),
"loser's partial tmp must be cleaned up, found: {:?}",
leftovers
);
std::fs::remove_dir_all(&tmp_dir).ok();
}
fn unique_tmp_dir(tag: &str) -> PathBuf {
std::env::temp_dir().join(format!(
"memd-{}-{}-{}",
tag,
std::process::id(),
SystemTime::now()
.duration_since(UNIX_EPOCH)
.unwrap()
.as_nanos()
))
}
#[test]
fn test_advisory_lock_path_preserves_full_filename() {
let p = PathBuf::from("/tmp/cache/model.onnx");
let lock = advisory_lock_path(&p);
assert_eq!(lock, PathBuf::from("/tmp/cache/model.onnx.lock"));
let p2 = PathBuf::from("/tmp/cache/tokenizer.json");
assert_eq!(
advisory_lock_path(&p2),
PathBuf::from("/tmp/cache/tokenizer.json.lock")
);
let p3 = PathBuf::from("/tmp/cache/blob");
assert_eq!(
advisory_lock_path(&p3),
PathBuf::from("/tmp/cache/blob.lock")
);
}
#[test]
fn test_advisory_lock_acquire_and_drop_cleans_up() {
let tmp_dir = unique_tmp_dir("lock-acquire");
std::fs::create_dir_all(&tmp_dir).unwrap();
let target = tmp_dir.join("target.bin");
let lock = advisory_lock_path(&target);
{
let outcome = try_acquire_advisory_lock(&lock);
match outcome {
AcquireLockOutcome::Acquired(_guard) => {
assert!(lock.exists(), "lock file should exist while guard is held");
}
_ => panic!("expected Acquired on first attempt"),
}
}
assert!(
!lock.exists(),
"lock file should be removed after guard drop"
);
std::fs::remove_dir_all(&tmp_dir).ok();
}
#[test]
fn test_advisory_lock_second_caller_is_contended() {
let tmp_dir = unique_tmp_dir("lock-contention");
std::fs::create_dir_all(&tmp_dir).unwrap();
let target = tmp_dir.join("target.bin");
let lock = advisory_lock_path(&target);
let first = try_acquire_advisory_lock(&lock);
let second = try_acquire_advisory_lock(&lock);
match (first, second) {
(AcquireLockOutcome::Acquired(_g1), AcquireLockOutcome::Contended) => {}
_ => panic!("expected Acquired then Contended on two racers"),
}
std::fs::remove_dir_all(&tmp_dir).ok();
}
#[test]
fn test_advisory_lock_stale_reclaim() {
let tmp_dir = unique_tmp_dir("lock-stale");
std::fs::create_dir_all(&tmp_dir).unwrap();
let target = tmp_dir.join("target.bin");
let lock = advisory_lock_path(&target);
std::fs::write(&lock, b"pid=99999 created_ms=0\n").unwrap();
assert!(lock.exists());
std::fs::remove_file(&lock).unwrap();
let outcome = try_acquire_advisory_lock(&lock);
assert!(matches!(outcome, AcquireLockOutcome::Acquired(_)));
std::fs::remove_dir_all(&tmp_dir).ok();
}
#[test]
fn test_lock_is_stale_by_mtime() {
let tmp_dir = unique_tmp_dir("lock-staleness");
std::fs::create_dir_all(&tmp_dir).unwrap();
let lock = tmp_dir.join("fresh.lock");
std::fs::write(&lock, b"x").unwrap();
assert!(
!lock_is_stale(&lock),
"fresh lock should not be classified stale"
);
let missing = tmp_dir.join("nope.lock");
assert!(!lock_is_stale(&missing));
std::fs::remove_dir_all(&tmp_dir).ok();
}
#[test]
fn test_wait_returns_published_when_target_appears() {
let tmp_dir = unique_tmp_dir("lock-wait-pub");
std::fs::create_dir_all(&tmp_dir).unwrap();
let target = tmp_dir.join("target.bin");
let lock = advisory_lock_path(&target);
std::fs::write(&lock, b"pid=1 created_ms=0").unwrap();
let target_clone = target.clone();
let lock_clone = lock.clone();
let t = std::thread::spawn(move || {
std::thread::sleep(Duration::from_millis(100));
std::fs::write(&target_clone, b"published").unwrap();
std::fs::remove_file(&lock_clone).ok();
});
let outcome = wait_for_publish_or_release(&target, &lock, Duration::from_secs(5));
t.join().unwrap();
assert!(matches!(outcome, WaitOutcome::Published));
std::fs::remove_dir_all(&tmp_dir).ok();
}
#[test]
fn test_wait_prefers_published_over_released_when_both_are_true() {
let tmp_dir = unique_tmp_dir("lock-wait-toctou");
std::fs::create_dir_all(&tmp_dir).unwrap();
let target = tmp_dir.join("target.bin");
let lock = advisory_lock_path(&target);
std::fs::write(&target, b"already-published").unwrap();
let outcome = wait_for_publish_or_release(&target, &lock, Duration::from_millis(200));
assert!(
matches!(outcome, WaitOutcome::Published),
"target present + lock absent must yield Published, got non-Published"
);
std::fs::remove_dir_all(&tmp_dir).ok();
}
#[test]
fn test_wait_returns_released_when_lock_disappears_without_target() {
let tmp_dir = unique_tmp_dir("lock-wait-rel");
std::fs::create_dir_all(&tmp_dir).unwrap();
let target = tmp_dir.join("target.bin");
let lock = advisory_lock_path(&target);
std::fs::write(&lock, b"pid=1 created_ms=0").unwrap();
let lock_clone = lock.clone();
let t = std::thread::spawn(move || {
std::thread::sleep(Duration::from_millis(100));
std::fs::remove_file(&lock_clone).ok();
});
let outcome = wait_for_publish_or_release(&target, &lock, Duration::from_secs(5));
t.join().unwrap();
assert!(
matches!(outcome, WaitOutcome::LockReleased),
"writer crash (lock gone, target absent) must yield LockReleased"
);
std::fs::remove_dir_all(&tmp_dir).ok();
}
#[test]
fn test_wait_returns_timeout_when_nothing_changes() {
let tmp_dir = unique_tmp_dir("lock-wait-to");
std::fs::create_dir_all(&tmp_dir).unwrap();
let target = tmp_dir.join("target.bin");
let lock = advisory_lock_path(&target);
std::fs::write(&lock, b"pid=1 created_ms=0").unwrap();
let outcome = wait_for_publish_or_release(&target, &lock, Duration::from_millis(200));
assert!(matches!(outcome, WaitOutcome::Timeout));
std::fs::remove_file(&lock).ok();
std::fs::remove_dir_all(&tmp_dir).ok();
}
#[test]
fn test_download_file_waiter_reuses_publication() {
use std::io::{Read, Write};
let body = b"one-download-shared".to_vec();
let Some(listener) =
loopback_listener_or_skip("test_download_file_waiter_reuses_publication")
else {
return;
};
let addr = listener.local_addr().unwrap();
let body_for_server = body.clone();
let server_thread = std::thread::spawn(move || {
if let Ok((mut stream, _)) = listener.accept() {
let mut buf = [0u8; 1024];
let _ = stream.read(&mut buf);
std::thread::sleep(Duration::from_millis(150));
let response = format!(
"HTTP/1.1 200 OK\r\nContent-Length: {}\r\nConnection: close\r\n\r\n",
body_for_server.len()
);
let _ = stream.write_all(response.as_bytes());
let _ = stream.write_all(&body_for_server);
}
});
let tmp_dir = unique_tmp_dir("download-coop");
std::fs::create_dir_all(&tmp_dir).unwrap();
let target = tmp_dir.join("shared.bin");
let target_clone = target.clone();
let url = format!("http://{}/x", addr);
let url_clone = url.clone();
let target_for_writer = target.clone();
let writer = std::thread::spawn(move || {
download_file(&url, &target_for_writer, "cooperative writer", None)
});
std::thread::sleep(Duration::from_millis(20));
let waiter = std::thread::spawn(move || {
download_file(&url_clone, &target_clone, "cooperative waiter", None)
});
let writer_res = writer.join().unwrap();
let waiter_res = waiter.join().unwrap();
server_thread.join().ok();
assert!(writer_res.is_ok(), "writer must succeed: {:?}", writer_res);
assert!(waiter_res.is_ok(), "waiter must succeed: {:?}", waiter_res);
let observed = std::fs::read(&target).unwrap();
assert_eq!(observed, body);
let leftovers: Vec<_> = std::fs::read_dir(&tmp_dir)
.unwrap()
.filter_map(|e| e.ok())
.map(|e| e.file_name().to_string_lossy().into_owned())
.filter(|n| n.contains("partial") || n.ends_with(".lock"))
.collect();
assert!(
leftovers.is_empty(),
"no partial or lock files should remain after coop download: {:?}",
leftovers
);
std::fs::remove_dir_all(&tmp_dir).ok();
}
}