use std::io::{Read, Write};
use std::path::{Path, PathBuf};
use std::time::{Duration, Instant};
use sha1::{Digest, Sha1};
use crate::catalog::ModelEntry;
use crate::error::Error;
use crate::paths::AppPaths;
const SHA1_HEX_LEN: usize = 40;
const IO_CHUNK: usize = 64 * 1024;
struct StoreKnobs {
proxy: String,
backoff_ms: u64,
io_chunk: usize,
}
fn store_knobs(paths: &AppPaths) -> StoreKnobs {
let seeds = crate::config::Store::seeds();
let store = crate::config::Store::load(&paths.config_file()).unwrap_or_default();
StoreKnobs {
proxy: store.http_proxy.or(seeds.http_proxy).unwrap_or_default(),
backoff_ms: store
.download_backoff_ms
.or(seeds.download_backoff_ms)
.unwrap_or_default(),
io_chunk: store
.io_chunk_bytes
.or(seeds.io_chunk_bytes)
.filter(|n| *n > 0)
.unwrap_or(IO_CHUNK),
}
}
pub fn model_path(paths: &AppPaths, entry: &ModelEntry) -> PathBuf {
paths.models_dir().join(&entry.filename)
}
pub fn is_present(paths: &AppPaths, entry: &ModelEntry) -> bool {
let path = model_path(paths, entry);
path.metadata()
.map(|m| m.len() == entry.size_bytes)
.unwrap_or(false)
}
pub fn is_verified(paths: &AppPaths, entry: &ModelEntry) -> bool {
verify(paths, entry).is_ok()
}
pub fn sha1_file(path: &Path) -> Result<String, Error> {
sha1_file_buffered(path, IO_CHUNK)
}
fn sha1_file_buffered(path: &Path, io_chunk: usize) -> Result<String, Error> {
let mut file = std::fs::File::open(path)?;
let mut hasher = Sha1::new();
let mut buf = vec![0u8; io_chunk];
loop {
let n = file.read(&mut buf)?;
if n == 0 {
break;
}
hasher.update(&buf[..n]);
}
Ok(hex::encode(hasher.finalize()))
}
fn ensure_digest_declared(entry: &ModelEntry) -> Result<(), Error> {
let digest = entry.sha1.trim();
let usable = digest.len() == SHA1_HEX_LEN && digest.bytes().all(|b| b.is_ascii_hexdigit());
if usable {
return Ok(());
}
Err(Error::Config(format!(
"catalog entry {} has no usable sha1 digest",
entry.name
)))
}
fn verify_file(path: &Path, entry: &ModelEntry, io_chunk: usize) -> Result<String, Error> {
ensure_digest_declared(entry)?;
let len = path.metadata()?.len();
if len != entry.size_bytes {
return Err(Error::ModelSizeMismatch {
name: entry.name.clone(),
expected: entry.size_bytes,
found: len,
});
}
let digest = sha1_file_buffered(path, io_chunk)?;
if digest != entry.sha1 {
return Err(Error::ModelDigestMismatch {
name: entry.name.clone(),
});
}
Ok(digest)
}
pub fn verify(paths: &AppPaths, entry: &ModelEntry) -> Result<String, Error> {
let path = model_path(paths, entry);
if !path.exists() {
return Err(Error::ModelNotFound {
name: entry.name.clone(),
});
}
verify_file(&path, entry, store_knobs(paths).io_chunk)
}
pub fn download(
paths: &AppPaths,
entry: &ModelEntry,
dl: &crate::config::DownloadKnobs,
offline: bool,
) -> Result<PathBuf, Error> {
let (retries, max_elapsed, timeout_secs) = (dl.retries, dl.max_elapsed_secs, dl.timeout_secs);
if offline {
return Err(Error::OfflineMissingModel {
name: entry.name.clone(),
});
}
ensure_digest_declared(entry)?;
if !entry.url.starts_with("https://") {
return Err(Error::InsecureOrigin {
url: entry.url.clone(),
});
}
paths.ensure()?;
let knobs = StoreKnobs {
proxy: dl.proxy.clone(),
backoff_ms: dl.backoff_ms,
io_chunk: dl.io_chunk,
};
let dest = model_path(paths, entry);
if dest.exists() && verify_file(&dest, entry, knobs.io_chunk).is_ok() {
return Ok(dest);
}
let started = Instant::now();
let mut last = anyhow::anyhow!("download failed");
let attempts = retries.max(1);
let part = dest.with_extension("bin.part");
for attempt in 0..attempts {
if started.elapsed() > Duration::from_secs(max_elapsed.max(1)) {
break;
}
match download_once(
&entry.url,
&part,
timeout_secs,
&knobs.proxy,
knobs.io_chunk,
) {
Ok(()) => match verify_file(&part, entry, knobs.io_chunk) {
Ok(_) => {
std::fs::rename(&part, &dest)?;
crate::signal::release_temp(&part);
return Ok(dest);
}
Err(e) => {
let _ = std::fs::remove_file(&part);
crate::signal::release_temp(&part);
if matches!(e, Error::ModelDigestMismatch { .. }) {
return Err(e);
}
last = anyhow::anyhow!(e.to_string());
}
},
Err(e) => {
let _ = std::fs::remove_file(&part);
crate::signal::release_temp(&part);
last = e;
}
}
let backoff = knobs.backoff_ms.saturating_mul(1u64 << attempt.min(4));
std::thread::sleep(Duration::from_millis(backoff));
}
Err(Error::ModelDownload(last))
}
fn download_once(
url: &str,
part: &Path,
timeout_secs: u64,
proxy: &str,
io_chunk: usize,
) -> Result<(), anyhow::Error> {
let mut builder = reqwest::blocking::Client::builder()
.use_rustls_tls()
.https_only(true)
.timeout(Duration::from_secs(timeout_secs.max(1)));
if !proxy.trim().is_empty() {
builder = builder.proxy(reqwest::Proxy::all(proxy.trim())?);
}
let client = builder.build()?;
let mut resp = client.get(url).send()?.error_for_status()?;
crate::signal::register_temp(part);
let mut file = std::fs::File::create(part)?;
let mut buf = vec![0u8; io_chunk];
loop {
let n = resp.read(&mut buf)?;
if n == 0 {
break;
}
file.write_all(&buf[..n])?;
}
file.flush()?;
Ok(())
}
pub fn remove(paths: &AppPaths, entry: &ModelEntry) -> Result<bool, Error> {
let path = model_path(paths, entry);
if !path.exists() {
return Ok(false);
}
std::fs::remove_file(path)?;
Ok(true)
}
#[cfg(test)]
mod tests {
fn test_knobs() -> crate::config::DownloadKnobs {
crate::config::DownloadKnobs {
retries: 1,
max_elapsed_secs: 1,
timeout_secs: 5,
backoff_ms: 1,
proxy: String::new(),
io_chunk: 4096,
}
}
use super::*;
fn paths_in(dir: &Path) -> AppPaths {
AppPaths::resolve(Some(dir), Some(dir)).expect("resolve")
}
fn entry(url: &str, sha1: &str, size: u64) -> ModelEntry {
ModelEntry {
name: "tiny".into(),
filename: "ggml-tiny.bin".into(),
url: url.into(),
size_bytes: size,
sha1: sha1.into(),
vram_bytes: 1,
}
}
const VALID_SHA1: &str = "bd577a113a864445d4c299885e0cb97d4ba92b5f";
#[test]
fn rejects_http_origin() {
let dir = tempfile::tempdir().expect("tempdir");
let paths = paths_in(dir.path());
let e = entry("http://example.invalid/ggml-tiny.bin", VALID_SHA1, 10);
let err = download(&paths, &e, &test_knobs(), false).expect_err("must refuse");
assert!(matches!(err, Error::InsecureOrigin { .. }));
}
#[test]
fn empty_digest_refuses_download_before_any_socket() {
let dir = tempfile::tempdir().expect("tempdir");
let paths = paths_in(dir.path());
let e = entry("https://example.invalid/ggml-tiny.bin", "", 10);
let err = download(&paths, &e, &test_knobs(), false).expect_err("must refuse");
assert_eq!(err.exit_code(), 78);
}
#[test]
fn short_digest_is_refused() {
let dir = tempfile::tempdir().expect("tempdir");
let paths = paths_in(dir.path());
let e = entry("https://example.invalid/ggml-tiny.bin", "abc123", 10);
assert!(download(&paths, &e, &test_knobs(), false).is_err());
}
#[test]
fn offline_refuses_before_digest_work() {
let dir = tempfile::tempdir().expect("tempdir");
let paths = paths_in(dir.path());
let e = entry("https://example.invalid/ggml-tiny.bin", VALID_SHA1, 10);
let err = download(&paths, &e, &test_knobs(), true).expect_err("must refuse");
assert!(matches!(err, Error::OfflineMissingModel { .. }));
}
#[test]
fn wrong_digest_never_commits_the_file() {
let dir = tempfile::tempdir().expect("tempdir");
let paths = paths_in(dir.path());
paths.ensure().expect("ensure");
let e = entry("https://example.invalid/ggml-tiny.bin", VALID_SHA1, 4);
let part = model_path(&paths, &e).with_extension("bin.part");
std::fs::write(&part, b"junk").expect("write part");
assert!(verify_file(&part, &e, IO_CHUNK).is_err());
assert!(!model_path(&paths, &e).exists());
}
#[test]
fn truncated_cache_file_fails_verify() {
let dir = tempfile::tempdir().expect("tempdir");
let paths = paths_in(dir.path());
paths.ensure().expect("ensure");
let e = entry("https://example.invalid/ggml-tiny.bin", VALID_SHA1, 1_000);
std::fs::write(model_path(&paths, &e), b"short").expect("write");
let err = verify(&paths, &e).expect_err("must refuse");
assert!(
matches!(
err,
Error::ModelSizeMismatch {
expected: 1_000,
found: 5,
..
}
),
"expected a size mismatch naming both numbers, got {err}"
);
assert_eq!(err.exit_code(), 78);
}
#[test]
fn right_size_wrong_bytes_is_a_digest_mismatch() {
let dir = tempfile::tempdir().expect("tempdir");
let paths = paths_in(dir.path());
paths.ensure().expect("ensure");
let body = b"exactly-twenty-bytes";
let e = entry(
"https://example.invalid/ggml-tiny.bin",
VALID_SHA1,
body.len() as u64,
);
std::fs::write(model_path(&paths, &e), body).expect("write");
let err = verify(&paths, &e).expect_err("must refuse");
assert!(
matches!(err, Error::ModelDigestMismatch { .. }),
"expected a digest mismatch, got {err}"
);
}
#[test]
fn half_size_file_is_not_present() {
let dir = tempfile::tempdir().expect("tempdir");
let paths = paths_in(dir.path());
paths.ensure().expect("ensure");
let e = entry("https://example.invalid/ggml-tiny.bin", VALID_SHA1, 100);
std::fs::write(model_path(&paths, &e), vec![0u8; 60]).expect("write");
assert!(!is_present(&paths, &e), "60 of 100 bytes is not present");
assert!(!is_verified(&paths, &e));
}
#[test]
fn exact_size_and_digest_verifies() {
let dir = tempfile::tempdir().expect("tempdir");
let paths = paths_in(dir.path());
paths.ensure().expect("ensure");
let body = b"whisper";
let digest = {
let mut h = Sha1::new();
h.update(body);
hex::encode(h.finalize())
};
let e = entry(
"https://example.invalid/ggml-tiny.bin",
&digest,
body.len() as u64,
);
std::fs::write(model_path(&paths, &e), body).expect("write");
assert!(is_present(&paths, &e));
assert_eq!(verify(&paths, &e).expect("verify"), digest);
}
}