use crate::ai_api::{
models::*,
storage::{validate_model_id, ModelStorage},
};
use anyhow::Result;
use reqwest::Client;
use serde::{Deserialize, Serialize};
use std::collections::HashMap;
use std::fs;
use std::path::Path;
use tokio::io::AsyncWriteExt;
use url::Url;
pub const MODEL_FILE_MAX: u64 = 512 * 1024 * 1024 * 1024;
pub struct ModelDownloader {
client: Client,
storage: ModelStorage,
cache_dir: std::path::PathBuf,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct DownloadRequest {
pub model_id: String,
pub source: ModelSource,
pub format_preference: Option<ModelFormat>,
pub quantization: Option<String>,
pub validate_checksum: bool,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct DownloadProgress {
pub model_id: String,
pub stage: DownloadStage,
pub progress: f64, pub bytes_downloaded: u64,
pub total_bytes: Option<u64>,
pub speed_bps: Option<u64>,
pub eta_seconds: Option<u64>,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub enum DownloadStage {
Initializing,
DownloadingMetadata,
DownloadingModel,
ValidatingChecksum,
ExtractingFiles,
StoringModel,
Complete,
Failed(String),
}
#[derive(Debug, Clone, Deserialize)]
pub struct HuggingFaceModelInfo {
pub id: String,
pub sha: String,
pub downloads: u64,
pub likes: u64,
pub tags: Vec<String>,
pub siblings: Vec<HuggingFaceFile>,
pub library_name: Option<String>,
pub pipeline_tag: Option<String>,
}
#[derive(Debug, Clone, Deserialize)]
pub struct HuggingFaceFile {
pub rfilename: String,
pub size: Option<u64>,
pub blob_id: String,
}
impl ModelDownloader {
pub fn new(storage: ModelStorage) -> Result<Self> {
let cache_dir = storage.get_cache_dir().to_path_buf();
fs::create_dir_all(&cache_dir)?;
crate::tls::install_provider();
let client =
crate::security::create_secure_async_client().unwrap_or_else(|_| Client::new());
Ok(Self {
client,
storage,
cache_dir,
})
}
pub async fn download_model(&mut self, request: DownloadRequest) -> Result<LocalModelMetadata> {
validate_model_id(&request.model_id)?;
match &request.source.origin {
origin if origin == "huggingface" => self.download_huggingface_model(request).await,
origin if origin == "url" => self.download_from_url(request).await,
_ => Err(anyhow::anyhow!(
"Unsupported model source: {}",
request.source.origin
)),
}
}
async fn download_huggingface_model(
&mut self,
request: DownloadRequest,
) -> Result<LocalModelMetadata> {
let model_id = &request.model_id;
let model_info = self.fetch_huggingface_model_info(model_id).await?;
let files_to_download = self.select_files_to_download(&model_info, &request)?;
if files_to_download.is_empty() {
return Err(anyhow::anyhow!(
"No suitable files found for model {}",
model_id
));
}
let temp_dir = self
.cache_dir
.join(format!("download_{}", uuid::Uuid::new_v4()));
fs::create_dir_all(&temp_dir)?;
let mut downloaded_files = Vec::new();
let mut total_size = 0u64;
for file in &files_to_download {
let file_url = format!(
"https://huggingface.co/{}/resolve/main/{}",
model_id, file.rfilename
);
check_listed_file_name(&file.rfilename)?;
let local_path = temp_dir.join(&file.rfilename);
if let Some(parent) = local_path.parent() {
fs::create_dir_all(parent)?;
}
let downloaded_size = self.download_file(&file_url, &local_path).await?;
total_size += downloaded_size;
downloaded_files.push((file.clone(), local_path));
}
let (_main_file, main_path, format) = self.identify_main_model_file(&downloaded_files)?;
let checksum = crate::ai_api::storage::sha256_file(&main_path)?;
let metadata = LocalModelMetadata {
id: model_id.clone(),
name: model_id.clone(),
description: Some(format!("Downloaded from Hugging Face: {}", model_id)),
version: model_info.sha.clone(),
format,
file_path: String::new(), config_path: self.find_config_file(&downloaded_files),
tokenizer_path: self.find_tokenizer_file(&downloaded_files),
size_bytes: total_size,
sha256: checksum,
downloaded_at: chrono::Utc::now(),
last_used: None,
usage_count: 0,
capabilities: self.infer_capabilities(&model_info),
parameters: self.extract_parameters(&model_info),
source: ModelSource {
origin: "huggingface".to_string(),
url: Some(format!("https://huggingface.co/{}", model_id)),
repository: Some(model_id.clone()),
commit: Some(model_info.sha),
license: None, },
};
self.storage
.store_model_file(&main_path, metadata.clone())
.await?;
self.store_additional_files(&downloaded_files, &metadata)
.await?;
if fs::remove_dir_all(&temp_dir).is_err() {
}
Ok(metadata)
}
async fn download_from_url(&mut self, request: DownloadRequest) -> Result<LocalModelMetadata> {
let url = request
.source
.url
.clone()
.ok_or_else(|| anyhow::anyhow!("URL required for URL source"))?;
crate::security::validate_http_url("download_model", &url)?;
let client = url_download_client()?;
let temp_path = self
.cache_dir
.join(format!("temp_{}", uuid::Uuid::new_v4()));
let downloaded_size = download_capped(&client, &url, &temp_path, MODEL_FILE_MAX).await?;
let checksum = crate::ai_api::storage::sha256_file(&temp_path)?;
let format = self.infer_format_from_url(&url)?;
let metadata = LocalModelMetadata {
id: request.model_id.clone(),
name: request.model_id.clone(),
description: Some(format!("Downloaded from URL: {}", url)),
version: "1.0".to_string(),
format,
file_path: String::new(),
config_path: None,
tokenizer_path: None,
size_bytes: downloaded_size,
sha256: checksum,
downloaded_at: chrono::Utc::now(),
last_used: None,
usage_count: 0,
capabilities: ModelCapabilities {
chat: true,
completions: true,
embeddings: false,
image_generation: false,
image_understanding: false,
audio_generation: false,
audio_understanding: false,
video_understanding: false,
function_calling: true,
streaming: true,
},
parameters: HashMap::new(),
source: request.source,
};
self.storage
.store_model_file(&temp_path, metadata.clone())
.await?;
if fs::remove_file(&temp_path).is_err() {
}
Ok(metadata)
}
async fn fetch_huggingface_model_info(&self, model_id: &str) -> Result<HuggingFaceModelInfo> {
let url = format!("https://huggingface.co/api/models/{}", model_id);
let response = self
.client
.get(&url)
.header("User-Agent", "ai-model-api/1.0")
.send()
.await?;
if !response.status().is_success() {
return Err(anyhow::anyhow!(
"Failed to fetch model info: HTTP {}",
response.status()
));
}
let model_info: HuggingFaceModelInfo =
crate::http_body::body_json_capped(response, crate::http_body::AI_BODY_MAX).await?;
Ok(model_info)
}
async fn download_file(&self, url: &str, local_path: &Path) -> Result<u64> {
download_capped(&self.client, url, local_path, MODEL_FILE_MAX).await
}
fn select_files_to_download(
&self,
model_info: &HuggingFaceModelInfo,
request: &DownloadRequest,
) -> Result<Vec<HuggingFaceFile>> {
let mut selected_files = Vec::new();
let format_extensions = match &request.format_preference {
Some(ModelFormat::GGUF) => vec!["gguf"],
Some(ModelFormat::SafeTensors) => vec!["safetensors"],
Some(ModelFormat::PyTorch) => vec!["bin", "pt", "pth"],
Some(ModelFormat::ONNX) => vec!["onnx"],
_ => vec!["gguf", "safetensors", "bin", "pt", "onnx"], };
for ext in &format_extensions {
for file in &model_info.siblings {
if file.rfilename.ends_with(&format!(".{}", ext)) {
if let Some(quant) = &request.quantization {
if file.rfilename.contains(quant) {
selected_files.push(file.clone());
break;
}
} else {
selected_files.push(file.clone());
break;
}
}
}
if !selected_files.is_empty() {
break;
}
}
for file in &model_info.siblings {
if file.rfilename == "config.json"
|| file.rfilename == "tokenizer.json"
|| file.rfilename == "tokenizer_config.json"
|| file.rfilename.starts_with("tokenizer")
{
selected_files.push(file.clone());
}
}
Ok(selected_files)
}
fn identify_main_model_file(
&self,
files: &[(HuggingFaceFile, std::path::PathBuf)],
) -> Result<(HuggingFaceFile, std::path::PathBuf, ModelFormat)> {
for (file, path) in files {
if file.rfilename.ends_with(".gguf") {
return Ok((file.clone(), path.clone(), ModelFormat::GGUF));
}
}
for (file, path) in files {
if file.rfilename.ends_with(".safetensors") {
return Ok((file.clone(), path.clone(), ModelFormat::SafeTensors));
}
}
for (file, path) in files {
if file.rfilename.ends_with(".bin") || file.rfilename.ends_with(".pt") {
return Ok((file.clone(), path.clone(), ModelFormat::PyTorch));
}
}
Err(anyhow::anyhow!("No suitable model file found"))
}
fn find_config_file(&self, files: &[(HuggingFaceFile, std::path::PathBuf)]) -> Option<String> {
for (file, path) in files {
if file.rfilename == "config.json" {
return Some(path.to_string_lossy().to_string());
}
}
None
}
fn find_tokenizer_file(
&self,
files: &[(HuggingFaceFile, std::path::PathBuf)],
) -> Option<String> {
for (file, path) in files {
if file.rfilename == "tokenizer.json" {
return Some(path.to_string_lossy().to_string());
}
}
None
}
async fn store_additional_files(
&self,
_files: &[(HuggingFaceFile, std::path::PathBuf)],
_metadata: &LocalModelMetadata,
) -> Result<()> {
Ok(())
}
fn infer_capabilities(&self, model_info: &HuggingFaceModelInfo) -> ModelCapabilities {
let mut capabilities = ModelCapabilities {
chat: false,
completions: true,
embeddings: false,
image_generation: false,
image_understanding: false,
audio_generation: false,
audio_understanding: false,
video_understanding: false,
function_calling: false,
streaming: true,
};
if let Some(pipeline) = &model_info.pipeline_tag {
match pipeline.as_str() {
"text-generation" => {
capabilities.chat = true;
capabilities.completions = true;
}
"feature-extraction" | "sentence-similarity" => {
capabilities.embeddings = true;
}
"text-to-image" => {
capabilities.image_generation = true;
}
"image-to-text" => {
capabilities.image_understanding = true;
}
_ => {}
}
}
for tag in &model_info.tags {
match tag.as_str() {
"conversational" | "chat" => capabilities.chat = true,
"function-calling" => capabilities.function_calling = true,
"multimodal" => capabilities.image_understanding = true,
_ => {}
}
}
capabilities
}
fn extract_parameters(
&self,
model_info: &HuggingFaceModelInfo,
) -> HashMap<String, serde_json::Value> {
let mut params = HashMap::new();
params.insert(
"downloads".to_string(),
serde_json::Value::Number(model_info.downloads.into()),
);
params.insert(
"likes".to_string(),
serde_json::Value::Number(model_info.likes.into()),
);
if let Some(library) = &model_info.library_name {
params.insert(
"library_name".to_string(),
serde_json::Value::String(library.clone()),
);
}
if let Some(pipeline) = &model_info.pipeline_tag {
params.insert(
"pipeline_tag".to_string(),
serde_json::Value::String(pipeline.clone()),
);
}
params.insert(
"tags".to_string(),
serde_json::Value::Array(
model_info
.tags
.iter()
.map(|t| serde_json::Value::String(t.clone()))
.collect(),
),
);
params
}
fn infer_format_from_url(&self, url: &str) -> Result<ModelFormat> {
let url_parsed = Url::parse(url)?;
let path = url_parsed.path();
if path.ends_with(".gguf") {
Ok(ModelFormat::GGUF)
} else if path.ends_with(".safetensors") {
Ok(ModelFormat::SafeTensors)
} else if path.ends_with(".bin") || path.ends_with(".pt") || path.ends_with(".pth") {
Ok(ModelFormat::PyTorch)
} else if path.ends_with(".onnx") {
Ok(ModelFormat::ONNX)
} else {
Err(anyhow::anyhow!("Cannot infer format from URL: {}", url))
}
}
pub async fn search_models(&self, query: &str, source: &str) -> Result<Vec<ModelSearchResult>> {
match source {
"huggingface" => self.search_huggingface_models(query).await,
_ => Err(anyhow::anyhow!("Unsupported search source: {}", source)),
}
}
async fn search_huggingface_models(&self, query: &str) -> Result<Vec<ModelSearchResult>> {
let url = format!(
"https://huggingface.co/api/models?search={}&limit=20",
urlencoding::encode(query)
);
let response = self
.client
.get(&url)
.header("User-Agent", "ai-model-api/1.0")
.send()
.await?;
if !response.status().is_success() {
return Err(anyhow::anyhow!("Search failed: HTTP {}", response.status()));
}
let models: Vec<HuggingFaceModelInfo> =
crate::http_body::body_json_capped(response, crate::http_body::AI_BODY_MAX).await?;
let results = models
.into_iter()
.map(|model| ModelSearchResult {
id: model.id.clone(),
name: model.id,
description: format!("Hugging Face model with {} downloads", model.downloads),
source: "huggingface".to_string(),
downloads: Some(model.downloads),
likes: Some(model.likes),
tags: model.tags,
library_name: model.library_name,
pipeline_tag: model.pipeline_tag,
})
.collect();
Ok(results)
}
}
fn url_download_client() -> Result<Client> {
crate::tls::install_provider();
Client::builder()
.timeout(std::time::Duration::from_secs(30))
.connect_timeout(std::time::Duration::from_secs(10))
.redirect(reqwest::redirect::Policy::custom(|attempt| {
if attempt.previous().len() >= 10 {
return attempt.error("too many redirects");
}
let next = attempt.url().to_string();
match redirect_hop_refusal(&next) {
None => attempt.follow(),
Some(reason) => attempt.error(reason),
}
}))
.build()
.map_err(Into::into)
}
fn redirect_hop_refusal(next: &str) -> Option<String> {
match crate::security::validate_http_url("download_model", next) {
Ok(_) => None,
Err(e) => Some(format!("redirect to {next} refused: {e}")),
}
}
async fn download_capped(client: &Client, url: &str, local_path: &Path, max: u64) -> Result<u64> {
let response = client
.get(url)
.header("User-Agent", "ai-model-api/1.0")
.send()
.await?;
if !response.status().is_success() {
return Err(anyhow::anyhow!(
"Failed to download file: HTTP {}",
response.status()
));
}
if let Some(declared) = response.content_length() {
if declared > max {
return Err(too_large(url, max));
}
}
let mut file = tokio::fs::File::create(local_path).await?;
let written = copy_capped(response, &mut file, url, max).await;
drop(file);
if written.is_err() {
let _ = tokio::fs::remove_file(local_path).await;
}
written
}
async fn copy_capped(
response: reqwest::Response,
file: &mut tokio::fs::File,
url: &str,
max: u64,
) -> Result<u64> {
let mut downloaded = 0u64;
let mut stream = response.bytes_stream();
use futures_util::StreamExt;
while let Some(chunk) = stream.next().await {
let chunk = chunk?;
downloaded += chunk.len() as u64;
if downloaded > max {
return Err(too_large(url, max));
}
file.write_all(&chunk).await?;
}
file.flush().await?;
Ok(downloaded)
}
fn check_listed_file_name(name: &str) -> Result<()> {
let plain = !name.is_empty()
&& !name.contains(['\\', ':', '\0'])
&& name
.split('/')
.all(|part| !part.is_empty() && part != "." && part != "..");
if plain {
Ok(())
} else {
anyhow::bail!("the model listing names an unsafe file: {name:?}")
}
}
fn too_large(url: &str, max: u64) -> anyhow::Error {
crate::safety::budget_exceeded(
"model_download",
format!("{url} is larger than {max} bytes"),
"download the file by other means if it really is this large",
)
}
#[cfg(test)]
mod listed_file_name_tests {
use super::check_listed_file_name;
#[test]
fn a_listed_name_cannot_leave_the_directory() {
for bad in [
"",
"..",
"../x",
"a/../../x",
"/etc/x",
"a//b",
"./a",
"a\\..\\b",
"C:x",
"C:/x",
] {
assert!(
check_listed_file_name(bad).is_err(),
"{bad:?} must be refused"
);
}
for good in [
"model.gguf",
"sub/dir/model-00001.safetensors",
"a b.bin",
"a..b",
] {
assert!(check_listed_file_name(good).is_ok(), "{good:?} must pass");
}
}
}
#[cfg(test)]
mod redirect_hop_tests {
use super::redirect_hop_refusal;
#[test]
fn a_hop_to_an_internal_address_is_refused() {
for next in [
"http://127.0.0.1/model.gguf",
"http://127.0.0.1:8080/",
"http://localhost/",
"http://[::1]/",
"http://[::ffff:127.0.0.1]/",
"http://169.254.169.254/latest/meta-data/",
"http://[::ffff:169.254.169.254]/",
"http://10.0.0.5/",
"http://192.168.1.1/",
"http://0.0.0.0/",
"file:///etc/passwd",
] {
let reason =
redirect_hop_refusal(next).unwrap_or_else(|| panic!("{next} must be refused"));
assert!(
reason.starts_with(&format!("redirect to {next} refused")),
"{reason}"
);
}
}
#[test]
fn a_hop_to_a_public_address_is_followed() {
assert_eq!(
redirect_hop_refusal("https://93.184.216.34/model.safetensors"),
None
);
}
}
#[cfg(test)]
mod tests {
use super::*;
use std::io::{Read, Write};
fn serve_once(head: String, body: Vec<u8>) -> String {
let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
let addr = listener.local_addr().unwrap();
std::thread::spawn(move || {
if let Ok((mut conn, _)) = listener.accept() {
let mut buf = [0u8; 4096];
let _ = conn.read(&mut buf);
let _ = conn.write_all(head.as_bytes());
let _ = conn.write_all(&body);
}
});
format!("http://{addr}/model.bin")
}
fn scratch(name: &str) -> std::path::PathBuf {
let d = std::env::temp_dir().join(format!("ae-dlcap-{}", std::process::id()));
std::fs::create_dir_all(&d).unwrap();
d.join(name)
}
fn is_budget(e: &anyhow::Error) -> bool {
e.downcast_ref::<crate::safety::SafetyError>()
.map(|s| s.code == crate::safety::ErrorCode::BudgetExceeded)
.unwrap_or(false)
}
#[tokio::test]
async fn a_file_past_the_cap_is_refused_and_leaves_nothing() {
crate::tls::install_provider();
let client = Client::new();
let url = serve_once(
"HTTP/1.1 200 OK\r\nContent-Length: 64\r\nConnection: close\r\n\r\n".into(),
vec![b'x'; 64],
);
let declared = scratch("declared.bin");
let err = download_capped(&client, &url, &declared, 16)
.await
.unwrap_err();
assert!(is_budget(&err), "{err:#}");
assert!(!declared.exists());
let url = serve_once(
"HTTP/1.1 200 OK\r\nConnection: close\r\n\r\n".into(),
vec![b'x'; 64],
);
let streamed = scratch("streamed.bin");
let err = download_capped(&client, &url, &streamed, 16)
.await
.unwrap_err();
assert!(is_budget(&err), "{err:#}");
assert!(!streamed.exists());
let url = serve_once(
"HTTP/1.1 200 OK\r\nContent-Length: 8\r\nConnection: close\r\n\r\n".into(),
vec![b'y'; 8],
);
let fits = scratch("fits.bin");
assert_eq!(download_capped(&client, &url, &fits, 16).await.unwrap(), 8);
assert_eq!(std::fs::read(&fits).unwrap(), vec![b'y'; 8]);
let _ = std::fs::remove_file(&fits);
}
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct ModelSearchResult {
pub id: String,
pub name: String,
pub description: String,
pub source: String,
pub downloads: Option<u64>,
pub likes: Option<u64>,
pub tags: Vec<String>,
pub library_name: Option<String>,
pub pipeline_tag: Option<String>,
}