#[derive(Clone, Debug, PartialEq, Eq)]
pub struct RemoteFile {
pub name: String,
pub size: Option<u64>,
}
const METADATA: &[&str] = &[
"config.json",
"params.json",
"generation_config.json",
"tokenizer.json",
"tokenizer_config.json",
"tokenizer.model",
"special_tokens_map.json",
"vocab.json",
"merges.txt",
"chat_template.jinja",
"preprocessor_config.json",
"processor_config.json",
];
fn is_excluded(name: &str) -> bool {
const SKIP_DIRS: &[&str] = &["original/", "onnx/", "openvino/", "coreml/", "tflite/"];
const SKIP_EXT: &[&str] = &[
".bin", ".pth", ".pt", ".msgpack", ".h5", ".onnx", ".gguf", ".tflite",
];
SKIP_DIRS.iter().any(|d| name.starts_with(d)) || SKIP_EXT.iter().any(|e| name.ends_with(e))
}
fn is_weight(name: &str) -> bool {
name.ends_with(".safetensors") || name.ends_with(".safetensors.index.json")
}
fn is_contained(name: &str) -> bool {
if name.is_empty() || name.starts_with('/') || name.contains('\\') {
return false;
}
let b = name.as_bytes();
if b.len() >= 2 && b[0].is_ascii_alphabetic() && b[1] == b':' {
return false;
}
name.split('/')
.all(|c| !c.is_empty() && c != "." && c != "..")
}
pub fn wanted(name: &str) -> bool {
if !is_contained(name) || is_excluded(name) {
return false;
}
is_weight(name) || (!name.contains('/') && METADATA.contains(&name))
}
pub fn select(files: &[RemoteFile]) -> Vec<RemoteFile> {
let mut out: Vec<RemoteFile> = files.iter().filter(|f| wanted(&f.name)).cloned().collect();
out.sort_by(|a, b| {
let key = |f: &RemoteFile| (is_weight(&f.name), f.size.unwrap_or(0), f.name.clone());
key(a).cmp(&key(b))
});
out
}
pub fn has_weights(plan: &[RemoteFile]) -> bool {
plan.iter().any(|f| f.name.ends_with(".safetensors"))
}
pub fn total_bytes(plan: &[RemoteFile]) -> u64 {
plan.iter().filter_map(|f| f.size).sum()
}
#[cfg(test)]
#[path = "plan_tests.rs"]
mod tests;