use anyhow::{Result, anyhow};
use std::io::{self, Read};
use std::path::Path;
use kjarni::ModelType;
pub fn resolve_input(input: Option<&str>) -> Result<String> {
match input {
Some(text) => {
let path = Path::new(text);
if path.exists() && path.is_file() {
std::fs::read_to_string(path)
.map_err(|e| anyhow!("Failed to read file '{}': {}", text, e))
} else {
Ok(text.to_string())
}
}
None => {
let stdin = io::stdin();
let mut buffer = String::new();
stdin.lock().read_to_string(&mut buffer)?;
if buffer.is_empty() {
return Err(anyhow!(
"No input provided. Pass text as argument, a file path, or pipe via stdin."
));
}
Ok(buffer)
}
}
}
pub fn resolve_model(name: &str, arch_hint: Option<&str>) -> anyhow::Result<ModelType> {
ModelType::from_cli_name(name)
.ok_or_else(|| anyhow::anyhow!(model_not_found_error(name, arch_hint)))
}
pub fn model_not_found_error(name: &str, arch_hint: Option<&str>) -> String {
let mut msg = format!("Unknown model: '{}'.", name);
if let Some(arch) = arch_hint {
msg.push_str(&format!(
" Run 'kjarni model list --arch {}' to see available models.",
arch
));
} else {
msg.push_str(" Run 'kjarni model list' to see available models.");
}
let suggestions = ModelType::find_similar(name);
if !suggestions.is_empty() {
msg.push_str("\n\nDid you mean?");
for (suggestion, _) in suggestions {
msg.push_str(&format!("\n - {}", suggestion));
}
}
msg
}
pub fn check_gpu_capacity(model_type: kjarni::ModelType, prefer_gguf: bool) -> anyhow::Result<()> {
let Some(free_bytes) = free_vram_bytes() else {
return Ok(());
};
let dir = model_type.cache_dir(&kjarni::registry::cache_dir());
let files: Vec<(String, u64)> = std::fs::read_dir(&dir)
.into_iter()
.flatten()
.flatten()
.filter_map(|e| {
let size = e.metadata().ok()?.len();
Some((e.file_name().to_string_lossy().into_owned(), size))
})
.collect();
let weights = weights_to_load(&files, prefer_gguf);
if weights.bytes == 0 {
return Ok(());
}
let needed = weights.bytes + weights.bytes / 5;
if needed <= free_bytes {
return Ok(());
}
let gb = |b: u64| b as f64 / 1_073_741_824.0;
let name = model_type.cli_name();
let quantized_hint = if weights.is_gguf {
String::new()
} else {
format!(
"The quantized build is roughly a quarter the size:\n \
kjarni model download {name} --gguf\n\n"
)
};
Err(anyhow::anyhow!(
"{name} needs about {:.1} GB of VRAM ({:.1} GB of weights plus working \
memory) and only {:.1} GB is free.\n\n\
{quantized_hint}Or run on the CPU by dropping --gpu.",
gb(needed),
gb(weights.bytes),
gb(free_bytes)
))
}
#[derive(Debug, PartialEq, Eq)]
struct WeightsToLoad {
bytes: u64,
is_gguf: bool,
}
fn weights_to_load(files: &[(String, u64)], prefer_gguf: bool) -> WeightsToLoad {
let total = |suffix: &str| -> u64 {
files
.iter()
.filter(|(name, _)| name.ends_with(suffix))
.map(|(_, size)| size)
.sum()
};
let (gguf, safetensors) = (total(".gguf"), total(".safetensors"));
let is_gguf = gguf > 0 && (prefer_gguf || safetensors == 0);
WeightsToLoad {
bytes: if is_gguf { gguf } else { safetensors },
is_gguf,
}
}
pub(crate) fn free_vram_bytes() -> Option<u64> {
let out = std::process::Command::new("nvidia-smi")
.args(["--query-gpu=memory.free", "--format=csv,noheader,nounits"])
.output()
.ok()?;
if !out.status.success() {
return None;
}
let mib: u64 = String::from_utf8_lossy(&out.stdout)
.lines()
.next()?
.trim()
.parse()
.ok()?;
Some(mib * 1_048_576)
}
#[cfg(test)]
mod tests {
use super::*;
fn both_formats() -> Vec<(String, u64)> {
vec![
("config.json".to_string(), 826),
(
"model-00001-of-000002.safetensors".to_string(),
8_667_826_246,
),
(
"model-00002-of-000002.safetensors".to_string(),
7_392_730_108,
),
("model.gguf".to_string(), 4_920_737_216),
("tokenizer.json".to_string(), 9_084_480),
]
}
#[test]
fn gguf_flag_counts_only_the_gguf_file() {
assert_eq!(
weights_to_load(&both_formats(), true),
WeightsToLoad {
bytes: 4_920_737_216,
is_gguf: true
}
);
}
#[test]
fn without_the_flag_only_safetensors_count() {
assert_eq!(
weights_to_load(&both_formats(), false),
WeightsToLoad {
bytes: 16_060_556_354,
is_gguf: false
}
);
}
#[test]
fn the_only_format_present_counts_whatever_the_flag() {
let gguf_only = vec![("model.gguf".to_string(), 100)];
let expected = WeightsToLoad {
bytes: 100,
is_gguf: true,
};
assert_eq!(weights_to_load(&gguf_only, false), expected);
let safetensors_only = vec![("model.safetensors".to_string(), 100)];
let expected = WeightsToLoad {
bytes: 100,
is_gguf: false,
};
assert_eq!(weights_to_load(&safetensors_only, true), expected);
}
#[test]
fn no_weights_count_as_nothing() {
let files = vec![("config.json".to_string(), 826)];
assert_eq!(weights_to_load(&files, true).bytes, 0);
}
}