use std::path::{Path, PathBuf};
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub struct Platform {
pub os: Os,
pub arch: Arch,
pub nvidia: bool,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum Os {
Linux,
Mac,
Windows,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum Arch {
X64,
Arm64,
}
impl Platform {
pub fn detect() -> Platform {
let os = if cfg!(target_os = "macos") {
Os::Mac
} else if cfg!(target_os = "windows") {
Os::Windows
} else {
Os::Linux
};
let arch = if cfg!(target_arch = "aarch64") {
Arch::Arm64
} else {
Arch::X64
};
let nvidia = std::process::Command::new("nvidia-smi")
.arg("-L")
.output()
.map(|o| o.status.success())
.unwrap_or(false);
Platform { os, arch, nvidia }
}
}
pub const HEARTH_CUDA_RELEASE: &str =
"https://github.com/Eth-Interchained/hearth/releases/download/runtime-cuda";
pub fn hearth_cuda_asset(p: Platform) -> Option<String> {
match (p.os, p.arch, p.nvidia) {
(Os::Linux, Arch::X64, true) => Some(format!(
"{HEARTH_CUDA_RELEASE}/llama-server-cuda-linux-x64.tar.gz"
)),
_ => None,
}
}
pub fn asset_pattern(p: Platform) -> &'static str {
match (p.os, p.arch, p.nvidia) {
(Os::Mac, Arch::Arm64, _) => "llama-{tag}-bin-macos-arm64.tar.gz",
(Os::Mac, Arch::X64, _) => "llama-{tag}-bin-macos-x64.tar.gz",
(Os::Linux, Arch::X64, true) => "llama-{tag}-bin-ubuntu-vulkan-x64.tar.gz",
(Os::Linux, Arch::X64, false) => "llama-{tag}-bin-ubuntu-x64.tar.gz",
(Os::Linux, Arch::Arm64, true) => "llama-{tag}-bin-ubuntu-vulkan-arm64.tar.gz",
(Os::Linux, Arch::Arm64, false) => "llama-{tag}-bin-ubuntu-arm64.tar.gz",
(Os::Windows, Arch::X64, true) => "llama-{tag}-bin-win-cuda-12.4-x64.zip",
(Os::Windows, Arch::X64, false) => "llama-{tag}-bin-win-cpu-x64.zip",
(Os::Windows, Arch::Arm64, _) => "llama-{tag}-bin-win-cpu-arm64.zip",
}
}
pub fn tradeoff_note(p: Platform) -> Option<&'static str> {
match (p.os, p.nvidia) {
(Os::Linux, true) => Some(
"this is the Vulkan build — zero compile, runs on the NVIDIA driver you \
already have. For maximum throughput, a CUDA source build is the \
optimization (cmake -DGGML_CUDA=ON); hearth will prefer it automatically \
if you install one on PATH.",
),
_ => None,
}
}
pub fn runtime_dir(hearth_home: &Path) -> PathBuf {
hearth_home.join("runtime")
}
pub fn fetched_server(hearth_home: &Path) -> Option<PathBuf> {
for rel in ["bin/llama-server", "build/bin/llama-server"] {
let p = runtime_dir(hearth_home).join(rel);
if p.exists() {
return Some(p);
}
}
None
}
pub fn resolve_server(hearth_home: &Path, path_has_server: bool) -> Resolved {
if let Ok(explicit) = std::env::var("HEARTH_LLAMA_SERVER") {
return Resolved::Explicit(PathBuf::from(explicit));
}
if path_has_server {
return Resolved::OnPath;
}
match fetched_server(hearth_home) {
Some(p) => Resolved::Fetched(p),
None => Resolved::Missing,
}
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum Resolved {
Explicit(PathBuf),
OnPath,
Fetched(PathBuf),
Missing,
}
#[cfg(test)]
mod tests {
use super::*;
fn plat(os: Os, arch: Arch, nvidia: bool) -> Platform {
Platform { os, arch, nvidia }
}
#[test]
fn linux_with_nvidia_gets_vulkan_because_no_linux_cuda_prebuilt_exists() {
assert_eq!(
asset_pattern(plat(Os::Linux, Arch::X64, true)),
"llama-{tag}-bin-ubuntu-vulkan-x64.tar.gz"
);
assert!(tradeoff_note(plat(Os::Linux, Arch::X64, true))
.unwrap()
.contains("CUDA source build is the optimization"));
}
#[test]
fn linux_without_a_gpu_gets_the_cpu_build_not_a_vulkan_it_cannot_use() {
assert_eq!(
asset_pattern(plat(Os::Linux, Arch::X64, false)),
"llama-{tag}-bin-ubuntu-x64.tar.gz"
);
assert!(tradeoff_note(plat(Os::Linux, Arch::X64, false)).is_none());
}
#[test]
fn macos_needs_no_gpu_flag_because_metal_is_in_the_build() {
for nvidia in [true, false] {
assert_eq!(
asset_pattern(plat(Os::Mac, Arch::Arm64, nvidia)),
"llama-{tag}-bin-macos-arm64.tar.gz"
);
}
}
#[test]
fn windows_with_nvidia_gets_the_cuda_build_that_actually_exists_there() {
assert_eq!(
asset_pattern(plat(Os::Windows, Arch::X64, true)),
"llama-{tag}-bin-win-cuda-12.4-x64.zip"
);
}
#[test]
fn resolution_order_lets_an_operators_cuda_build_win() {
let dir = std::env::temp_dir().join("hearth-rt-test");
let _ = std::fs::remove_dir_all(&dir);
assert_eq!(resolve_server(&dir, false), Resolved::Missing);
std::fs::create_dir_all(runtime_dir(&dir).join("bin")).unwrap();
std::fs::write(runtime_dir(&dir).join("bin/llama-server"), b"x").unwrap();
assert!(matches!(resolve_server(&dir, false), Resolved::Fetched(_)));
assert_eq!(resolve_server(&dir, true), Resolved::OnPath);
std::fs::remove_dir_all(&dir).ok();
}
#[test]
fn the_explicit_env_var_beats_everything() {
std::env::set_var("HEARTH_LLAMA_SERVER", "/opt/llama/llama-server");
let r = resolve_server(Path::new("/nowhere"), true);
std::env::remove_var("HEARTH_LLAMA_SERVER");
assert_eq!(
r,
Resolved::Explicit(PathBuf::from("/opt/llama/llama-server"))
);
}
}