use std::path::PathBuf;
use anyhow::{Context, Result};
use clap::ValueEnum;
use crate::container::{ContainerEngine, ContainerManager};
use crate::llama_release::DownloadMarker;
#[derive(Debug, Clone, Copy, PartialEq, Eq, ValueEnum)]
#[value(rename_all = "lower")]
pub enum Runtime {
Auto,
Docker,
Podman,
Bin,
Path,
}
impl Runtime {
pub fn ociman(self) -> Option<ContainerManager> {
match self {
Runtime::Docker => Some(ContainerManager::Docker),
Runtime::Podman => Some(ContainerManager::Podman),
Runtime::Auto | Runtime::Bin | Runtime::Path => None,
}
}
pub fn as_str(self) -> &'static str {
match self {
Runtime::Auto => "auto",
Runtime::Docker => "docker",
Runtime::Podman => "podman",
Runtime::Bin => "bin",
Runtime::Path => "path",
}
}
fn from_ociman(ociman: ContainerManager) -> Runtime {
match ociman {
ContainerManager::Docker => Runtime::Docker,
ContainerManager::Podman => Runtime::Podman,
}
}
}
#[derive(Debug, Clone)]
pub enum Resolved {
Container(ContainerManager),
Local { source: Runtime, bin: PathBuf },
}
impl std::fmt::Display for Resolved {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
match self {
Resolved::Container(m) => write!(f, "{} containers", m.binary()),
Resolved::Local { source, bin } => write!(f, "{} ({})", source.as_str(), bin.display()),
}
}
}
impl Resolved {
pub fn runtime(&self) -> Runtime {
match self {
Resolved::Container(m) => Runtime::from_ociman(*m),
Resolved::Local { source, .. } => *source,
}
}
pub fn ociman(&self) -> Option<ContainerManager> {
match self {
Resolved::Container(m) => Some(*m),
Resolved::Local { .. } => None,
}
}
pub fn llama_server_bin(&self) -> Option<&PathBuf> {
match self {
Resolved::Container(_) => None,
Resolved::Local { bin, .. } => Some(bin),
}
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum Workload {
General,
CpuOnly,
}
impl Workload {
fn engine(self) -> ContainerEngine {
match self {
Workload::General => ContainerEngine::LlamaServer,
Workload::CpuOnly => ContainerEngine::Speech,
}
}
fn backend(self) -> crate::container::GpuBackend {
match self {
Workload::General => crate::container::detect_backend(),
Workload::CpuOnly => crate::container::SPEECH_BACKEND,
}
}
fn label(self) -> &'static str {
match self {
Workload::General => "",
Workload::CpuOnly => " (cpu worker)",
}
}
}
struct Slot {
resolved: std::sync::Mutex<Option<Resolved>>,
resolving: tokio::sync::Mutex<()>,
}
impl Slot {
fn new(resolved: Option<Resolved>) -> Slot {
Slot {
resolved: std::sync::Mutex::new(resolved),
resolving: tokio::sync::Mutex::new(()),
}
}
fn known(&self) -> Option<Resolved> {
self.resolved
.lock()
.unwrap_or_else(|e| e.into_inner())
.clone()
}
fn set(&self, resolved: Resolved) {
*self.resolved.lock().unwrap_or_else(|e| e.into_inner()) = Some(resolved);
}
}
pub struct Lazy {
requested: Runtime,
llama_cpp_version: Option<String>,
hardened: bool,
general: Slot,
cpu_only: Slot,
}
impl Lazy {
pub fn new(
requested: Runtime,
llama_cpp_version: Option<String>,
hardened: bool,
resolved: Option<Resolved>,
) -> Self {
Lazy {
requested,
llama_cpp_version,
hardened,
general: Slot::new(resolved),
cpu_only: Slot::new(None),
}
}
fn slot(&self, workload: Workload) -> &Slot {
match workload {
Workload::General => &self.general,
Workload::CpuOnly => &self.cpu_only,
}
}
pub fn requested(&self) -> Runtime {
self.requested
}
pub fn known(&self) -> Option<Resolved> {
self.general.known()
}
fn set(&self, resolved: Resolved) {
self.general.set(resolved);
}
pub async fn resolve(&self) -> Result<Resolved> {
self.resolve_for(Workload::General).await
}
pub async fn resolve_for(&self, workload: Workload) -> Result<Resolved> {
let slot = self.slot(workload);
if let Some(resolved) = slot.known() {
return Ok(resolved);
}
let _one_at_a_time = slot.resolving.lock().await;
if let Some(resolved) = slot.known() {
return Ok(resolved);
}
eprintln!(
"[llmman] runtime {}{}: not set up at startup; resolving it now",
self.requested.as_str(),
workload.label()
);
let requested = self.requested;
let pin = self.llama_cpp_version.clone();
let hardened = self.hardened;
let resolved = tokio::task::spawn_blocking(move || {
resolve_for(workload, requested, pin.as_deref(), hardened)
})
.await
.context("resolve runtime task panicked")?
.context("the llama.cpp runtime could not be set up (this daemon started without it)")?;
slot.set(resolved.clone());
Ok(resolved)
}
pub async fn ociman(&self) -> Result<Option<ContainerManager>> {
self.ociman_for(Workload::General).await
}
pub async fn ociman_for(&self, workload: Workload) -> Result<Option<ContainerManager>> {
if let Some(resolved) = self.slot(workload).known() {
return Ok(resolved.ociman());
}
if !candidates(self.requested, false)
.iter()
.any(|c| c.ociman().is_some())
{
return Ok(None);
}
Ok(self.resolve_for(workload).await?.ociman())
}
pub async fn local_llama_server_bin(&self) -> Result<PathBuf> {
let local = |resolved: Resolved| match resolved {
Resolved::Local { source, bin } => Ok((source, bin)),
Resolved::Container(m) => anyhow::bail!(
"no local llama-server binary: --runtime {} runs it in a container",
m.binary()
),
};
let (_, bin) = local(self.resolve().await?)?;
if bin.exists() {
return Ok(bin);
}
let _one_at_a_time = self.general.resolving.lock().await;
let (source, bin) = local(self.known().expect("resolved above"))?;
if bin.exists() {
return Ok(bin);
}
eprintln!(
"[llmman] llama-server at {} no longer exists; re-resolving",
bin.display()
);
let pin = self.llama_cpp_version.clone();
let (_, bin) = tokio::task::spawn_blocking(move || resolve_local(source, pin.as_deref()))
.await
.context("resolve llama-server task panicked")??;
self.set(Resolved::Local {
source,
bin: bin.clone(),
});
Ok(bin)
}
}
pub fn resolve(
runtime: Runtime,
llama_cpp_version: Option<&str>,
hardened: bool,
) -> Result<Resolved> {
resolve_for(Workload::General, runtime, llama_cpp_version, hardened)
}
pub fn resolve_for(
workload: Workload,
runtime: Runtime,
llama_cpp_version: Option<&str>,
hardened: bool,
) -> Result<Resolved> {
resolve_with(workload, runtime, llama_cpp_version, hardened, false)
}
fn resolve_with(
workload: Workload,
runtime: Runtime,
llama_cpp_version: Option<&str>,
hardened: bool,
local_only: bool,
) -> Result<Resolved> {
let explicit = runtime != Runtime::Auto;
resolve_from(
&candidates(runtime, local_only),
runtime,
workload,
|candidate| try_one(workload, candidate, llama_cpp_version, hardened, explicit),
)
}
pub fn resolve_local(
runtime: Runtime,
llama_cpp_version: Option<&str>,
) -> Result<(Runtime, PathBuf)> {
if let Some(m) = runtime.ociman() {
anyhow::bail!(
"--runtime {} runs llama-server in a container; there is no local binary",
m.binary()
);
}
match resolve_with(Workload::General, runtime, llama_cpp_version, false, true)? {
Resolved::Local { source, bin } => Ok((source, bin)),
Resolved::Container(_) => unreachable!("bin and path resolve to a local binary"),
}
}
fn candidates(runtime: Runtime, local_only: bool) -> Vec<Runtime> {
match runtime {
Runtime::Auto if cfg!(target_os = "linux") && !local_only => {
vec![
Runtime::Docker,
Runtime::Podman,
Runtime::Bin,
Runtime::Path,
]
}
Runtime::Auto => vec![Runtime::Bin, Runtime::Path],
one => vec![one],
}
}
fn resolve_from(
candidates: &[Runtime],
requested: Runtime,
workload: Workload,
mut step: impl FnMut(Runtime) -> Result<Resolved>,
) -> Result<Resolved> {
let explicit = requested != Runtime::Auto;
let mut failures = Vec::new();
for &candidate in candidates {
match step(candidate) {
Ok(resolved) => {
eprintln!(
"[llmman] runtime {}{}: using {resolved}",
requested.as_str(),
workload.label()
);
return Ok(resolved);
}
Err(e) if explicit => return Err(e),
Err(e) => {
eprintln!(
"[llmman] runtime auto{}: skipping {}: {e:#}",
workload.label(),
candidate.as_str()
);
failures.push(format!("{}: {e:#}", candidate.as_str()));
}
}
}
anyhow::bail!(
"no usable llama.cpp runtime (tried {}); set --runtime/LLMMAN_RUNTIME to pick one \
and see its own error",
failures.join("; ")
)
}
fn try_one(
workload: Workload,
candidate: Runtime,
llama_cpp_version: Option<&str>,
hardened: bool,
explicit: bool,
) -> Result<Resolved> {
match candidate {
Runtime::Auto => unreachable!("auto is expanded by candidates()"),
Runtime::Docker | Runtime::Podman => {
let ociman = candidate.ociman().expect("container runtime");
if !cfg!(target_os = "linux") {
anyhow::bail!(
"--runtime {} is only supported on Linux",
candidate.as_str()
);
}
if !explicit {
match workload {
Workload::General => ociman.probe()?,
Workload::CpuOnly => ociman.probe_cpu_only()?,
}
}
prime_docker_identity(candidate, explicit, hardened, || {
crate::container::identity::docker_is_rootless();
});
with_download_marker(|marker| {
prepare_container(
marker,
|| {
crate::container::pull_image(
ociman,
workload.engine(),
llama_cpp_version,
None,
)
},
(!explicit).then_some(|| {
crate::container::verify_llama_server_runs(
ociman,
workload.backend(),
llama_cpp_version,
hardened,
)
}),
)
})?;
Ok(Resolved::Container(ociman))
}
Runtime::Bin => {
let bin = crate::llama_release::ensure_llama_server(llama_cpp_version)
.context("download of llama.cpp's prebuilt llama-server failed")?
.bin;
Ok(Resolved::Local {
source: Runtime::Bin,
bin,
})
}
Runtime::Path => {
let bin = crate::find_on_path("llama-server").context("no llama-server on PATH")?;
Ok(Resolved::Local {
source: Runtime::Path,
bin,
})
}
}
}
fn prime_docker_identity(candidate: Runtime, explicit: bool, hardened: bool, probe: impl FnOnce()) {
if hardened && explicit && candidate == Runtime::Docker && cfg!(target_os = "linux") {
probe();
}
}
fn with_download_marker<T>(setup: impl FnOnce(&DownloadMarker) -> T) -> T {
let marker = DownloadMarker::create();
marker.keep_alive_during(|| setup(&marker))
}
fn prepare_container(
marker: &DownloadMarker,
pull: impl FnOnce() -> Result<()>,
verify: Option<impl FnOnce() -> Result<()>>,
) -> Result<()> {
marker.set_status(crate::container::PULL_STATUS);
pull()?;
if let Some(verify) = verify {
marker.set_status(crate::container::VERIFY_STATUS);
verify()?;
}
Ok(())
}
pub fn llama_cpp_pin(arg: Option<&str>) -> Option<String> {
match arg {
Some("latest") => None,
Some(v) => Some(v.to_owned()),
None => Some(crate::llama_release::default_release().to_owned()),
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn runtime_spellings_round_trip() {
for (s, r) in [
("auto", Runtime::Auto),
("docker", Runtime::Docker),
("podman", Runtime::Podman),
("bin", Runtime::Bin),
("path", Runtime::Path),
] {
assert_eq!(Runtime::from_str(s, true).unwrap(), r);
assert_eq!(r.as_str(), s);
}
}
#[test]
fn only_container_runtimes_have_an_ociman() {
assert_eq!(Runtime::Docker.ociman(), Some(ContainerManager::Docker));
assert_eq!(Runtime::Podman.ociman(), Some(ContainerManager::Podman));
assert_eq!(Runtime::Auto.ociman(), None);
assert_eq!(Runtime::Bin.ociman(), None);
assert_eq!(Runtime::Path.ociman(), None);
}
#[test]
fn pin_defaults_to_the_ci_release_and_latest_unpins() {
let default = crate::llama_release::default_release();
assert!(default.starts_with('v'), "{default}");
assert_eq!(llama_cpp_pin(None).as_deref(), Some(default));
assert_eq!(llama_cpp_pin(Some("latest")), None);
assert_eq!(llama_cpp_pin(Some("b9994")).as_deref(), Some("b9994"));
}
fn marker_at_temp_path(name: &str) -> (DownloadMarker, PathBuf) {
let path = std::env::temp_dir().join(format!(
"llmman-runtime-marker-{name}-{}-{}",
std::process::id(),
std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.unwrap()
.as_nanos()
));
(DownloadMarker::create_at(path.clone()), path)
}
#[test]
fn the_marker_is_live_and_says_what_it_is_waiting_on_through_pull_and_verify() {
let (marker, path) = marker_at_temp_path("verify");
let seen = std::cell::RefCell::new(Vec::new());
let note = || {
seen.borrow_mut()
.push(std::fs::read_to_string(&path).expect("the marker exists"));
Ok(())
};
prepare_container(&marker, note, Some(note)).unwrap();
assert_eq!(
*seen.borrow(),
[
crate::container::PULL_STATUS,
crate::container::VERIFY_STATUS
]
);
}
#[test]
fn verify_is_skipped_for_an_explicit_runtime_and_after_a_failed_pull() {
let (marker, _path) = marker_at_temp_path("skip");
let verified = std::cell::Cell::new(false);
let verify = || {
verified.set(true);
Ok(())
};
prepare_container(&marker, || Ok(()), None::<fn() -> Result<()>>).unwrap();
assert!(!verified.get());
let err = prepare_container(&marker, || anyhow::bail!("pull failed"), Some(verify))
.unwrap_err()
.to_string();
assert_eq!(err, "pull failed");
assert!(!verified.get());
}
#[test]
fn auto_skips_a_refused_container_for_the_general_answer_and_takes_it_cpu_only() {
let order = [
Runtime::Docker,
Runtime::Podman,
Runtime::Bin,
Runtime::Path,
];
let tried = std::cell::RefCell::new(Vec::new());
let step = |workload: Workload| {
let tried = &tried;
move |candidate: Runtime| -> Result<Resolved> {
tried.borrow_mut().push(candidate);
match (candidate, workload) {
(Runtime::Docker | Runtime::Podman, Workload::General) => {
anyhow::bail!("no NVIDIA Container Toolkit")
}
(Runtime::Docker | Runtime::Podman, Workload::CpuOnly) => {
Ok(Resolved::Container(candidate.ociman().expect("container")))
}
(Runtime::Bin, _) => Ok(Resolved::Local {
source: Runtime::Bin,
bin: PathBuf::from("/tmp/llama-server"),
}),
(other, _) => anyhow::bail!("{} not reached", other.as_str()),
}
}
};
let general = resolve_from(
&order,
Runtime::Auto,
Workload::General,
step(Workload::General),
)
.unwrap();
assert_eq!(general.runtime(), Runtime::Bin);
assert_eq!(
*tried.borrow(),
[Runtime::Docker, Runtime::Podman, Runtime::Bin]
);
tried.borrow_mut().clear();
let cpu_only = resolve_from(
&order,
Runtime::Auto,
Workload::CpuOnly,
step(Workload::CpuOnly),
)
.unwrap();
assert_eq!(cpu_only.ociman(), Some(ContainerManager::Docker));
assert_eq!(*tried.borrow(), [Runtime::Docker]);
}
#[test]
fn explicit_docker_primes_the_rootless_answer_before_any_spawn() {
let probed = |candidate, explicit, hardened| {
let hit = std::cell::Cell::new(false);
prime_docker_identity(candidate, explicit, hardened, || hit.set(true));
hit.get()
};
let linux = cfg!(target_os = "linux");
assert_eq!(probed(Runtime::Docker, true, true), linux);
assert!(!probed(Runtime::Docker, true, false));
assert!(!probed(Runtime::Docker, false, true));
assert!(!probed(Runtime::Podman, true, true));
assert!(!probed(Runtime::Bin, true, true));
}
#[test]
fn an_explicit_runtime_returns_its_own_error_and_auto_lists_every_skip() {
let err = resolve_from(
&[Runtime::Docker],
Runtime::Docker,
Workload::General,
|_| anyhow::bail!("docker info failed"),
)
.unwrap_err()
.to_string();
assert_eq!(err, "docker info failed");
let err = resolve_from(
&[Runtime::Bin, Runtime::Path],
Runtime::Auto,
Workload::CpuOnly,
|candidate| anyhow::bail!("{} failed", candidate.as_str()),
)
.unwrap_err()
.to_string();
assert!(err.contains("bin: bin failed; path: path failed"), "{err}");
}
#[tokio::test]
async fn the_two_answers_are_kept_apart() {
let lazy = Lazy::new(
Runtime::Path,
None,
true,
Some(Resolved::Local {
source: Runtime::Path,
bin: PathBuf::from("/usr/bin/llama-server"),
}),
);
assert_eq!(lazy.known().map(|r| r.runtime()), Some(Runtime::Path));
assert!(lazy.slot(Workload::CpuOnly).known().is_none());
lazy.slot(Workload::CpuOnly)
.set(Resolved::Container(ContainerManager::Docker));
assert_eq!(
lazy.ociman_for(Workload::CpuOnly).await.unwrap(),
Some(ContainerManager::Docker)
);
assert_eq!(lazy.ociman().await.unwrap(), None);
assert_eq!(lazy.known().map(|r| r.runtime()), Some(Runtime::Path));
}
#[test]
fn container_runtimes_are_refused_off_linux() {
if cfg!(target_os = "linux") {
return;
}
let err = resolve(Runtime::Docker, None, true)
.unwrap_err()
.to_string();
assert!(err.contains("only supported on Linux"), "{err}");
}
#[test]
fn resolved_reports_its_concrete_runtime() {
let local = Resolved::Local {
source: Runtime::Path,
bin: PathBuf::from("/usr/bin/llama-server"),
};
assert_eq!(local.runtime(), Runtime::Path);
assert_eq!(local.ociman(), None);
assert_eq!(
local.llama_server_bin(),
Some(&PathBuf::from("/usr/bin/llama-server"))
);
let container = Resolved::Container(ContainerManager::Podman);
assert_eq!(container.runtime(), Runtime::Podman);
assert_eq!(container.ociman(), Some(ContainerManager::Podman));
assert_eq!(container.llama_server_bin(), None);
}
#[tokio::test(flavor = "multi_thread")]
async fn lazy_keeps_a_startup_resolution_as_is() {
let startup = Resolved::Local {
source: Runtime::Path,
bin: PathBuf::from("/nonexistent/llama-server"),
};
let lazy = Lazy::new(Runtime::Bin, Some("b0".into()), true, Some(startup));
assert_eq!(lazy.known().unwrap().runtime(), Runtime::Path);
assert_eq!(lazy.ociman().await.unwrap(), None);
assert_eq!(
lazy.resolve().await.unwrap().llama_server_bin(),
Some(&PathBuf::from("/nonexistent/llama-server"))
);
}
#[tokio::test(flavor = "multi_thread")]
async fn lazy_answers_ociman_without_fetching_under_a_local_runtime() {
for requested in [Runtime::Bin, Runtime::Path] {
let lazy = Lazy::new(requested, Some("b0".into()), true, None);
assert_eq!(lazy.ociman().await.unwrap(), None);
assert!(lazy.known().is_none(), "{requested:?}");
}
if !cfg!(target_os = "linux") {
let lazy = Lazy::new(Runtime::Auto, Some("b0".into()), true, None);
assert_eq!(lazy.ociman().await.unwrap(), None);
assert!(lazy.known().is_none());
}
}
#[tokio::test(flavor = "multi_thread")]
async fn lazy_does_not_cache_a_failed_resolution() {
if cfg!(target_os = "linux") {
return;
}
let lazy = Lazy::new(Runtime::Docker, None, true, None);
let err = lazy.ociman().await.unwrap_err().to_string();
assert!(err.contains("could not be set up"), "{err}");
assert!(lazy.known().is_none(), "a failure must not be remembered");
assert!(
lazy.resolve().await.is_err(),
"and the next attempt retries"
);
}
#[tokio::test(flavor = "multi_thread")]
async fn lazy_set_replaces_the_resolution() {
let lazy = Lazy::new(Runtime::Path, None, true, None);
lazy.set(Resolved::Container(ContainerManager::Docker));
assert_eq!(lazy.ociman().await.unwrap(), Some(ContainerManager::Docker));
lazy.set(Resolved::Local {
source: Runtime::Path,
bin: PathBuf::from("/x/llama-server"),
});
assert_eq!(lazy.resolve().await.unwrap().runtime(), Runtime::Path);
}
}