use std::path::PathBuf;
pub fn context_length_from_env() -> Option<u32> {
parse_context_length(std::env::var("LLMMAN_CONTEXT_LENGTH").ok().as_deref())
}
fn parse_context_length(value: Option<&str>) -> Option<u32> {
value?.trim().parse().ok()
}
pub(super) fn embedding_context_length_from_env() -> Option<u32> {
parse_context_length(
std::env::var("LLMMAN_EMBEDDING_CONTEXT_LENGTH")
.ok()
.as_deref(),
)
}
pub const DEFAULT_CTX_SIZE: u32 = 262144;
pub(super) fn flash_attention_from_env() -> Option<String> {
parse_flash_attention(std::env::var("LLMMAN_FLASH_ATTENTION").ok().as_deref())
}
fn parse_flash_attention(value: Option<&str>) -> Option<String> {
let v = value?.trim();
if v.is_empty() {
return None;
}
Some(match v.to_ascii_lowercase().as_str() {
"1" | "true" | "yes" => "on".to_string(),
"0" | "false" | "no" => "off".to_string(),
other => other.to_string(),
})
}
pub(super) fn kv_cache_type_from_env() -> Option<String> {
parse_kv_cache_type(std::env::var("LLMMAN_KV_CACHE_TYPE").ok().as_deref())
}
fn parse_kv_cache_type(value: Option<&str>) -> Option<String> {
let v = value?.trim();
(!v.is_empty()).then(|| v.to_string())
}
pub(super) fn sched_spread_from_env() -> Option<&'static str> {
parse_sched_spread(std::env::var("LLMMAN_SCHED_SPREAD").ok().as_deref())
}
fn parse_sched_spread(value: Option<&str>) -> Option<&'static str> {
let v = value?.trim();
if v.is_empty() {
return None;
}
match v.to_ascii_lowercase().as_str() {
"1" | "true" | "yes" | "on" | "layer" => Some("layer"),
"0" | "false" | "no" | "off" | "none" => Some("none"),
_ => None,
}
}
pub(super) fn num_parallel_from_env() -> Option<u32> {
parse_num_parallel(std::env::var("LLMMAN_NUM_PARALLEL").ok().as_deref())
}
fn parse_num_parallel(value: Option<&str>) -> Option<u32> {
let n: u32 = value?.trim().parse().ok()?;
(n != 0).then_some(n)
}
pub(super) fn threads_from_env_or_host() -> Option<u32> {
if std::env::var_os("LLAMA_ARG_THREADS").is_some() {
return None;
}
let available = std::thread::available_parallelism().ok()?.get();
Some(default_threads(
u32::try_from(available).unwrap_or(u32::MAX),
))
}
const HALVE_THREADS_ABOVE: u32 = 8;
fn default_threads(available: u32) -> u32 {
if available > HALVE_THREADS_ABOVE {
available / 2
} else {
available
}
}
#[cfg(target_os = "linux")]
fn host_cpu_limit() -> Option<u32> {
let allowed = std::thread::available_parallelism().ok()?.get() as u32;
let online = online_cpu_count()?;
(allowed < online).then_some(allowed)
}
pub(super) fn container_cpu_limit() -> Option<f64> {
#[cfg(target_os = "linux")]
{
let online = online_cpu_count()?;
let limit = match cgroup_v2_cpu_quota() {
Some(q) => q.min(f64::from(affinity_cpu_count().unwrap_or(online))),
None => f64::from(host_cpu_limit()?),
};
(limit < f64::from(online)).then_some(limit)
}
#[cfg(not(target_os = "linux"))]
None
}
#[cfg(target_os = "linux")]
fn affinity_cpu_count() -> Option<u32> {
let status = std::fs::read_to_string("/proc/self/status").ok()?;
let list = status
.lines()
.find_map(|l| l.strip_prefix("Cpus_allowed_list:"))?;
cpu_list_count(list)
}
#[cfg(target_os = "linux")]
fn cgroup_v2_cpu_quota() -> Option<f64> {
let cgroup = std::fs::read_to_string("/proc/self/cgroup").ok()?;
let rel = cgroup.lines().find_map(|l| l.strip_prefix("0::"))?.trim();
let root = std::path::Path::new("/sys/fs/cgroup");
root.join(rel.trim_start_matches('/'))
.ancestors()
.take_while(|d| d.starts_with(root))
.filter_map(|d| std::fs::read_to_string(d.join("cpu.max")).ok())
.filter_map(|s| parse_cpu_max(&s))
.reduce(f64::min)
}
#[cfg(target_os = "linux")]
fn parse_cpu_max(content: &str) -> Option<f64> {
let mut parts = content.split_whitespace();
let quota: f64 = parts.next()?.parse().ok()?;
let period: f64 = parts.next()?.parse().ok()?;
(quota > 0.0 && period > 0.0).then(|| quota / period)
}
#[cfg(target_os = "linux")]
fn online_cpu_count() -> Option<u32> {
cpu_list_count(&std::fs::read_to_string("/sys/devices/system/cpu/online").ok()?)
}
#[cfg(target_os = "linux")]
fn cpu_list_count(list: &str) -> Option<u32> {
let mut count: u32 = 0;
for part in list.trim().split(',') {
match part.split_once('-') {
Some((lo, hi)) => {
let (lo, hi): (u32, u32) = (lo.trim().parse().ok()?, hi.trim().parse().ok()?);
if lo > hi {
return None;
}
count = count.checked_add(hi.checked_sub(lo)?.checked_add(1)?)?;
}
None => {
let _: u32 = part.trim().parse().ok()?;
count = count.checked_add(1)?;
}
}
}
(count > 0).then_some(count)
}
pub(super) fn backend_ctx_size(ctx_size: Option<u32>, num_parallel: Option<u32>) -> Option<u32> {
ctx_size.map(|c| c.saturating_mul(num_parallel.unwrap_or(1)))
}
pub(super) fn effective_num_parallel(
ctx_size: Option<u32>,
num_parallel: Option<u32>,
embedding: bool,
) -> Option<u32> {
ctx_size.and(num_parallel).or(embedding.then_some(1))
}
const DEFAULT_MAX_QUEUE: usize = 512;
pub(super) fn max_queue_from_env() -> usize {
parse_max_queue(std::env::var("LLMMAN_MAX_QUEUE").ok().as_deref())
}
fn parse_max_queue(value: Option<&str>) -> usize {
match value.map(str::trim) {
Some(v) if !v.is_empty() => v.parse().unwrap_or(DEFAULT_MAX_QUEUE),
_ => DEFAULT_MAX_QUEUE,
}
}
pub(super) fn tls_from_env() -> anyhow::Result<Option<(PathBuf, PathBuf)>> {
parse_tls(
std::env::var_os("LLMMAN_TLS_CERT"),
std::env::var_os("LLMMAN_TLS_KEY"),
)
}
fn parse_tls(
cert: Option<std::ffi::OsString>,
key: Option<std::ffi::OsString>,
) -> anyhow::Result<Option<(PathBuf, PathBuf)>> {
let present = |v: Option<std::ffi::OsString>| v.filter(|v| !v.is_empty()).map(PathBuf::from);
match (present(cert), present(key)) {
(Some(cert), Some(key)) => Ok(Some((cert, key))),
(None, None) => Ok(None),
(Some(_), None) => anyhow::bail!("LLMMAN_TLS_CERT is set but LLMMAN_TLS_KEY is not"),
(None, Some(_)) => anyhow::bail!("LLMMAN_TLS_KEY is set but LLMMAN_TLS_CERT is not"),
}
}
pub(super) fn metrics_enabled_from_env() -> bool {
parse_metrics_enabled(std::env::var("LLMMAN_METRICS").ok().as_deref())
}
fn parse_metrics_enabled(value: Option<&str>) -> bool {
matches!(
value.map(|v| v.trim().to_ascii_lowercase()).as_deref(),
Some("1" | "true" | "yes" | "on")
)
}
pub(super) fn max_loaded_models_from_env() -> usize {
parse_max_loaded_models(std::env::var("LLMMAN_MAX_LOADED_MODELS").ok().as_deref())
}
fn parse_max_loaded_models(value: Option<&str>) -> usize {
value
.map(str::trim)
.and_then(|v| v.parse().ok())
.unwrap_or(0)
}
pub(super) fn supports_context_shift(model_ref: &str) -> bool {
!model_ref.to_ascii_lowercase().contains("deepseek")
}
pub(super) fn gguf_trained_ctx(info: &crate::gguf::Info) -> Option<u32> {
trained_ctx(info.context_length())
}
pub(super) fn trained_ctx(context_length: Option<u64>) -> Option<u32> {
context_length
.and_then(|n| u32::try_from(n).ok())
.filter(|n| *n > 0)
}
pub(super) fn embedding_model_ctx(info: &crate::gguf::Info) -> Option<Option<u32>> {
info.is_embedding().then(|| gguf_trained_ctx(info))
}
pub(super) const DECISION_DEFAULT_BATCH: u32 = 8192;
pub(super) fn whole_prompt_batch(
embedding: bool,
decision: bool,
ctx_size: Option<u32>,
explicit: bool,
trained: Option<u32>,
) -> Option<u32> {
if embedding {
return ctx_size.filter(|n| *n > 0);
}
if !decision {
return None;
}
if !explicit {
let wanted = ctx_size.filter(|n| *n > 0);
return Some(wanted.map_or(DECISION_DEFAULT_BATCH, |n| n.min(DECISION_DEFAULT_BATCH)));
}
let trained = trained.filter(|n| *n > 0);
match (ctx_size.unwrap_or(0), trained) {
(0, trained) => trained,
(n, Some(trained)) => Some(n.min(trained)),
(n, None) => Some(n),
}
}
pub(super) fn initial_ctx_size(
configured: Option<u32>,
explicit: bool,
trained: Option<u32>,
embedding: bool,
) -> Option<u32> {
let Some(trained) = trained else {
return configured;
};
if explicit && !embedding {
return configured;
}
Some(
configured
.filter(|n| *n > 0)
.map_or(trained, |n| n.min(trained)),
)
}
pub(super) const MAX_CTX_SHRINK_ATTEMPTS: u32 = 4;
const MIN_CTX_SIZE_FOR_RETRY: u32 = 16384;
pub(super) fn next_ctx_size_after_oom(current: u32) -> Option<u32> {
let next = (current / 2).max(MIN_CTX_SIZE_FOR_RETRY);
(next < current).then_some(next)
}
pub(super) fn looks_like_oom(detail: &str) -> bool {
let d = detail.to_ascii_lowercase();
[
"failed to allocate",
"out of memory",
"not enough memory",
"insufficient memory",
"cudamalloc failed",
"std::bad_alloc",
]
.iter()
.any(|needle| d.contains(needle))
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn default_threads_halves_only_above_eight() {
assert_eq!(default_threads(1), 1);
assert_eq!(default_threads(2), 2);
assert_eq!(default_threads(8), 8);
assert_eq!(default_threads(9), 4);
assert_eq!(default_threads(16), 8);
assert_eq!(default_threads(128), 64);
}
#[test]
fn parse_max_queue_defaults_to_ollamas_own_512_on_anything_unparseable() {
assert_eq!(parse_max_queue(None), 512);
assert_eq!(parse_max_queue(Some("")), 512);
assert_eq!(parse_max_queue(Some("garbage")), 512);
assert_eq!(parse_max_queue(Some("10")), 10);
assert_eq!(parse_max_queue(Some("0")), 0);
}
#[test]
fn parse_max_loaded_models_defaults_to_unbounded_on_anything_unparseable() {
assert_eq!(parse_max_loaded_models(None), 0);
assert_eq!(parse_max_loaded_models(Some("")), 0);
assert_eq!(parse_max_loaded_models(Some("garbage")), 0);
assert_eq!(parse_max_loaded_models(Some("3")), 3);
}
#[test]
fn parse_metrics_enabled_accepts_the_truthy_spellings_and_nothing_else() {
for on in ["1", "true", "TRUE", "yes", "on", " 1 "] {
assert!(parse_metrics_enabled(Some(on)), "{on:?} should enable");
}
for off in ["0", "false", "no", "off", "", " ", "2", "enabled"] {
assert!(!parse_metrics_enabled(Some(off)), "{off:?} should not");
}
assert!(
!parse_metrics_enabled(None),
"unset is the default, and the default is off"
);
}
#[test]
fn tls_takes_both_paths_or_neither() {
let os = |v: &str| Some(std::ffi::OsString::from(v));
assert_eq!(parse_tls(None, None).unwrap(), None);
assert_eq!(parse_tls(os(""), os("")).unwrap(), None, "blank is unset");
assert_eq!(
parse_tls(os("/c.pem"), os("/k.pem")).unwrap(),
Some((PathBuf::from("/c.pem"), PathBuf::from("/k.pem")))
);
assert!(parse_tls(os("/c.pem"), None)
.unwrap_err()
.to_string()
.contains("LLMMAN_TLS_KEY"));
assert!(parse_tls(None, os("/k.pem"))
.unwrap_err()
.to_string()
.contains("LLMMAN_TLS_CERT"));
}
#[test]
fn parse_context_length_accepts_a_plain_number_and_rejects_everything_else() {
assert_eq!(parse_context_length(Some("32768")), Some(32768));
assert_eq!(parse_context_length(Some(" 32768 \n")), Some(32768));
assert_eq!(parse_context_length(Some("0")), Some(0));
assert_eq!(parse_context_length(None), None);
assert_eq!(parse_context_length(Some("")), None);
assert_eq!(parse_context_length(Some("not-a-number")), None);
assert_eq!(parse_context_length(Some("-1")), None);
}
#[test]
fn parse_flash_attention_accepts_llama_server_and_ollama_spellings() {
assert_eq!(parse_flash_attention(Some("on")), Some("on".into()));
assert_eq!(parse_flash_attention(Some("off")), Some("off".into()));
assert_eq!(parse_flash_attention(Some("auto")), Some("auto".into()));
assert_eq!(parse_flash_attention(Some("1")), Some("on".into()));
assert_eq!(parse_flash_attention(Some("true")), Some("on".into()));
assert_eq!(parse_flash_attention(Some("0")), Some("off".into()));
assert_eq!(parse_flash_attention(Some("false")), Some("off".into()));
assert_eq!(parse_flash_attention(Some(" ON \n")), Some("on".into()));
assert_eq!(parse_flash_attention(None), None);
assert_eq!(parse_flash_attention(Some("")), None);
assert_eq!(parse_flash_attention(Some(" ")), None);
}
#[test]
fn parse_kv_cache_type_trims_whitespace_and_treats_empty_as_unset() {
assert_eq!(parse_kv_cache_type(Some("q8_0")), Some("q8_0".into()));
assert_eq!(parse_kv_cache_type(Some(" q4_0 \n")), Some("q4_0".into()));
assert_eq!(parse_kv_cache_type(None), None);
assert_eq!(parse_kv_cache_type(Some("")), None);
assert_eq!(parse_kv_cache_type(Some(" ")), None);
}
#[test]
fn parse_sched_spread_maps_truthy_and_falsey_spellings_to_split_mode() {
for truthy in ["1", "true", "yes", "on", "layer", " ON \n"] {
assert_eq!(
parse_sched_spread(Some(truthy)),
Some("layer"),
"input {truthy:?}"
);
}
for falsey in ["0", "false", "no", "off", "none", " OFF \n"] {
assert_eq!(
parse_sched_spread(Some(falsey)),
Some("none"),
"input {falsey:?}"
);
}
}
#[test]
fn parse_sched_spread_leaves_llama_servers_own_default_untouched_when_unset_or_unparseable() {
assert_eq!(parse_sched_spread(None), None);
assert_eq!(parse_sched_spread(Some("")), None);
assert_eq!(parse_sched_spread(Some(" ")), None);
assert_eq!(parse_sched_spread(Some("garbage")), None);
}
#[test]
fn parse_num_parallel_accepts_a_positive_integer() {
assert_eq!(parse_num_parallel(Some("4")), Some(4));
assert_eq!(parse_num_parallel(Some(" 1 ")), Some(1));
}
#[test]
fn parse_num_parallel_rejects_zero_and_unparseable_values() {
assert_eq!(parse_num_parallel(Some("0")), None);
assert_eq!(parse_num_parallel(None), None);
assert_eq!(parse_num_parallel(Some("")), None);
assert_eq!(parse_num_parallel(Some("-1")), None);
assert_eq!(parse_num_parallel(Some("garbage")), None);
}
#[cfg(target_os = "linux")]
#[test]
fn cpu_list_count_counts_ids_and_inclusive_ranges() {
let cases = [
("0-15\n", Some(16)),
("0", Some(1)),
("0,4-7\n", Some(5)),
("0-3,8-11\n", Some(8)),
("", None),
("\n", None),
("3-1", None),
("0-x", None),
("a", None),
("0,,2", None),
("0-4294967295", None),
("0-4294967295,4", None),
];
for (list, expected) in &cases {
assert_eq!(&cpu_list_count(list), expected, "list={list:?}");
}
}
#[cfg(target_os = "linux")]
#[test]
fn parse_cpu_max_reads_quota_over_period_and_treats_max_as_unlimited() {
assert_eq!(parse_cpu_max("200000 100000\n"), Some(2.0));
assert_eq!(parse_cpu_max("50000 100000\n"), Some(0.5));
assert_eq!(parse_cpu_max("150000 100000"), Some(1.5));
for bad in ["max 100000\n", "", "100000", "0 100000", "100000 0", "x y"] {
assert_eq!(parse_cpu_max(bad), None, "{bad:?}");
}
}
#[test]
fn backend_ctx_size_scales_by_num_parallel_so_each_slot_keeps_the_full_context() {
assert_eq!(backend_ctx_size(Some(4096), Some(4)), Some(16384));
assert_eq!(backend_ctx_size(Some(4096), Some(1)), Some(4096));
assert_eq!(backend_ctx_size(Some(4096), None), Some(4096));
assert_eq!(backend_ctx_size(None, Some(4)), None);
assert_eq!(backend_ctx_size(None, None), None);
}
#[test]
fn backend_ctx_size_saturates_instead_of_overflowing() {
assert_eq!(backend_ctx_size(Some(u32::MAX), Some(2)), Some(u32::MAX));
}
#[test]
fn effective_num_parallel_drops_to_none_without_a_ctx_size_to_scale() {
assert_eq!(effective_num_parallel(None, Some(4), false), None);
assert_eq!(effective_num_parallel(Some(4096), Some(4), false), Some(4));
assert_eq!(effective_num_parallel(Some(4096), None, false), None);
assert_eq!(effective_num_parallel(None, None, false), None);
}
#[test]
fn an_embedding_model_defaults_to_one_slot() {
assert_eq!(effective_num_parallel(Some(2048), None, true), Some(1));
assert_eq!(effective_num_parallel(None, None, true), Some(1));
assert_eq!(effective_num_parallel(Some(2048), Some(4), true), Some(4));
assert_eq!(effective_num_parallel(None, Some(4), true), Some(1));
assert_eq!(effective_num_parallel(Some(2048), None, false), None);
}
#[test]
fn embedding_model_ctx_keys_off_pooling_type_and_reports_the_trained_context() {
let chat = crate::gguf::write_test_gguf_with(&[]);
let embed = crate::gguf::write_test_gguf_with(&[("llama.pooling_type", 1)]);
let chat_info = crate::gguf::read_info(&chat).unwrap();
let embed_info = crate::gguf::read_info(&embed).unwrap();
std::fs::remove_file(&chat).ok();
std::fs::remove_file(&embed).ok();
assert_eq!(embedding_model_ctx(&chat_info), None);
assert_eq!(embedding_model_ctx(&embed_info), Some(Some(4096)));
assert_eq!(gguf_trained_ctx(&chat_info), Some(4096));
assert_eq!(gguf_trained_ctx(&embed_info), Some(4096));
}
#[test]
fn whole_prompt_batch_sizes_the_ubatch_for_a_model_that_reads_its_input_at_once() {
let embed = |ctx| whole_prompt_batch(true, false, ctx, false, Some(4096));
assert_eq!(embed(Some(32768)), Some(32768));
assert_eq!(embed(Some(0)), None);
assert_eq!(embed(None), None);
assert_eq!(
whole_prompt_batch(false, false, Some(4096), true, None),
None
);
}
#[test]
fn whole_prompt_batch_gives_a_decision_model_its_context_up_to_what_memory_allows() {
let default = |ctx, trained| whole_prompt_batch(false, true, ctx, false, trained);
assert_eq!(default(Some(4096), Some(262144)), Some(4096));
assert_eq!(
default(Some(262144), Some(262144)),
Some(DECISION_DEFAULT_BATCH)
);
assert_eq!(default(None, None), Some(DECISION_DEFAULT_BATCH));
assert_eq!(default(Some(0), Some(262144)), Some(DECISION_DEFAULT_BATCH));
let explicit = |ctx, trained| whole_prompt_batch(false, true, ctx, true, trained);
assert_eq!(explicit(Some(65536), Some(262144)), Some(65536));
assert_eq!(explicit(Some(1 << 20), Some(262144)), Some(262144));
assert_eq!(explicit(Some(2048), Some(262144)), Some(2048));
assert_eq!(explicit(Some(65536), None), Some(65536));
assert_eq!(explicit(Some(0), Some(262144)), Some(262144));
assert_eq!(explicit(None, Some(262144)), Some(262144));
assert_eq!(explicit(Some(0), None), None);
}
#[test]
fn initial_ctx_size_caps_the_auto_default_at_the_trained_context() {
let auto = Some(DEFAULT_CTX_SIZE);
assert_eq!(
initial_ctx_size(auto, false, Some(32768), false),
Some(32768)
);
assert_eq!(initial_ctx_size(auto, false, Some(1 << 20), false), auto);
assert_eq!(initial_ctx_size(auto, false, None, false), auto);
}
#[test]
fn initial_ctx_size_forwards_an_explicit_value_unless_embedding() {
assert_eq!(
initial_ctx_size(Some(65536), true, Some(4096), false),
Some(65536)
);
assert_eq!(initial_ctx_size(Some(0), true, Some(4096), false), Some(0));
assert_eq!(
initial_ctx_size(Some(65536), true, Some(4096), true),
Some(4096)
);
assert_eq!(
initial_ctx_size(Some(2048), true, Some(4096), true),
Some(2048)
);
assert_eq!(
initial_ctx_size(Some(0), false, Some(4096), true),
Some(4096)
);
assert_eq!(initial_ctx_size(None, false, Some(4096), true), Some(4096));
}
#[test]
fn supports_context_shift_disables_only_for_deepseek_family_models() {
assert!(!supports_context_shift("deepseek-v3:latest"));
assert!(!supports_context_shift("deepseek-r1:70b"));
assert!(!supports_context_shift("DeepSeek-V2.5:latest")); assert!(supports_context_shift("qwen3.5:latest"));
assert!(supports_context_shift("gpt-oss:20b"));
}
#[test]
fn next_ctx_size_after_oom_halves_from_the_default_down_to_the_floor() {
assert_eq!(next_ctx_size_after_oom(DEFAULT_CTX_SIZE), Some(131072));
assert_eq!(next_ctx_size_after_oom(131072), Some(65536));
assert_eq!(next_ctx_size_after_oom(65536), Some(32768));
assert_eq!(next_ctx_size_after_oom(32768), Some(16384));
assert_eq!(next_ctx_size_after_oom(16384), None);
assert_eq!(next_ctx_size_after_oom(8192), None);
}
#[test]
fn default_ctx_size_reaches_the_floor_within_the_shrink_budget() {
let mut ctx = DEFAULT_CTX_SIZE;
let mut attempts = 0;
while let Some(next) = next_ctx_size_after_oom(ctx) {
ctx = next;
attempts += 1;
}
assert_eq!(ctx, MIN_CTX_SIZE_FOR_RETRY);
assert!(attempts <= MAX_CTX_SHRINK_ATTEMPTS);
}
#[test]
fn looks_like_oom_matches_known_allocation_failure_phrasings() {
for msg in [
"ggml_backend_alloc_ctx_tensors_from_buft: failed to allocate CUDA0 buffer of size 123",
"llama_kv_cache: failed to allocate buffer for kv cache",
"CUDA error: out of memory",
"terminate called after throwing an instance of 'std::bad_alloc'",
"cudaMalloc failed: out of memory",
] {
assert!(looks_like_oom(msg), "expected OOM match for {msg:?}");
}
}
#[test]
fn looks_like_oom_does_not_flag_unrelated_startup_failures() {
for msg in [
"error while loading shared libraries: libcuda.so.1: cannot open shared object file",
"error loading model: unknown architecture 'not-a-real-arch'",
"error: unknown argument: --not-a-real-flag",
] {
assert!(!looks_like_oom(msg), "unexpected OOM match for {msg:?}");
}
}
}