use std::fs::File;
use std::io::{Cursor, Read};
use std::path::{Path, PathBuf};
use std::sync::atomic::{AtomicBool, AtomicU32, AtomicU64, AtomicUsize, Ordering};
use std::sync::{Arc, Mutex};
use std::time::Instant;
use candle_core::quantized::gguf_file;
use candle_core::{Device, Tensor};
use memmap2::Mmap;
use rand::distributions::{Distribution, WeightedIndex};
use rand::thread_rng;
use tokenizers::Tokenizer;
use crate::embedding::EmbeddingModel;
use crate::model::{Architecture, QuantizedModel};
use crate::npu::{NpuBackend, NpuSession};
use crate::template::ChatTemplate;
use crate::error::{JoshuaError, Result};
use crate::types::{ChatMessage, GenerationOptions, Tool, UsageInfo};
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
pub enum PageSize {
#[default]
Default,
TwoMiB,
OneGiB,
}
impl PageSize {
fn params(self) -> (Option<u8>, usize) {
match self {
Self::Default => (None, default_hugepage_bytes()),
Self::TwoMiB => (Some(21), 2 * 1024 * 1024),
Self::OneGiB => (Some(30), 1024 * 1024 * 1024),
}
}
}
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
pub enum HugePages {
#[default]
Off,
Transparent,
Explicit(PageSize),
}
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
pub enum MmapMode {
#[default]
Auto,
Required,
}
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
pub enum MlockMode {
#[default]
Off,
On,
Required,
}
#[derive(Debug, Clone, Default)]
pub struct EngineOptions {
pub n_ctx: u32,
pub huge_pages: HugePages,
pub lazy_weights: bool,
pub mmap: MmapMode,
pub pin_hot_weights: bool,
pub mlock_hot_weights: MlockMode,
}
impl EngineOptions {
pub fn with_n_ctx(n_ctx: u32) -> Self {
Self {
n_ctx,
..Self::default()
}
}
pub fn huge_pages(mut self, huge_pages: HugePages) -> Self {
self.huge_pages = huge_pages;
self
}
pub fn lazy_weights(mut self, lazy: bool) -> Self {
self.lazy_weights = lazy;
self
}
pub fn mmap(mut self, mmap: MmapMode) -> Self {
self.mmap = mmap;
self
}
pub fn pin_hot_weights(mut self, pin: bool) -> Self {
self.pin_hot_weights = pin;
self
}
pub fn mlock_hot_weights(mut self, mode: MlockMode) -> Self {
self.mlock_hot_weights = mode;
self
}
}
pub struct Engine {
model_path: PathBuf,
mmap: Arc<Mmap>,
tokenizer: Arc<Tokenizer>,
eos_token_ids: Vec<u32>,
chat_template: Option<ChatTemplate>,
embed_model: Mutex<Option<Arc<EmbeddingModel>>>,
model_cache: Mutex<Vec<CachedModel>>,
kv_reuses: AtomicU64,
npu: Option<NpuState>,
in_flight: AtomicUsize,
max_concurrency: usize,
max_output_tokens: u32,
model_name: String,
n_ctx: u32,
device: Device,
arch_error: Option<String>,
}
const MAX_CACHED_MODELS: usize = 2;
const NPU_MAX_FAILURES: u32 = 3;
const DEFAULT_MAX_OUTPUT_TOKENS: u32 = 4096;
struct InFlightGuard<'a> {
counter: &'a AtomicUsize,
}
impl<'a> InFlightGuard<'a> {
fn acquire(counter: &'a AtomicUsize, max: usize) -> Result<Self> {
let prev = counter.fetch_add(1, Ordering::AcqRel);
if prev >= max {
counter.fetch_sub(1, Ordering::AcqRel);
return Err(JoshuaError::Overloaded(format!(
"at capacity ({max} concurrent requests); retry shortly"
)));
}
Ok(Self { counter })
}
}
impl Drop for InFlightGuard<'_> {
fn drop(&mut self) {
self.counter.fetch_sub(1, Ordering::AcqRel);
}
}
struct CachedModel {
session: GenSession,
tokens: Vec<u32>,
}
enum GenSession {
Candle(Box<QuantizedModel>),
Npu(Box<dyn NpuSession>),
}
impl GenSession {
fn forward_tokens(&mut self, tokens: &[u32], pos: usize, device: &Device) -> Result<Vec<f32>> {
match self {
Self::Candle(model) => {
let input = Tensor::new(tokens, device)
.and_then(|t| t.unsqueeze(0))
.map_err(|e| JoshuaError::Inference(e.to_string()))?;
let logits = model
.forward(&input, pos)
.map_err(|e| JoshuaError::Inference(e.to_string()))?;
squeeze_batch_logits(&logits)
}
Self::Npu(session) => session.forward(tokens, pos).map_err(JoshuaError::Inference),
}
}
fn clear_state(&mut self) -> bool {
match self {
Self::Candle(model) => model.clear_kv_cache(),
Self::Npu(session) => session.reset(),
}
}
fn is_npu(&self) -> bool {
matches!(self, Self::Npu(_))
}
fn supports_media(&self) -> bool {
match self {
Self::Candle(_) => false,
Self::Npu(session) => session.supports_media(),
}
}
fn media_prefill(&mut self, prompt: &str, images: &[Vec<u8>]) -> Result<(usize, Vec<f32>)> {
match self {
Self::Candle(_) => Err(JoshuaError::InvalidRequest(
"the candle path does not support multimodal input".to_string(),
)),
Self::Npu(session) => session
.media_prefill(prompt, images)
.map_err(JoshuaError::Inference),
}
}
}
struct DecodeOutcome {
response: String,
n_decoded: u32,
fed_tokens: Vec<u32>,
decode_tps: f64,
}
struct NpuState {
backend: Arc<dyn NpuBackend>,
failures: AtomicU32,
disabled: AtomicBool,
}
impl NpuState {
fn record_failure(&self, backend_name: &str, error: &str) {
let failures = self.failures.fetch_add(1, Ordering::Relaxed) + 1;
tracing::warn!("NPU backend {backend_name} failure {failures}/{NPU_MAX_FAILURES}: {error}");
if failures >= NPU_MAX_FAILURES && !self.disabled.swap(true, Ordering::Relaxed) {
tracing::error!(
"NPU backend {backend_name} disabled after {failures} failures; \
all requests will use the candle CPU/GPU path"
);
}
}
fn usable(&self) -> bool {
!self.disabled.load(Ordering::Relaxed)
}
}
impl Engine {
pub fn new(model_path: impl AsRef<Path>) -> Result<Self> {
Self::with_options(model_path, EngineOptions::default())
}
pub fn with_n_ctx(model_path: impl AsRef<Path>, n_ctx: u32) -> Result<Self> {
Self::with_options(model_path, EngineOptions::with_n_ctx(n_ctx))
}
pub fn with_options(model_path: impl AsRef<Path>, options: EngineOptions) -> Result<Self> {
let n_ctx = if options.n_ctx == 0 {
4096
} else {
options.n_ctx
};
let raw_path = model_path.as_ref().to_path_buf();
let gguf_path = if raw_path.is_dir() {
find_gguf_in_dir(&raw_path)?
} else {
raw_path
};
tracing::info!("Loading model from {:?}", gguf_path);
let model_dir = gguf_path
.parent()
.unwrap_or_else(|| Path::new("."))
.to_path_buf();
let tokenizer_path = model_dir.join("tokenizer.json");
if !tokenizer_path.exists() {
return Err(JoshuaError::ModelLoad(format!(
"tokenizer.json not found at {:?}.\n\
Place it alongside the .gguf file \
(download from the model's HuggingFace repository).",
tokenizer_path
)));
}
let tokenizer = Tokenizer::from_file(&tokenizer_path)
.map_err(|e| JoshuaError::ModelLoad(format!("tokenizer load failed: {e}")))?;
let hot_pinning = options.pin_hot_weights || options.mlock_hot_weights != MlockMode::Off;
let mmap = map_model(
&gguf_path,
options.huge_pages,
options.lazy_weights,
options.mmap,
hot_pinning,
)?;
let gguf = read_gguf_header(&mmap)
.map_err(|e| JoshuaError::ModelLoad(format!("GGUF read failed: {e}")))?;
if hot_pinning {
if matches!(options.huge_pages, HugePages::Explicit(_)) {
tracing::warn!(
"ignoring hot-weight pinning: explicit huge pages already copy the whole \
model into anonymous RAM, so every range is resident. Use --huge-pages \
transparent (or none) for pinning to matter."
);
} else {
let raw = crate::gguf_ext::read_header(&mut Cursor::new(&mmap[..]))
.map_err(|e| JoshuaError::ModelLoad(format!("GGUF header re-read failed: {e}")))?;
apply_hot_weight_pinning(
&mmap,
&raw,
options.pin_hot_weights,
options.mlock_hot_weights,
)?; }
}
let model_name = gguf_path
.file_stem()
.and_then(|s| s.to_str())
.unwrap_or("unknown")
.to_string();
let display_name = gguf
.metadata
.get("general.name")
.and_then(|v| v.to_string().ok().cloned())
.and_then(|s| sanitize_model_name(&s))
.unwrap_or_else(|| model_name.clone());
let (arch, arch_error) = match Architecture::detect(&gguf.metadata) {
Ok(arch) => (Some(arch), None),
Err(e) => {
tracing::warn!("{e} (continuing; an NPU backend may still serve this model)");
(None, Some(e))
}
};
let eos_token_ids = extract_eos_ids(&gguf, &tokenizer);
let chat_template = extract_chat_template(&gguf, &tokenizer);
let device = Self::default_device();
tracing::info!(
"Model '{}' ready (arch={}, ctx={}, eos_ids={:?}, chat_template={}, device={:?})",
display_name,
arch.as_ref()
.map(|a| a.display_name())
.unwrap_or("unknown (NPU-only)"),
n_ctx,
eos_token_ids,
if chat_template.is_some() {
"from GGUF"
} else {
"ChatML fallback"
},
device
);
let max_concurrency = std::thread::available_parallelism()
.map(|n| n.get())
.unwrap_or(4);
Ok(Self {
model_path: gguf_path,
mmap: Arc::new(mmap),
tokenizer: Arc::new(tokenizer),
eos_token_ids,
chat_template,
embed_model: Mutex::new(None),
model_cache: Mutex::new(Vec::new()),
kv_reuses: AtomicU64::new(0),
npu: None,
in_flight: AtomicUsize::new(0),
max_concurrency,
max_output_tokens: DEFAULT_MAX_OUTPUT_TOKENS,
model_name,
n_ctx,
device,
arch_error,
})
}
pub fn with_max_concurrency(mut self, max: usize) -> Self {
self.max_concurrency = max.max(1);
self
}
pub fn with_max_output_tokens(mut self, max: u32) -> Self {
self.max_output_tokens = max.max(1);
self
}
fn default_device() -> Device {
#[cfg(feature = "cuda")]
{
match Device::new_cuda(0) {
Ok(device) => return device,
Err(e) => tracing::warn!("CUDA unavailable, falling back to CPU: {e}"),
}
}
#[cfg(feature = "metal")]
{
match Device::new_metal(0) {
Ok(device) => return device,
Err(e) => tracing::warn!("Metal unavailable, falling back to CPU: {e}"),
}
}
Device::Cpu
}
pub fn model_name(&self) -> &str {
&self.model_name
}
pub fn model_path(&self) -> &Path {
&self.model_path
}
pub fn n_ctx(&self) -> u32 {
self.n_ctx
}
pub fn has_chat_template(&self) -> bool {
self.chat_template.is_some()
}
fn format_chatml_prompt(messages: &[ChatMessage], tools: Option<&[Tool]>) -> String {
let mut prompt = String::new();
if let Some(tools) = tools.filter(|t| !t.is_empty()) {
prompt.push_str(
"<|im_start|>system\n\
# Tools\n\n\
You may call one or more functions to assist with the user query.\n\n\
You are provided with function signatures within <tools></tools> XML tags:\n\
<tools>\n",
);
for tool in tools {
if let Ok(json) = serde_json::to_string(tool) {
prompt.push_str(&json);
prompt.push('\n');
}
}
prompt.push_str(
"</tools>\n\n\
For each function call, return a json object with function name and arguments \
within <tool_call></tool_call> XML tags:\n\
<tool_call>\n\
{\"name\": <function-name>, \"arguments\": <args-json-object>}\n\
</tool_call><|im_end|>\n",
);
}
for msg in messages {
prompt.push_str("<|im_start|>");
prompt.push_str(&msg.role);
prompt.push('\n');
prompt.push_str(&msg.content);
prompt.push_str("<|im_end|>\n");
}
prompt.push_str("<|im_start|>assistant\n");
prompt
}
fn format_prompt(&self, messages: &[ChatMessage], tools: Option<&[Tool]>) -> (String, bool) {
if let Some(template) = &self.chat_template {
match template.render(messages, tools) {
Ok(prompt) => return (prompt, false),
Err(e) => {
tracing::warn!("GGUF chat template unusable, falling back to ChatML: {e}");
}
}
}
(Self::format_chatml_prompt(messages, tools), true)
}
pub fn complete(
&self,
messages: &[ChatMessage],
options: &GenerationOptions,
) -> Result<(String, UsageInfo, f64, f64)> {
self.complete_chat(messages, None, options)
}
pub fn complete_chat(
&self,
messages: &[ChatMessage],
tools: Option<&[Tool]>,
options: &GenerationOptions,
) -> Result<(String, UsageInfo, f64, f64)> {
if messages
.iter()
.any(|m| m.images.as_ref().is_some_and(|i| !i.is_empty()))
{
let (marked_messages, images) = resolve_message_media(messages)?;
let (prompt, _) = self.format_prompt(&marked_messages, tools);
return self.complete_media(&prompt, &images, options);
}
let (prompt, add_special_tokens) = self.format_prompt(messages, tools);
self.complete_with(&prompt, add_special_tokens, options)
}
fn complete_media(
&self,
prompt: &str,
images: &[Vec<u8>],
options: &GenerationOptions,
) -> Result<(String, UsageInfo, f64, f64)> {
let _permit = InFlightGuard::acquire(&self.in_flight, self.max_concurrency)?;
let options = &self.clamp_options(options, None);
let (mut session, _) = self.acquire_session(&[], true)?;
if !session.supports_media() {
if session.clear_state() {
self.release_model(session, Vec::new());
}
return Err(JoshuaError::InvalidRequest(
"this request contains images, which require a multimodal NPU plugin — \
run with --npu-plugin pointing at the llama.cpp adapter built with an \
mmproj (JOSHUA_LLAMA_MMPROJ) or another media-capable plugin"
.to_string(),
));
}
let prefill_start = Instant::now();
let result = session
.media_prefill(prompt, images)
.and_then(|(n_past, logits)| {
let prefill_ms = prefill_start.elapsed().as_secs_f64() * 1000.0;
let outcome = self.decode_loop(&mut session, logits, n_past, &[], options)?;
Ok((n_past, prefill_ms, outcome))
});
match result {
Ok((n_past, prefill_ms, outcome)) => {
if session.clear_state() {
self.release_model(session, Vec::new());
}
let prefill_tps = if prefill_ms > 0.0 {
n_past as f64 / (prefill_ms / 1000.0)
} else {
0.0
};
let usage = UsageInfo {
prompt_tokens: n_past as u32,
completion_tokens: outcome.n_decoded,
total_tokens: n_past as u32 + outcome.n_decoded,
};
Ok((outcome.response, usage, prefill_tps, outcome.decode_tps))
}
Err(e) => {
if let Some(npu) = &self.npu {
npu.record_failure(&npu.backend.name(), &e.to_string());
}
Err(e)
}
}
}
pub fn complete_raw(
&self,
prompt: &str,
options: &GenerationOptions,
) -> Result<(String, UsageInfo, f64, f64)> {
self.complete_with(prompt, true, options)
}
fn clamp_options(
&self,
options: &GenerationOptions,
prompt_len: Option<usize>,
) -> GenerationOptions {
let mut clamped = options.clone();
let mut cap = clamped.max_tokens.min(self.max_output_tokens);
if let Some(n_prompt) = prompt_len {
let remaining = (self.n_ctx as usize).saturating_sub(n_prompt).max(1) as u32;
cap = cap.min(remaining);
}
clamped.max_tokens = cap;
clamped
}
fn complete_with(
&self,
prompt: &str,
add_special_tokens: bool,
options: &GenerationOptions,
) -> Result<(String, UsageInfo, f64, f64)> {
let _permit = InFlightGuard::acquire(&self.in_flight, self.max_concurrency)?;
let encoding = self
.tokenizer
.encode(prompt, add_special_tokens)
.map_err(|e| JoshuaError::Tokenization(e.to_string()))?;
let prompt_tokens: Vec<u32> = encoding.get_ids().to_vec();
let n_prompt = prompt_tokens.len();
if n_prompt >= self.n_ctx as usize {
return Err(JoshuaError::PromptTooLong(n_prompt, self.n_ctx as usize));
}
let options = &self.clamp_options(options, Some(n_prompt));
let (mut session, n_reused) = self.acquire_session(&prompt_tokens, true)?;
let was_npu = session.is_npu();
let result = self.run_generation(&mut session, &prompt_tokens, n_reused, options);
match result {
Ok((response, usage, prefill_tps, decode_tps, kv_tokens)) => {
self.release_model(session, kv_tokens);
Ok((response, usage, prefill_tps, decode_tps))
}
Err(e) if was_npu => {
if let Some(npu) = &self.npu {
npu.record_failure(&npu.backend.name(), &e.to_string());
}
if session.clear_state() {
self.release_model(session, Vec::new());
}
if self.arch_error.is_some() {
return Err(e);
}
tracing::warn!("Retrying request on the candle path after NPU failure: {e}");
let (mut session, n_reused) = self.acquire_session(&prompt_tokens, false)?;
match self.run_generation(&mut session, &prompt_tokens, n_reused, options) {
Ok((response, usage, prefill_tps, decode_tps, kv_tokens)) => {
self.release_model(session, kv_tokens);
Ok((response, usage, prefill_tps, decode_tps))
}
Err(e) => {
if session.clear_state() {
self.release_model(session, Vec::new());
}
Err(e)
}
}
}
Err(e) => {
if session.clear_state() {
self.release_model(session, Vec::new());
}
Err(e)
}
}
}
fn run_generation(
&self,
model: &mut GenSession,
prompt_tokens: &[u32],
n_reused: usize,
options: &GenerationOptions,
) -> Result<(String, UsageInfo, f64, f64, Vec<u32>)> {
let n_prompt = prompt_tokens.len();
let new_tokens = &prompt_tokens[n_reused..];
let mut kv_tokens = prompt_tokens.to_vec();
let prefill_start = Instant::now();
let logits_vec = model.forward_tokens(new_tokens, n_reused, &self.device)?;
let prefill_ms = prefill_start.elapsed().as_secs_f64() * 1000.0;
let outcome = self.decode_loop(model, logits_vec, n_prompt, prompt_tokens, options)?;
kv_tokens.extend_from_slice(&outcome.fed_tokens);
let n_prefilled = new_tokens.len();
let prefill_tps = if prefill_ms > 0.0 {
n_prefilled as f64 / (prefill_ms / 1000.0)
} else {
0.0
};
tracing::debug!(
prompt_tokens = n_prompt,
prefill_tokens = n_prefilled,
reused_tokens = n_reused,
prefill_tps,
decode_tokens = outcome.n_decoded,
decode_tps = outcome.decode_tps,
"Completion finished"
);
let usage = UsageInfo {
prompt_tokens: n_prompt as u32,
completion_tokens: outcome.n_decoded,
total_tokens: n_prompt as u32 + outcome.n_decoded,
};
Ok((
outcome.response,
usage,
prefill_tps,
outcome.decode_tps,
kv_tokens,
))
}
fn decode_loop(
&self,
model: &mut GenSession,
mut logits_vec: Vec<f32>,
start_pos: usize,
penalty_seed: &[u32],
options: &GenerationOptions,
) -> Result<DecodeOutcome> {
const REP_WINDOW: usize = 64;
let mut recent_tokens: Vec<u32> = penalty_seed
.iter()
.rev()
.take(REP_WINDOW)
.copied()
.collect::<Vec<_>>()
.into_iter()
.rev()
.collect();
let mut rng = thread_rng();
let mut response = String::new();
let mut fed_tokens: Vec<u32> = Vec::new();
let mut n_decoded: u32 = 0;
let mut n_cur = start_pos;
let decode_start = Instant::now();
loop {
if n_decoded >= options.max_tokens {
break;
}
if n_cur >= self.n_ctx as usize {
break;
}
let next_token = sample_token(&logits_vec, options, &mut rng, &recent_tokens)?;
if self.eos_token_ids.contains(&next_token) {
break;
}
let piece = self
.tokenizer
.decode(&[next_token], false)
.map_err(|e| JoshuaError::Inference(e.to_string()))?;
response.push_str(&piece);
n_decoded += 1;
if recent_tokens.len() >= REP_WINDOW {
recent_tokens.remove(0);
}
recent_tokens.push(next_token);
if Self::check_stop_sequences(&mut response, &options.stop_sequences) {
break;
}
logits_vec = model.forward_tokens(&[next_token], n_cur, &self.device)?;
fed_tokens.push(next_token);
n_cur += 1;
}
let decode_ms = decode_start.elapsed().as_secs_f64() * 1000.0;
let decode_tps = if decode_ms > 0.0 && n_decoded > 0 {
n_decoded as f64 / (decode_ms / 1000.0)
} else {
0.0
};
Ok(DecodeOutcome {
response,
n_decoded,
fed_tokens,
decode_tps,
})
}
pub fn embed(&self, texts: &[String]) -> Result<Vec<Vec<f32>>> {
Ok(self.embed_with_usage(texts)?.0)
}
pub fn embed_with_usage(&self, texts: &[String]) -> Result<(Vec<Vec<f32>>, u32)> {
let _permit = InFlightGuard::acquire(&self.in_flight, self.max_concurrency)?;
let model = self.embedding_model()?;
let mut vectors = Vec::with_capacity(texts.len());
let mut total_tokens: u32 = 0;
for text in texts {
let encoding = self
.tokenizer
.encode(text.as_str(), true)
.map_err(|e| JoshuaError::Tokenization(e.to_string()))?;
let tokens = encoding.get_ids();
if tokens.len() >= self.n_ctx as usize {
return Err(JoshuaError::PromptTooLong(
tokens.len(),
self.n_ctx as usize,
));
}
total_tokens += tokens.len() as u32;
let vector = model
.embed_tokens(tokens)
.map_err(|e| JoshuaError::Inference(e.to_string()))?;
vectors.push(vector);
}
Ok((vectors, total_tokens))
}
fn embedding_model(&self) -> Result<Arc<EmbeddingModel>> {
let mut slot = self
.embed_model
.lock()
.unwrap_or_else(|poisoned| poisoned.into_inner());
if let Some(model) = slot.as_ref() {
return Ok(Arc::clone(model));
}
let mut cursor = Cursor::new(&self.mmap[..]);
let gguf = gguf_file::Content::read(&mut cursor)
.map_err(|e| JoshuaError::ModelLoad(format!("GGUF read failed: {e}")))?;
let model = EmbeddingModel::from_gguf(gguf, &mut cursor, &self.device)
.map_err(|e| JoshuaError::InvalidRequest(e.to_string()))?;
let model = Arc::new(model);
*slot = Some(Arc::clone(&model));
Ok(model)
}
pub fn kv_reuse_count(&self) -> u64 {
self.kv_reuses.load(Ordering::Relaxed)
}
pub fn with_npu_backend(mut self, backend: Arc<dyn NpuBackend>) -> Self {
tracing::info!("NPU backend configured: {}", backend.name());
self.npu = Some(NpuState {
backend,
failures: AtomicU32::new(0),
disabled: AtomicBool::new(false),
});
self
}
pub fn npu_active(&self) -> bool {
self.npu.as_ref().is_some_and(|n| n.usable())
}
fn acquire_session(
&self,
prompt_tokens: &[u32],
allow_npu: bool,
) -> Result<(GenSession, usize)> {
let want_npu = allow_npu && self.npu.as_ref().is_some_and(|n| n.usable());
{
let mut pool = self.model_pool();
let best = pool
.iter()
.enumerate()
.filter(|(_, c)| {
c.session.is_npu() == want_npu
&& c.tokens.len() < prompt_tokens.len()
&& prompt_tokens.starts_with(&c.tokens)
})
.max_by_key(|(_, c)| c.tokens.len())
.map(|(i, _)| i);
if let Some(i) = best {
let cached = pool.swap_remove(i);
self.kv_reuses.fetch_add(1, Ordering::Relaxed);
tracing::debug!(
reused_tokens = cached.tokens.len(),
prompt_tokens = prompt_tokens.len(),
npu = want_npu,
"Continuing from cached KV prefix"
);
return Ok((cached.session, cached.tokens.len()));
}
let resettable = pool.iter().position(|c| c.session.is_npu() == want_npu);
if let Some(i) = resettable {
let mut cached = pool.swap_remove(i);
if cached.session.clear_state() {
tracing::debug!(npu = want_npu, "Reusing pooled session with cleared state");
return Ok((cached.session, 0));
}
}
}
if want_npu {
let npu = self.npu.as_ref().expect("checked above");
match npu.backend.create_session(&self.model_path, self.n_ctx) {
Ok(session) => return Ok((GenSession::Npu(session), 0)),
Err(e) => {
npu.record_failure(&npu.backend.name(), &e);
if self.arch_error.is_some() {
return Err(JoshuaError::ModelLoad(format!(
"{} session creation failed: {e}",
npu.backend.name()
)));
}
tracing::warn!("NPU session creation failed, using candle path: {e}");
}
}
}
Ok((GenSession::Candle(Box::new(self.load_model()?)), 0))
}
fn model_pool(&self) -> std::sync::MutexGuard<'_, Vec<CachedModel>> {
self.model_cache
.lock()
.unwrap_or_else(|poisoned| poisoned.into_inner())
}
fn release_model(&self, session: GenSession, tokens: Vec<u32>) {
{
let mut pool = self.model_pool();
pool.push(CachedModel { session, tokens });
while pool.len() > MAX_CACHED_MODELS {
pool.remove(0);
}
}
}
fn load_model(&self) -> Result<QuantizedModel> {
if let Some(err) = &self.arch_error {
let msg = match &self.npu {
Some(npu) if !npu.usable() => format!(
"{err}; the {} backend serving this model was disabled after \
{NPU_MAX_FAILURES} failures",
npu.backend.name()
),
_ => err.clone(),
};
return Err(JoshuaError::ModelLoad(msg));
}
let mut cursor = Cursor::new(&self.mmap[..]);
let gguf = read_gguf_header(&self.mmap)
.map_err(|e| JoshuaError::ModelLoad(format!("GGUF read failed: {e}")))?;
QuantizedModel::from_gguf_mmap(
gguf,
&mut cursor,
&self.device,
Some(Arc::clone(&self.mmap)),
)
.map_err(|e| JoshuaError::ModelLoad(format!("model init failed: {e}")))
}
fn check_stop_sequences(response: &mut String, stops: &[String]) -> bool {
for stop in stops {
if stop.is_empty() {
continue;
}
if response.ends_with(stop.as_str()) {
response.truncate(response.len() - stop.len());
return true;
}
}
false
}
}
fn resolve_message_media(messages: &[ChatMessage]) -> Result<(Vec<ChatMessage>, Vec<Vec<u8>>)> {
let mut marked = messages.to_vec();
let mut images = Vec::new();
for msg in &mut marked {
let Some(attached) = msg.images.take() else {
continue;
};
let mut markers = String::new();
for source in &attached {
images.push(load_image_bytes(source)?);
markers.push_str(crate::npu::MEDIA_MARKER);
markers.push('\n');
}
msg.content = format!("{markers}{}", msg.content);
}
Ok((marked, images))
}
const MAX_IMAGE_BYTES: usize = 16 * 1024 * 1024;
fn load_image_bytes(source: &str) -> Result<Vec<u8>> {
let Some(rest) = source.strip_prefix("data:") else {
return Err(JoshuaError::InvalidRequest(
"image sources must be inline base64 `data:` URLs; \
filesystem paths and remote URLs are not accepted"
.to_string(),
));
};
let b64 = rest.split_once("base64,").map(|(_, b)| b).ok_or_else(|| {
JoshuaError::InvalidRequest("only base64 data: URLs are supported".to_string())
})?;
use base64::Engine as _;
let bytes = base64::engine::general_purpose::STANDARD
.decode(b64.trim())
.map_err(|e| JoshuaError::InvalidRequest(format!("invalid base64 image data: {e}")))?;
if bytes.len() > MAX_IMAGE_BYTES {
return Err(JoshuaError::InvalidRequest(format!(
"image is {} bytes, exceeding the {MAX_IMAGE_BYTES}-byte limit",
bytes.len()
)));
}
Ok(bytes)
}
fn map_model(
path: &Path,
huge: HugePages,
lazy: bool,
mmap_mode: MmapMode,
per_range_advice: bool,
) -> Result<Mmap> {
let file = File::open(path)?;
if let HugePages::Explicit(size) = huge {
if lazy {
tracing::warn!(
"ignoring the explicit huge-page request: it copies the whole model into \
anonymous memory, which cannot work for a model flagged larger than RAM. \
Using the file-backed mapping so weights page in on demand. Use \
--huge-pages transparent for huge pages that keep the mapping file-backed."
);
} else {
check_mappable(path, &file, mmap_mode, false)?;
return map_model_hugetlb(path, &file, size);
}
}
check_mappable(path, &file, mmap_mode, true)?;
let mmap = unsafe { Mmap::map(&file) }
.map_err(|e| JoshuaError::ModelLoad(format!("mmap of GGUF file failed: {e}")))?;
#[cfg(unix)]
if !per_range_advice {
let _ = mmap.advise(if lazy {
memmap2::Advice::Random
} else {
memmap2::Advice::Sequential
});
}
if huge == HugePages::Transparent {
#[cfg(target_os = "linux")]
match mmap.advise(memmap2::Advice::HugePage) {
Ok(()) => tracing::info!("requested transparent huge pages for the model mapping"),
Err(e) => {
tracing::warn!("transparent huge pages unavailable; using normal pages: {e}")
}
}
#[cfg(not(target_os = "linux"))]
tracing::warn!("transparent huge pages are Linux-only; using normal pages");
}
Ok(mmap)
}
type ByteRange = (usize, usize);
fn is_routed_expert(name: &str) -> bool {
name.contains(".ffn_gate_exps")
|| name.contains(".ffn_down_exps")
|| name.contains(".ffn_up_exps")
}
fn weight_ranges(
header: &crate::gguf_ext::GgufHeader,
file_size: u64,
) -> (Vec<ByteRange>, Vec<ByteRange>) {
let mut all: Vec<(u64, &str)> = header
.tensors
.iter()
.map(|(n, t)| (header.tensor_data_offset + t.offset, n.as_str()))
.collect();
all.sort_by_key(|(o, _)| *o);
let mut dense: Vec<ByteRange> = Vec::new();
let mut experts: Vec<ByteRange> = Vec::new();
for i in 0..all.len() {
let (off, name) = all[i];
let end = if i + 1 < all.len() {
all[i + 1].0
} else {
file_size
};
if end <= off {
continue; }
let range = (off as usize, (end - off) as usize);
if is_routed_expert(name) {
experts.push(range);
} else {
dense.push(range);
}
}
fn merge(mut ranges: Vec<ByteRange>) -> Vec<ByteRange> {
ranges.sort_by_key(|(o, _)| *o);
let mut out: Vec<ByteRange> = Vec::new();
for (off, len) in ranges {
if let Some(last) = out.last_mut() {
if off <= last.0 + last.1 + 4096 {
let end = (off + len).max(last.0 + last.1);
last.1 = end - last.0;
continue;
}
}
out.push((off, len));
}
out
}
(merge(dense), merge(experts))
}
#[cfg(unix)]
fn apply_hot_weight_pinning(
mmap: &Mmap,
header: &crate::gguf_ext::GgufHeader,
prefetch: bool,
mlock: MlockMode,
) -> Result<()> {
let (dense, experts) = weight_ranges(header, mmap.len() as u64);
let gib = |r: &[ByteRange]| r.iter().map(|(_, l)| *l as u64).sum::<u64>() as f64 / 2f64.powi(30);
tracing::info!(
"hot-weight pinning: {} dense range(s) ({:.2} GiB) to keep resident, {} expert range(s) ({:.2} GiB) on demand",
dense.len(),
gib(&dense),
experts.len(),
gib(&experts),
);
if prefetch {
let mut failed = 0usize;
for &(off, len) in &dense {
if mmap.advise_range(memmap2::Advice::WillNeed, off, len).is_err() {
failed += 1;
}
}
if failed > 0 {
tracing::warn!("{failed}/{} dense ranges ignored the WILLNEED prefetch hint", dense.len());
}
}
{
let mut failed = 0usize;
for &(off, len) in &experts {
if mmap.advise_range(memmap2::Advice::Random, off, len).is_err() {
failed += 1;
}
}
if failed > 0 {
tracing::warn!("{failed}/{} expert ranges ignored the RANDOM access hint", experts.len());
}
}
if mlock != MlockMode::Off {
let page = unsafe { libc::sysconf(libc::_SC_PAGESIZE) } as usize;
if page == 0 || !page.is_power_of_two() {
tracing::warn!("mlock skipped: could not determine the system page size");
} else {
let required = aligned_range_bytes(&dense, page, mmap.len());
let limit = memlock_limit_bytes();
match mlock_decision(mlock, limit, required) {
MlockDecision::Proceed => {
let failed = mlock_ranges(mmap, &dense, page);
if failed > 0 && mlock == MlockMode::Required {
return Err(JoshuaError::ModelLoad(format!(
"mlock of the hot weight set failed ({failed}/{} ranges) despite \
RLIMIT_MEMLOCK appearing sufficient — see the warnings above",
dense.len()
)));
}
}
MlockDecision::Degrade => tracing::warn!(
"RLIMIT_MEMLOCK is {limit}, below the {:.2} GiB hot set — skipping the lock \
and degrading to advisory pinning. Raise it with /etc/security/limits.conf \
(`{user} - memlock unlimited`), `LimitMEMLOCK=infinity` (systemd), or \
`ulimit -l unlimited`; for a live systemd user session apply it without \
re-login: `sudo prlimit --pid <user manager pid> --memlock=-1:-1`.",
required as f64 / 2f64.powi(30),
limit = display_memlock_limit(limit),
user = whoami(),
),
MlockDecision::Fail => {
return Err(JoshuaError::ModelLoad(format!(
"RLIMIT_MEMLOCK is {limit}, below the {:.2} GiB hot set that \
--mlock-hot-weights=required demands be locked. Raise it with \
/etc/security/limits.conf (`{user} - memlock unlimited`), \
`LimitMEMLOCK=infinity` (systemd), or `ulimit -l unlimited`; for a \
live systemd user session: `sudo prlimit --pid <user manager pid> \
--memlock=-1:-1`.",
required as f64 / 2f64.powi(30),
limit = display_memlock_limit(limit),
user = whoami(),
)))
}
}
}
}
Ok(())
}
#[cfg(not(unix))]
fn apply_hot_weight_pinning(
_mmap: &Mmap,
_header: &crate::gguf_ext::GgufHeader,
_prefetch: bool,
_mlock: MlockMode,
) -> Result<()> {
tracing::warn!("hot-weight pinning is unix-only; ignoring the request");
Ok(())
}
#[cfg(unix)]
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
enum MlockDecision {
Proceed,
Degrade,
Fail,
}
#[cfg(unix)]
fn mlock_decision(mode: MlockMode, limit: Option<u64>, required: u64) -> MlockDecision {
match mode {
MlockMode::Off => MlockDecision::Proceed, MlockMode::On => match limit {
Some(l) if l < required => MlockDecision::Degrade,
_ => MlockDecision::Proceed,
},
MlockMode::Required => match limit {
Some(l) if l < required => MlockDecision::Fail,
_ => MlockDecision::Proceed,
},
}
}
#[cfg(unix)]
fn memlock_limit_bytes() -> Option<u64> {
let mut lim = libc::rlimit {
rlim_cur: 0,
rlim_max: 0,
};
if unsafe { libc::getrlimit(libc::RLIMIT_MEMLOCK, &mut lim) } != 0 {
return None;
}
if lim.rlim_cur == libc::RLIM_INFINITY {
return None; }
Some(lim.rlim_cur)
}
#[cfg(unix)]
fn display_memlock_limit(limit: Option<u64>) -> String {
match limit {
None => "unlimited".to_string(),
Some(b) if b >= 1024 * 1024 && b % (1024 * 1024) == 0 => {
format!("{} MiB", b / (1024 * 1024))
}
Some(b) => format!("{b} bytes"),
}
}
#[cfg(unix)]
fn whoami() -> String {
std::env::var("USER").unwrap_or_else(|_| "USER".to_string())
}
#[cfg(unix)]
fn aligned_range_bytes(ranges: &[ByteRange], page: usize, map_len: usize) -> u64 {
let map_end = map_len.saturating_add(page - 1) & !(page - 1);
let mut total: u64 = 0;
let mut covered_end: usize = 0; for &(off, len) in ranges {
let start = off & !(page - 1);
let end = ((off + len).saturating_add(page - 1)) & !(page - 1);
let end = end.min(map_end);
if end <= start {
continue;
}
if start >= covered_end {
total += (end - start) as u64;
} else if end > covered_end {
total += (end - covered_end) as u64;
}
covered_end = covered_end.max(end);
}
total
}
#[cfg(unix)]
fn mlock_ranges(mmap: &Mmap, ranges: &[ByteRange], page: usize) -> usize {
let total: u64 = ranges.iter().map(|(_, l)| *l as u64).sum();
let mut locked: u64 = 0;
let mut failed = 0usize;
for &(off, len) in ranges {
let start = off & !(page - 1);
let end = ((off + len).saturating_add(page - 1)) & !(page - 1);
let end = end.min(mmap.len().saturating_add(page - 1) & !(page - 1));
if end <= start {
continue;
}
let rc = unsafe {
libc::mlock(
mmap.as_ptr().add(start).cast::<libc::c_void>(),
end - start,
)
};
if rc == 0 {
locked += (end - start) as u64;
} else {
failed += 1;
if failed <= 3 {
tracing::warn!(
"mlock of hot weight range at offset {off:#x} failed: {}",
std::io::Error::last_os_error()
);
}
}
}
if failed > 0 {
tracing::warn!(
"mlock failed for {failed}/{} hot ranges ({:.2} of {:.2} GiB locked). \
Raise the process memlock limit to pin them: /etc/security/limits.conf \
(`{user} - memlock unlimited`), `LimitMEMLOCK=infinity` (systemd), or \
`ulimit -l unlimited`; for a live systemd user session: \
`sudo prlimit --pid <user manager pid> --memlock=-1:-1`.",
ranges.len(),
locked as f64 / 2f64.powi(30),
total as f64 / 2f64.powi(30),
user = whoami(),
);
}
failed
}
fn check_mappable(path: &Path, file: &File, mode: MmapMode, file_backed: bool) -> Result<()> {
let Some(found) = crate::compression::detect_gguf(file) else {
return Ok(());
};
if !file_backed && matches!(found, crate::compression::Compression::Filesystem { .. }) {
return Ok(());
}
let what = format!("{path:?} cannot be memory-mapped usefully: {found}");
match mode {
MmapMode::Required => Err(JoshuaError::ModelLoad(format!(
"{what}. Memory mapping was requested explicitly, so this is an error; \
drop the explicit mmap request to load the model anyway."
))),
MmapMode::Auto => {
tracing::warn!("{what}.");
Ok(())
}
}
}
#[cfg(target_os = "linux")]
fn map_model_hugetlb(path: &Path, file: &File, size: PageSize) -> Result<Mmap> {
let len = file.metadata()?.len() as usize;
let (page_bits, page_len) = size.params();
let mapped_len = len
.div_ceil(page_len)
.checked_mul(page_len)
.ok_or_else(|| JoshuaError::ModelLoad("model too large for huge-page mapping".into()))?;
let mut anon = memmap2::MmapOptions::new()
.len(mapped_len)
.huge(page_bits)
.map_anon()
.map_err(|e| {
JoshuaError::ModelLoad(format!(
"could not allocate {} MiB of {}-byte huge pages — is the pool configured \
(e.g. `sysctl vm.nr_hugepages`)?: {e}",
mapped_len / (1024 * 1024),
page_len
))
})?;
File::open(path)?
.read_exact(&mut anon[..len])
.map_err(|e| {
JoshuaError::ModelLoad(format!("reading model into huge pages failed: {e}"))
})?;
let mmap = anon
.make_read_only()
.map_err(|e| JoshuaError::ModelLoad(format!("freezing huge-page mapping failed: {e}")))?;
tracing::info!(
"loaded model into {} MiB of explicit huge pages ({}-byte pages, anonymous — \
not shared through the page cache)",
mapped_len / (1024 * 1024),
page_len
);
Ok(mmap)
}
#[cfg(not(target_os = "linux"))]
fn map_model_hugetlb(_path: &Path, file: &File, _size: PageSize) -> Result<Mmap> {
tracing::warn!("explicit huge pages are Linux-only; using a normal file mapping");
unsafe { Mmap::map(file) }
.map_err(|e| JoshuaError::ModelLoad(format!("mmap of GGUF file failed: {e}")))
}
fn default_hugepage_bytes() -> usize {
const FALLBACK: usize = 2 * 1024 * 1024;
let Ok(meminfo) = std::fs::read_to_string("/proc/meminfo") else {
return FALLBACK;
};
for line in meminfo.lines() {
if let Some(rest) = line.strip_prefix("Hugepagesize:") {
let mut it = rest.split_whitespace();
if let (Some(kb), Some(_unit)) = (it.next(), it.next()) {
if let Ok(kb) = kb.parse::<usize>() {
return kb * 1024;
}
}
}
}
FALLBACK
}
pub fn find_gguf_in_dir(dir: &Path) -> Result<PathBuf> {
for entry in std::fs::read_dir(dir)? {
let path = entry?.path();
if path.extension().and_then(|e| e.to_str()) == Some("gguf") {
return Ok(path);
}
}
Err(JoshuaError::ModelLoad(format!(
"No .gguf file found in {:?}",
dir
)))
}
fn token_str_from_metadata(
gguf: &gguf_file::Content,
key: &str,
tokenizer: &Tokenizer,
) -> Option<String> {
let id = match gguf.metadata.get(key)? {
gguf_file::Value::U32(id) => *id,
gguf_file::Value::I32(id) => *id as u32,
gguf_file::Value::U64(id) => *id as u32,
_ => return None,
};
tokenizer.id_to_token(id)
}
fn read_gguf_header(mmap: &[u8]) -> Result<gguf_file::Content> {
let header = crate::gguf_ext::read_header(&mut Cursor::new(mmap))?;
if let Ok(arch) = Architecture::detect(&header.metadata) {
if arch != Architecture::DeepSeek4 {
let unsupported = header.unsupported_tensors();
if !unsupported.is_empty() {
let names = unsupported
.iter()
.map(|(n, d)| format!("`{n}` (GGUF dtype id {d})"))
.collect::<Vec<_>>()
.join(", ");
return Err(crate::JoshuaError::ModelLoad(format!(
"GGUF header: model architecture '{}' contains {} tensor(s) in a GGUF \
dtype the engine cannot decode: {}. Only the deepseek4 loader reads \
tensors by raw dtype id; re-quantize this model to a candle-supported \
format (F32/F16/Q8_0/Q4_K, …) or use a loader that decodes these dtypes.",
arch.display_name(),
unsupported.len(),
names
)));
}
}
}
header.to_candle_content()
}
fn sanitize_model_name(raw: &str) -> Option<String> {
const MAX_BYTES: usize = 128;
let mut out = String::with_capacity(raw.len().min(MAX_BYTES));
let mut seen_non_space = false;
let mut pending_space = false;
for c in raw.chars() {
if out.len() >= MAX_BYTES {
break;
}
if c.is_control() {
continue;
}
if c.is_whitespace() {
pending_space = seen_non_space;
continue;
}
if pending_space {
out.push(' ');
pending_space = false;
}
out.push(c);
seen_non_space = true;
}
if out.is_empty() {
None
} else {
Some(out)
}
}
fn extract_chat_template(gguf: &gguf_file::Content, tokenizer: &Tokenizer) -> Option<ChatTemplate> {
let source = gguf
.metadata
.get("tokenizer.chat_template")?
.to_string()
.ok()?
.clone();
if source.trim().is_empty() {
return None;
}
let bos =
token_str_from_metadata(gguf, "tokenizer.ggml.bos_token_id", tokenizer).unwrap_or_default();
let eos =
token_str_from_metadata(gguf, "tokenizer.ggml.eos_token_id", tokenizer).unwrap_or_default();
Some(ChatTemplate::new(source, bos, eos))
}
fn extract_eos_ids(gguf: &gguf_file::Content, tokenizer: &Tokenizer) -> Vec<u32> {
let mut ids: Vec<u32> = Vec::new();
let eos_key = "tokenizer.ggml.eos_token_id";
match gguf.metadata.get(eos_key) {
Some(gguf_file::Value::U32(id)) => ids.push(*id),
Some(gguf_file::Value::I32(id)) => ids.push(*id as u32),
Some(gguf_file::Value::U64(id)) => ids.push(*id as u32),
_ => {}
}
for token_str in &[
"</s>",
"<|endoftext|>",
"<|im_end|>",
"<end_of_turn>",
"<eos>",
"<|eot_id|>",
"<|end|>",
] {
if let Some(id) = tokenizer.token_to_id(token_str) {
if !ids.contains(&id) {
ids.push(id);
}
}
}
ids
}
fn squeeze_batch_logits(logits: &Tensor) -> Result<Vec<f32>> {
logits
.squeeze(0)
.and_then(|t| t.to_vec1::<f32>())
.map_err(|e| JoshuaError::Inference(e.to_string()))
}
fn sample_token(
logits: &[f32],
opts: &GenerationOptions,
rng: &mut impl rand::Rng,
recent_tokens: &[u32],
) -> Result<u32> {
if logits.is_empty() {
return Ok(0);
}
let logits: Vec<f32> = if opts.repetition_penalty != 1.0 {
let mut v = logits.to_vec();
for &token in recent_tokens {
if let Some(l) = v.get_mut(token as usize) {
if *l > 0.0 {
*l /= opts.repetition_penalty;
} else {
*l *= opts.repetition_penalty;
}
}
}
v
} else {
logits.to_vec()
};
if opts.temperature <= 0.0 {
return Ok(logits
.iter()
.enumerate()
.max_by(|(_, a), (_, b)| a.partial_cmp(b).unwrap_or(std::cmp::Ordering::Equal))
.map(|(i, _)| i as u32)
.unwrap_or(0));
}
let inv_temp = 1.0_f32 / opts.temperature;
let max_logit = logits.iter().cloned().fold(f32::NEG_INFINITY, f32::max);
let mut probs: Vec<f32> = logits
.iter()
.map(|&l| ((l - max_logit) * inv_temp).exp())
.collect();
let k = opts.top_k as usize;
if k > 0 && k < probs.len() {
let mut indexed: Vec<(usize, f32)> = probs.iter().copied().enumerate().collect();
indexed.sort_unstable_by(|(_, a), (_, b)| {
b.partial_cmp(a).unwrap_or(std::cmp::Ordering::Equal)
});
for &(idx, _) in indexed.iter().skip(k) {
probs[idx] = 0.0;
}
}
if opts.min_p > 0.0 {
let max_p = probs.iter().cloned().fold(f32::NEG_INFINITY, f32::max);
let threshold = max_p * opts.min_p;
for p in &mut probs {
if *p < threshold {
*p = 0.0;
}
}
}
if opts.top_p < 1.0 && opts.top_p > 0.0 {
let sum: f32 = probs.iter().sum();
if sum > 0.0 {
let mut sorted_idx: Vec<usize> = (0..probs.len()).collect();
sorted_idx.sort_unstable_by(|&a, &b| {
probs[b]
.partial_cmp(&probs[a])
.unwrap_or(std::cmp::Ordering::Equal)
});
let mut cumsum = 0.0_f32;
let mut cut_from = probs.len();
for (rank, &idx) in sorted_idx.iter().enumerate() {
cumsum += probs[idx] / sum;
if cumsum > opts.top_p {
cut_from = rank + 1;
break;
}
}
for &idx in sorted_idx.iter().skip(cut_from) {
probs[idx] = 0.0;
}
}
}
let total: f32 = probs.iter().sum();
if total <= 0.0 {
return Ok(logits
.iter()
.enumerate()
.max_by(|(_, a), (_, b)| a.partial_cmp(b).unwrap_or(std::cmp::Ordering::Equal))
.map(|(i, _)| i as u32)
.unwrap_or(0));
}
for p in &mut probs {
*p /= total;
}
let dist = WeightedIndex::new(&probs).map_err(|e| JoshuaError::Inference(e.to_string()))?;
Ok(dist.sample(rng) as u32)
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn sanitize_model_name_keeps_plain_names() {
assert_eq!(
sanitize_model_name("DeepSeek-V4-Flash"),
Some("DeepSeek-V4-Flash".into())
);
assert_eq!(
sanitize_model_name(" Llama 3.1 8B "),
Some("Llama 3.1 8B".into())
);
}
#[test]
fn sanitize_model_name_strips_control_chars_and_newlines() {
assert_eq!(
sanitize_model_name("evil\n[2Jname"),
Some("evil[2Jname".into())
);
assert_eq!(sanitize_model_name("a\u{1b}[31mb"), Some("a[31mb".into()));
assert_eq!(sanitize_model_name("\u{0}"), None);
assert_eq!(sanitize_model_name(" "), None);
}
#[test]
fn sanitize_model_name_caps_length() {
let long = "x".repeat(1000);
assert_eq!(sanitize_model_name(&long), Some("x".repeat(128)));
}
#[test]
fn page_size_params_are_correct() {
assert_eq!(PageSize::TwoMiB.params(), (Some(21), 2 * 1024 * 1024));
assert_eq!(PageSize::OneGiB.params(), (Some(30), 1024 * 1024 * 1024));
let (bits, bytes) = PageSize::Default.params();
assert_eq!(bits, None);
assert!(bytes >= 4096 && bytes.is_power_of_two(), "got {bytes}");
}
#[test]
fn default_hugepage_bytes_is_sane() {
let bytes = default_hugepage_bytes();
assert!(
bytes >= 2 * 1024 * 1024 && bytes.is_power_of_two(),
"got {bytes}"
);
}
#[test]
fn engine_options_builder() {
let o = EngineOptions::with_n_ctx(2048).huge_pages(HugePages::Transparent);
assert_eq!(o.n_ctx, 2048);
assert_eq!(o.huge_pages, HugePages::Transparent);
assert_eq!(EngineOptions::default().huge_pages, HugePages::Off);
assert_eq!(EngineOptions::default().mmap, MmapMode::Auto);
assert_eq!(
EngineOptions::default().mmap(MmapMode::Required).mmap,
MmapMode::Required
);
assert!(!EngineOptions::default().pin_hot_weights);
assert_eq!(EngineOptions::default().mlock_hot_weights, MlockMode::Off);
let o = EngineOptions::default()
.pin_hot_weights(true)
.mlock_hot_weights(MlockMode::On);
assert!(o.pin_hot_weights);
assert_eq!(o.mlock_hot_weights, MlockMode::On);
assert_eq!(
EngineOptions::default()
.mlock_hot_weights(MlockMode::Required)
.mlock_hot_weights,
MlockMode::Required
);
}
fn header_with_tensors(
tensor_data_offset: u64,
tensors: &[(&str, u64)],
) -> crate::gguf_ext::GgufHeader {
use std::collections::HashMap;
crate::gguf_ext::GgufHeader {
version: 3,
metadata: HashMap::new(),
tensors: tensors
.iter()
.map(|(n, off)| {
(
(*n).to_string(),
crate::gguf_ext::RawTensorInfo {
dtype: 0,
dims: vec![1],
offset: *off,
},
)
})
.collect(),
tensor_data_offset,
}
}
#[test]
fn is_routed_expert_matches_moe_names() {
for dense in [
"token_embd.weight",
"output.weight",
"blk.0.attn_q.weight",
"blk.0.attn_output.weight",
"blk.0.ffn_norm.weight",
"blk.0.ffn_gate_inp.weight",
"blk.0.ffn_gate_shexp.weight",
"blk.0.ffn_up_shexp.weight",
"blk.0.indexer.proj.weight",
"blk.0.indexer_compressor_gate.weight",
"blk.0.hc_attn_fn.weight",
] {
assert!(!is_routed_expert(dense), "{dense} should be dense");
}
for expert in [
"blk.0.ffn_gate_exps.weight",
"blk.0.ffn_down_exps.weight",
"blk.0.ffn_up_exps.weight",
"blk.42.ffn_gate_exps.weight",
] {
assert!(is_routed_expert(expert), "{expert} should be an expert");
}
}
#[test]
fn weight_ranges_classifies_and_sizes_interleaved_tensors() {
let header = header_with_tensors(
4096,
&[
("blk.0.attn_q.weight", 0),
("blk.0.ffn_gate_exps.weight", 100),
("blk.0.ffn_norm.weight", 5 * 1024 * 1024 + 100),
("blk.0.ffn_up_exps.weight", 5 * 1024 * 1024 + 200),
],
);
let (dense, experts) = weight_ranges(&header, 4096 + 5 * 1024 * 1024 + 300);
assert_eq!(dense, vec![(4096, 100), (4096 + 5 * 1024 * 1024 + 100, 100)]);
assert_eq!(experts, vec![(4196, 5243080)]);
}
#[test]
fn weight_ranges_merges_dense_ranges_across_small_expert_gaps() {
let header = header_with_tensors(
0,
&[
("blk.0.attn_q.weight", 0),
("blk.0.ffn_gate_exps.weight", 100), ("blk.0.ffn_norm.weight", 1100),
],
);
let (dense, experts) = weight_ranges(&header, 1200);
assert_eq!(dense, vec![(0, 1200)]);
assert_eq!(experts, vec![(100, 1000)]);
let header = header_with_tensors(
0,
&[
("blk.0.attn_q.weight", 0),
("blk.0.ffn_gate_exps.weight", 100),
("blk.0.ffn_norm.weight", 5 * 1024 * 1024 + 100),
],
);
let (dense, _) = weight_ranges(&header, 5 * 1024 * 1024 + 200);
assert_eq!(dense, vec![(0, 100), (5 * 1024 * 1024 + 100, 100)]);
}
#[test]
fn weight_ranges_adds_tensor_data_offset() {
let header = header_with_tensors(
0x1_0000,
&[
("token_embd.weight", 0),
("blk.0.ffn_down_exps.weight", 64),
],
);
let (dense, experts) = weight_ranges(&header, 0x1_0000 + 128);
assert_eq!(dense, vec![(0x1_0000, 64)]);
assert_eq!(experts, vec![(0x1_0000 + 64, 64)]);
}
#[cfg(unix)]
#[test]
fn apply_hot_weight_pinning_is_best_effort_on_a_real_mapping() {
let (path, _) = model_fixture("pin.gguf", &vec![0u8; 16 * 4096]);
let file = File::open(&path).expect("reopen fixture");
let mmap = unsafe { Mmap::map(&file) }.expect("mmap fixture");
let empty = crate::gguf_ext::GgufHeader {
version: 3,
metadata: std::collections::HashMap::new(),
tensors: std::collections::HashMap::new(),
tensor_data_offset: 0,
};
apply_hot_weight_pinning(&mmap, &empty, true, MlockMode::On).expect("no tensors is fine");
let header = header_with_tensors(
0,
&[
("token_embd.weight", 0),
("blk.0.ffn_gate_exps.weight", 4096),
],
);
apply_hot_weight_pinning(&mmap, &header, true, MlockMode::On)
.expect("advice/mlock is best effort");
let _ = std::fs::remove_file(&path);
}
#[cfg(unix)]
#[test]
fn mlock_decision_degrades_or_fails_on_short_limit() {
let required = 8 * 1024 * 1024 * 1024; assert_eq!(
mlock_decision(MlockMode::On, Some(8 * 1024 * 1024), required),
MlockDecision::Degrade
);
assert_eq!(
mlock_decision(MlockMode::On, Some(required), required),
MlockDecision::Proceed
);
assert_eq!(
mlock_decision(MlockMode::On, Some(required + 1), required),
MlockDecision::Proceed
);
assert_eq!(
mlock_decision(MlockMode::On, None, required),
MlockDecision::Proceed
);
assert_eq!(
mlock_decision(MlockMode::Required, Some(8 * 1024 * 1024), required),
MlockDecision::Fail
);
assert_eq!(
mlock_decision(MlockMode::Required, Some(required), required),
MlockDecision::Proceed
);
assert_eq!(
mlock_decision(MlockMode::Required, None, required),
MlockDecision::Proceed
);
}
#[cfg(unix)]
#[test]
fn memlock_limit_bytes_reads_without_panicking() {
if let Some(bytes) = memlock_limit_bytes() {
assert!(bytes > 0, "a finite memlock limit must be positive");
}
}
#[cfg(unix)]
#[test]
fn aligned_range_bytes_rounds_to_pages_and_clamps() {
let page = 4096;
assert_eq!(aligned_range_bytes(&[(0, 4096)], page, 1 << 20), 4096);
assert_eq!(aligned_range_bytes(&[(100, 100)], page, 1 << 20), 4096);
assert_eq!(
aligned_range_bytes(&[(0, 100), (8192, 100)], page, 1 << 20),
2 * 4096
);
assert_eq!(
aligned_range_bytes(&[(0, 100), (100, 100)], page, 1 << 20),
4096
);
assert_eq!(aligned_range_bytes(&[(0, 1 << 20)], page, 4096 + 10), 8192);
}
fn model_fixture(name: &str, bytes: &[u8]) -> (PathBuf, File) {
use std::io::Write;
let path =
std::env::temp_dir().join(format!("joshua-mapcheck-{}-{name}", std::process::id()));
let mut f = File::create(&path).expect("create fixture");
f.write_all(bytes).expect("write fixture");
drop(f);
let opened = File::open(&path).expect("open fixture");
(path, opened)
}
#[test]
fn compressed_model_warns_by_default_and_errors_when_mmap_is_explicit() {
let (path, file) = model_fixture("gz.gguf", b"\x1f\x8b\x08\x00\x00\x00\x00\x00\x00\x03");
check_mappable(&path, &file, MmapMode::Auto, true).expect("warn, not fail");
let err = check_mappable(&path, &file, MmapMode::Required, true).unwrap_err();
assert!(matches!(err, JoshuaError::ModelLoad(_)));
let msg = err.to_string();
assert!(msg.contains("gzip"), "got: {msg}");
assert!(msg.contains("gunzip"), "got: {msg}");
let _ = std::fs::remove_file(&path);
}
#[test]
fn plain_model_file_passes_the_mappability_check() {
let (path, file) = model_fixture("plain.gguf", b"GGUF\x03\x00\x00\x00");
check_mappable(&path, &file, MmapMode::Auto, true).expect("plain GGUF is fine");
check_mappable(&path, &file, MmapMode::Required, true).expect("plain GGUF is fine");
let _ = std::fs::remove_file(&path);
}
#[cfg(unix)]
#[test]
fn filesystem_compression_is_ignored_for_the_anonymous_copy() {
let path = std::env::temp_dir().join(format!(
"joshua-mapcheck-{}-sparse.gguf",
std::process::id()
));
let mut f = File::create(&path).expect("create sparse fixture");
std::io::Write::write_all(&mut f, &[0x5Au8; 64 * 1024]).expect("write prefix");
f.set_len(64 * 1024 * 1024).expect("set_len");
drop(f);
let file = File::open(&path).expect("open sparse fixture");
if !matches!(
crate::compression::detect_gguf(&file),
Some(crate::compression::Compression::Filesystem { .. })
) {
let _ = std::fs::remove_file(&path);
return;
}
check_mappable(&path, &file, MmapMode::Required, false).expect("copy path is unaffected");
check_mappable(&path, &file, MmapMode::Auto, true).expect("warn, not fail");
assert!(check_mappable(&path, &file, MmapMode::Required, true).is_err());
let _ = std::fs::remove_file(&path);
}
#[test]
fn in_flight_guard_caps_and_releases() {
let counter = AtomicUsize::new(0);
let a = InFlightGuard::acquire(&counter, 2).expect("first permit");
let b = InFlightGuard::acquire(&counter, 2).expect("second permit");
match InFlightGuard::acquire(&counter, 2) {
Err(JoshuaError::Overloaded(_)) => {}
Err(e) => panic!("expected Overloaded, got {e:?}"),
Ok(_) => panic!("expected Overloaded, got a permit"),
}
assert_eq!(counter.load(Ordering::Acquire), 2);
drop(a);
let _c = InFlightGuard::acquire(&counter, 2).expect("permit after release");
drop(b);
drop(_c);
assert_eq!(counter.load(Ordering::Acquire), 0);
}
#[test]
fn image_sources_must_be_data_urls() {
let ok = load_image_bytes("data:image/png;base64,AQID").expect("data url");
assert_eq!(ok, vec![1, 2, 3]);
let err = load_image_bytes("/etc/passwd").unwrap_err();
assert!(matches!(err, JoshuaError::InvalidRequest(_)));
assert!(err.to_string().contains("data:"), "got: {err}");
assert!(load_image_bytes("src/engine.rs").is_err());
assert!(load_image_bytes("http://169.254.169.254/").is_err());
assert!(load_image_bytes("https://example.com/x.png").is_err());
}
}