joshua 0.1.0

An mmap-based LLM inference engine for the Rust ecosystem — a Rust clone of Cactus
Documentation
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
//! Architecture-aware model dispatch for Joshua.
//!
//! Reads `general.architecture` from the GGUF metadata and routes to the
//! correct candle quantized model type.  This covers every GGUF architecture
//! that candle ships a pure-Rust quantized loader for:
//!
//! | `general.architecture`                          | candle loader
//! |-------------------------------------------------|---------------------------
//! | `llama` (Llama 1-3, Mistral, Mixtral, TinyLlama, SmolLM, Vicuna, Zephyr, Yi, …) | `quantized_llama`
//! | `gemma` / `gemma2` / `gemma3` / `gemma-embedding` | `quantized_gemma3`
//! | `glm4`                                          | `quantized_glm4`
//! | `lfm2`                                          | `quantized_lfm2`
//! | `phi2`                                          | `quantized_phi`
//! | `phi3`                                          | `quantized_phi3`
//! | `qwen2`                                         | `quantized_qwen2`
//! | `qwen3`                                         | `quantized_qwen3`
//! | `qwen3moe`                                      | `quantized_qwen3_moe`
//! | `deepseek2` (DeepSeek-V2/V3, Kimi-K2)            | `quantized_deepseek2` (Joshua)
//! | `deepseek4` (DeepSeek-V4)                        | `quantized_deepseek4` (Joshua)
//!
//! Every other architecture name in llama.cpp's registry is recognised and
//! reported with a clear "known but not yet loadable in pure Rust" error, so
//! users can tell the difference between an unsupported model and a corrupt
//! or mislabelled file.

use std::collections::HashMap;
use std::io::{Read, Seek, SeekFrom};

use candle_core::quantized::gguf_file;
use candle_core::{DType, Device, Result, Tensor};
use candle_transformers::models::{
    quantized_gemma3, quantized_glm4, quantized_lfm2, quantized_llama, quantized_phi,
    quantized_phi3, quantized_qwen2, quantized_qwen3, quantized_qwen3_moe,
};

// ─── Architecture enum ──────────────────────────────────────────────────────

/// GGUF model architectures with a pure-Rust quantized candle loader.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum Architecture {
    /// `llama` — Llama 1/2/3, Mistral, Mixtral, TinyLlama, SmolLM, Yi, and
    /// every other model that llama.cpp's converters emit as `llama`.
    Llama,
    /// `gemma`, `gemma2`, `gemma3`, `gemma-embedding`.
    Gemma,
    /// `glm4` — GLM-4 dense models.
    Glm4,
    /// `lfm2` — Liquid LFM2 hybrid (attention + short-conv) models.
    Lfm2,
    /// `phi2` — Phi-1, Phi-1.5, Phi-2.
    Phi2,
    /// `phi3` — Phi-3 / Phi-3.5.
    Phi3,
    /// `qwen2` — Qwen1.5 / Qwen2 / Qwen2.5 dense models.
    Qwen2,
    /// `qwen3` — Qwen3 dense models.
    Qwen3,
    /// `qwen3moe` — Qwen3 mixture-of-experts models.
    Qwen3Moe,
    /// `deepseek2` — DeepSeek-V2/V3 and Kimi-K2 (MLA + fine-grained MoE).
    DeepSeek2,
    /// `deepseek4` — DeepSeek-V4 (sliding-window MLA + KV compression +
    /// indexer-selected sparse attention + Hyper-Connections).
    DeepSeek4,
}

/// Architecture names understood by llama.cpp but without a pure-Rust
/// quantized loader in candle yet.  Kept in sync with llama.cpp's
/// `llama-arch.cpp` registry so we can give a precise error instead of a
/// generic "unknown architecture".
const KNOWN_UNSUPPORTED_ARCHS: &[&str] = &[
    "afmoe",
    "apertus",
    "arcee",
    "arctic",
    "arwkv7",
    "baichuan",
    "bailingmoe",
    "bailingmoe2",
    "bert",
    "bitnet",
    "bloom",
    "chameleon",
    "chatglm",
    "codeshell",
    "cogvlm",
    "cohere2",
    "command-r",
    "dbrx",
    "deci",
    "deepseek",
    "dots1",
    "dream",
    "ernie4_5",
    "ernie4_5-moe",
    "exaone",
    "exaone4",
    "falcon",
    "falcon-h1",
    "gemma3n",
    "glm4moe",
    "gpt2",
    "gpt-oss",
    "gptj",
    "gptneox",
    "granite",
    "granitehybrid",
    "granitemoe",
    "grok",
    "grovemoe",
    "hunyuan-dense",
    "hunyuan-moe",
    "jais",
    "jamba",
    "jina-bert-v2",
    // Kimi K3. The architecture primitives (KDA, attention residuals, the
    // latent MoE) live in `crate::kimi_k3`, but the end-to-end loader is not
    // finished, so the model is still reported as unsupported rather than
    // half-loading.
    "kimi-k3",
    "kimi-linear",
    "llada",
    "llada-moe",
    "llama4",
    "lfm2moe",
    "mamba",
    "mamba2",
    "minicpm",
    "minicpm3",
    "minimax-m2",
    "mpt",
    "nemotron",
    "nemotron-h",
    "neo-bert",
    "nomic-bert",
    "nomic-bert-moe",
    "olmo",
    "olmo2",
    "olmoe",
    "openelm",
    "orion",
    "phimoe",
    "plamo",
    "plamo2",
    "plm",
    "qwen",
    "qwen2moe",
    "qwen2vl",
    "refact",
    "rwkv6",
    "rwkv6qwen2",
    "rwkv7",
    "seed-oss",
    "smallthinker",
    "smollm3",
    "stablelm",
    "starcoder",
    "starcoder2",
    "t5",
    "t5encoder",
    "wavtokenizer-dec",
    "xverse",
    "internlm2",
];

impl Architecture {
    /// Parse an architecture from its GGUF `general.architecture` name.
    ///
    /// Returns `None` if candle has no quantized loader for it — use
    /// [`Architecture::is_known_llama_cpp_arch`] to distinguish "known to
    /// llama.cpp but unimplemented" from "never heard of it".
    pub fn from_name(name: &str) -> Option<Self> {
        Some(match name {
            "llama" => Self::Llama,
            "gemma" | "gemma2" | "gemma3" | "gemma-embedding" => Self::Gemma,
            "glm4" => Self::Glm4,
            "lfm2" => Self::Lfm2,
            "phi2" => Self::Phi2,
            "phi3" => Self::Phi3,
            "qwen2" => Self::Qwen2,
            "qwen3" => Self::Qwen3,
            "qwen3moe" => Self::Qwen3Moe,
            "deepseek2" => Self::DeepSeek2,
            "deepseek4" => Self::DeepSeek4,
            _ => return None,
        })
    }

    /// Parse an architecture from the GGUF `general.architecture` metadata.
    pub fn from_gguf_metadata(metadata: &HashMap<String, gguf_file::Value>) -> Option<Self> {
        Self::from_name(Self::arch_name(metadata)?.as_str())
    }

    /// Extract the raw `general.architecture` string from GGUF metadata.
    pub fn arch_name(metadata: &HashMap<String, gguf_file::Value>) -> Option<String> {
        metadata
            .get("general.architecture")?
            .to_string()
            .ok()
            .cloned()
    }

    /// Detect the architecture, or return a human-readable explanation of why
    /// the model cannot be loaded.
    pub fn detect(
        metadata: &HashMap<String, gguf_file::Value>,
    ) -> std::result::Result<Self, String> {
        let Some(name) = Self::arch_name(metadata) else {
            return Err(
                "GGUF metadata has no `general.architecture` key — the file is corrupt \
                 or is not a model file"
                    .to_string(),
            );
        };
        if let Some(arch) = Self::from_name(&name) {
            return Ok(arch);
        }
        if Self::is_known_llama_cpp_arch(&name) {
            Err(format!(
                "Model architecture '{name}' is a known llama.cpp architecture, but no \
                 pure-Rust quantized loader exists for it in candle yet. \
                 Joshua currently supports: {}.",
                Self::list_known()
            ))
        } else {
            Err(format!(
                "Unrecognised GGUF architecture '{name}'. \
                 Joshua currently supports: {}.",
                Self::list_known()
            ))
        }
    }

    /// Whether `name` appears in llama.cpp's architecture registry (either
    /// supported here or known-but-unimplemented).
    pub fn is_known_llama_cpp_arch(name: &str) -> bool {
        Self::from_name(name).is_some() || KNOWN_UNSUPPORTED_ARCHS.contains(&name)
    }

    /// Human-readable name.
    pub fn display_name(&self) -> &'static str {
        match self {
            Self::Llama => "Llama (also Mistral, Mixtral, TinyLlama, SmolLM, Yi, …)",
            Self::Gemma => "Gemma / Gemma 2 / Gemma 3",
            Self::Glm4 => "GLM-4",
            Self::Lfm2 => "LFM2",
            Self::Phi2 => "Phi-1 / Phi-1.5 / Phi-2",
            Self::Phi3 => "Phi-3",
            Self::Qwen2 => "Qwen2 / Qwen2.5",
            Self::Qwen3 => "Qwen3",
            Self::Qwen3Moe => "Qwen3-MoE",
            Self::DeepSeek2 => "DeepSeek-V2 / DeepSeek-V3 / Kimi-K2",
            Self::DeepSeek4 => "DeepSeek-V4",
        }
    }

    /// Comma-separated list of supported architecture names for error messages.
    pub fn list_known() -> &'static str {
        "llama (incl. Mistral/Mixtral), gemma, gemma2, gemma3, gemma-embedding, \
         glm4, lfm2, phi2, phi3, qwen2, qwen3, qwen3moe, deepseek2, deepseek4"
    }
}

// ─── Dispatched model ───────────────────────────────────────────────────────

/// A quantized model loaded from a GGUF file, wrapping the correct candle
/// model type for the detected architecture.
///
/// This hides the concrete model type behind a uniform `forward()` API so the
/// engine can handle multiple architectures without code duplication.
// Model structs are built once per model load and matched by reference; the
// size difference between variants is irrelevant.
#[allow(clippy::large_enum_variant)]
pub enum QuantizedModel {
    Llama(quantized_llama::ModelWeights),
    Gemma(quantized_gemma3::ModelWeights),
    Glm4(quantized_glm4::ModelWeights),
    Lfm2(quantized_lfm2::ModelWeights),
    Phi2(quantized_phi::ModelWeights),
    Phi3(quantized_phi3::ModelWeights),
    Qwen2(quantized_qwen2::ModelWeights),
    Qwen3(quantized_qwen3::ModelWeights),
    Qwen3Moe(quantized_qwen3_moe::GGUFQWenMoE),
    DeepSeek2(crate::quantized_deepseek2::ModelWeights),
    DeepSeek4(crate::quantized_deepseek4::ModelWeights),
}

impl QuantizedModel {
    /// Load a model from a GGUF file, dispatching to the correct quantized
    /// loader based on `general.architecture`.
    pub fn from_gguf<R: Read + Seek>(
        gguf: gguf_file::Content,
        reader: &mut R,
        device: &Device,
    ) -> Result<Self> {
        Self::from_gguf_mmap(gguf, reader, device, None)
    }

    /// Load, borrowing weights in place from `mmap` for the architectures
    /// whose loaders Joshua owns.
    ///
    /// candle's own quantized loaders read through the `reader` and copy, so
    /// for those architectures the mapping is ignored — they are small enough
    /// that it does not matter.  Joshua's own loaders (currently `deepseek2`)
    /// borrow, which is what makes the very large MoE models tractable.
    pub fn from_gguf_mmap<R: Read + Seek>(
        gguf: gguf_file::Content,
        reader: &mut R,
        device: &Device,
        mmap: Option<std::sync::Arc<memmap2::Mmap>>,
    ) -> Result<Self> {
        let arch = Architecture::detect(&gguf.metadata).map_err(candle_core::Error::Msg)?;

        tracing::info!("Detected model architecture: {}", arch.display_name());

        // Re-read the header the way `gguf_ext` does — with each tensor's
        // dtype kept as its raw GGUF id — but only for deepseek4, the one
        // loader that consumes the raw table (candle's `Content` cannot name
        // IQ2_XXS or I32, so the deepseek4 loader needs this to decode those
        // tensors).  The reader is rewound to where candle's header parse
        // left it (the aligned data start) afterwards, so nothing downstream
        // shifts.
        //
        // The re-read must not run for other architectures: none of them
        // consume the raw table, so the second parse would be pure cost (a
        // full re-parse and clone of the metadata + tensor maps for every
        // warm-pool instance) and any difference between the two parsers
        // could reject a file candle accepts.  `read_header` is a superset of
        // candle's parser for every file that can actually load (nested
        // arrays, depth cap, lenient strings; its per-string/array resource
        // caps are deliberately tighter than candle's theoretical 1 GiB, on
        // which candle itself eagerly allocates and fails), but other
        // architectures should stay on exactly the behaviour candle gave
        // them.
        //
        // A failed re-read is a real header problem (truncated/corrupt file,
        // implausible counts, non-UTF-8 key) — surface it as the load error
        // rather than silently treating the file as having no raw table, which
        // would surface much later as a misleading "cannot find tensor".
        let raw = if arch == Architecture::DeepSeek4 {
            let data_pos = reader.stream_position()?;
            let h = (|| -> std::result::Result<_, crate::JoshuaError> {
                reader.seek(SeekFrom::Start(0))?;
                let h = crate::gguf_ext::read_header(reader)?;
                reader.seek(SeekFrom::Start(data_pos))?;
                Ok(h)
            })()
            .map_err(|e| {
                // Restore the reader even on failure so callers see a
                // deterministic position, then report the actual problem.
                let _ = reader.seek(SeekFrom::Start(data_pos));
                candle_core::Error::Msg(format!("GGUF header re-read failed: {e}"))
            })?;
            Some(h)
        } else {
            None
        };
        let raw = raw.as_ref();

        match arch {
            Architecture::Llama => {
                quantized_llama::ModelWeights::from_gguf(gguf, reader, device).map(Self::Llama)
            }
            Architecture::Gemma => {
                quantized_gemma3::ModelWeights::from_gguf(gguf, reader, device).map(Self::Gemma)
            }
            Architecture::Glm4 => {
                // F32 activations: fastest/most accurate compute dtype on CPU.
                quantized_glm4::ModelWeights::from_gguf(gguf, reader, device, DType::F32)
                    .map(Self::Glm4)
            }
            Architecture::Lfm2 => {
                quantized_lfm2::ModelWeights::from_gguf(gguf, reader, device).map(Self::Lfm2)
            }
            Architecture::Phi2 => {
                quantized_phi::ModelWeights::from_gguf(gguf, reader, device).map(Self::Phi2)
            }
            Architecture::Phi3 => {
                // CPU-only: flash attention not available.
                quantized_phi3::ModelWeights::from_gguf(false, gguf, reader, device).map(Self::Phi3)
            }
            Architecture::Qwen2 => {
                quantized_qwen2::ModelWeights::from_gguf(gguf, reader, device).map(Self::Qwen2)
            }
            Architecture::Qwen3 => {
                quantized_qwen3::ModelWeights::from_gguf(gguf, reader, device).map(Self::Qwen3)
            }
            Architecture::Qwen3Moe => {
                quantized_qwen3_moe::GGUFQWenMoE::from_gguf(gguf, reader, device, DType::F32)
                    .map(Self::Qwen3Moe)
            }
            Architecture::DeepSeek2 => {
                crate::quantized_deepseek2::ModelWeights::from_gguf_mmap(gguf, reader, device, mmap)
                    .map(Self::DeepSeek2)
            }
            Architecture::DeepSeek4 => crate::quantized_deepseek4::ModelWeights::from_gguf_mmap(
                gguf, raw, reader, device, mmap,
            )
            .map(Self::DeepSeek4),
        }
    }

    /// Clear the KV cache so the instance can serve an unrelated prompt,
    /// where the underlying candle model supports it.
    ///
    /// Returns `false` when the architecture has no reset hook — the caller
    /// must build a fresh instance instead.
    pub fn clear_kv_cache(&mut self) -> bool {
        match self {
            Self::Llama(m) => {
                m.clear_kv_cache();
                true
            }
            Self::Qwen2(m) => {
                m.clear_kv_cache();
                true
            }
            Self::Qwen3(m) => {
                m.clear_kv_cache();
                true
            }
            Self::DeepSeek2(m) => {
                m.clear_kv_cache();
                true
            }
            Self::DeepSeek4(m) => {
                m.clear_kv_cache();
                true
            }
            _ => false,
        }
    }

    /// Whether [`QuantizedModel::clear_kv_cache`] can reset this instance.
    pub fn supports_kv_clear(&self) -> bool {
        matches!(
            self,
            Self::Llama(_)
                | Self::Qwen2(_)
                | Self::Qwen3(_)
                | Self::DeepSeek2(_)
                | Self::DeepSeek4(_)
        )
    }

    /// Unified forward pass.
    ///
    /// `input` has shape `[1, seq_len]` for the initial prefill, or `[1, 1]`
    /// for single-token decode steps.  `index_pos` is the absolute position in
    /// the KV cache of the first token in `input`.
    pub fn forward(&mut self, input: &Tensor, index_pos: usize) -> Result<Tensor> {
        match self {
            Self::Llama(m) => m.forward(input, index_pos),
            Self::Gemma(m) => m.forward(input, index_pos),
            Self::Glm4(m) => m.forward(input, index_pos),
            Self::Lfm2(m) => m.forward(input, index_pos),
            Self::Phi2(m) => m.forward(input, index_pos),
            Self::Phi3(m) => m.forward(input, index_pos),
            Self::Qwen2(m) => m.forward(input, index_pos),
            Self::Qwen3(m) => m.forward(input, index_pos),
            Self::Qwen3Moe(m) => m.forward(input, index_pos),
            Self::DeepSeek2(m) => m.forward(input, index_pos),
            Self::DeepSeek4(m) => m.forward(input, index_pos),
        }
    }
}

// ─── Tests ──────────────────────────────────────────────────────────────────

#[cfg(test)]
mod tests {
    use super::*;

    fn metadata_with_arch(arch: &str) -> HashMap<String, gguf_file::Value> {
        let mut m = HashMap::new();
        m.insert(
            "general.architecture".to_string(),
            gguf_file::Value::String(arch.to_string()),
        );
        m
    }

    #[test]
    fn supported_architectures_resolve() {
        for (name, expected) in [
            ("llama", Architecture::Llama),
            ("gemma", Architecture::Gemma),
            ("gemma2", Architecture::Gemma),
            ("gemma3", Architecture::Gemma),
            ("gemma-embedding", Architecture::Gemma),
            ("glm4", Architecture::Glm4),
            ("lfm2", Architecture::Lfm2),
            ("phi2", Architecture::Phi2),
            ("phi3", Architecture::Phi3),
            ("qwen2", Architecture::Qwen2),
            ("qwen3", Architecture::Qwen3),
            ("qwen3moe", Architecture::Qwen3Moe),
            ("deepseek2", Architecture::DeepSeek2),
            ("deepseek4", Architecture::DeepSeek4),
        ] {
            assert_eq!(Architecture::from_name(name), Some(expected), "arch {name}");
            assert_eq!(
                Architecture::from_gguf_metadata(&metadata_with_arch(name)),
                Some(expected),
                "metadata arch {name}"
            );
            assert!(Architecture::is_known_llama_cpp_arch(name));
        }
    }

    #[test]
    fn known_unsupported_architectures_give_specific_error() {
        for name in ["mamba", "gpt2", "deepseek", "rwkv7", "starcoder2"] {
            assert_eq!(Architecture::from_name(name), None);
            assert!(Architecture::is_known_llama_cpp_arch(name), "arch {name}");
            let err = Architecture::detect(&metadata_with_arch(name)).unwrap_err();
            assert!(
                err.contains("known llama.cpp architecture"),
                "error for {name}: {err}"
            );
        }
    }

    #[test]
    fn unknown_architecture_gives_generic_error() {
        assert!(!Architecture::is_known_llama_cpp_arch("not-a-real-arch"));
        let err = Architecture::detect(&metadata_with_arch("not-a-real-arch")).unwrap_err();
        assert!(err.contains("Unrecognised"), "error: {err}");
    }

    #[test]
    fn missing_architecture_key_gives_corrupt_file_error() {
        let err = Architecture::detect(&HashMap::new()).unwrap_err();
        assert!(err.contains("general.architecture"), "error: {err}");
    }
}