1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
//! Candle inference backend — loads GGUF models, runs on Metal/CUDA/CPU.
//!
//! Supports both standard Qwen3 and Qwen3-MoE (Mixture of Experts) architectures.
//! Auto-detects the architecture from GGUF metadata.
use std::path::Path;
use candle_core::quantized::gguf_file;
use candle_core::{Device as CandleDevice, Tensor};
use candle_transformers::models::quantized_qwen3 as qwen;
use tokenizers::Tokenizer;
use super::moe::Qwen3MoeModel;
use crate::{Device, InferenceError};
/// Loaded model — either standard Qwen3 or MoE variant.
enum QwenModel {
Standard(qwen::ModelWeights),
/// MoE model using our naive (non-fused) implementation that works on Metal/CPU.
Moe(Qwen3MoeModel),
}
/// A loaded model ready for inference.
pub struct CandleBackend {
model: QwenModel,
pub tokenizer: Tokenizer,
pub device: CandleDevice,
}
impl CandleBackend {
/// Load a GGUF model + tokenizer from a model directory.
///
/// Expects `model.gguf` and `tokenizer.json` in `model_dir`.
/// Auto-detects standard vs MoE architecture from GGUF metadata.
pub fn load(model_dir: &Path, device: Device) -> Result<Self, InferenceError> {
let candle_device = to_candle_device(device)?;
// Load GGUF weights
let model_path = model_dir.join("model.gguf");
let mut file = std::fs::File::open(&model_path)
.map_err(|e| InferenceError::InferenceFailed(format!("open model: {e}")))?;
let gguf = gguf_file::Content::read(&mut file)
.map_err(|e| InferenceError::InferenceFailed(format!("read gguf: {e}")))?;
// Detect architecture from GGUF metadata
let arch = gguf
.metadata
.get("general.architecture")
.and_then(|v| v.to_string().ok())
.map(|s| s.to_string())
.unwrap_or_default();
let model = if arch == "qwen3moe" {
let moe = Qwen3MoeModel::from_gguf(gguf, &mut file, &candle_device)
.map_err(|e| InferenceError::InferenceFailed(format!("load moe weights: {e}")))?;
QwenModel::Moe(moe)
} else {
let std = qwen::ModelWeights::from_gguf(gguf, &mut file, &candle_device)
.map_err(|e| InferenceError::InferenceFailed(format!("load weights: {e}")))?;
QwenModel::Standard(std)
};
// Load tokenizer
let tokenizer_path = model_dir.join("tokenizer.json");
let tokenizer = Tokenizer::from_file(&tokenizer_path)
.map_err(|e| InferenceError::TokenizationError(format!("load tokenizer: {e}")))?;
Ok(Self {
model,
tokenizer,
device: candle_device,
})
}
/// Clear the KV cache so the next forward pass starts fresh.
pub fn clear_kv_cache(&mut self) {
match &mut self.model {
QwenModel::Standard(m) => m.clear_kv_cache(),
QwenModel::Moe(m) => m.clear_kv_cache(),
}
}
/// Run a forward pass for a sequence of token IDs. Returns logits.
pub fn forward(&mut self, tokens: &[u32], pos: usize) -> Result<Tensor, InferenceError> {
let input = Tensor::new(tokens, &self.device)
.map_err(|e| InferenceError::InferenceFailed(format!("tensor: {e}")))?
.unsqueeze(0)
.map_err(|e| InferenceError::InferenceFailed(format!("unsqueeze: {e}")))?;
match &mut self.model {
QwenModel::Standard(m) => m
.forward(&input, pos)
.map_err(|e| InferenceError::InferenceFailed(format!("forward: {e}"))),
QwenModel::Moe(m) => m
.forward(&input, pos)
.map_err(|e| InferenceError::InferenceFailed(format!("forward moe: {e}"))),
}
}
/// Encode text to token IDs.
pub fn encode(&self, text: &str) -> Result<Vec<u32>, InferenceError> {
let encoding = self
.tokenizer
.encode(text, true)
.map_err(|e| InferenceError::TokenizationError(e.to_string()))?;
Ok(encoding.get_ids().to_vec())
}
/// Decode token IDs back to text.
pub fn decode(&self, tokens: &[u32]) -> Result<String, InferenceError> {
self.tokenizer
.decode(tokens, true)
.map_err(|e| InferenceError::TokenizationError(e.to_string()))
}
/// Encode text to token IDs *without* adding tokenizer special tokens
/// (BOS, etc.). Pair with [`Self::detokenize_raw`] for the round-trip
/// property `detokenize_raw(tokenize_raw(s)) == s` that downstream
/// validation harnesses (e.g. tokhn) check.
pub fn tokenize_raw(&self, text: &str) -> Result<Vec<u32>, InferenceError> {
let encoding = self
.tokenizer
.encode(text, false)
.map_err(|e| InferenceError::TokenizationError(e.to_string()))?;
Ok(encoding.get_ids().to_vec())
}
/// Decode token IDs back to text *without* skipping special tokens, so
/// the caller sees exactly what's in the token sequence (matching the
/// raw-tokenize path).
pub fn detokenize_raw(&self, tokens: &[u32]) -> Result<String, InferenceError> {
self.tokenizer
.decode(tokens, false)
.map_err(|e| InferenceError::TokenizationError(e.to_string()))
}
/// Get the EOS token ID.
pub fn eos_token_id(&self) -> Option<u32> {
self.tokenizer
.token_to_id("<|endoftext|>")
.or_else(|| self.tokenizer.token_to_id("</s>"))
}
/// Look up any token's ID by string.
pub fn token_id(&self, token: &str) -> Option<u32> {
self.tokenizer.token_to_id(token)
}
/// Get the model's maximum context length (tokens).
/// Returns None if not determinable from GGUF metadata.
pub fn context_length(&self) -> Option<usize> {
// Qwen3 models default to 32768 context
Some(32768)
}
}
/// Convert our Device enum to candle's Device. Public for use by vision backend.
pub fn to_candle_device_pub(device: Device) -> Result<CandleDevice, InferenceError> {
to_candle_device(device)
}
fn to_candle_device(device: Device) -> Result<CandleDevice, InferenceError> {
match device {
Device::Cpu => Ok(CandleDevice::Cpu),
Device::Metal => {
#[cfg(feature = "metal")]
{
Ok(CandleDevice::new_metal(0)
.map_err(|e| InferenceError::DeviceError(format!("metal: {e}")))?)
}
#[cfg(not(feature = "metal"))]
{
Err(InferenceError::DeviceError(
"metal feature not enabled".to_string(),
))
}
}
Device::Cuda(ordinal) => {
// x86_64 Linux + Windows are compiled with candle CUDA (see
// car-inference/Cargo.toml). `cuda_if_available` returns the
// GPU device when an NVIDIA card + driver are present and
// transparently yields CPU otherwise — so one binary both
// GPU-accelerates and runs on GPU-less hosts (and GPU-less
// CI runners). macOS (MLX), aarch64 Linux, and mobile targets
// don't compile the CUDA path, so the arm there is an explicit
// error.
#[cfg(all(
any(target_os = "linux", target_os = "windows"),
target_arch = "x86_64",
not(car_skip_cuda)
))]
{
// `cuda_if_available` doesn't just probe the driver — candle
// eagerly creates a cuBLAS handle at CUDA-device construction,
// which PANICS inside cudarc (uncatchably — it poisons a lazy
// static) when the CUDA *runtime* (cuBLAS/cudart, from the
// toolkit) isn't loadable even though the *driver* is present.
// So we must decide BEFORE calling it: if an NVIDIA driver is
// present (a CUDA device would be selected) but the runtime
// isn't loadable, return a clean, actionable error instead.
// Deliberately an ERROR, not a silent CPU fallback — CPU-only
// local inference isn't the intended posture on an NVIDIA box,
// and a silent downgrade would hide the fix from the user (the
// caller surfaces the guidance). When no driver is present
// (GPU-less host, CI runner, AMD/Intel box) we fall through, and
// `cuda_if_available` correctly yields CPU.
if crate::hardware::nvidia_driver_present() == Some(true)
&& crate::hardware::cuda_runtime_available() == Some(false)
{
return Err(InferenceError::DeviceError(
"an NVIDIA GPU is present but the CUDA runtime libraries \
(cuBLAS/cudart) are not installed, so GPU inference can't \
start. Install the CUDA 12 toolkit (it provides \
cublas64_12.dll / libcublas.so.12); a bundled runtime is \
planned so this becomes automatic."
.to_string(),
));
}
CandleDevice::cuda_if_available(ordinal)
.map_err(|e| InferenceError::DeviceError(format!("cuda({ordinal}): {e}")))
}
#[cfg(not(all(
any(target_os = "linux", target_os = "windows"),
target_arch = "x86_64",
not(car_skip_cuda)
)))]
{
let _ = ordinal;
Err(InferenceError::DeviceError(
"CUDA is only available on Linux/Windows (macOS uses Metal/MLX)".to_string(),
))
}
}
}
}