rig_candle/types.rs
1//! Public errors and response metadata.
2
3use rig_core::completion::{CompletionError, GetTokenUsage, Usage};
4use serde::{Deserialize, Serialize};
5use thiserror::Error;
6
7use crate::profile::ModelFamily;
8
9/// Why a local Candle completion failed.
10#[derive(Debug, Error, Clone)]
11#[non_exhaustive]
12pub enum CandleError {
13 /// A required artifact buffer was empty.
14 #[error("the {artifact} buffer is empty")]
15 EmptyBuffer { artifact: &'static str },
16 /// The Hugging Face configuration could not be parsed or is internally invalid.
17 #[error("invalid model configuration: {0}")]
18 Configuration(String),
19 /// A parsed configuration field is incompatible with Candle's Llama implementation.
20 #[error("invalid model configuration field `{field}`: {reason}")]
21 InvalidConfigurationValue {
22 /// Configuration field name.
23 field: &'static str,
24 /// Explanation of the invalid value or relationship.
25 reason: String,
26 },
27 /// The tokenizer bytes could not be loaded.
28 #[error("invalid tokenizer: {0}")]
29 TokenizerLoading(String),
30 /// The tokenizer metadata does not identify a supported prompt family.
31 #[error("unsupported model family: {0}")]
32 UnsupportedModelFamily(String),
33 /// An explicitly selected model family disagrees with validated artifacts.
34 #[error("selected model family {selected:?} does not match detected family {detected:?}")]
35 ModelFamilyMismatch {
36 /// Family requested by the caller.
37 selected: ModelFamily,
38 /// Family detected from the tokenizer.
39 detected: ModelFamily,
40 },
41 /// Independently supplied model artifacts disagree with one another.
42 #[error("{artifact} does not match the selected model artifacts: {reason}")]
43 ArtifactMismatch {
44 /// Artifact or metadata field that disagreed.
45 artifact: &'static str,
46 /// Human-readable mismatch details.
47 reason: String,
48 },
49 /// The tokenizer vocabulary does not agree with the model configuration.
50 #[error("tokenizer vocabulary size {actual} does not match config.vocab_size {expected}")]
51 TokenizerVocabularyMismatch {
52 /// Vocabulary size required by the model configuration.
53 expected: usize,
54 /// Vocabulary size reported by the tokenizer, including added tokens.
55 actual: usize,
56 },
57 /// A selected prompt-format token is absent from the tokenizer.
58 #[error("tokenizer is missing required prompt-format special token `{token}`")]
59 MissingSpecialToken {
60 /// Required special-token string.
61 token: &'static str,
62 },
63 /// A prompt-format token exists but is not registered as special.
64 #[error("tokenizer token `{token}` must be registered as a special token")]
65 SpecialTokenNotMarked {
66 /// Formatting token that must be treated atomically by the tokenizer.
67 token: &'static str,
68 },
69 /// A configured or tokenizer-provided token ID lies outside the model vocabulary.
70 #[error("token `{token}` has ID {id}, outside vocabulary size {vocab_size}")]
71 TokenIdOutOfRange {
72 /// Configuration field or special-token name.
73 token: String,
74 /// Invalid token ID.
75 id: u32,
76 /// Model vocabulary size.
77 vocab_size: usize,
78 },
79 /// The tokenizer failed to encode the rendered prompt.
80 #[error("tokenizer encoding failed: {0}")]
81 TokenizerEncoding(String),
82 /// The tokenizer failed to decode generated token IDs.
83 #[error("tokenizer decoding failed: {0}")]
84 TokenizerDecoding(String),
85 /// The safetensors checkpoint was malformed or incompatible with the configuration.
86 #[error("invalid or incompatible safetensors checkpoint: {0}")]
87 InvalidCheckpoint(String),
88 /// A GGUF checkpoint was malformed or inconsistent with its configuration.
89 #[error("invalid or incompatible GGUF checkpoint: {0}")]
90 InvalidQuantizedCheckpoint(String),
91 /// The GGUF checkpoint does not use a supported production quantization.
92 #[error("unsupported GGUF quantization: {0}")]
93 UnsupportedQuantization(String),
94 /// A message contains content that the selected text-only prompt renderer cannot represent.
95 #[error("unsupported prompt content: {0}")]
96 UnsupportedPromptContent(&'static str),
97 /// Caller-controlled content contains a delimiter reserved by the selected chat template.
98 #[error("{field} contains reserved protocol marker `{marker}`")]
99 ReservedProtocolMarker {
100 /// Kind of prompt content containing the marker.
101 field: &'static str,
102 /// Structural delimiter that was rejected.
103 marker: &'static str,
104 },
105 /// A tensor required by the configured architecture was absent.
106 #[error("checkpoint is missing expected tensor `{0}`")]
107 MissingTensor(String),
108 /// A checkpoint tensor has an incompatible shape.
109 #[error("tensor `{tensor}` has shape {actual:?}, expected {expected:?}")]
110 TensorShapeMismatch {
111 /// Tensor name in the checkpoint.
112 tensor: String,
113 /// Shape required by Candle's implementation.
114 expected: Vec<usize>,
115 /// Shape stored in the checkpoint.
116 actual: Vec<usize>,
117 },
118 /// A checkpoint tensor uses a dtype outside the portable CPU scope.
119 #[error("tensor `{tensor}` uses safetensors dtype `{dtype}`; expected F32, F16, or BF16")]
120 UnsupportedTensorDtype {
121 /// Tensor name in the checkpoint.
122 tensor: String,
123 /// Unsupported safetensors dtype.
124 dtype: String,
125 },
126 /// Neither the configuration nor tokenizer supplied a usable stop token.
127 #[error("unable to determine a valid EOS or model end-of-turn token")]
128 MissingStopToken,
129 /// Candle could not load the model tensors.
130 #[error("Candle model loading failed: {0}")]
131 ModelLoading(String),
132 /// Candle inference failed.
133 #[error("Candle inference failed: {0}")]
134 Inference(String),
135 /// A generation setting was invalid.
136 #[error("invalid generation setting: {0}")]
137 InvalidGeneration(String),
138 /// The encoded prompt itself exceeds the model context window.
139 #[error("prompt length {prompt_tokens} exceeds the model context limit {context_limit}")]
140 PromptTooLong {
141 /// Number of prompt tokens.
142 prompt_tokens: usize,
143 /// Configured model context limit.
144 context_limit: usize,
145 },
146 /// The prompt fills the context window, leaving no capacity for generation.
147 #[error(
148 "prompt length {prompt_tokens} leaves no generation capacity in context limit {context_limit}"
149 )]
150 NoGenerationCapacity {
151 /// Number of prompt tokens.
152 prompt_tokens: usize,
153 /// Configured model context limit.
154 context_limit: usize,
155 },
156 /// A numeric generation value cannot be represented on the current platform.
157 #[error("generation value `{field}`={value} cannot be represented on this platform")]
158 NumericConversion {
159 /// Name of the value being converted.
160 field: &'static str,
161 /// Original portable value.
162 value: u64,
163 },
164 /// The native inference concurrency limit is invalid.
165 #[error("max_concurrent_requests must be greater than zero")]
166 InvalidConcurrencyLimit,
167 /// The native inference admission controller was closed unexpectedly.
168 #[cfg(not(target_family = "wasm"))]
169 #[error("Candle inference concurrency controller is closed")]
170 ConcurrencyControllerClosed,
171 /// Local inference was cooperatively cancelled.
172 #[error("Candle inference was cancelled")]
173 Cancelled,
174 /// The request uses a feature outside the selected local-model protocol.
175 #[error("unsupported Candle request feature: {0}")]
176 UnsupportedFeature(String),
177 /// A tool definition cannot be represented safely by the selected protocol.
178 #[error("invalid tool definition `{tool}`: {reason}")]
179 InvalidToolDefinition {
180 /// Tool name, or a placeholder when the name itself is invalid.
181 tool: String,
182 /// Validation failure.
183 reason: String,
184 },
185 /// A model-generated tool call was malformed or incomplete.
186 #[error("malformed Qwen3 tool call: {0}")]
187 MalformedToolCall(String),
188 /// The generated response violated the requested tool-choice policy.
189 #[error("Qwen3 tool-choice violation: {0}")]
190 ToolChoiceViolation(String),
191 /// A tool result could not be correlated with a preceding assistant call.
192 #[error("tool result `{result_id}` does not match a preceding unresolved tool call")]
193 UnmatchedToolResult {
194 /// Result/call identifier supplied in history.
195 result_id: String,
196 },
197 /// `CompletionModel::make` cannot load a byte-backed model.
198 #[error(
199 "`CompletionModel::make` is unsupported for rig-candle; use a byte-backed `CandleModel` constructor or builder"
200 )]
201 UnsupportedMake,
202 /// A native blocking inference task could not be joined.
203 #[cfg(not(target_family = "wasm"))]
204 #[error("Candle blocking task failed: {0}")]
205 BlockingTaskJoin(String),
206 /// The consumer of a native streaming response closed its bounded channel.
207 #[cfg(not(target_family = "wasm"))]
208 #[error("Candle streaming response channel was closed")]
209 StreamingChannelClosed,
210}
211
212impl From<CandleError> for CompletionError {
213 fn from(error: CandleError) -> Self {
214 CompletionError::ProviderError(error.to_string())
215 }
216}
217
218/// The reason local generation ended.
219#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
220#[serde(rename_all = "snake_case")]
221pub enum FinishReason {
222 /// A configured EOS or family-specific end-of-turn token was sampled.
223 Eos,
224 /// The configured maximum output length was reached.
225 MaxTokens,
226}
227
228/// Serializable details returned alongside a Rig completion response.
229#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
230pub struct CandleCompletionResponse {
231 /// Decoded generated text, excluding the prompt and stop token.
232 pub text: String,
233 /// Number of encoded prompt tokens.
234 pub prompt_tokens: u64,
235 /// Number of sampled output tokens, including an EOS token when sampled.
236 pub generated_tokens: u64,
237 /// Maximum output tokens selected by request/default precedence before context clamping.
238 pub requested_max_tokens: u64,
239 /// Maximum output tokens available after applying the model context limit.
240 pub effective_max_tokens: u64,
241 /// Why generation ended.
242 pub finish_reason: FinishReason,
243 /// Time spent preparing the prompt tensor and running its initial forward pass.
244 pub prefill_duration_ms: u64,
245 /// Time from generation start until the first sampled token, in milliseconds.
246 pub time_to_first_token_ms: Option<u64>,
247 /// Total time spent in prefill and token generation, in milliseconds.
248 pub generation_duration_ms: u64,
249 /// Generated tokens per second when the measured duration is nonzero.
250 pub tokens_per_second: Option<f64>,
251}
252
253impl GetTokenUsage for CandleCompletionResponse {
254 fn token_usage(&self) -> Usage {
255 Usage {
256 input_tokens: self.prompt_tokens,
257 output_tokens: self.generated_tokens,
258 total_tokens: self.prompt_tokens.saturating_add(self.generated_tokens),
259 ..Usage::new()
260 }
261 }
262}