lattice-embed 0.16.0

SIMD-accelerated vector operations and embedding generation
Documentation
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
//! Image (and image+text) embedding through the Qwen3.5 vision-language
//! pooled-embedding pipeline (ADR-069 S5, #1007).
//!
//! This is a wire-through: [`VisionEmbeddingModel::embed_image`] and
//! [`VisionEmbeddingModel::embed_text`] call straight into
//! `lattice_model_qwen35::vision::pooled_embed::embed_image_from_bytes_f16` /
//! `lattice_model_qwen35::forward::cpu_f16::embed_text_vlm_f16`, the same
//! pooling + L2-normalization contract #1007 established. No new math lives
//! here — only checkpoint loading (mirroring the directory-loading pattern
//! `service::native` uses for the BERT/Qwen text models) and error mapping.
//!
//! [`VisionEmbeddingModel::from_directory`] accepts either a
//! `model.safetensors.index.json` naming exactly one decoder shard (the
//! Qwen3.5-0.8B layout), or an unindexed directory containing exactly one
//! safetensors file named `model.safetensors`. The indexed path is
//! authoritative when both are present. The unindexed path parses the
//! safetensors header and validates the exact `model.visual.*` inventory
//! before tensor payloads are materialized. Decoder and visual tensors are
//! materialized from the same mmap-backed open file, so a path replacement
//! during loading cannot mix two checkpoint versions. For an indexed
//! one-shard layout, the authoritative `weight_map` must exactly match that
//! opened shard's header inventory. `quantize_index.json` is rejected: this
//! loader constructs an f16 decoder and cannot bind a quantized visual file
//! set coherently. Callers with pre-loaded components, or a multi-shard
//! decoder checkpoint, can assemble their own weights and call
//! [`VisionEmbeddingModel::new`] directly.

use crate::error::{EmbedError, Result};
use lattice_arch::qwen35_config::Qwen35Config;
use lattice_inference_core::InferenceError;
use lattice_model_qwen35::vision::checkpoint::{
    Qwen35VisionWeights, load_qwen35_vision_weights_from_safetensors,
    open_qwen35_single_decoder_safetensors,
};
use lattice_model_qwen35::vision::pooled_embed::{
    embed_image_from_bytes_f16, embed_image_from_bytes_f16_metal,
};
use lattice_model_qwen35::vision::qwen35_vit::preprocess_qwen35_image;
use lattice_model_qwen35::weights::f16_weights::load_f16_weights;
use lattice_tokenizer::bpe::BpeTokenizer;
use lattice_tokenizer::common::Tokenizer as _;
use lattice_weights::f16_weights::F16ModelWeights;
use std::path::Path;

pub use lattice_model_qwen35::forward::cpu_f16::PoolingStrategy;

/// Raises `tokenizer`'s own truncation cap to `config`'s
/// `max_position_embeddings.min(8192)` decode window when it sits below it,
/// otherwise returns `tokenizer` unchanged. Only ever raises the cap, never
/// lowers it.
///
/// Mirrors `lattice_inference::serve::embeddings`'s identically-named
/// guard (issue #1408): without it, a caller-supplied `BpeTokenizer` left at
/// its constructor default (4096) truncates any prompt between 4097 tokens
/// and this checkpoint's real window before [`embed_text_vlm_f16`] or
/// [`embed_image_from_bytes_f16`] ever see it, silently discarding content
/// the model could otherwise process. Unlike the `lattice_inference` sibling,
/// this crate has no admission guard ahead of the forward pass to reject an
/// over-window input outright -- the tokenizer's own cap is the only thing
/// standing between "process it" and "silently shorten it", so it must never
/// sit below the checkpoint's window.
fn capped_tokenizer(tokenizer: BpeTokenizer, config: &Qwen35Config) -> BpeTokenizer {
    let max_context = max_context(config);
    if tokenizer.max_seq_len() < max_context {
        tokenizer.with_max_seq_len(max_context)
    } else {
        tokenizer
    }
}

/// This checkpoint's usable decoder window: `max_position_embeddings.min(8192)`,
/// the same value [`capped_tokenizer`] raises the tokenizer cap to. Shared
/// source of truth for the admission checks below.
fn max_context(config: &Qwen35Config) -> usize {
    config.max_position_embeddings.min(8192)
}

/// Rejects `prompt` whose PRE-truncation token count, plus `reserved`
/// non-text scaffold tokens (image delimiters + pad tokens; `0` for a
/// text-only call), exceeds `max_context` -- i.e. would otherwise be
/// silently shortened by the tokenizer's own truncation before the forward
/// pass ever sees it (issue #1408). Mirrors
/// `lattice_inference::serve::embeddings::check_item_fits_window`'s
/// pre-truncation-count admission for the sibling wire route: `real_length`
/// can never exceed the tokenizer's own cap, so comparing against it (as
/// opposed to `pre_truncation_len`) can never observe an over-window input
/// once the cap sits at or above `max_context` -- the rejection would become
/// unreachable and the input silently embedded from a truncated prefix.
///
/// # Errors
///
/// Returns [`EmbedError::InvalidInput`] naming the pre-truncation count, the
/// reserved scaffold count, and `max_context`.
fn check_prompt_admission(
    tokenizer: &BpeTokenizer,
    prompt: &str,
    reserved: usize,
    max_context: usize,
) -> Result<()> {
    let pre_truncation_len = tokenizer.tokenize(prompt).pre_truncation_len;
    let total = pre_truncation_len.saturating_add(reserved);
    if total > max_context {
        return Err(EmbedError::InvalidInput(format!(
            "prompt tokenizes to {pre_truncation_len} tokens before truncation ({total} \
             including {reserved} scaffold tokens), exceeding the model's context window of \
             {max_context} tokens"
        )));
    }
    Ok(())
}

/// Real (pre-decode) scaffold token count an image consumes:
/// `vision_start_token_id` + `vision_end_token_id` + the checkpoint's
/// per-image pad-token count derived from the patch grid, mirroring
/// `lattice_inference::serve::embeddings::EmbeddingModel::image_scaffold_token_count`'s
/// identical arithmetic. Computed via a separate, cheap preprocessing call
/// (not the caller's own forward pass) so the admission check below can run
/// before the (expensive) ViT forward.
///
/// # Errors
///
/// Returns [`EmbedError::InvalidInput`] if the checkpoint has no
/// `vision_config`, or if the image cannot be decoded/preprocessed.
fn image_scaffold_reserve(config: &Qwen35Config, image_bytes: &[u8]) -> Result<usize> {
    let vision_cfg = config.vision_config.as_ref().ok_or_else(|| {
        EmbedError::InvalidInput(
            "checkpoint has no vision_config; not a vision-language checkpoint".to_string(),
        )
    })?;
    let (_pixel_values, grid) = preprocess_qwen35_image(image_bytes, vision_cfg)
        .map_err(|e| EmbedError::InvalidInput(format!("image preprocessing failed: {e}")))?;
    let merge_sq = vision_cfg.spatial_merge_size * vision_cfg.spatial_merge_size;
    if merge_sq == 0 || !grid.num_patches().is_multiple_of(merge_sq) {
        return Err(EmbedError::InvalidInput(format!(
            "image grid {grid:?} patch count is not a multiple of spatial_merge_size^2"
        )));
    }
    Ok(2 + grid.num_patches() / merge_sq)
}

#[cfg(test)]
thread_local! {
    static AFTER_VISUAL_LOAD_HOOK: std::cell::RefCell<Option<Box<dyn FnOnce()>>> =
        std::cell::RefCell::new(None);
}

#[cfg(test)]
fn run_after_visual_load_hook() {
    let hook = AFTER_VISUAL_LOAD_HOOK.with(|slot| slot.borrow_mut().take());
    if let Some(hook) = hook {
        hook();
    }
}

#[cfg(test)]
fn with_after_visual_load_hook<T>(hook: impl FnOnce() + 'static, action: impl FnOnce() -> T) -> T {
    struct ClearHookOnDrop;
    impl Drop for ClearHookOnDrop {
        fn drop(&mut self) {
            AFTER_VISUAL_LOAD_HOOK.with(|slot| {
                slot.borrow_mut().take();
            });
        }
    }

    AFTER_VISUAL_LOAD_HOOK.with(|slot| {
        let previous = slot.borrow_mut().replace(Box::new(hook));
        assert!(
            previous.is_none(),
            "visual-load test hook already installed"
        );
    });
    let _clear_on_drop = ClearHookOnDrop;
    let result = action();
    AFTER_VISUAL_LOAD_HOOK.with(|slot| {
        assert!(
            slot.borrow().is_none(),
            "VisionEmbeddingModel::from_directory did not traverse the visual-load test hook"
        );
    });
    result
}

/// A loaded Qwen3.5 vision-language checkpoint, ready to pool image (and
/// image+text) embeddings.
///
/// See [`docs/model.md`](../docs/model.md) for the general model-loading design; this type
/// follows the same "load once, reuse" shape as `NativeEmbeddingService`'s wrapped models.
pub struct VisionEmbeddingModel {
    weights: F16ModelWeights,
    config: Qwen35Config,
    vision_weights: Qwen35VisionWeights,
    tokenizer: BpeTokenizer,
}

impl VisionEmbeddingModel {
    /// Compose a model from already-loaded components (no I/O).
    ///
    /// Use this when the checkpoint spans multiple safetensors shards (not
    /// supported by [`Self::from_directory`]) or when components are shared
    /// across other in-process model instances.
    ///
    /// Raises `tokenizer`'s own truncation cap to `config`'s context window
    /// when it sits below it (see `capped_tokenizer`) -- see that
    /// function's docs for why: without this, a caller-supplied tokenizer
    /// left at its default cap silently truncates any prompt between 4097
    /// tokens and the checkpoint's real window before it ever reaches the
    /// forward pass. [`Self::from_directory`] goes through this constructor
    /// too, so it gets the same guarantee.
    pub fn new(
        weights: F16ModelWeights,
        config: Qwen35Config,
        vision_weights: Qwen35VisionWeights,
        tokenizer: BpeTokenizer,
    ) -> Self {
        let tokenizer = capped_tokenizer(tokenizer, &config);
        Self {
            weights,
            config,
            vision_weights,
            tokenizer,
        }
    }

    /// Load a Qwen3.5 vision-language checkpoint directory: `config.json`,
    /// `tokenizer.json`, the `model.visual.*` vision-encoder tensors, and a
    /// single-shard f16 decoder checkpoint. An existing
    /// `model.safetensors.index.json` is authoritative. Without an index, the
    /// directory must contain exactly one `*.safetensors` file, named
    /// `model.safetensors`; its header must carry the exact `model.visual.*`
    /// inventory implied by `vision_config`. A `quantize_index.json` is
    /// rejected because this constructor has no quantized decoder loader. The
    /// chosen safetensors file is opened once and that same reader supplies
    /// both visual and decoder tensors.
    ///
    /// # Errors
    ///
    /// Returns [`EmbedError::ModelInitialization`] if `config.json` is
    /// missing or invalid, if the checkpoint has no `vision_config`, if a
    /// quantized checkpoint is present, if the unindexed safetensors layout is
    /// missing or ambiguous, if the decoder weights are sharded across more
    /// than one file, or if any component tensor fails to load.
    pub fn from_directory(dir: &Path) -> Result<Self> {
        let quantized_index = dir.join("quantize_index.json");
        match std::fs::symlink_metadata(&quantized_index) {
            Ok(_) => {
                return Err(EmbedError::ModelInitialization(format!(
                    "{} is present, but quantized checkpoints are not supported by \
                     VisionEmbeddingModel::from_directory's f16 decoder loader",
                    quantized_index.display()
                )));
            }
            Err(err) if err.kind() == std::io::ErrorKind::NotFound => {}
            Err(err) => {
                return Err(EmbedError::ModelInitialization(format!(
                    "failed to inspect {}: {err}",
                    quantized_index.display()
                )));
            }
        }

        let config = Qwen35Config::from_model_dir(dir)
            .map_err(|e| EmbedError::ModelInitialization(format!("config.json: {e}")))?;
        let vision_cfg = config.vision_config.clone().ok_or_else(|| {
            EmbedError::ModelInitialization(format!(
                "{} has no vision_config; not a vision-language checkpoint",
                dir.display()
            ))
        })?;

        // Tokenizer parsing is independent of tensor payloads, so reject a
        // missing or malformed tokenizer before materializing multi-GB model
        // weights. The checkpoint itself is still opened only once below so
        // visual and decoder tensors stay bound to one file descriptor.
        let tokenizer_path = dir.join("tokenizer.json");
        let tokenizer = BpeTokenizer::from_tokenizer_json(&tokenizer_path).map_err(|e| {
            EmbedError::ModelInitialization(format!("{}: {e}", tokenizer_path.display()))
        })?;

        let (mut sf, shard_path) = open_qwen35_single_decoder_safetensors(dir)
            .map_err(|e| EmbedError::ModelInitialization(format!("decoder checkpoint: {e}")))?;
        let vision_weights =
            load_qwen35_vision_weights_from_safetensors(&mut sf, &shard_path, &vision_cfg)
                .map_err(|e| EmbedError::ModelInitialization(format!("vision weights: {e}")))?;
        #[cfg(test)]
        run_after_visual_load_hook();
        let weights = load_f16_weights(&sf, &config)
            .map_err(|e| EmbedError::ModelInitialization(format!("decoder weights: {e}")))?;

        Ok(Self::new(weights, config, vision_weights, tokenizer))
    }

    /// Pool an image (plus an optional text prompt) into a single
    /// L2-normalized `[dimensions()]` embedding vector.
    ///
    /// Same scaffold and pooling contract as
    /// [`lattice_model_qwen35::vision::pooled_embed::embed_image_from_bytes_f16`] (see that
    /// function's docs for the exact prompt-assembly layout).
    ///
    /// # Errors
    ///
    /// Returns [`EmbedError::InvalidInput`] if `image_bytes` cannot be
    /// decoded, its dimensions are not compatible with the checkpoint's
    /// patch/merge geometry, the assembled request otherwise fails
    /// validation (the error message names the offending field), or the
    /// image's scaffold tokens plus `prompt`'s pre-truncation token count
    /// exceed the checkpoint's context window (issue #1408: rejected before
    /// the tokenizer's own truncation could silently shorten `prompt`).
    /// Returns [`EmbedError::InferenceFailed`] for every other underlying
    /// failure.
    pub fn embed_image(
        &self,
        image_bytes: &[u8],
        prompt: &str,
        pooling: PoolingStrategy,
    ) -> Result<Vec<f32>> {
        let reserved = image_scaffold_reserve(&self.config, image_bytes)?;
        check_prompt_admission(&self.tokenizer, prompt, reserved, max_context(&self.config))?;
        embed_image_from_bytes_f16(
            &self.weights,
            &self.config,
            &self.vision_weights,
            &self.tokenizer,
            image_bytes,
            prompt,
            pooling,
        )
        .map_err(map_inference_error)
    }

    /// Metal-dispatching sibling of [`Self::embed_image`]: runs the ViT
    /// forward pass on the Metal GPU instead of the CPU (see
    /// [`lattice_model_qwen35::vision::pooled_embed::embed_image_from_bytes_f16_metal`]).
    /// Same scaffold, pooling contract, and error semantics as
    /// [`Self::embed_image`], with one addition: on a build/platform with no
    /// Metal support, this returns [`EmbedError::InferenceFailed`] (Metal
    /// unavailability is a runtime-backend failure, not caller-input
    /// validation) rather than silently falling back to
    /// [`Self::embed_image`]'s CPU path — callers that want a fallback must
    /// call [`Self::embed_image`] themselves.
    ///
    /// # Errors
    ///
    /// See [`Self::embed_image`]'s docs.
    pub fn embed_image_metal(
        &self,
        image_bytes: &[u8],
        prompt: &str,
        pooling: PoolingStrategy,
    ) -> Result<Vec<f32>> {
        let reserved = image_scaffold_reserve(&self.config, image_bytes)?;
        check_prompt_admission(&self.tokenizer, prompt, reserved, max_context(&self.config))?;
        embed_image_from_bytes_f16_metal(
            &self.weights,
            &self.config,
            &self.vision_weights,
            &self.tokenizer,
            image_bytes,
            prompt,
            pooling,
        )
        .map_err(map_inference_error)
    }

    /// Pool a text-only prompt through the same decoder + pooling path as
    /// [`Self::embed_image`], landing in the same vector space.
    ///
    /// # Errors
    ///
    /// Returns [`EmbedError::InvalidInput`] if the prompt is empty, tokenizes
    /// to an out-of-vocabulary id, or its pre-truncation token count exceeds
    /// the checkpoint's context window (issue #1408: rejected before the
    /// tokenizer's own truncation could silently shorten it). Returns
    /// [`EmbedError::InferenceFailed`] for every other underlying failure.
    pub fn embed_text(&self, prompt: &str, pooling: PoolingStrategy) -> Result<Vec<f32>> {
        check_prompt_admission(&self.tokenizer, prompt, 0, max_context(&self.config))?;
        lattice_model_qwen35::forward::cpu_f16::embed_text_vlm_f16(
            &self.weights,
            &self.config,
            &self.tokenizer,
            prompt,
            pooling,
        )
        .map_err(map_inference_error)
    }

    /// Output embedding dimension (the checkpoint's decoder hidden size).
    pub fn dimensions(&self) -> usize {
        self.config.hidden_size
    }
}

/// Map an inference-layer error to the embed crate's two-variant contract:
/// caller-supplied-input problems stay distinguishable from every other
/// (model/runtime) failure, so callers can tell "fix your request" apart
/// from "retry or report a bug" (see `embed_image`/`embed_text` docs).
fn map_inference_error(e: InferenceError) -> EmbedError {
    match e {
        InferenceError::InvalidInput(msg) => EmbedError::InvalidInput(msg),
        other => EmbedError::InferenceFailed(other.to_string()),
    }
}

#[cfg(test)]
mod tests;