use crate::error::{EmbedError, Result};
use lattice_arch::qwen35_config::Qwen35Config;
use lattice_inference_core::InferenceError;
use lattice_model_qwen35::vision::checkpoint::{
Qwen35VisionWeights, load_qwen35_vision_weights_from_safetensors,
open_qwen35_single_decoder_safetensors,
};
use lattice_model_qwen35::vision::pooled_embed::{
embed_image_from_bytes_f16, embed_image_from_bytes_f16_metal,
};
use lattice_model_qwen35::vision::qwen35_vit::preprocess_qwen35_image;
use lattice_model_qwen35::weights::f16_weights::load_f16_weights;
use lattice_tokenizer::bpe::BpeTokenizer;
use lattice_tokenizer::common::Tokenizer as _;
use lattice_weights::f16_weights::F16ModelWeights;
use std::path::Path;
pub use lattice_model_qwen35::forward::cpu_f16::PoolingStrategy;
fn capped_tokenizer(tokenizer: BpeTokenizer, config: &Qwen35Config) -> BpeTokenizer {
let max_context = max_context(config);
if tokenizer.max_seq_len() < max_context {
tokenizer.with_max_seq_len(max_context)
} else {
tokenizer
}
}
fn max_context(config: &Qwen35Config) -> usize {
config.max_position_embeddings.min(8192)
}
fn check_prompt_admission(
tokenizer: &BpeTokenizer,
prompt: &str,
reserved: usize,
max_context: usize,
) -> Result<()> {
let pre_truncation_len = tokenizer.tokenize(prompt).pre_truncation_len;
let total = pre_truncation_len.saturating_add(reserved);
if total > max_context {
return Err(EmbedError::InvalidInput(format!(
"prompt tokenizes to {pre_truncation_len} tokens before truncation ({total} \
including {reserved} scaffold tokens), exceeding the model's context window of \
{max_context} tokens"
)));
}
Ok(())
}
fn image_scaffold_reserve(config: &Qwen35Config, image_bytes: &[u8]) -> Result<usize> {
let vision_cfg = config.vision_config.as_ref().ok_or_else(|| {
EmbedError::InvalidInput(
"checkpoint has no vision_config; not a vision-language checkpoint".to_string(),
)
})?;
let (_pixel_values, grid) = preprocess_qwen35_image(image_bytes, vision_cfg)
.map_err(|e| EmbedError::InvalidInput(format!("image preprocessing failed: {e}")))?;
let merge_sq = vision_cfg.spatial_merge_size * vision_cfg.spatial_merge_size;
if merge_sq == 0 || !grid.num_patches().is_multiple_of(merge_sq) {
return Err(EmbedError::InvalidInput(format!(
"image grid {grid:?} patch count is not a multiple of spatial_merge_size^2"
)));
}
Ok(2 + grid.num_patches() / merge_sq)
}
#[cfg(test)]
thread_local! {
static AFTER_VISUAL_LOAD_HOOK: std::cell::RefCell<Option<Box<dyn FnOnce()>>> =
std::cell::RefCell::new(None);
}
#[cfg(test)]
fn run_after_visual_load_hook() {
let hook = AFTER_VISUAL_LOAD_HOOK.with(|slot| slot.borrow_mut().take());
if let Some(hook) = hook {
hook();
}
}
#[cfg(test)]
fn with_after_visual_load_hook<T>(hook: impl FnOnce() + 'static, action: impl FnOnce() -> T) -> T {
struct ClearHookOnDrop;
impl Drop for ClearHookOnDrop {
fn drop(&mut self) {
AFTER_VISUAL_LOAD_HOOK.with(|slot| {
slot.borrow_mut().take();
});
}
}
AFTER_VISUAL_LOAD_HOOK.with(|slot| {
let previous = slot.borrow_mut().replace(Box::new(hook));
assert!(
previous.is_none(),
"visual-load test hook already installed"
);
});
let _clear_on_drop = ClearHookOnDrop;
let result = action();
AFTER_VISUAL_LOAD_HOOK.with(|slot| {
assert!(
slot.borrow().is_none(),
"VisionEmbeddingModel::from_directory did not traverse the visual-load test hook"
);
});
result
}
pub struct VisionEmbeddingModel {
weights: F16ModelWeights,
config: Qwen35Config,
vision_weights: Qwen35VisionWeights,
tokenizer: BpeTokenizer,
}
impl VisionEmbeddingModel {
pub fn new(
weights: F16ModelWeights,
config: Qwen35Config,
vision_weights: Qwen35VisionWeights,
tokenizer: BpeTokenizer,
) -> Self {
let tokenizer = capped_tokenizer(tokenizer, &config);
Self {
weights,
config,
vision_weights,
tokenizer,
}
}
pub fn from_directory(dir: &Path) -> Result<Self> {
let quantized_index = dir.join("quantize_index.json");
match std::fs::symlink_metadata(&quantized_index) {
Ok(_) => {
return Err(EmbedError::ModelInitialization(format!(
"{} is present, but quantized checkpoints are not supported by \
VisionEmbeddingModel::from_directory's f16 decoder loader",
quantized_index.display()
)));
}
Err(err) if err.kind() == std::io::ErrorKind::NotFound => {}
Err(err) => {
return Err(EmbedError::ModelInitialization(format!(
"failed to inspect {}: {err}",
quantized_index.display()
)));
}
}
let config = Qwen35Config::from_model_dir(dir)
.map_err(|e| EmbedError::ModelInitialization(format!("config.json: {e}")))?;
let vision_cfg = config.vision_config.clone().ok_or_else(|| {
EmbedError::ModelInitialization(format!(
"{} has no vision_config; not a vision-language checkpoint",
dir.display()
))
})?;
let tokenizer_path = dir.join("tokenizer.json");
let tokenizer = BpeTokenizer::from_tokenizer_json(&tokenizer_path).map_err(|e| {
EmbedError::ModelInitialization(format!("{}: {e}", tokenizer_path.display()))
})?;
let (mut sf, shard_path) = open_qwen35_single_decoder_safetensors(dir)
.map_err(|e| EmbedError::ModelInitialization(format!("decoder checkpoint: {e}")))?;
let vision_weights =
load_qwen35_vision_weights_from_safetensors(&mut sf, &shard_path, &vision_cfg)
.map_err(|e| EmbedError::ModelInitialization(format!("vision weights: {e}")))?;
#[cfg(test)]
run_after_visual_load_hook();
let weights = load_f16_weights(&sf, &config)
.map_err(|e| EmbedError::ModelInitialization(format!("decoder weights: {e}")))?;
Ok(Self::new(weights, config, vision_weights, tokenizer))
}
pub fn embed_image(
&self,
image_bytes: &[u8],
prompt: &str,
pooling: PoolingStrategy,
) -> Result<Vec<f32>> {
let reserved = image_scaffold_reserve(&self.config, image_bytes)?;
check_prompt_admission(&self.tokenizer, prompt, reserved, max_context(&self.config))?;
embed_image_from_bytes_f16(
&self.weights,
&self.config,
&self.vision_weights,
&self.tokenizer,
image_bytes,
prompt,
pooling,
)
.map_err(map_inference_error)
}
pub fn embed_image_metal(
&self,
image_bytes: &[u8],
prompt: &str,
pooling: PoolingStrategy,
) -> Result<Vec<f32>> {
let reserved = image_scaffold_reserve(&self.config, image_bytes)?;
check_prompt_admission(&self.tokenizer, prompt, reserved, max_context(&self.config))?;
embed_image_from_bytes_f16_metal(
&self.weights,
&self.config,
&self.vision_weights,
&self.tokenizer,
image_bytes,
prompt,
pooling,
)
.map_err(map_inference_error)
}
pub fn embed_text(&self, prompt: &str, pooling: PoolingStrategy) -> Result<Vec<f32>> {
check_prompt_admission(&self.tokenizer, prompt, 0, max_context(&self.config))?;
lattice_model_qwen35::forward::cpu_f16::embed_text_vlm_f16(
&self.weights,
&self.config,
&self.tokenizer,
prompt,
pooling,
)
.map_err(map_inference_error)
}
pub fn dimensions(&self) -> usize {
self.config.hidden_size
}
}
fn map_inference_error(e: InferenceError) -> EmbedError {
match e {
InferenceError::InvalidInput(msg) => EmbedError::InvalidInput(msg),
other => EmbedError::InferenceFailed(other.to_string()),
}
}
#[cfg(test)]
mod tests;