use anyhow::{anyhow, Result};
use image::{imageops::FilterType, GenericImageView, ImageBuffer, ImageFormat, Rgb, RgbImage};
#[derive(Debug, Clone, PartialEq)]
pub struct PreprocessConfig {
pub target_size: u32,
pub mean: [f32; 3],
pub std: [f32; 3],
}
pub const GEMMA4_VISION_CONFIG: PreprocessConfig = PreprocessConfig {
target_size: 896,
mean: [0.5, 0.5, 0.5],
std: [0.5, 0.5, 0.5],
};
pub fn preprocess_rgb_chw(bytes: &[u8], config: &PreprocessConfig) -> Result<Vec<f32>> {
if config.target_size == 0 || config.target_size > u16::MAX as u32 {
return Err(anyhow!(
"invalid target_size {}: must be in 1..=65535",
config.target_size
));
}
let fmt = image::guess_format(bytes).map_err(|e| anyhow!("guess_format: {e}"))?;
match fmt {
ImageFormat::Png | ImageFormat::Jpeg => {}
other => {
return Err(anyhow!(
"image format {:?} is not supported by this build (only PNG + JPEG)",
other
));
}
}
let img = image::load_from_memory(bytes).map_err(|e| anyhow!("decode image: {e}"))?;
let (_w, _h) = img.dimensions();
let resized = img.resize_exact(config.target_size, config.target_size, FilterType::Triangle);
let rgb = resized.to_rgb8();
let size = config.target_size as usize;
let hw = size * size;
let mut out = vec![0f32; 3 * hw];
for (y, row) in rgb.rows().enumerate() {
for (x, pix) in row.enumerate() {
let channels = [
pix[0] as f32 / 255.0,
pix[1] as f32 / 255.0,
pix[2] as f32 / 255.0,
];
let idx = y * size + x;
for (c, &channel) in channels.iter().enumerate() {
out[c * hw + idx] = (channel - config.mean[c]) / config.std[c];
}
}
}
Ok(out)
}
#[derive(Debug, Clone, PartialEq)]
pub struct Gemma4vPreprocessConfig {
pub patch_size: u32,
pub n_merge: u32,
pub token_min: u32,
pub token_max: u32,
}
pub const GEMMA4V_PREPROCESS_DEFAULT: Gemma4vPreprocessConfig = Gemma4vPreprocessConfig {
patch_size: 16,
n_merge: 3,
token_min: 252,
token_max: 280,
};
#[derive(Debug, Clone, PartialEq)]
pub struct Gemma4vPreprocessed {
pub patches: Vec<f32>,
pub pos_x: Vec<u32>,
pub pos_y: Vec<u32>,
pub n_x: u32,
pub n_y: u32,
}
impl Gemma4vPreprocessed {
pub fn n_patches(&self) -> u32 {
self.n_x.saturating_mul(self.n_y)
}
}
pub fn preprocess_gemma4v(
bytes: &[u8],
cfg: &Gemma4vPreprocessConfig,
) -> Result<Gemma4vPreprocessed> {
if cfg.patch_size == 0 {
return Err(anyhow!("gemma4v: patch_size must be > 0"));
}
if cfg.n_merge == 0 {
return Err(anyhow!("gemma4v: n_merge must be > 0"));
}
if cfg.token_max == 0 {
return Err(anyhow!("gemma4v: token_max must be > 0"));
}
if cfg.token_min > cfg.token_max {
return Err(anyhow!(
"gemma4v: token_min ({}) > token_max ({})",
cfg.token_min,
cfg.token_max
));
}
let fmt = image::guess_format(bytes).map_err(|e| anyhow!("guess_format: {e}"))?;
match fmt {
ImageFormat::Png | ImageFormat::Jpeg => {}
other => {
return Err(anyhow!(
"image format {:?} is not supported by gemma4v preprocess (only PNG + JPEG)",
other
));
}
}
let img = image::load_from_memory(bytes).map_err(|e| anyhow!("decode image: {e}"))?;
let (orig_w, orig_h) = img.dimensions();
if orig_w == 0 || orig_h == 0 {
return Err(anyhow!("gemma4v: image has zero dimension"));
}
let p = cfg.patch_size;
let (n_x, n_y) =
compute_gemma4v_patch_grid(orig_w, orig_h, p, cfg.n_merge, cfg.token_min, cfg.token_max)?;
let target_w = n_x * p;
let target_h = n_y * p;
let src_rgb = img.to_rgb8();
let rgb = resize_bilinear_pad_llama_cpp(&src_rgb, target_w, target_h, [0, 0, 0]);
let n_patches = (n_x as usize) * (n_y as usize);
let p_us = p as usize;
let inner = p_us * p_us * 3;
let mut patches = vec![0f32; n_patches * inner];
let mut pos_x = Vec::with_capacity(n_patches);
let mut pos_y = Vec::with_capacity(n_patches);
let p2 = p_us * p_us; for py in 0..n_y {
for px in 0..n_x {
let patch_idx = (py as usize) * (n_x as usize) + (px as usize);
let row_base = patch_idx * inner;
pos_x.push(px);
pos_y.push(py);
for dy in 0..p {
for dx in 0..p {
let img_x = px * p + dx;
let img_y = py * p + dy;
let pix = rgb.get_pixel(img_x, img_y);
let pos_in_plane = (dy as usize) * p_us + (dx as usize);
#[allow(clippy::erasing_op, clippy::identity_op)]
{
patches[row_base + 0 * p2 + pos_in_plane] =
(pix[0] as f32 / 255.0) * 4.0 - 3.0;
patches[row_base + 1 * p2 + pos_in_plane] =
(pix[1] as f32 / 255.0) * 4.0 - 3.0;
patches[row_base + 2 * p2 + pos_in_plane] =
(pix[2] as f32 / 255.0) * 4.0 - 3.0;
}
}
}
}
}
Ok(Gemma4vPreprocessed {
patches,
pos_x,
pos_y,
n_x,
n_y,
})
}
fn compute_gemma4v_patch_grid(
orig_w: u32,
orig_h: u32,
p: u32,
n_merge: u32,
token_min: u32,
token_max: u32,
) -> Result<(u32, u32)> {
let align_size: u64 = (p as u64) * (n_merge as u64);
if align_size == 0 {
return Err(anyhow!(
"gemma4v patch grid: align_size = patch_size ({p}) * n_merge ({n_merge}) is zero"
));
}
let patch_area: u64 = (p as u64) * (p as u64) * (n_merge as u64) * (n_merge as u64);
let min_pixels: u64 = (token_min as u64) * patch_area;
let max_pixels: u64 = (token_max as u64) * patch_area;
let round_by =
|x: f64| -> u64 { ((x / align_size as f64).round() as i64).max(0) as u64 * align_size };
let ceil_by =
|x: f64| -> u64 { ((x / align_size as f64).ceil() as i64).max(0) as u64 * align_size };
let floor_by =
|x: f64| -> u64 { ((x / align_size as f64).floor() as i64).max(0) as u64 * align_size };
let width = orig_w as u64;
let height = orig_h as u64;
let mut h_bar: u64 = align_size.max(round_by(height as f64));
let mut w_bar: u64 = align_size.max(round_by(width as f64));
if h_bar * w_bar > max_pixels {
let beta = ((height * width) as f64 / max_pixels as f64).sqrt();
h_bar = align_size.max(floor_by(height as f64 / beta));
w_bar = align_size.max(floor_by(width as f64 / beta));
} else if h_bar * w_bar < min_pixels {
let beta = (min_pixels as f64 / (height * width) as f64).sqrt();
h_bar = ceil_by(height as f64 * beta);
w_bar = ceil_by(width as f64 * beta);
}
let n_x_u64 = w_bar / (p as u64);
let n_y_u64 = h_bar / (p as u64);
if n_x_u64 == 0 || n_y_u64 == 0 || n_x_u64 > u32::MAX as u64 || n_y_u64 > u32::MAX as u64 {
return Err(anyhow!(
"gemma4v patch grid: degenerate output ({} x {}) for input ({} x {})",
n_x_u64,
n_y_u64,
orig_w,
orig_h
));
}
let n_x = n_x_u64 as u32;
let n_y = n_y_u64 as u32;
debug_assert!(
n_x % n_merge == 0 && n_y % n_merge == 0,
"gemma4v patch grid: ({n_x},{n_y}) not aligned to n_merge={n_merge}"
);
Ok((n_x, n_y))
}
fn resize_bilinear_llama_cpp(src: &RgbImage, target_w: u32, target_h: u32) -> RgbImage {
let src_w = src.width();
let src_h = src.height();
if target_w == 0 || target_h == 0 || src_w == 0 || src_h == 0 {
return ImageBuffer::new(target_w.max(1), target_h.max(1));
}
if src_w == target_w && src_h == target_h {
return src.clone();
}
let x_ratio = if target_w > 1 {
(src_w as f32 - 1.0) / (target_w as f32 - 1.0)
} else {
0.0
};
let y_ratio = if target_h > 1 {
(src_h as f32 - 1.0) / (target_h as f32 - 1.0)
} else {
0.0
};
let mut dst: RgbImage = ImageBuffer::new(target_w, target_h);
let src_w_i = src_w as i32;
let src_h_i = src_h as i32;
for y in 0..target_h {
for x in 0..target_w {
let px = x as f32 * x_ratio;
let py = y as f32 * y_ratio;
let x0 = (px as i32).min(src_w_i - 1).max(0);
let y0 = (py as i32).min(src_h_i - 1).max(0);
let x1 = (x0 + 1).min(src_w_i - 1);
let y1 = (y0 + 1).min(src_h_i - 1);
let xf = px - (x0 as f32);
let yf = py - (y0 as f32);
let p00 = src.get_pixel(x0 as u32, y0 as u32).0;
let p10 = src.get_pixel(x1 as u32, y0 as u32).0;
let p01 = src.get_pixel(x0 as u32, y1 as u32).0;
let p11 = src.get_pixel(x1 as u32, y1 as u32).0;
let mut out = [0u8; 3];
for c in 0..3 {
let top = (p00[c] as f32) + ((p10[c] as f32) - (p00[c] as f32)) * xf;
let bottom = (p01[c] as f32) + ((p11[c] as f32) - (p01[c] as f32)) * xf;
let v = top + (bottom - top) * yf;
out[c] = v.clamp(0.0, 255.0) as u8;
}
dst.put_pixel(x, y, Rgb(out));
}
}
dst
}
fn resize_bilinear_pad_llama_cpp(
src: &RgbImage,
target_w: u32,
target_h: u32,
pad_color: [u8; 3],
) -> RgbImage {
let src_w = src.width();
let src_h = src.height();
if src_w == target_w && src_h == target_h {
return src.clone();
}
if target_w == 0 || target_h == 0 || src_w == 0 || src_h == 0 {
return ImageBuffer::new(target_w.max(1), target_h.max(1));
}
let scale_w = (target_w as f32) / (src_w as f32);
let scale_h = (target_h as f32) / (src_h as f32);
let scale = scale_w.min(scale_h);
let new_w_f = (src_w as f32) * scale;
let new_h_f = (src_h as f32) * scale;
let new_w = (new_w_f.ceil() as i64).min(target_w as i64).max(1) as u32;
let new_h = (new_h_f.ceil() as i64).min(target_h as i64).max(1) as u32;
let resized = resize_bilinear_llama_cpp(src, new_w, new_h);
let mut dst: RgbImage = ImageBuffer::from_pixel(target_w, target_h, Rgb(pad_color));
let offset_x = ((target_w - new_w) / 2) as i32;
let offset_y = ((target_h - new_h) / 2) as i32;
for y in 0..new_h {
for x in 0..new_w {
let dx = (x as i32) + offset_x;
let dy = (y as i32) + offset_y;
if dx < 0 || dy < 0 || dx >= target_w as i32 || dy >= target_h as i32 {
continue;
}
let p = *resized.get_pixel(x, y);
dst.put_pixel(dx as u32, dy as u32, p);
}
}
dst
}
#[derive(Debug, Clone, PartialEq)]
pub struct Qwen3VlPreprocessConfig {
pub patch_size: u32,
pub spatial_merge_size: u32,
pub image_mean: [f32; 3],
pub image_std: [f32; 3],
pub image_min_pixels: u64,
pub image_max_pixels: u64,
}
impl Qwen3VlPreprocessConfig {
pub fn from_mmproj(cfg: &super::mmproj::MmprojConfig) -> anyhow::Result<Self> {
let sm = cfg.spatial_merge_size.ok_or_else(|| {
anyhow!(
"Qwen3VlPreprocessConfig::from_mmproj: MmprojConfig.spatial_merge_size is None \
— Qwen3-VL mmproj must carry `clip.vision.spatial_merge_size`"
)
})?;
if cfg.patch_size == 0 || sm == 0 {
return Err(anyhow!(
"Qwen3VlPreprocessConfig::from_mmproj: patch_size ({}) and \
spatial_merge_size ({}) must be > 0",
cfg.patch_size,
sm
));
}
let patch_area: u64 = (cfg.patch_size as u64).pow(2) * (sm as u64).pow(2);
let image_min_pixels: u64 = 8 * patch_area;
let image_max_pixels: u64 = 4096 * patch_area;
Ok(Self {
patch_size: cfg.patch_size,
spatial_merge_size: sm,
image_mean: cfg.image_mean,
image_std: cfg.image_std,
image_min_pixels,
image_max_pixels,
})
}
}
#[derive(Debug, Clone, PartialEq)]
pub struct Qwen3VlPreprocessed {
pub pixel_values: Vec<f32>,
pub target_size: u32,
pub target_w: u32,
pub target_h: u32,
pub n_x_token: u32,
pub n_y_token: u32,
pub n_image_tokens: u32,
}
impl Qwen3VlPreprocessed {
pub fn target_pixel_grid(&self) -> (u32, u32) {
(self.target_w, self.target_h)
}
}
pub fn preprocess_qwen3vl(
bytes: &[u8],
cfg: &Qwen3VlPreprocessConfig,
image_size: u32,
) -> Result<Qwen3VlPreprocessed> {
if cfg.patch_size == 0 || cfg.spatial_merge_size == 0 {
return Err(anyhow!(
"qwen3vl preprocess: patch_size ({}) and spatial_merge_size ({}) \
must both be > 0",
cfg.patch_size,
cfg.spatial_merge_size
));
}
let stride = cfg.patch_size * cfg.spatial_merge_size;
if image_size == 0 || image_size % stride != 0 {
return Err(anyhow!(
"qwen3vl preprocess: image_size ({}) must be a positive multiple \
of patch_size ({}) * spatial_merge_size ({}) = {}",
image_size,
cfg.patch_size,
cfg.spatial_merge_size,
stride
));
}
if cfg.image_min_pixels > cfg.image_max_pixels {
return Err(anyhow!(
"qwen3vl preprocess: image_min_pixels ({}) > image_max_pixels ({})",
cfg.image_min_pixels,
cfg.image_max_pixels
));
}
let fmt = image::guess_format(bytes).map_err(|e| anyhow!("guess_format: {e}"))?;
match fmt {
ImageFormat::Png | ImageFormat::Jpeg => {}
other => {
return Err(anyhow!(
"image format {:?} is not supported by qwen3vl preprocess (only PNG + JPEG)",
other
));
}
}
let img = image::load_from_memory(bytes).map_err(|e| anyhow!("decode image: {e}"))?;
let (orig_w, orig_h) = img.dimensions();
if orig_w == 0 || orig_h == 0 {
return Err(anyhow!("qwen3vl preprocess: image has zero dimension"));
}
let canvas_pixels = (image_size as u64) * (image_size as u64);
let (smart_w, smart_h) = qwen3vl_calc_size_preserved_ratio(
orig_w,
orig_h,
stride,
cfg.image_min_pixels,
cfg.image_max_pixels,
)?;
let (target_w, target_h) = if smart_w > image_size || smart_h > image_size {
let max_dim = smart_w.max(smart_h) as f64;
let scale = (image_size as f64) / max_dim;
let scaled_w = ((smart_w as f64 * scale) as u32 / stride) * stride;
let scaled_h = ((smart_h as f64 * scale) as u32 / stride) * stride;
let final_w = scaled_w.max(stride);
let final_h = scaled_h.max(stride);
(final_w, final_h)
} else {
(smart_w, smart_h)
};
if target_w == 0 || target_h == 0 {
return Err(anyhow!(
"qwen3vl preprocess: smart-resize produced degenerate target \
({target_w}x{target_h}) for input ({orig_w}x{orig_h})"
));
}
if target_w % stride != 0 || target_h % stride != 0 {
return Err(anyhow!(
"qwen3vl preprocess: post-clamp ({target_w}x{target_h}) not \
stride-aligned (stride={stride})"
));
}
debug_assert!(
(target_w as u64) * (target_h as u64) <= canvas_pixels,
"post-clamp area {}x{} exceeds canvas {canvas_pixels}",
target_w,
target_h,
);
let src_rgb = img.to_rgb8();
let resized = resize_bilinear_llama_cpp(&src_rgb, target_w, target_h);
let tw_us = target_w as usize;
let th_us = target_h as usize;
let hw = th_us * tw_us;
let mut pixel_values = vec![0f32; 3 * hw];
for (y, row) in resized.rows().enumerate() {
for (x, pix) in row.enumerate() {
let idx = y * tw_us + x;
for c in 0..3 {
let v = (pix[c] as f32 / 255.0 - cfg.image_mean[c]) / cfg.image_std[c];
pixel_values[c * hw + idx] = v;
}
}
}
let n_x_token = target_w / stride;
let n_y_token = target_h / stride;
let n_image_tokens = n_x_token * n_y_token;
Ok(Qwen3VlPreprocessed {
pixel_values,
target_size: image_size,
target_w,
target_h,
n_x_token,
n_y_token,
n_image_tokens,
})
}
fn qwen3vl_calc_size_preserved_ratio(
orig_w: u32,
orig_h: u32,
align_size: u32,
min_pixels: u64,
max_pixels: u64,
) -> Result<(u32, u32)> {
let align: u64 = align_size as u64;
if align == 0 {
return Err(anyhow!("qwen3vl smart_resize: align_size must be > 0"));
}
let width = orig_w as u64;
let height = orig_h as u64;
let round_by = |x: f64| -> u64 { ((x / align as f64).round() as i64).max(0) as u64 * align };
let ceil_by = |x: f64| -> u64 { ((x / align as f64).ceil() as i64).max(0) as u64 * align };
let floor_by = |x: f64| -> u64 { ((x / align as f64).floor() as i64).max(0) as u64 * align };
let mut h_bar: u64 = align.max(round_by(height as f64));
let mut w_bar: u64 = align.max(round_by(width as f64));
if h_bar * w_bar > max_pixels {
let beta = ((height * width) as f64 / max_pixels as f64).sqrt();
h_bar = align.max(floor_by(height as f64 / beta));
w_bar = align.max(floor_by(width as f64 / beta));
} else if h_bar * w_bar < min_pixels {
let beta = (min_pixels as f64 / (height * width) as f64).sqrt();
h_bar = ceil_by(height as f64 * beta);
w_bar = ceil_by(width as f64 * beta);
}
if h_bar == 0 || w_bar == 0 || h_bar > u32::MAX as u64 || w_bar > u32::MAX as u64 {
return Err(anyhow!(
"qwen3vl smart_resize: degenerate output ({} x {}) for input ({} x {})",
w_bar,
h_bar,
orig_w,
orig_h
));
}
Ok((w_bar as u32, h_bar as u32))
}
#[cfg(test)]
mod tests {
use super::*;
use image::{ImageBuffer, Rgb, RgbImage};
use std::io::Cursor;
fn encode_png(img: &RgbImage) -> Vec<u8> {
let mut buf: Vec<u8> = Vec::new();
img.write_to(&mut Cursor::new(&mut buf), ImageFormat::Png)
.expect("encode png");
buf
}
#[test]
fn preprocess_solid_gray_image_produces_expected_shape() {
let img: RgbImage = ImageBuffer::from_fn(4, 4, |_x, _y| Rgb([127u8, 127, 127]));
let png = encode_png(&img);
let cfg = PreprocessConfig {
target_size: 4,
mean: [0.5, 0.5, 0.5],
std: [0.5, 0.5, 0.5],
};
let out = preprocess_rgb_chw(&png, &cfg).unwrap();
assert_eq!(out.len(), 3 * 4 * 4);
for v in &out {
assert!(
(*v + 0.01).abs() < 0.02,
"expected ~-0.004 per pixel, got {}",
v
);
}
}
#[test]
fn preprocess_resizes_to_target_size() {
let img: RgbImage = ImageBuffer::from_fn(8, 8, |_x, _y| Rgb([0u8, 0, 255]));
let png = encode_png(&img);
let cfg = PreprocessConfig {
target_size: 2,
mean: [0.0, 0.0, 0.0],
std: [1.0, 1.0, 1.0],
};
let out = preprocess_rgb_chw(&png, &cfg).unwrap();
assert_eq!(out.len(), 3 * 2 * 2);
for i in 0..4 {
assert!((out[i] - 0.0).abs() < 1e-5, "R[{}] = {}", i, out[i]);
assert!((out[4 + i] - 0.0).abs() < 1e-5, "G[{}] = {}", i, out[4 + i]);
assert!((out[8 + i] - 1.0).abs() < 1e-5, "B[{}] = {}", i, out[8 + i]);
}
}
#[test]
fn preprocess_normalizes_with_configured_mean_std() {
let img: RgbImage = ImageBuffer::from_fn(1, 1, |_x, _y| Rgb([200u8, 100, 50]));
let png = encode_png(&img);
let cfg = PreprocessConfig {
target_size: 1,
mean: [0.1, 0.2, 0.3],
std: [0.5, 0.5, 0.5],
};
let out = preprocess_rgb_chw(&png, &cfg).unwrap();
assert_eq!(out.len(), 3);
assert!((out[0] - 1.3686).abs() < 1e-3, "R={}", out[0]);
assert!((out[1] - 0.3843).abs() < 1e-3, "G={}", out[1]);
assert!((out[2] - (-0.2078)).abs() < 1e-3, "B={}", out[2]);
}
#[test]
fn preprocess_layout_is_chw_not_hwc() {
let img: RgbImage = ImageBuffer::from_fn(2, 2, |x, y| match (x, y) {
(0, 0) => Rgb([255, 0, 0]),
(1, 0) => Rgb([0, 255, 0]),
(0, 1) => Rgb([0, 0, 255]),
_ => Rgb([0, 0, 0]),
});
let png = encode_png(&img);
let cfg = PreprocessConfig {
target_size: 2,
mean: [0.0, 0.0, 0.0],
std: [1.0, 1.0, 1.0],
};
let out = preprocess_rgb_chw(&png, &cfg).unwrap();
assert!((out[0] - 1.0).abs() < 1e-5, "R[TL] = {}", out[0]);
assert!(out[1].abs() < 1e-5, "R[TR] = {}", out[1]);
assert!(out[2].abs() < 1e-5, "R[BL] = {}", out[2]);
assert!(out[3].abs() < 1e-5, "R[BR] = {}", out[3]);
assert!(out[4].abs() < 1e-5);
assert!((out[5] - 1.0).abs() < 1e-5, "G[TR] = {}", out[5]);
assert!(out[6].abs() < 1e-5);
assert!((out[10] - 1.0).abs() < 1e-5, "B[BL] = {}", out[10]);
}
#[test]
fn preprocess_rejects_unsupported_format() {
let gibberish = vec![0xABu8; 64];
let cfg = GEMMA4_VISION_CONFIG.clone();
let err = preprocess_rgb_chw(&gibberish, &cfg).unwrap_err();
let msg = format!("{err}");
assert!(
msg.contains("guess_format") || msg.contains("not supported"),
"unexpected error: {msg}"
);
}
#[test]
fn preprocess_rejects_zero_target_size() {
let img: RgbImage = ImageBuffer::from_pixel(1, 1, Rgb([0u8, 0, 0]));
let png = encode_png(&img);
let cfg = PreprocessConfig {
target_size: 0,
mean: [0.0; 3],
std: [1.0; 3],
};
let err = preprocess_rgb_chw(&png, &cfg).unwrap_err();
assert!(format!("{err}").contains("invalid target_size"));
}
#[test]
fn preprocess_accepts_jpeg_input() {
let img: RgbImage = ImageBuffer::from_pixel(16, 16, Rgb([128u8, 128, 128]));
let mut buf: Vec<u8> = Vec::new();
img.write_to(&mut Cursor::new(&mut buf), ImageFormat::Jpeg)
.expect("encode jpeg");
let cfg = PreprocessConfig {
target_size: 4,
mean: [0.0; 3],
std: [1.0; 3],
};
let out = preprocess_rgb_chw(&buf, &cfg).unwrap();
assert_eq!(out.len(), 3 * 4 * 4);
}
#[test]
fn gemma4_vision_config_constants() {
assert_eq!(GEMMA4_VISION_CONFIG.target_size, 896);
assert_eq!(GEMMA4_VISION_CONFIG.mean, [0.5, 0.5, 0.5]);
assert_eq!(GEMMA4_VISION_CONFIG.std, [0.5, 0.5, 0.5]);
}
fn encode_solid_png(w: u32, h: u32, rgb: [u8; 3]) -> Vec<u8> {
let img: RgbImage = ImageBuffer::from_pixel(w, h, Rgb(rgb));
let mut buf: Vec<u8> = Vec::new();
img.write_to(&mut Cursor::new(&mut buf), ImageFormat::Png)
.expect("encode png");
buf
}
#[test]
fn gemma4v_preprocess_default_constants_match_llama_cpp() {
assert_eq!(GEMMA4V_PREPROCESS_DEFAULT.patch_size, 16);
assert_eq!(GEMMA4V_PREPROCESS_DEFAULT.n_merge, 3);
assert_eq!(GEMMA4V_PREPROCESS_DEFAULT.token_min, 252);
assert_eq!(GEMMA4V_PREPROCESS_DEFAULT.token_max, 280);
}
#[test]
fn gemma4v_preprocess_token_budget_post_pool() {
let n_merge = GEMMA4V_PREPROCESS_DEFAULT.n_merge;
for (w, h) in [(64u32, 64), (256, 256), (1024, 1024)] {
let png = encode_solid_png(w, h, [128, 128, 128]);
let out = preprocess_gemma4v(&png, &GEMMA4V_PREPROCESS_DEFAULT)
.unwrap_or_else(|e| panic!("({w},{h}): {e}"));
assert_eq!(
out.n_x % n_merge,
0,
"({w},{h}) n_x={} not mul of {n_merge}",
out.n_x
);
assert_eq!(
out.n_y % n_merge,
0,
"({w},{h}) n_y={} not mul of {n_merge}",
out.n_y
);
let post_pool = (out.n_x / n_merge) * (out.n_y / n_merge);
assert!(
(252..=280).contains(&post_pool),
"({w},{h}) → pre-pool ({},{}) → post-pool {post_pool} tokens, expected [252, 280]",
out.n_x,
out.n_y
);
let n = out.n_patches();
assert_eq!(out.patches.len(), (n as usize) * 16 * 16 * 3);
assert_eq!(out.pos_x.len(), n as usize);
assert_eq!(out.pos_y.len(), n as usize);
}
}
#[test]
fn gemma4v_preprocess_pixel_scaling_4x_minus_3() {
for (rgb, expect) in [([0u8, 0, 0], -3.0_f32), ([255, 255, 255], 1.0)] {
let png = encode_solid_png(256, 256, rgb);
let out = preprocess_gemma4v(&png, &GEMMA4V_PREPROCESS_DEFAULT).unwrap();
for &i in &[0, out.patches.len() / 2, out.patches.len() - 1] {
assert!(
(out.patches[i] - expect).abs() < 1e-3,
"rgb={:?} idx={} got={} expect={}",
rgb,
i,
out.patches[i],
expect
);
}
}
let png_mid = encode_solid_png(256, 256, [128, 128, 128]);
let out_mid = preprocess_gemma4v(&png_mid, &GEMMA4V_PREPROCESS_DEFAULT).unwrap();
let v = out_mid.patches[0];
let expect_mid = (128.0_f32 / 255.0) * 4.0 - 3.0; assert!(
(v - expect_mid).abs() < 1e-3,
"mid-gray got {v}, expected ≈ {expect_mid}"
);
}
#[test]
fn gemma4v_preprocess_pixel_range_in_minus_three_plus_one() {
let img: RgbImage = ImageBuffer::from_fn(128, 128, |x, y| {
Rgb([
(x as u8).wrapping_mul(2),
(y as u8).wrapping_mul(2),
((x ^ y) as u8).wrapping_mul(2),
])
});
let mut buf: Vec<u8> = Vec::new();
img.write_to(&mut Cursor::new(&mut buf), ImageFormat::Png)
.expect("encode png");
let out = preprocess_gemma4v(&buf, &GEMMA4V_PREPROCESS_DEFAULT).unwrap();
let max_v = out.patches.iter().cloned().fold(f32::MIN, f32::max);
let min_v = out.patches.iter().cloned().fold(f32::MAX, f32::min);
assert!(
min_v >= -3.0 - 1e-6 && max_v <= 1.0 + 1e-6,
"range out of bounds: [{min_v}, {max_v}]"
);
}
#[test]
fn gemma4v_preprocess_pos_indices_are_dense_grid() {
let png = encode_solid_png(256, 256, [10, 20, 30]);
let out = preprocess_gemma4v(&png, &GEMMA4V_PREPROCESS_DEFAULT).unwrap();
for idx in 0..out.n_patches() as usize {
let exp_x = (idx as u32) % out.n_x;
let exp_y = (idx as u32) / out.n_x;
assert_eq!(out.pos_x[idx], exp_x, "pos_x[{idx}]");
assert_eq!(out.pos_y[idx], exp_y, "pos_y[{idx}]");
}
}
#[test]
fn gemma4v_preprocess_rejects_unknown_format() {
let gibberish = vec![0xABu8; 64];
let err = preprocess_gemma4v(&gibberish, &GEMMA4V_PREPROCESS_DEFAULT).unwrap_err();
let msg = format!("{err}");
assert!(
msg.contains("guess_format") || msg.contains("not supported"),
"unexpected: {msg}"
);
}
#[test]
fn gemma4v_preprocess_rejects_zero_patch_size() {
let png = encode_solid_png(64, 64, [0, 0, 0]);
let cfg = Gemma4vPreprocessConfig {
patch_size: 0,
..GEMMA4V_PREPROCESS_DEFAULT.clone()
};
let err = preprocess_gemma4v(&png, &cfg).unwrap_err();
assert!(format!("{err}").contains("patch_size"));
}
#[test]
fn resize_bilinear_llama_cpp_corner_aligned_identity_2x2_to_3x3() {
let mut src: RgbImage = ImageBuffer::new(2, 2);
src.put_pixel(0, 0, Rgb([0, 0, 0])); src.put_pixel(1, 0, Rgb([100, 100, 100])); src.put_pixel(0, 1, Rgb([200, 200, 200])); src.put_pixel(1, 1, Rgb([255, 255, 255]));
let dst = resize_bilinear_llama_cpp(&src, 3, 3);
assert_eq!(dst.get_pixel(0, 0).0[0], 0, "corner (0,0)");
assert_eq!(dst.get_pixel(2, 2).0[0], 255, "corner (2,2)");
assert_eq!(dst.get_pixel(2, 0).0[0], 100, "corner (2,0)");
assert_eq!(dst.get_pixel(0, 2).0[0], 200, "corner (0,2)");
assert_eq!(dst.get_pixel(1, 1).0[0], 138, "center (1,1)");
}
#[test]
fn resize_bilinear_llama_cpp_truncates_not_rounds() {
let mut src: RgbImage = ImageBuffer::new(1, 2);
src.put_pixel(0, 0, Rgb([0, 0, 0]));
src.put_pixel(0, 1, Rgb([1, 1, 1]));
let dst = resize_bilinear_llama_cpp(&src, 1, 3);
assert_eq!(dst.get_pixel(0, 0).0, [0, 0, 0]);
assert_eq!(dst.get_pixel(0, 1).0, [0, 0, 0], "trunc(0.5)=0");
assert_eq!(dst.get_pixel(0, 2).0, [1, 1, 1]);
}
#[test]
fn resize_bilinear_pad_llama_cpp_no_pad_for_square_input() {
let src: RgbImage = ImageBuffer::from_fn(4, 4, |x, _y| Rgb([(x * 50) as u8; 3]));
let padded = resize_bilinear_pad_llama_cpp(&src, 8, 8, [0, 0, 0]);
let plain = resize_bilinear_llama_cpp(&src, 8, 8);
for y in 0..8 {
for x in 0..8 {
assert_eq!(
padded.get_pixel(x, y).0,
plain.get_pixel(x, y).0,
"({x},{y})"
);
}
}
}
#[test]
fn resize_bilinear_pad_llama_cpp_pads_non_square_input() {
let src: RgbImage = ImageBuffer::from_fn(4, 2, |_x, _y| Rgb([200, 100, 50]));
let dst = resize_bilinear_pad_llama_cpp(&src, 4, 4, [0, 0, 0]);
for x in 0..4 {
assert_eq!(dst.get_pixel(x, 0).0, [0, 0, 0], "pad top ({x},0)");
}
for x in 0..4 {
assert_eq!(dst.get_pixel(x, 1).0, [200, 100, 50]);
assert_eq!(dst.get_pixel(x, 2).0, [200, 100, 50]);
}
for x in 0..4 {
assert_eq!(dst.get_pixel(x, 3).0, [0, 0, 0], "pad bot ({x},3)");
}
}
#[test]
fn gemma4v_preprocess_uses_llama_cpp_resize_for_four_corner_dots() {
let img: RgbImage = ImageBuffer::from_fn(8, 8, |x, y| {
if (x == 0 || x == 7) && (y == 0 || y == 7) {
Rgb([255u8, 255, 255])
} else {
Rgb([0, 0, 0])
}
});
let mut buf: Vec<u8> = Vec::new();
img.write_to(&mut Cursor::new(&mut buf), ImageFormat::Png)
.expect("encode png");
let out = preprocess_gemma4v(&buf, &GEMMA4V_PREPROCESS_DEFAULT).unwrap();
let inner = (16 * 16 * 3) as usize;
let first_patch = &out.patches[0..inner];
assert!(
(first_patch[0] - 1.0).abs() < 1e-3,
"first patch (R, 0, 0) should be +1.0 (corner-aligned exact src), got {}",
first_patch[0]
);
let n_patches = out.n_patches() as usize;
let last_patch = &out.patches[(n_patches - 1) * inner..n_patches * inner];
assert!(
(last_patch[255] - 1.0).abs() < 1e-3,
"last patch (R, 15, 15) should be +1.0, got {}",
last_patch[255]
);
}
#[test]
fn gemma4v_preprocess_rejects_inverted_token_bounds() {
let png = encode_solid_png(64, 64, [0, 0, 0]);
let cfg = Gemma4vPreprocessConfig {
patch_size: 16,
n_merge: 3,
token_min: 300,
token_max: 100,
};
let err = preprocess_gemma4v(&png, &cfg).unwrap_err();
assert!(format!("{err}").contains("token_min"));
}
fn qwen3vl_test_cfg() -> Qwen3VlPreprocessConfig {
Qwen3VlPreprocessConfig {
patch_size: 16,
spatial_merge_size: 2,
image_mean: [0.48145466, 0.4578275, 0.40821073],
image_std: [0.26862954, 0.26130258, 0.27577711],
image_min_pixels: 8 * 16 * 16 * 2 * 2, image_max_pixels: 4096 * 16 * 16 * 2 * 2, }
}
#[test]
fn qwen3vl_preprocess_pixel_shape_matches_smart_resize_grid() {
let png = encode_solid_png(256, 256, [127, 127, 127]);
let cfg = qwen3vl_test_cfg();
let image_size = 768; let out = preprocess_qwen3vl(&png, &cfg, image_size).unwrap();
assert_eq!(out.target_w, 256);
assert_eq!(out.target_h, 256);
assert_eq!(out.pixel_values.len(), 3 * 256 * 256);
assert_eq!(out.target_size, image_size); assert_eq!(out.target_pixel_grid(), (256, 256));
assert_eq!(out.n_x_token, 8);
assert_eq!(out.n_y_token, 8);
assert_eq!(out.n_image_tokens, 8 * 8);
}
#[test]
fn qwen3vl_preprocess_smart_resize_aligned_to_stride() {
let stride: u32 = 16 * 2;
let cfg = qwen3vl_test_cfg();
for &(orig_w, orig_h) in &[
(100, 100),
(200, 50),
(50, 200),
(1, 1), (8000, 4000), (1024, 768), ] {
let (tw, th) = qwen3vl_calc_size_preserved_ratio(
orig_w,
orig_h,
stride,
cfg.image_min_pixels,
cfg.image_max_pixels,
)
.expect("smart_resize ok");
assert!(
tw % stride == 0 && th % stride == 0,
"({orig_w}x{orig_h}) → ({tw}x{th}) not aligned to stride={stride}"
);
let area = tw as u64 * th as u64;
assert!(
area >= cfg.image_min_pixels && area <= cfg.image_max_pixels,
"({tw}x{th}) area={area} not in [{}, {}]",
cfg.image_min_pixels,
cfg.image_max_pixels
);
}
}
#[test]
fn qwen3vl_preprocess_per_axis_clamp_keeps_output_within_canvas() {
let cfg = qwen3vl_test_cfg();
let image_size = 768u32;
let stride: u32 = cfg.patch_size * cfg.spatial_merge_size;
for &(orig_w, orig_h, label) in &[
(1920u32, 1080u32, "1080p landscape"),
(1080, 1920, "1080p portrait"),
(2560, 1440, "2K landscape"),
(3840, 2160, "4K landscape"),
(4096, 4096, "4K square (matches max_pixels exactly)"),
(8000, 4000, "extreme landscape"),
] {
let png = encode_solid_png(orig_w.min(64), orig_h.min(64), [80, 90, 100]);
let (smart_w, smart_h) = qwen3vl_calc_size_preserved_ratio(
orig_w,
orig_h,
stride,
cfg.image_min_pixels,
cfg.image_max_pixels,
)
.unwrap_or_else(|e| panic!("smart_resize ok for {label}: {e}"));
let (expected_w, expected_h) = if smart_w > image_size || smart_h > image_size {
let max_dim = smart_w.max(smart_h) as f64;
let scale = (image_size as f64) / max_dim;
let scaled_w = ((smart_w as f64 * scale) as u32 / stride) * stride;
let scaled_h = ((smart_h as f64 * scale) as u32 / stride) * stride;
(scaled_w.max(stride), scaled_h.max(stride))
} else {
(smart_w, smart_h)
};
assert!(
expected_w <= image_size && expected_h <= image_size,
"{label} ({orig_w}x{orig_h}) post-clamp ({expected_w}x{expected_h}) \
exceeds canvas={image_size} — per-axis clamp violated"
);
assert!(
expected_w % stride == 0 && expected_h % stride == 0,
"{label} post-clamp ({expected_w}x{expected_h}) not stride={stride} aligned"
);
let _ = png; }
}
#[test]
fn qwen3vl_preprocess_aspect_ratio_preserved_in_smart_resize() {
let stride: u32 = 32;
let cfg = qwen3vl_test_cfg();
let (tw, th) = qwen3vl_calc_size_preserved_ratio(
200,
50,
stride,
cfg.image_min_pixels,
cfg.image_max_pixels,
)
.unwrap();
let orig_ratio = 200.0_f64 / 50.0;
let new_ratio = tw as f64 / th as f64;
assert!(
(orig_ratio - new_ratio).abs() / orig_ratio < 0.5,
"smart_resize aspect drift: orig={orig_ratio}, new={new_ratio} ({tw}x{th})"
);
}
#[test]
fn qwen3vl_preprocess_rejects_misaligned_image_size() {
let png = encode_solid_png(100, 100, [0, 0, 0]);
let cfg = qwen3vl_test_cfg();
let err = preprocess_qwen3vl(&png, &cfg, 100).unwrap_err();
let msg = format!("{err}");
assert!(msg.contains("must be a positive multiple"), "got: {msg}");
}
#[test]
fn qwen3vl_preprocess_normalization_mean_std_applied() {
let png = encode_solid_png(64, 64, [128, 128, 128]);
let cfg = Qwen3VlPreprocessConfig {
patch_size: 16,
spatial_merge_size: 2,
image_mean: [0.5, 0.5, 0.5],
image_std: [0.5, 0.5, 0.5],
image_min_pixels: 64 * 64, image_max_pixels: 768u64.pow(2), };
let out = preprocess_qwen3vl(&png, &cfg, 768).unwrap();
assert_eq!(out.target_w, 64);
assert_eq!(out.target_h, 64);
assert_eq!(out.pixel_values.len(), 3 * 64 * 64);
let center_idx = 64 * 32 + 32; let v = out.pixel_values[center_idx];
assert!(
v.abs() < 0.05,
"center pixel of normalized 128-gray image should be ~0, got {v}"
);
let corner_v = out.pixel_values[0];
assert!(
corner_v.abs() < 0.05,
"Phase-2 corner is REAL content (no pad), expected ~0.004 from \
128-gray input, got {corner_v}"
);
}
#[test]
fn qwen3vl_preprocess_from_mmproj_picks_up_canonical_pixel_bounds() {
let mmcfg = super::super::mmproj::MmprojConfig {
image_size: 768,
patch_size: 16,
num_patches_side: 48,
hidden_size: 1024,
intermediate_size: 4096,
num_attention_heads: 16,
num_hidden_layers: 24,
layer_norm_eps: 1e-6,
projector: super::super::mmproj::ProjectorType::Qwen3VlMerger,
image_mean: [0.5, 0.5, 0.5],
image_std: [0.5, 0.5, 0.5],
spatial_merge_size: Some(2),
projection_dim: Some(2048),
deepstack_indexes: Some(vec![5, 11, 17]),
};
let cfg = Qwen3VlPreprocessConfig::from_mmproj(&mmcfg).unwrap();
assert_eq!(cfg.image_min_pixels, 8192);
assert_eq!(cfg.image_max_pixels, 4_194_304);
assert_eq!(cfg.patch_size, 16);
assert_eq!(cfg.spatial_merge_size, 2);
}
#[test]
fn qwen3vl_preprocess_from_mmproj_rejects_missing_spatial_merge() {
let mmcfg = super::super::mmproj::MmprojConfig {
image_size: 768,
patch_size: 16,
num_patches_side: 48,
hidden_size: 1024,
intermediate_size: 4096,
num_attention_heads: 16,
num_hidden_layers: 24,
layer_norm_eps: 1e-6,
projector: super::super::mmproj::ProjectorType::Qwen3VlMerger,
image_mean: [0.5, 0.5, 0.5],
image_std: [0.5, 0.5, 0.5],
spatial_merge_size: None, projection_dim: Some(2048),
deepstack_indexes: Some(vec![5, 11, 17]),
};
let err = Qwen3VlPreprocessConfig::from_mmproj(&mmcfg).unwrap_err();
assert!(format!("{err}").contains("spatial_merge_size"));
}
#[test]
fn qwen3vl_preprocess_rejects_non_image_bytes() {
let cfg = qwen3vl_test_cfg();
let err = preprocess_qwen3vl(&[1, 2, 3, 4, 5], &cfg, 768).unwrap_err();
let msg = format!("{err}");
assert!(
msg.contains("guess_format") || msg.contains("not supported"),
"got: {msg}"
);
}
#[test]
fn qwen3vl_preprocess_phase2_square_768_matches_phase1_grid() {
let png = encode_solid_png(768, 768, [80, 90, 100]);
let cfg = qwen3vl_test_cfg();
let out = preprocess_qwen3vl(&png, &cfg, 768).unwrap();
assert_eq!(out.target_w, 768);
assert_eq!(out.target_h, 768);
assert_eq!(out.pixel_values.len(), 3 * 768 * 768);
assert_eq!(out.n_x_token, 24);
assert_eq!(out.n_y_token, 24);
assert_eq!(out.n_image_tokens, 576);
assert_eq!(out.target_pixel_grid(), (768, 768));
}
#[test]
fn qwen3vl_preprocess_phase2_landscape_1024x576_aspect_preserved() {
let cfg = Qwen3VlPreprocessConfig {
patch_size: 16,
spatial_merge_size: 2,
image_mean: [0.5, 0.5, 0.5],
image_std: [0.5, 0.5, 0.5],
image_min_pixels: 8 * 16 * 16 * 2 * 2, image_max_pixels: 4096 * 16 * 16 * 2 * 2, };
let png = encode_solid_png(1024, 576, [128, 128, 128]); let out = preprocess_qwen3vl(&png, &cfg, 768).unwrap();
assert_eq!(out.target_w, 768);
assert_eq!(out.target_h, 416);
assert_eq!(out.pixel_values.len(), 3 * 768 * 416);
assert_eq!(out.n_x_token, 24);
assert_eq!(out.n_y_token, 13);
assert_eq!(out.n_image_tokens, 24 * 13);
let mut min_v = f32::INFINITY;
for &v in out.pixel_values.iter() {
if v < min_v {
min_v = v;
}
}
assert!(
min_v > -0.5,
"Phase-2 should have NO pad region; mid-gray input must \
normalize to ~0.004 everywhere (NO -1.0 pad signature). \
observed min={min_v}"
);
}
#[test]
fn qwen3vl_preprocess_phase2_portrait_576x1024_aspect_preserved() {
let png = encode_solid_png(576, 1024, [200, 50, 80]);
let cfg = qwen3vl_test_cfg();
let out = preprocess_qwen3vl(&png, &cfg, 768).unwrap();
assert_eq!(out.target_w, 416);
assert_eq!(out.target_h, 768);
assert_eq!(out.pixel_values.len(), 3 * 768 * 416);
assert_eq!(out.n_x_token, 13);
assert_eq!(out.n_y_token, 24);
assert_eq!(out.n_image_tokens, 13 * 24);
assert!(out.n_y_token > out.n_x_token, "portrait → n_y > n_x");
}
}