pub mod pil_resample;
pub mod staff_detect;
use std::path::Path;
use image::{DynamicImage, GenericImageView, ImageDecoder, ImageReader, imageops::FilterType};
use crate::error::{FocrError, FocrResult};
pub const IMAGE_MEAN: [f32; 3] = [0.5, 0.5, 0.5];
pub const IMAGE_STD: [f32; 3] = [0.5, 0.5, 0.5];
pub const PATCH_SIZE: usize = 16;
pub const DOWNSAMPLE_RATIO: usize = 4;
pub const BASE_SIZE: usize = 1024;
pub const GUNDAM_TILE_SIZE: usize = 640;
pub const MIN_NUM: usize = 2;
pub const MAX_NUM: usize = 32;
pub const CROP_THRESHOLD: u32 = 640;
pub const PAD_FILL: u8 = 127;
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum PreprocessMode {
Base {
base_size: usize,
},
Gundam {
base_size: usize,
tile_size: usize,
},
}
impl Default for PreprocessMode {
fn default() -> Self {
Self::Base { base_size: 1024 }
}
}
impl PreprocessMode {
#[must_use]
pub fn base() -> Self {
PreprocessMode::Base {
base_size: BASE_SIZE,
}
}
#[must_use]
pub fn gundam() -> Self {
PreprocessMode::Gundam {
base_size: BASE_SIZE,
tile_size: GUNDAM_TILE_SIZE,
}
}
#[must_use]
pub fn base_size(self) -> usize {
match self {
PreprocessMode::Base { base_size } | PreprocessMode::Gundam { base_size, .. } => {
base_size
}
}
}
}
#[must_use]
pub fn num_queries(size: usize) -> usize {
(size / PATCH_SIZE).div_ceil(DOWNSAMPLE_RATIO)
}
#[derive(Debug, Clone, PartialEq, Default)]
pub struct ViewTensor {
pub pixels: crate::native_engine::tensor::Mat,
pub height: usize,
pub width: usize,
}
impl ViewTensor {
#[must_use]
pub fn shape(&self) -> (usize, usize) {
(self.height, self.width)
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
pub struct CropGrid {
pub width_crop_num: usize,
pub height_crop_num: usize,
}
impl CropGrid {
#[must_use]
pub fn single() -> Self {
CropGrid {
width_crop_num: 1,
height_crop_num: 1,
}
}
#[must_use]
pub fn blocks(self) -> usize {
self.width_crop_num
.checked_mul(self.height_crop_num)
.expect("CropGrid::blocks: width_crop_num*height_crop_num overflow")
}
#[must_use]
pub fn is_tiled(self) -> bool {
self.width_crop_num > 1 || self.height_crop_num > 1
}
}
#[derive(Debug, Clone, PartialEq, Default)]
pub struct Preprocessed {
pub mode: PreprocessMode,
pub global: ViewTensor,
pub tiles: Vec<ViewTensor>,
pub crop_grid: CropGrid,
pub original_size: (u32, u32),
}
impl Preprocessed {
#[must_use]
pub fn num_views(&self) -> usize {
1 + self.tiles.len()
}
#[must_use]
pub fn placeholder_token_count(&self) -> usize {
let q_base = num_queries(self.mode.base_size());
let mut total = q_base
.checked_add(1)
.and_then(|cols| cols.checked_mul(q_base))
.and_then(|tokens| tokens.checked_add(1))
.expect("Preprocessed::placeholder_token_count: global placeholder count overflow");
if let PreprocessMode::Gundam { tile_size, .. } = self.mode
&& self.crop_grid.is_tiled()
{
let q_local = num_queries(tile_size);
let w = self.crop_grid.width_crop_num;
let h = self.crop_grid.height_crop_num;
let local_cols = q_local
.checked_mul(w)
.and_then(|cols| cols.checked_add(1))
.expect("Preprocessed::placeholder_token_count: local column count overflow");
let local_rows = q_local
.checked_mul(h)
.expect("Preprocessed::placeholder_token_count: local row count overflow");
let local = local_cols
.checked_mul(local_rows)
.expect("Preprocessed::placeholder_token_count: local placeholder count overflow");
total = total
.checked_add(local)
.expect("Preprocessed::placeholder_token_count: total placeholder count overflow");
}
total
}
}
pub fn preprocess_image(path: &Path, mode: PreprocessMode) -> FocrResult<Preprocessed> {
let img = decode_path(path)?;
preprocess_dynamic(img, mode)
}
pub fn preprocess_bytes(bytes: &[u8], mode: PreprocessMode) -> FocrResult<Preprocessed> {
let img = decode_bytes(bytes)?;
preprocess_dynamic(img, mode)
}
pub fn preprocess_dynamic(img: DynamicImage, mode: PreprocessMode) -> FocrResult<Preprocessed> {
let original_size = img.dimensions();
let validated = validate_mode(mode)?;
let global_img = pad_to_square(&img, validated.base_size);
let global = view_tensor(&global_img);
let (tiles, crop_grid) = match mode {
PreprocessMode::Base { .. } => (Vec::new(), CropGrid::single()),
PreprocessMode::Gundam { .. } => build_gundam_tiles(&img, validated.tile_size)?,
};
Ok(Preprocessed {
mode,
global,
tiles,
crop_grid,
original_size,
})
}
pub fn preprocess_dynamic_squash(img: DynamicImage, base_size: usize) -> FocrResult<Preprocessed> {
let mode = PreprocessMode::Base { base_size };
let validated = validate_mode(mode)?;
let original_size = img.dimensions();
let squashed =
pil_resample::resize_bicubic(&img.to_rgb8(), validated.base_size, validated.base_size);
Ok(Preprocessed {
mode,
global: view_tensor(&squashed.into()),
tiles: Vec::new(),
crop_grid: CropGrid::single(),
original_size,
})
}
#[derive(Debug, Clone, Copy)]
struct ValidatedMode {
base_size: u32,
tile_size: u32,
}
fn validate_mode(mode: PreprocessMode) -> FocrResult<ValidatedMode> {
let base_size = validate_edge("base_size", mode.base_size(), BASE_SIZE)?;
let tile_size = match mode {
PreprocessMode::Base { .. } => 0,
PreprocessMode::Gundam { tile_size, .. } => {
validate_edge("tile_size", tile_size, GUNDAM_TILE_SIZE)?
}
};
Ok(ValidatedMode {
base_size,
tile_size,
})
}
fn validate_edge(name: &str, value: usize, max: usize) -> FocrResult<u32> {
if value < PATCH_SIZE {
return Err(FocrError::Usage(format!(
"preprocess {name} must be at least {PATCH_SIZE} pixels, got {value}"
)));
}
if value > max {
return Err(FocrError::Usage(format!(
"preprocess {name} must be <= {max} pixels for this model, got {value}"
)));
}
if !value.is_multiple_of(PATCH_SIZE) {
return Err(FocrError::Usage(format!(
"preprocess {name} must be a multiple of patch size {PATCH_SIZE}, got {value}"
)));
}
u32::try_from(value).map_err(|_| {
FocrError::Usage(format!(
"preprocess {name} exceeds u32 pixel edge limit: {value}"
))
})
}
pub(crate) fn decode_path(path: &Path) -> FocrResult<DynamicImage> {
let reader = ImageReader::open(path)
.map_err(|e| FocrError::InputDecode(format!("open {}: {e}", path.display())))?
.with_guessed_format()
.map_err(|e| FocrError::InputDecode(format!("sniff {}: {e}", path.display())))?;
decode_reader(reader)
.map_err(|e| FocrError::InputDecode(format!("decode {}: {e}", path.display())))
}
fn decode_bytes(bytes: &[u8]) -> FocrResult<DynamicImage> {
let reader = ImageReader::new(std::io::Cursor::new(bytes))
.with_guessed_format()
.map_err(|e| FocrError::InputDecode(format!("sniff bytes: {e}")))?;
decode_reader(reader).map_err(|e| FocrError::InputDecode(format!("decode bytes: {e}")))
}
fn decode_reader<R: std::io::BufRead + std::io::Seek>(
reader: ImageReader<R>,
) -> image::ImageResult<DynamicImage> {
let mut decoder = reader.into_decoder()?;
const MAX_PIXELS: u64 = 1 << 30; let (w, h) = decoder.dimensions();
if u64::from(w) * u64::from(h) > MAX_PIXELS {
return Err(image::ImageError::Limits(
image::error::LimitError::from_kind(image::error::LimitErrorKind::DimensionError),
));
}
let mut limits = image::Limits::default();
limits.max_image_width = Some(1 << 17);
limits.max_image_height = Some(1 << 17);
limits.max_alloc = Some(4 << 30);
decoder.set_limits(limits)?;
let orientation = decoder.orientation()?;
let mut img = DynamicImage::from_decoder(decoder)?;
img.apply_orientation(orientation);
Ok(img)
}
pub const RESAMPLE_ENV: &str = "FOCR_RESAMPLE";
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum ResampleKind {
CatmullRom,
PilBicubic,
}
#[must_use]
pub fn resample_kind() -> ResampleKind {
resample_kind_from(std::env::var(RESAMPLE_ENV).ok().as_deref())
}
fn resample_kind_from(raw: Option<&str>) -> ResampleKind {
match raw.map(str::trim) {
Some("pil-bicubic" | "pil_bicubic") => ResampleKind::PilBicubic,
_ => ResampleKind::CatmullRom,
}
}
fn resample_exact(img: &DynamicImage, w: u32, h: u32) -> DynamicImage {
resample_exact_with(resample_kind(), img, w, h)
}
fn resample_exact_with(kind: ResampleKind, img: &DynamicImage, w: u32, h: u32) -> DynamicImage {
match kind {
ResampleKind::CatmullRom => img.resize_exact(w, h, FilterType::CatmullRom),
ResampleKind::PilBicubic => {
DynamicImage::ImageRgb8(pil_resample::resize_bicubic(&img.to_rgb8(), w, h))
}
}
}
fn pad_to_square(img: &DynamicImage, size: u32) -> DynamicImage {
let (w, h) = img.dimensions();
let (rw, rh) = if w == 0 || h == 0 {
(size, size)
} else if w >= h {
let rh = pillow_fit_edge(h, w, size);
(size, rh)
} else {
let rw = pillow_fit_edge(w, h, size);
(rw, size)
};
let resized = resample_exact(img, rw, rh).to_rgb8();
let mut canvas = image::RgbImage::from_pixel(size, size, image::Rgb([PAD_FILL; 3]));
let ox = pillow_center_offset(size, rw);
let oy = pillow_center_offset(size, rh);
for y in 0..rh {
for x in 0..rw {
let p = *resized.get_pixel(x, y);
canvas.put_pixel(ox + x, oy + y, p);
}
}
DynamicImage::ImageRgb8(canvas)
}
fn pillow_fit_edge(short: u32, long: u32, size: u32) -> u32 {
let scaled = f64::from(short) / f64::from(long) * f64::from(size);
round_ties_even_positive(scaled).max(1)
}
fn pillow_center_offset(size: u32, resized: u32) -> u32 {
round_ties_even_positive(f64::from(size - resized) * 0.5)
}
fn round_ties_even_positive(value: f64) -> u32 {
debug_assert!(value.is_finite());
debug_assert!(value >= 0.0);
let floor = value.floor();
let frac = value - floor;
let rounded = if frac < 0.5 {
floor
} else if frac > 0.5 {
floor + 1.0
} else {
let floor_u = floor as u64;
if floor_u.is_multiple_of(2) {
floor
} else {
floor + 1.0
}
};
rounded as u32
}
fn build_gundam_tiles(img: &DynamicImage, tile: u32) -> FocrResult<(Vec<ViewTensor>, CropGrid)> {
let (w, h) = img.dimensions();
if w <= CROP_THRESHOLD && h <= CROP_THRESHOLD {
return Ok((Vec::new(), CropGrid::single()));
}
let ratios = candidate_ratios(MIN_NUM, MAX_NUM);
let (wc, hc) = find_closest_aspect_ratio(w as f64 / h as f64, &ratios, w, h, tile);
let target_w = checked_tile_extent(tile, wc, "width")?;
let target_h = checked_tile_extent(tile, hc, "height")?;
let resized = resample_exact(img, target_w, target_h);
let cols = wc; let blocks = wc * hc;
let mut tiles = Vec::with_capacity(blocks);
for i in 0..blocks {
let col = (i % cols) as u32;
let row = (i / cols) as u32;
let split = resized.crop_imm(col * tile, row * tile, tile, tile);
tiles.push(view_tensor(&split));
}
Ok((
tiles,
CropGrid {
width_crop_num: wc,
height_crop_num: hc,
},
))
}
fn checked_tile_extent(tile: u32, count: usize, axis: &str) -> FocrResult<u32> {
let count = u32::try_from(count).map_err(|_| {
FocrError::Usage(format!(
"preprocess Gundam {axis} tile count exceeds u32: {count}"
))
})?;
tile.checked_mul(count).ok_or_else(|| {
FocrError::Usage(format!(
"preprocess Gundam {axis} extent overflows u32: tile_size={tile}, tile_count={count}"
))
})
}
#[must_use]
pub fn candidate_ratios(min_num: usize, max_num: usize) -> Vec<(usize, usize)> {
let mut set = std::collections::BTreeSet::new();
for i in 1..=max_num {
for j in 1..=max_num {
let prod = i * j;
if (min_num..=max_num).contains(&prod) {
set.insert((i, j));
}
}
}
let mut out: Vec<(usize, usize)> = set.into_iter().collect();
out.sort_by_key(|&(i, j)| i * j);
out
}
#[must_use]
pub fn find_closest_aspect_ratio(
aspect_ratio: f64,
target_ratios: &[(usize, usize)],
width: u32,
height: u32,
tile: u32,
) -> (usize, usize) {
let mut best_diff = f64::INFINITY;
let mut best = (1usize, 1usize);
let area = f64::from(width) * f64::from(height);
let tile_f = f64::from(tile);
for &(i, j) in target_ratios {
let target = i as f64 / j as f64;
let diff = (aspect_ratio - target).abs();
if diff < best_diff {
best_diff = diff;
best = (i, j);
} else if diff == best_diff && area > 0.5 * tile_f * tile_f * (i as f64) * (j as f64) {
best = (i, j);
}
}
best
}
pub const CLIP_MEAN: [f32; 3] = [0.481_454_66, 0.457_827_5, 0.408_210_73];
pub const CLIP_STD: [f32; 3] = [0.268_629_54, 0.261_302_6, 0.275_777_1];
pub const GOT_SIZE: u32 = 1024;
pub fn preprocess_got(path: &Path) -> FocrResult<crate::native_engine::tensor::Mat> {
Ok(got_view_tensor(&decode_path(path)?))
}
const SMOLVLM2_FRAME: u32 = 512;
const SMOLVLM2_LONGEST: u32 = 2048;
#[derive(Debug, Clone)]
pub struct Smolvlm2Preprocessed {
pub frames: Vec<f32>,
pub n_frames: usize,
pub rows: usize,
pub cols: usize,
}
pub fn preprocess_smolvlm2(img: &DynamicImage) -> FocrResult<Smolvlm2Preprocessed> {
let rgb = img.to_rgb8();
let (w0, h0) = rgb.dimensions();
if w0 == 0 || h0 == 0 {
return Err(FocrError::Other(anyhow::anyhow!(
"smolvlm2 preprocess: degenerate {w0}x{h0} input image"
)));
}
let aspect = f64::from(w0) / f64::from(h0);
let (w2, h2) = if w0 >= h0 {
let w = SMOLVLM2_LONGEST;
let mut h = (f64::from(w) / aspect) as u32; if !h.is_multiple_of(2) {
h += 1;
}
(w, h.max(1))
} else {
let h = SMOLVLM2_LONGEST;
let mut w = (f64::from(h) * aspect) as u32;
if !w.is_multiple_of(2) {
w += 1;
}
(w.max(1), h)
};
let long2048 = pil_resample::resize_lanczos(&rgb, w2, h2);
let ceil512 = |v: u32| v.div_ceil(SMOLVLM2_FRAME) * SMOLVLM2_FRAME;
let aspect2 = f64::from(w2) / f64::from(h2);
let (w3, h3) = if w2 >= h2 {
let w = ceil512(w2);
let h = ceil512((f64::from(w) / aspect2) as u32);
(w, h)
} else {
let h = ceil512(h2);
let w = ceil512((f64::from(h) * aspect2) as u32);
(w, h)
};
let ceiled512 = if (w3, h3) == (w2, h2) {
long2048.clone()
} else {
pil_resample::resize_lanczos(&long2048, w3, h3)
};
let rows = (h3 / SMOLVLM2_FRAME) as usize;
let cols = (w3 / SMOLVLM2_FRAME) as usize;
let n_frames = rows * cols + 1;
let side = SMOLVLM2_FRAME as usize;
let frame_len = 3 * side * side;
let mut frames = vec![0.0f32; n_frames * frame_len];
let norm = |px: u8| -> f32 {
let r = (f64::from(px) * (1.0 / 255.0)) as f32;
(r - 0.5) / 0.5
};
let mut write_frame = |idx: usize, tile: &image::RgbImage, ox: u32, oy: u32| {
let dst = &mut frames[idx * frame_len..(idx + 1) * frame_len];
for y in 0..side {
for x in 0..side {
let px = tile.get_pixel(ox + x as u32, oy + y as u32).0;
let s = y * side + x;
dst[s] = norm(px[0]);
dst[side * side + s] = norm(px[1]);
dst[2 * side * side + s] = norm(px[2]);
}
}
};
for r in 0..rows {
for c in 0..cols {
write_frame(
r * cols + c,
&ceiled512,
c as u32 * SMOLVLM2_FRAME,
r as u32 * SMOLVLM2_FRAME,
);
}
}
let global = pil_resample::resize_lanczos(&ceiled512, SMOLVLM2_FRAME, SMOLVLM2_FRAME);
write_frame(n_frames - 1, &global, 0, 0);
Ok(Smolvlm2Preprocessed {
frames,
n_frames,
rows,
cols,
})
}
pub fn preprocess_smolvlm2_path(path: &Path) -> FocrResult<Smolvlm2Preprocessed> {
preprocess_smolvlm2(&decode_path(path)?)
}
pub fn onechart_view_tensor(img: &DynamicImage) -> crate::native_engine::tensor::Mat {
let rgb = resample_exact(img, GOT_SIZE, GOT_SIZE).to_rgb8();
let side = GOT_SIZE as usize;
let n = side * side;
let mut data = vec![0.0f32; 3 * n];
for y in 0..side {
for x in 0..side {
let px = rgb.get_pixel(x as u32, y as u32).0;
let s = y * side + x;
for c in 0..3 {
data[c * n + s] = f32::from(px[c]) / 255.0;
}
}
}
crate::native_engine::tensor::Mat::from_vec(3, n, data)
}
pub fn got_view_tensor(img: &DynamicImage) -> crate::native_engine::tensor::Mat {
let rgb = resample_exact(img, GOT_SIZE, GOT_SIZE).to_rgb8();
let side = GOT_SIZE as usize;
let n = side * side;
let mut data = vec![0.0f32; 3 * n];
for y in 0..side {
for x in 0..side {
let px = rgb.get_pixel(x as u32, y as u32).0;
let s = y * side + x;
for c in 0..3 {
let v = f32::from(px[c]) / 255.0;
data[c * n + s] = (v - CLIP_MEAN[c]) / CLIP_STD[c];
}
}
}
crate::native_engine::tensor::Mat::from_vec(3, n, data)
}
fn view_tensor(img: &DynamicImage) -> ViewTensor {
let rgb = img.to_rgb8();
let (w, h) = rgb.dimensions();
let (wi, hi) = (w as usize, h as usize);
let n = wi * hi;
let mut data = vec![0.0f32; 3 * n];
for y in 0..hi {
for x in 0..wi {
let px = rgb.get_pixel(x as u32, y as u32).0;
let s = y * wi + x;
for c in 0..3 {
let v = f32::from(px[c]) / 255.0;
data[c * n + s] = (v - IMAGE_MEAN[c]) / IMAGE_STD[c];
}
}
}
ViewTensor {
pixels: crate::native_engine::tensor::Mat::from_vec(3, n, data),
height: hi,
width: wi,
}
}
const TROMR_MEAN: f32 = 0.7931 * 255.0;
const TROMR_STD: f32 = 0.1738 * 255.0;
fn bilinear_u8(src: &[u8], w: usize, h: usize, nw: usize, nh: usize) -> Vec<u8> {
let mut out = vec![0u8; nw * nh];
let sx_ratio = w as f32 / nw as f32;
let sy_ratio = h as f32 / nh as f32;
for dy in 0..nh {
let fy = ((dy as f32 + 0.5) * sy_ratio - 0.5).max(0.0);
let y0 = (fy as usize).min(h - 1);
let y1 = (y0 + 1).min(h - 1);
let wy = fy - y0 as f32;
for dx in 0..nw {
let fx = ((dx as f32 + 0.5) * sx_ratio - 0.5).max(0.0);
let x0 = (fx as usize).min(w - 1);
let x1 = (x0 + 1).min(w - 1);
let wx = fx - x0 as f32;
let top = f32::from(src[y0 * w + x0]) * (1.0 - wx) + f32::from(src[y0 * w + x1]) * wx;
let bot = f32::from(src[y1 * w + x0]) * (1.0 - wx) + f32::from(src[y1 * w + x1]) * wx;
out[dy * nw + dx] = (top * (1.0 - wy) + bot * wy).round().clamp(0.0, 255.0) as u8;
}
}
out
}
pub fn tromr_staff_tensor(img: &DynamicImage) -> FocrResult<(Vec<f32>, usize)> {
let (w, h) = (img.width() as usize, img.height() as usize);
if w == 0 || h == 0 {
return Err(FocrError::Other(anyhow::anyhow!(
"tromr preprocess: degenerate {w}x{h} input"
)));
}
let alpha_is_ink = img.color().has_alpha() && img.to_rgba8().pixels().any(|p| p.0[3] < 255);
let gray: Vec<u8> = if alpha_is_ink {
img.to_rgba8().pixels().map(|p| 255 - p.0[3]).collect()
} else {
img.to_rgb8()
.pixels()
.map(|p| {
let [r, g, b] = p.0;
((4899 * u32::from(r) + 9617 * u32::from(g) + 1868 * u32::from(b) + 8192) >> 14)
.min(255) as u8
})
.collect()
};
let new_h = crate::native_engine::tromr::IMG_H;
let new_w = ((new_h as f64 / h as f64 * w as f64) as usize) / 16 * 16;
if new_w == 0 {
return Err(FocrError::Other(anyhow::anyhow!(
"tromr preprocess: {w}x{h} resizes to zero width (image too narrow)"
)));
}
if new_w > crate::native_engine::tromr::POS_COLS * crate::native_engine::tromr::PATCH {
return Err(FocrError::Other(anyhow::anyhow!(
"tromr preprocess: resized width {new_w} exceeds the 1280 position clamp — \
pass a single-staff crop (aspect ≤ 10:1 at h=128; the staff-detection \
front end enforces this)"
)));
}
let resized = bilinear_u8(&gray, w, h, new_w, new_h);
let pixels = resized
.iter()
.map(|&v| (f32::from(v) - TROMR_MEAN) / TROMR_STD)
.collect();
Ok((pixels, new_w))
}
#[cfg(test)]
mod tests {
#[test]
fn palette_png_decodes_expanded_to_rgb() {
const PALETTE_PNG: [u8; 92] = [
0x89, 0x50, 0x4E, 0x47, 0x0D, 0x0A, 0x1A, 0x0A, 0x00, 0x00, 0x00, 0x0D, 0x49, 0x48,
0x44, 0x52, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x02, 0x08, 0x03, 0x00, 0x00,
0x00, 0x45, 0x68, 0xFD, 0x16, 0x00, 0x00, 0x00, 0x09, 0x50, 0x4C, 0x54, 0x45, 0xFF,
0x00, 0x00, 0x00, 0xFF, 0x00, 0x00, 0x00, 0xFF, 0x2D, 0x4A, 0xCD, 0x8A, 0x00, 0x00,
0x00, 0x0E, 0x49, 0x44, 0x41, 0x54, 0x78, 0x9C, 0x63, 0x60, 0x60, 0x64, 0x60, 0x62,
0x00, 0x00, 0x00, 0x0E, 0x00, 0x04, 0xC6, 0x88, 0x7C, 0xF8, 0x00, 0x00, 0x00, 0x00,
0x49, 0x45, 0x4E, 0x44, 0xAE, 0x42, 0x60, 0x82,
];
let img = super::decode_bytes(&PALETTE_PNG).expect("palette PNG decodes");
assert_eq!((img.width(), img.height()), (2, 2));
let rgb = img.to_rgb8();
assert_eq!(rgb.get_pixel(0, 0).0, [255, 0, 0]);
assert_eq!(rgb.get_pixel(1, 0).0, [0, 255, 0]);
assert_eq!(rgb.get_pixel(0, 1).0, [0, 0, 255]);
assert_eq!(rgb.get_pixel(1, 1).0, [255, 0, 0]);
}
#[test]
fn tromr_alpha_ink_path_fires_only_when_alpha_varies() {
use image::{DynamicImage, Rgba, RgbaImage};
let mut var = RgbaImage::from_pixel(64, 128, Rgba([9, 9, 9, 0]));
for y in 60..68 {
for x in 0..64 {
var.put_pixel(x, y, Rgba([200, 200, 200, 255]));
}
}
let (px, w) = super::tromr_staff_tensor(&DynamicImage::ImageRgba8(var))
.expect("varying-alpha preprocess runs");
assert_eq!(w, 64);
let dark = (0.0f32 - 0.7931 * 255.0) / (0.1738 * 255.0);
let light = (255.0f32 - 0.7931 * 255.0) / (0.1738 * 255.0);
let mid = px[64 * 64 + 32]; let top = px[10 * 64 + 32];
assert!((mid - dark).abs() < 1e-4, "strip is ink: {mid} vs {dark}");
assert!(
(top - light).abs() < 1e-4,
"background is paper: {top} vs {light}"
);
let mut opaque = RgbaImage::from_pixel(64, 128, Rgba([250, 250, 250, 255]));
for y in 60..68 {
for x in 0..64 {
opaque.put_pixel(x, y, Rgba([10, 10, 10, 255]));
}
}
let (px, _) = super::tromr_staff_tensor(&DynamicImage::ImageRgba8(opaque))
.expect("opaque-alpha preprocess runs");
let mid = px[64 * 64 + 32];
let top = px[10 * 64 + 32];
assert!(
mid < top,
"opaque path reads RGB ink: strip {mid} vs paper {top}"
);
assert!(mid < -3.0, "the dark strip is strongly negative: {mid}");
}
use super::*;
use image::{Rgb, RgbImage};
#[test]
fn got_preprocess_matches_oracle_l0b() {
let path = concat!(
env!("CARGO_MANIFEST_DIR"),
"/tests/fixtures/got/sample_text.png"
);
let m = preprocess_got(std::path::Path::new(path)).expect("got preprocess");
assert_eq!(m.rows, 3);
assert_eq!(m.cols, (GOT_SIZE * GOT_SIZE) as usize);
let d: Vec<f64> = m.data.iter().map(|&v| f64::from(v)).collect();
let mean = d.iter().sum::<f64>() / d.len() as f64;
let var = d.iter().map(|&v| (v - mean).powi(2)).sum::<f64>() / d.len() as f64;
let std = var.sqrt();
let min = d.iter().copied().fold(f64::INFINITY, f64::min);
let max = d.iter().copied().fold(f64::NEG_INFINITY, f64::max);
eprintln!("[L0b] mean={mean:.4} std={std:.4} min={min:.4} max={max:.4}");
let (o_mean, o_std, o_min, o_max) = (2.046_326_7, 0.138_841_3, -1.777_664_1, 2.145_897);
assert!((mean - o_mean).abs() < 5e-3, "mean {mean} vs {o_mean}");
assert!((std - o_std).abs() < 5e-3, "std {std} vs {o_std}");
assert!((min - o_min).abs() < 1e-2, "min {min} vs {o_min}");
assert!((max - o_max).abs() < 1e-2, "max {max} vs {o_max}");
}
fn solid(w: u32, h: u32, color: [u8; 3]) -> DynamicImage {
DynamicImage::ImageRgb8(RgbImage::from_pixel(w, h, Rgb(color)))
}
#[test]
fn num_queries_matches_census() {
assert_eq!(num_queries(1024), 16);
assert_eq!(num_queries(640), 10);
}
#[test]
fn base_global_placeholder_is_273() {
let p = Preprocessed {
mode: PreprocessMode::base(),
global: view_tensor(&solid(8, 8, [0, 0, 0])),
tiles: Vec::new(),
crop_grid: CropGrid::single(),
original_size: (8, 8),
};
assert_eq!(p.placeholder_token_count(), 273);
assert_eq!(p.num_views(), 1);
}
#[test]
fn multi_page_base_640_placeholder_is_111() {
assert_eq!(num_queries(640), 10);
let p = Preprocessed {
mode: PreprocessMode::Base { base_size: 640 },
global: view_tensor(&solid(8, 8, [0, 0, 0])),
tiles: Vec::new(),
crop_grid: CropGrid::single(),
original_size: (8, 8),
};
assert_eq!(p.placeholder_token_count(), 111);
assert_eq!(p.num_views(), 1);
println!(r#"{{"check":"multi_page_census_640","per_page":111,"result":"pass"}}"#);
}
#[test]
fn gundam_placeholder_census_matches_table() {
let cases = [
((2, 1), 483usize),
((1, 2), 493),
((2, 2), 693),
((3, 2), 893),
((4, 4), 1913),
];
for ((w, h), expected) in cases {
let grid = CropGrid {
width_crop_num: w,
height_crop_num: h,
};
let p = Preprocessed {
mode: PreprocessMode::gundam(),
global: view_tensor(&solid(4, 4, [0, 0, 0])),
tiles: vec![view_tensor(&solid(4, 4, [0, 0, 0])); grid.blocks()],
crop_grid: grid,
original_size: (4, 4),
};
assert_eq!(
p.placeholder_token_count(),
expected,
"grid {w}x{h} census mismatch"
);
}
}
#[test]
fn gundam_no_crop_grid_is_273() {
let p = Preprocessed {
mode: PreprocessMode::gundam(),
global: view_tensor(&solid(4, 4, [0, 0, 0])),
tiles: Vec::new(),
crop_grid: CropGrid::single(),
original_size: (4, 4),
};
assert_eq!(p.placeholder_token_count(), 273);
}
#[test]
fn candidate_ratios_count_and_bounds() {
let r = candidate_ratios(MIN_NUM, MAX_NUM);
assert_eq!(r.len(), 118);
assert!(!r.contains(&(1, 1)));
for &(i, j) in &r {
assert!((MIN_NUM..=MAX_NUM).contains(&(i * j)));
}
for pair in r.windows(2) {
assert!(pair[0].0 * pair[0].1 <= pair[1].0 * pair[1].1);
}
assert!(r.contains(&(1, 2)));
assert!(r.contains(&(2, 1)));
assert!(r.contains(&(4, 8)));
}
#[test]
fn closest_ratio_picks_documented_grids() {
let ratios = candidate_ratios(MIN_NUM, MAX_NUM);
let g = find_closest_aspect_ratio(1280.0 / 640.0, &ratios, 1280, 640, 640);
assert_eq!(g, (2, 1));
let g = find_closest_aspect_ratio(640.0 / 1280.0, &ratios, 640, 1280, 640);
assert_eq!(g, (1, 2));
let g = find_closest_aspect_ratio(1300.0 / 1280.0, &ratios, 1300, 1280, 640);
assert_eq!(g.0, g.1, "near-square should pick a square grid, got {g:?}");
}
#[test]
fn closest_ratio_tie_break_prefers_larger_area() {
let ratios = vec![(2usize, 2usize), (3usize, 3usize)];
let g_big = find_closest_aspect_ratio(1.0, &ratios, 4000, 4000, 640);
assert_eq!(g_big, (3, 3));
let g_small = find_closest_aspect_ratio(1.0, &ratios, 100, 100, 640);
assert_eq!(g_small, (2, 2));
}
#[test]
fn normalize_maps_to_minus_one_one() {
let mut img = RgbImage::new(2, 2);
img.put_pixel(0, 0, Rgb([0, 0, 0]));
img.put_pixel(1, 0, Rgb([255, 255, 255]));
img.put_pixel(0, 1, Rgb([128, 128, 128]));
img.put_pixel(1, 1, Rgb([255, 255, 255]));
let vt = view_tensor(&DynamicImage::ImageRgb8(img));
assert_eq!(vt.shape(), (2, 2));
assert_eq!(vt.pixels.rows, 3);
assert_eq!(vt.pixels.cols, 4);
for c in 0..3 {
assert!((vt.pixels.get(c, 0) - (-1.0)).abs() < 1e-6);
}
for c in 0..3 {
assert!((vt.pixels.get(c, 1) - 1.0).abs() < 1e-6);
}
let expected_mid = 2.0 * (128.0f32 / 255.0) - 1.0;
for c in 0..3 {
assert!((vt.pixels.get(c, 2) - expected_mid).abs() < 1e-6);
}
}
#[test]
fn base_mode_single_padded_square_view() {
let img = solid(100, 40, [200, 100, 50]);
let p = preprocess_dynamic(img, PreprocessMode::Base { base_size: 64 }).unwrap();
assert_eq!(p.num_views(), 1);
assert!(p.tiles.is_empty());
assert_eq!(p.crop_grid, CropGrid::single());
assert_eq!(p.global.shape(), (64, 64));
assert_eq!(p.global.pixels.rows, 3);
assert_eq!(p.global.pixels.cols, 64 * 64);
assert_eq!(p.original_size, (100, 40));
let gray = 2.0 * (f32::from(PAD_FILL) / 255.0) - 1.0;
assert!((p.global.pixels.get(0, 0) - gray).abs() < 1e-6);
}
#[test]
fn decompression_bomb_png_is_rejected_before_allocation() {
const BOMB: [u8; 100] = [
0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a, 0x00, 0x00, 0x00, 0x0d, 0x49, 0x48,
0x44, 0x52, 0x00, 0x00, 0x00, 0x06, 0x80, 0x00, 0x00, 0x04, 0x08, 0x02, 0x00, 0x00,
0x00, 0x22, 0x66, 0xd9, 0x14, 0x00, 0x00, 0x00, 0x56, 0x49, 0x44, 0x41, 0x54, 0x78,
0x01, 0xed, 0xc0, 0x03, 0xa0, 0x24, 0x3b, 0x6b, 0xd5, 0xaf, 0xc2, 0x67, 0x3f, 0x1a,
0x1e, 0x0d, 0x8f, 0x86, 0x47, 0xc3, 0xa3, 0xa1, 0xf2, 0xea, 0x3c, 0x10, 0x50, 0x79,
0x75, 0x1e, 0x08, 0xa8, 0xbc, 0x3a, 0x0f, 0x04, 0xfc, 0x23, 0x74, 0xa4, 0x02, 0x0d,
0x6a, 0x18, 0x6a, 0x6a, 0x00, 0x00, 0x00, 0x00, 0x49, 0x45, 0x4e, 0x44, 0xae, 0x42,
0x60, 0x82,
];
let err = decode_bytes(&BOMB).expect_err("2^31-height PNG must be rejected");
assert!(
matches!(err, crate::FocrError::InputDecode(_)),
"bomb must be a typed InputDecode, got {err:?}"
);
let err = preprocess_bytes(&BOMB, PreprocessMode::base()).expect_err("rejected");
assert!(matches!(err, crate::FocrError::InputDecode(_)));
println!(r#"{{"check":"decompression_bomb_bound","bytes":100,"result":"pass"}}"#);
}
#[test]
fn pad_to_square_matches_pillow_rounding_geometry() {
let color = [200, 0, 0];
let pad = [PAD_FILL; 3];
let rounded_size = pad_to_square(&solid(100, 40, color), 64).to_rgb8();
assert_eq!(rounded_size.get_pixel(0, 18).0, pad);
assert_eq!(rounded_size.get_pixel(0, 19).0, color);
assert_eq!(rounded_size.get_pixel(0, 44).0, color);
assert_eq!(rounded_size.get_pixel(0, 45).0, pad);
let rounded_offset = pad_to_square(&solid(101, 40, color), 64).to_rgb8();
assert_eq!(rounded_offset.get_pixel(0, 19).0, pad);
assert_eq!(rounded_offset.get_pixel(0, 20).0, color);
assert_eq!(rounded_offset.get_pixel(0, 44).0, color);
assert_eq!(rounded_offset.get_pixel(0, 45).0, pad);
}
#[test]
fn resample_kind_default_and_kill_switch_parse() {
assert_eq!(resample_kind_from(None), ResampleKind::CatmullRom);
assert_eq!(resample_kind_from(Some("")), ResampleKind::CatmullRom);
assert_eq!(
resample_kind_from(Some("catmullrom")),
ResampleKind::CatmullRom
);
assert_eq!(resample_kind_from(Some("bogus")), ResampleKind::CatmullRom);
assert_eq!(
resample_kind_from(Some("pil-bicubic")),
ResampleKind::PilBicubic
);
assert_eq!(
resample_kind_from(Some("pil_bicubic")),
ResampleKind::PilBicubic
);
assert_eq!(
resample_kind_from(Some(" pil-bicubic ")),
ResampleKind::PilBicubic
);
}
#[test]
fn default_resample_is_catmullrom_byte_identical() {
let mut rgba = image::RgbaImage::new(13, 7);
for (x, y, p) in rgba.enumerate_pixels_mut() {
*p = image::Rgba([(x * 19 + y * 3) as u8, (x * 7) as u8, (y * 31) as u8, 255]);
}
let img = DynamicImage::ImageRgba8(rgba);
let via_dispatch = resample_exact_with(ResampleKind::CatmullRom, &img, 8, 5);
let direct = img.resize_exact(8, 5, FilterType::CatmullRom);
assert_eq!(
via_dispatch.color(),
direct.color(),
"default resample changed the color type"
);
assert_eq!(
via_dispatch.as_bytes(),
direct.as_bytes(),
"default resample output moved (doctrine #2 violation)"
);
}
#[test]
fn pil_kill_switch_dispatch_routes_to_pil_resampler() {
let mut rgb = RgbImage::new(5, 4);
for (x, y, p) in rgb.enumerate_pixels_mut() {
*p = Rgb([(x * 40) as u8, (y * 60) as u8, (x * y * 13) as u8]);
}
let img = DynamicImage::ImageRgb8(rgb.clone());
let via_dispatch = resample_exact_with(ResampleKind::PilBicubic, &img, 3, 6);
let direct = pil_resample::resize_bicubic(&rgb, 3, 6);
assert_eq!(via_dispatch.color(), image::ColorType::Rgb8);
assert_eq!(via_dispatch.as_bytes(), direct.as_raw().as_slice());
}
#[test]
fn gundam_small_image_short_circuits_to_no_crop() {
let img = solid(320, 200, [10, 20, 30]);
let p = preprocess_dynamic(img, PreprocessMode::gundam()).unwrap();
assert!(p.tiles.is_empty());
assert_eq!(p.crop_grid, CropGrid::single());
assert_eq!(p.global.shape(), (1024, 1024));
assert_eq!(p.placeholder_token_count(), 273);
}
#[test]
fn gundam_wide_image_tiles_into_grid() {
let img = solid(1000, 500, [50, 60, 70]);
let ratios = candidate_ratios(MIN_NUM, MAX_NUM);
let expected = find_closest_aspect_ratio(1000.0 / 500.0, &ratios, 1000, 500, 640);
assert_eq!(expected, (2, 1), "pinned config: 2:1 wide -> (2,1)");
let p = preprocess_dynamic(img, PreprocessMode::gundam()).unwrap();
assert_eq!(
p.crop_grid,
CropGrid {
width_crop_num: expected.0,
height_crop_num: expected.1,
}
);
assert_eq!(p.tiles.len(), expected.0 * expected.1);
for t in &p.tiles {
assert_eq!(t.shape(), (GUNDAM_TILE_SIZE, GUNDAM_TILE_SIZE));
assert_eq!(t.pixels.rows, 3);
assert_eq!(t.pixels.cols, GUNDAM_TILE_SIZE * GUNDAM_TILE_SIZE);
}
assert_eq!(p.global.shape(), (BASE_SIZE, BASE_SIZE));
assert_eq!(p.num_views(), 1 + expected.0 * expected.1);
}
#[test]
fn gundam_tile_count_equals_grid_blocks() {
let img = solid(700, 2100, [1, 2, 3]); let mode = PreprocessMode::Gundam {
base_size: 128,
tile_size: 64,
};
let p = preprocess_dynamic(img, mode).unwrap();
assert_eq!(p.tiles.len(), p.crop_grid.blocks());
assert!(p.crop_grid.is_tiled());
let q_base = num_queries(128);
let q_local = num_queries(64);
let w = p.crop_grid.width_crop_num;
let h = p.crop_grid.height_crop_num;
let expected = (q_base + 1) * q_base + 1 + (q_local * w + 1) * (q_local * h);
assert_eq!(p.placeholder_token_count(), expected);
}
#[test]
fn preprocess_rejects_invalid_mode_sizes() {
let img = solid(32, 32, [1, 2, 3]);
let cases = [
PreprocessMode::Base { base_size: 0 },
PreprocessMode::Base { base_size: 15 },
PreprocessMode::Base { base_size: 2048 },
PreprocessMode::Gundam {
base_size: BASE_SIZE,
tile_size: 0,
},
PreprocessMode::Gundam {
base_size: BASE_SIZE,
tile_size: 1024,
},
PreprocessMode::Gundam {
base_size: BASE_SIZE,
tile_size: 15,
},
PreprocessMode::Gundam {
base_size: 2048,
tile_size: GUNDAM_TILE_SIZE,
},
];
for mode in cases {
let err = preprocess_dynamic(img.clone(), mode).unwrap_err();
assert!(
matches!(err, FocrError::Usage(_)),
"mode {mode:?} should be a usage error, got {err:?}"
);
}
}
#[test]
fn malformed_crop_grid_blocks_rejects_overflow() {
let grid = CropGrid {
width_crop_num: usize::MAX,
height_crop_num: usize::MAX,
};
let panic = std::panic::catch_unwind(|| grid.blocks()).expect_err("overflow must panic");
let message = panic_message(panic);
assert!(message.contains("CropGrid::blocks"));
}
#[test]
fn placeholder_count_rejects_malformed_grid_overflow() {
let grid = CropGrid {
width_crop_num: usize::MAX,
height_crop_num: usize::MAX,
};
let p = Preprocessed {
mode: PreprocessMode::gundam(),
global: view_tensor(&solid(4, 4, [0, 0, 0])),
tiles: Vec::new(),
crop_grid: grid,
original_size: (4, 4),
};
let panic = std::panic::catch_unwind(|| p.placeholder_token_count())
.expect_err("overflow must panic");
let message = panic_message(panic);
assert!(message.contains("Preprocessed::placeholder_token_count"));
}
fn panic_message(panic: Box<dyn std::any::Any + Send>) -> String {
if let Some(s) = panic.downcast_ref::<&str>() {
(*s).to_owned()
} else if let Some(s) = panic.downcast_ref::<String>() {
s.clone()
} else {
"<non-string panic>".to_owned()
}
}
#[test]
fn preprocess_missing_file_is_input_decode_error() {
let r = preprocess_image(
Path::new("/definitely/not/a/real/image.png"),
PreprocessMode::base(),
);
assert!(matches!(r, Err(FocrError::InputDecode(_))));
}
#[test]
fn preprocess_garbage_bytes_is_input_decode_error() {
let r = preprocess_bytes(&[0u8, 1, 2, 3, 4, 5, 6, 7], PreprocessMode::base());
assert!(matches!(r, Err(FocrError::InputDecode(_))));
}
#[test]
fn preprocess_bytes_decodes_real_png() {
let img = solid(50, 30, [123, 45, 67]);
let mut buf = std::io::Cursor::new(Vec::new());
img.write_to(&mut buf, image::ImageFormat::Png).unwrap();
let bytes = buf.into_inner();
let p = preprocess_bytes(&bytes, PreprocessMode::Base { base_size: 32 }).unwrap();
assert_eq!(p.global.shape(), (32, 32));
assert_eq!(p.original_size, (50, 30));
}
#[test]
fn view_tensor_layout_is_channel_major() {
let mut img = RgbImage::new(2, 1);
img.put_pixel(0, 0, Rgb([0, 128, 255]));
img.put_pixel(1, 0, Rgb([255, 128, 0]));
let vt = view_tensor(&DynamicImage::ImageRgb8(img));
assert!((vt.pixels.get(0, 0) - (-1.0)).abs() < 1e-6);
assert!((vt.pixels.get(0, 1) - 1.0).abs() < 1e-6);
assert!((vt.pixels.get(2, 0) - 1.0).abs() < 1e-6);
assert!((vt.pixels.get(2, 1) - (-1.0)).abs() < 1e-6);
}
#[test]
fn smolvlm2_layout_across_aspects() {
let mk = |w, h| DynamicImage::ImageRgb8(RgbImage::new(w, h));
let p = preprocess_smolvlm2(&mk(1024, 768)).unwrap();
assert_eq!((p.rows, p.cols, p.n_frames), (3, 4, 13));
assert_eq!(p.frames.len(), 13 * 3 * 512 * 512);
let p = preprocess_smolvlm2(&mk(768, 1024)).unwrap();
assert_eq!((p.rows, p.cols, p.n_frames), (4, 3, 13));
let p = preprocess_smolvlm2(&mk(640, 640)).unwrap();
assert_eq!((p.rows, p.cols, p.n_frames), (4, 4, 17));
let p = preprocess_smolvlm2(&mk(999, 500)).unwrap();
assert_eq!((p.rows, p.cols), (3, 4));
let p = preprocess_smolvlm2(&mk(10, 10)).unwrap();
assert_eq!((p.rows, p.cols, p.n_frames), (4, 4, 17));
}
#[test]
fn smolvlm2_normalize_rail() {
let img = DynamicImage::ImageRgb8(RgbImage::from_pixel(800, 600, Rgb([128, 0, 255])));
let p = preprocess_smolvlm2(&img).unwrap();
let want_r = ((128.0f64 * (1.0 / 255.0)) as f32 - 0.5) / 0.5;
let n = 512 * 512;
for f in 0..p.n_frames {
let fr = &p.frames[f * 3 * n..(f + 1) * 3 * n];
assert!((fr[0] - want_r).abs() < 1e-7, "frame {f} R rail");
assert!((fr[n] - (-1.0)).abs() < 1e-7, "frame {f} G rail");
assert!((fr[2 * n] - 1.0).abs() < 1e-7, "frame {f} B rail");
}
}
#[test]
fn smolvlm2_degenerate_image_errors() {
let img = DynamicImage::ImageRgb8(RgbImage::new(0, 5));
assert!(preprocess_smolvlm2(&img).is_err());
}
#[test]
fn smolvlm2_preprocess_matches_torch_oracle() {
let Ok(dir) = std::env::var("FOCR_SMOLVLM2_DIR") else {
return;
};
let pv_path = format!("{dir}/smolvlm2_pixel_values.bin");
if !std::path::Path::new(&pv_path).is_file() {
eprintln!("skip-with-SUCCESS: {pv_path} absent (run the vision oracle script)");
return;
}
let want: Vec<f32> = std::fs::read(&pv_path)
.expect("oracle blob reads")
.as_chunks::<4>()
.0
.iter()
.map(|c| f32::from_le_bytes(*c))
.collect();
let photo = concat!(
env!("CARGO_MANIFEST_DIR"),
"/tests/fixtures/smolvlm2/sample_photo.png"
);
let p = preprocess_smolvlm2_path(std::path::Path::new(photo)).expect("preprocess");
assert_eq!((p.rows, p.cols, p.n_frames), (3, 4, 13), "tile layout");
assert_eq!(p.frames.len(), want.len(), "frame count/shape");
let mut max_abs = 0.0f32;
let mut n_diff = 0usize;
for (a, b) in p.frames.iter().zip(&want) {
let d = (a - b).abs();
if d > 0.0 {
n_diff += 1;
}
max_abs = max_abs.max(d);
}
eprintln!(
"[C7 L0b] maxabs={max_abs:.3e} n_diff={n_diff}/{}",
want.len()
);
assert!(
max_abs <= 1e-6,
"preprocess maxabs {max_abs:.3e} > 1e-6 — the resample or normalize drifted"
);
}
}