#[cfg(feature = "image")]
use base64::Engine;
#[cfg(feature = "image")]
use ndarray::Array3;
use serde::{Deserialize, Serialize};
#[cfg(feature = "image")]
use std::path::Path;
use crate::errors::{Result, TokenizerError};
#[cfg(feature = "image")]
pub const DATASET_MEAN: [f64; 3] = [0.481_454_66, 0.457_827_5, 0.408_210_73];
#[cfg(feature = "image")]
pub const DATASET_STD: [f64; 3] = [0.268_629_54, 0.261_302_58, 0.275_777_11];
const fn default_spatial_merge_size() -> usize {
1
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct ImageConfig {
pub image_patch_size: usize,
pub max_image_size: usize,
#[serde(default = "default_spatial_merge_size")]
pub spatial_merge_size: usize,
}
impl ImageConfig {
pub fn new(
image_patch_size: usize,
max_image_size: usize,
spatial_merge_size: usize,
) -> Result<Self> {
if image_patch_size == 0 {
return Err(TokenizerError::InvalidConfig(
"image_patch_size must be > 0".to_string(),
));
}
if max_image_size == 0 {
return Err(TokenizerError::InvalidConfig(
"max_image_size must be > 0".to_string(),
));
}
if spatial_merge_size == 0 {
return Err(TokenizerError::InvalidConfig(
"spatial_merge_size must be > 0".to_string(),
));
}
Ok(Self {
image_patch_size,
max_image_size,
spatial_merge_size,
})
}
#[must_use]
pub const fn pixels_per_token(&self) -> usize {
self.image_patch_size * self.spatial_merge_size
}
pub fn validate(&self) -> Result<()> {
Self::new(
self.image_patch_size,
self.max_image_size,
self.spatial_merge_size,
)
.map(|_| ())
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub struct SpecialImageIds {
pub img: u32,
pub img_break: u32,
pub img_end: u32,
}
#[derive(Debug, Clone, PartialEq, Eq)]
#[cfg(feature = "image")]
pub struct Image {
width: usize,
height: usize,
pixels: Vec<u8>,
}
#[cfg(feature = "image")]
impl Image {
pub fn new(width: usize, height: usize, pixels: Vec<u8>) -> Result<Self> {
if width == 0 || height == 0 {
return Err(TokenizerError::Image(format!(
"Image dimensions must be > 0, got {width}x{height}"
)));
}
let expected = width
.checked_mul(height)
.and_then(|n| n.checked_mul(3))
.ok_or_else(|| {
TokenizerError::Image(format!("Image dimensions overflow: {width}x{height}"))
})?;
if pixels.len() != expected {
return Err(TokenizerError::Image(format!(
"Expected {expected} bytes for a {width}x{height} RGB image, got {}",
pixels.len()
)));
}
Ok(Self {
width,
height,
pixels,
})
}
pub fn from_file<P: AsRef<Path>>(path: P) -> Result<Self> {
let bytes = std::fs::read(path)?;
Self::from_bytes(&bytes)
}
pub fn from_bytes(bytes: &[u8]) -> Result<Self> {
let decoded = ::image::load_from_memory(bytes)
.map_err(|e| TokenizerError::Image(format!("Failed to decode image: {e}")))?;
Self::from_dynamic_image(&decoded)
}
pub fn from_base64(data: &str) -> Result<Self> {
let payload = data
.strip_prefix("data:")
.and_then(|rest| rest.split_once(";base64,"))
.map_or(data, |(_, encoded)| encoded);
let bytes = base64::engine::general_purpose::STANDARD.decode(payload.trim())?;
Self::from_bytes(&bytes)
}
fn from_dynamic_image(decoded: &::image::DynamicImage) -> Result<Self> {
use ::image::GenericImageView;
let (width, height) = decoded.dimensions();
let pixels = if decoded.color().has_alpha() {
let rgba = decoded.to_rgba8();
let mut pixels = Vec::with_capacity(rgba.len() / 4 * 3);
for pixel in rgba.pixels() {
let [red, green, blue, alpha] = pixel.0;
pixels.push(blend_onto_white(red, alpha));
pixels.push(blend_onto_white(green, alpha));
pixels.push(blend_onto_white(blue, alpha));
}
pixels
} else {
decoded.to_rgb8().into_raw()
};
Self::new(width as usize, height as usize, pixels)
}
#[must_use]
pub const fn width(&self) -> usize {
self.width
}
#[must_use]
pub const fn height(&self) -> usize {
self.height
}
#[must_use]
pub fn pixels(&self) -> &[u8] {
&self.pixels
}
}
#[cfg(feature = "image")]
const fn blend_onto_white(channel: u8, alpha: u8) -> u8 {
muldiv255(255, 255 - alpha) + muldiv255(channel, alpha)
}
#[allow(clippy::cast_possible_truncation)]
#[cfg(feature = "image")]
const fn muldiv255(a: u8, b: u8) -> u8 {
let tmp = a as u32 * b as u32 + 128;
(((tmp >> 8) + tmp) >> 8) as u8
}
#[derive(Debug, Clone)]
#[cfg(feature = "image")]
pub struct ImageEncoding {
pub tokens: Vec<u32>,
pub image: Array3<f32>,
}
#[derive(Debug, Clone)]
pub struct ImageEncoder {
pub config: ImageConfig,
pub special_ids: SpecialImageIds,
}
impl ImageEncoder {
#[must_use]
pub const fn new(config: ImageConfig, special_ids: SpecialImageIds) -> Self {
Self {
config,
special_ids,
}
}
#[must_use]
pub const fn image_token(&self) -> u32 {
self.special_ids.img
}
#[allow(clippy::cast_precision_loss, clippy::cast_sign_loss)]
pub fn image_to_num_tokens(&self, width: usize, height: usize) -> Result<(usize, usize)> {
let (width, height) = self.resized_dimensions(width, height)?;
let pixels_per_token = self.config.pixels_per_token();
Ok((
(width - 1) / pixels_per_token + 1,
(height - 1) / pixels_per_token + 1,
))
}
#[allow(
clippy::cast_precision_loss,
clippy::cast_sign_loss,
clippy::cast_possible_truncation
)]
fn resized_dimensions(&self, width: usize, height: usize) -> Result<(usize, usize)> {
if width == 0 || height == 0 {
return Err(TokenizerError::Image(format!(
"Image dimensions must be > 0, got {width}x{height}"
)));
}
let max_size = self.config.max_image_size as f64;
let ratio = f64::max(height as f64 / max_size, width as f64 / max_size);
if ratio <= 1.0 {
return Ok((width, height));
}
let width = (width as f64 / ratio).round_ties_even() as usize;
let height = (height as f64 / ratio).round_ties_even() as usize;
if width == 0 || height == 0 {
return Err(TokenizerError::Image(
"Image is too elongated to be resized to a whole number of pixels".to_string(),
));
}
Ok((width, height))
}
pub fn encode_dimensions(&self, width: usize, height: usize) -> Result<Vec<u32>> {
let (width_tokens, height_tokens) = self.image_to_num_tokens(width, height)?;
let mut tokens = Vec::with_capacity((width_tokens + 1) * height_tokens);
for _ in 0..height_tokens {
tokens.extend(std::iter::repeat_n(self.special_ids.img, width_tokens));
tokens.push(self.special_ids.img_break);
}
if let Some(last) = tokens.last_mut() {
*last = self.special_ids.img_end;
}
Ok(tokens)
}
#[cfg(feature = "image")]
pub fn encode(&self, image: &Image) -> Result<ImageEncoding> {
let (width_tokens, height_tokens) =
self.image_to_num_tokens(image.width(), image.height())?;
let tokens = self.encode_dimensions(image.width(), image.height())?;
let pixels_per_token = self.config.pixels_per_token();
let processed = transform_image(
image,
width_tokens * pixels_per_token,
height_tokens * pixels_per_token,
)?;
Ok(ImageEncoding {
tokens,
image: processed,
})
}
}
#[cfg(feature = "image")]
pub fn transform_image(image: &Image, new_width: usize, new_height: usize) -> Result<Array3<f32>> {
if new_width == 0 || new_height == 0 {
return Err(TokenizerError::Image(format!(
"Target dimensions must be > 0, got {new_width}x{new_height}"
)));
}
let pixels = image.pixels();
let stride = image.width() * 3;
let resized = resize_bicubic_with(
|row, index| f32::from(pixels[row * stride + index]),
image.width(),
image.height(),
new_width,
new_height,
3,
);
Ok(normalize(
&resized,
new_width,
new_height,
DATASET_MEAN,
DATASET_STD,
))
}
#[must_use]
#[allow(clippy::cast_possible_truncation)]
#[cfg(feature = "image")]
pub fn normalize(
samples: &[f32],
width: usize,
height: usize,
mean: [f64; 3],
std: [f64; 3],
) -> Array3<f32> {
assert_eq!(
samples.len(),
width * height * 3,
"expected {} samples for a {width}x{height} RGB image, got {}",
width * height * 3,
samples.len()
);
Array3::from_shape_fn((3, height, width), |(channel, y, x)| {
let scaled = samples[(y * width + x) * 3 + channel] / 255.0;
((f64::from(scaled) - mean[channel]) / std[channel]) as f32
})
}
#[cfg(feature = "image")]
const TAPS: [isize; 4] = [-1, 0, 1, 2];
#[cfg(feature = "image")]
struct CubicTaps {
positions: [usize; 4],
weights: [f32; 4],
}
#[allow(clippy::cast_precision_loss, clippy::cast_possible_truncation)]
#[cfg(feature = "image")]
fn cubic_taps(src_len: usize, dst_len: usize) -> Vec<CubicTaps> {
let scale = src_len as f64 / dst_len as f64;
(0..dst_len)
.map(|index| {
let mapped = ((index as f64 + 0.5) * scale - 0.5) as f32;
let base = mapped.floor();
CubicTaps {
positions: TAPS.map(|tap| clamp_index(base as isize + tap, src_len)),
weights: cubic_weights(mapped - base),
}
})
.collect()
}
#[must_use]
#[cfg(feature = "image")]
pub fn resize_bicubic(
source: &[f32],
src_width: usize,
src_height: usize,
dst_width: usize,
dst_height: usize,
channels: usize,
) -> Vec<f32> {
assert_eq!(
source.len(),
src_width * src_height * channels,
"source length does not match the given dimensions"
);
let stride = src_width * channels;
resize_bicubic_with(
|row, index| source[row * stride + index],
src_width,
src_height,
dst_width,
dst_height,
channels,
)
}
#[cfg(feature = "image")]
fn resize_bicubic_with<F>(
sample: F,
src_width: usize,
src_height: usize,
dst_width: usize,
dst_height: usize,
channels: usize,
) -> Vec<f32>
where
F: Fn(usize, usize) -> f32,
{
assert!(
src_width > 0 && src_height > 0 && dst_width > 0 && dst_height > 0 && channels > 0,
"resize dimensions must be > 0"
);
let columns = cubic_taps(src_width, dst_width);
let rows = cubic_taps(src_height, dst_height);
let row_len = dst_width * channels;
let mut window: [Vec<f32>; 4] = std::array::from_fn(|_| vec![0.0; row_len]);
let mut window_rows = [usize::MAX; 4];
let mut resized = vec![0.0f32; dst_height * row_len];
for (y, row_taps) in rows.iter().enumerate() {
let mut slots = [usize::MAX; 4];
let mut taken = [false; 4];
for (tap, &row) in row_taps.positions.iter().enumerate() {
if let Some(slot) = window_rows.iter().position(|&cached| cached == row) {
slots[tap] = slot;
taken[slot] = true;
}
}
for (tap, &row) in row_taps.positions.iter().enumerate() {
if slots[tap] != usize::MAX {
continue;
}
if let Some(slot) = window_rows.iter().position(|&cached| cached == row) {
slots[tap] = slot;
continue;
}
let slot = taken
.iter()
.position(|&used| !used)
.expect("four taps never need more than four rows");
taken[slot] = true;
window_rows[slot] = row;
resample_row(&sample, row, &columns, channels, &mut window[slot]);
slots[tap] = slot;
}
let resized_row = &mut resized[y * row_len..(y + 1) * row_len];
for (slot, weight) in slots.iter().zip(&row_taps.weights) {
for (destination, source) in resized_row.iter_mut().zip(&window[*slot]) {
*destination += source * weight;
}
}
}
resized
}
#[cfg(feature = "image")]
fn resample_row<F>(
sample: &F,
row: usize,
columns: &[CubicTaps],
channels: usize,
resampled: &mut [f32],
) where
F: Fn(usize, usize) -> f32,
{
for (x, taps) in columns.iter().enumerate() {
for channel in 0..channels {
let mut value = 0.0f32;
for (column, weight) in taps.positions.iter().zip(&taps.weights) {
value += sample(row, column * channels + channel) * weight;
}
resampled[x * channels + channel] = value;
}
}
}
#[allow(clippy::cast_sign_loss)]
#[cfg(feature = "image")]
const fn clamp_index(index: isize, len: usize) -> usize {
if index < 0 {
0
} else if index as usize >= len {
len - 1
} else {
index as usize
}
}
#[cfg(feature = "image")]
fn cubic_weights(fraction: f32) -> [f32; 4] {
const A: f32 = -0.75;
let x = fraction;
let w0 = ((A * (x + 1.0) - 5.0 * A) * (x + 1.0) + 8.0 * A) * (x + 1.0) - 4.0 * A;
let w1 = ((A + 2.0) * x - (A + 3.0)) * x * x + 1.0;
let w2 = ((A + 2.0) * (1.0 - x) - (A + 3.0)) * (1.0 - x) * (1.0 - x) + 1.0;
let w3 = 1.0 - w0 - w1 - w2;
[w0, w1, w2, w3]
}