#![allow(clippy::cast_precision_loss)]
#![allow(clippy::cast_possible_truncation)]
#![allow(clippy::cast_sign_loss)]
use std::fmt;
use rayon::prelude::*;
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub struct FrameRate {
pub num: u32,
pub den: u32,
}
impl FrameRate {
pub fn new(num: u32, den: u32) -> Self {
Self {
num,
den: den.max(1),
}
}
pub fn ntsc_film() -> Self {
Self {
num: 24_000,
den: 1_001,
}
}
pub fn film() -> Self {
Self { num: 24, den: 1 }
}
pub fn pal() -> Self {
Self { num: 25, den: 1 }
}
pub fn ntsc() -> Self {
Self {
num: 30_000,
den: 1_001,
}
}
pub fn fps30() -> Self {
Self { num: 30, den: 1 }
}
pub fn fps50() -> Self {
Self { num: 50, den: 1 }
}
pub fn fps59_94() -> Self {
Self {
num: 60_000,
den: 1_001,
}
}
pub fn fps60() -> Self {
Self { num: 60, den: 1 }
}
pub fn to_f64(self) -> f64 {
self.num as f64 / self.den as f64
}
pub fn frame_duration_secs(self) -> f64 {
self.den as f64 / self.num as f64
}
}
impl fmt::Display for FrameRate {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
if self.den == 1 {
write!(f, "{} fps", self.num)
} else {
write!(f, "{}/{} fps", self.num, self.den)
}
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum InterpolationMode {
FrameBlend,
MotionCompensated,
}
impl fmt::Display for InterpolationMode {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
match self {
Self::FrameBlend => write!(f, "FrameBlend"),
Self::MotionCompensated => write!(f, "MotionCompensated"),
}
}
}
#[derive(Debug, Clone)]
pub struct TemporalScalingConfig {
pub src_rate: FrameRate,
pub dst_rate: FrameRate,
pub mode: InterpolationMode,
pub block_size: u32,
pub search_radius: u32,
}
impl TemporalScalingConfig {
pub fn new(src_rate: FrameRate, dst_rate: FrameRate) -> Self {
Self {
src_rate,
dst_rate,
mode: InterpolationMode::FrameBlend,
block_size: 16,
search_radius: 8,
}
}
pub fn with_mode(mut self, mode: InterpolationMode) -> Self {
self.mode = mode;
self
}
pub fn with_block_size(mut self, block_size: u32) -> Self {
self.block_size = block_size.max(4);
self
}
pub fn with_search_radius(mut self, radius: u32) -> Self {
self.search_radius = radius;
self
}
}
#[derive(Debug, Clone)]
pub struct VideoFrame {
pub pixels: Vec<u8>,
pub width: u32,
pub height: u32,
pub pts_secs: f64,
}
impl VideoFrame {
pub fn new(pixels: Vec<u8>, width: u32, height: u32, pts_secs: f64) -> Option<Self> {
if pixels.len() < (width as usize) * (height as usize) * 3 {
return None;
}
Some(Self {
pixels,
width,
height,
pts_secs,
})
}
#[inline]
pub fn sample(&self, x: i32, y: i32) -> [u8; 3] {
let cx = x.clamp(0, self.width as i32 - 1) as usize;
let cy = y.clamp(0, self.height as i32 - 1) as usize;
let base = (cy * self.width as usize + cx) * 3;
[
self.pixels[base],
self.pixels[base + 1],
self.pixels[base + 2],
]
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub struct MotionVector {
pub dx: i32,
pub dy: i32,
}
impl MotionVector {
pub const ZERO: Self = Self { dx: 0, dy: 0 };
pub fn new(dx: i32, dy: i32) -> Self {
Self { dx, dy }
}
}
pub fn estimate_motion(
reference: &VideoFrame,
target: &VideoFrame,
block_size: u32,
search_radius: u32,
) -> MotionVector {
if reference.width == 0 || reference.height == 0 {
return MotionVector::ZERO;
}
let bs = block_size as usize;
let sr = search_radius as i32;
let w = reference.width as usize;
let h = reference.height as usize;
let blocks_x = (w / bs).max(1);
let blocks_y = (h / bs).max(1);
let vectors: Vec<MotionVector> = (0..blocks_y)
.into_par_iter()
.flat_map_iter(|by| {
let oy = by * bs;
(0..blocks_x).map(move |bx| {
let ox = bx * bs;
let mut best_dx = 0i32;
let mut best_dy = 0i32;
let mut best_sad = block_sad(reference, target, ox, oy, bs, 0, 0);
for dy in -sr..=sr {
for dx in -sr..=sr {
if dx == 0 && dy == 0 {
continue; }
let sad = block_sad(reference, target, ox, oy, bs, dx, dy);
if sad < best_sad {
best_sad = sad;
best_dx = dx;
best_dy = dy;
}
}
}
MotionVector::new(best_dx, best_dy)
})
})
.collect();
median_motion_vector(&vectors)
}
fn block_sad(
reference: &VideoFrame,
target: &VideoFrame,
ox: usize,
oy: usize,
block_size: usize,
dx: i32,
dy: i32,
) -> u64 {
let mut sad = 0u64;
for row in 0..block_size {
for col in 0..block_size {
let rx = ox + col;
let ry = oy + row;
let tx = rx as i32 + dx;
let ty = ry as i32 + dy;
let rp = reference.sample(rx as i32, ry as i32);
let tp = target.sample(tx, ty);
for c in 0..3 {
sad += (rp[c] as i64 - tp[c] as i64).unsigned_abs() as u64;
}
}
}
sad
}
fn median_motion_vector(vectors: &[MotionVector]) -> MotionVector {
if vectors.is_empty() {
return MotionVector::ZERO;
}
let mut dxs: Vec<i32> = vectors.iter().map(|v| v.dx).collect();
let mut dys: Vec<i32> = vectors.iter().map(|v| v.dy).collect();
dxs.sort_unstable();
dys.sort_unstable();
let mid = dxs.len() / 2;
MotionVector::new(dxs[mid], dys[mid])
}
pub fn blend_frames(frame_a: &VideoFrame, frame_b: &VideoFrame, alpha: f64) -> Option<Vec<u8>> {
if frame_a.width != frame_b.width || frame_a.height != frame_b.height {
return None;
}
let count = (frame_a.width as usize) * (frame_a.height as usize) * 3;
if frame_a.pixels.len() < count || frame_b.pixels.len() < count {
return None;
}
let a = alpha.clamp(0.0, 1.0) as f32;
let b = 1.0 - a;
let blended: Vec<u8> = frame_a.pixels[..count]
.iter()
.zip(frame_b.pixels[..count].iter())
.map(|(&pa, &pb)| (pa as f32 * a + pb as f32 * b).round().clamp(0.0, 255.0) as u8)
.collect();
Some(blended)
}
pub fn motion_compensated_interpolate(
frame_a: &VideoFrame,
frame_b: &VideoFrame,
t: f64,
block_size: u32,
search_radius: u32,
) -> Option<Vec<u8>> {
if frame_a.width != frame_b.width || frame_a.height != frame_b.height {
return None;
}
let w = frame_a.width as usize;
let h = frame_a.height as usize;
let count = w * h * 3;
if frame_a.pixels.len() < count || frame_b.pixels.len() < count {
return None;
}
let t = t.clamp(0.0, 1.0);
let mv = estimate_motion(frame_a, frame_b, block_size, search_radius);
let warped_a = warp_frame(
frame_a,
(mv.dx as f64 * t) as i32,
(mv.dy as f64 * t) as i32,
);
let warped_b = warp_frame(
frame_b,
-(mv.dx as f64 * (1.0 - t)) as i32,
-(mv.dy as f64 * (1.0 - t)) as i32,
);
let wa = VideoFrame {
pixels: warped_a,
width: frame_a.width,
height: frame_a.height,
pts_secs: frame_a.pts_secs,
};
let wb = VideoFrame {
pixels: warped_b,
width: frame_b.width,
height: frame_b.height,
pts_secs: frame_b.pts_secs,
};
blend_frames(&wa, &wb, 1.0 - t)
}
fn warp_frame(frame: &VideoFrame, dx: i32, dy: i32) -> Vec<u8> {
let w = frame.width as usize;
let h = frame.height as usize;
let mut out = vec![0u8; w * h * 3];
for oy in 0..h {
for ox in 0..w {
let sx = ox as i32 - dx;
let sy = oy as i32 - dy;
let p = frame.sample(sx, sy);
let base = (oy * w + ox) * 3;
out[base] = p[0];
out[base + 1] = p[1];
out[base + 2] = p[2];
}
}
out
}
#[derive(Debug)]
pub struct TemporalScaler {
config: TemporalScalingConfig,
}
impl TemporalScaler {
pub fn new(config: TemporalScalingConfig) -> Self {
Self { config }
}
pub fn config(&self) -> &TemporalScalingConfig {
&self.config
}
pub fn compute_schedule(&self, num_input_frames: usize) -> Vec<(usize, usize, f64)> {
if num_input_frames < 2 {
return Vec::new();
}
let src_fps = self.config.src_rate.to_f64();
let dst_fps = self.config.dst_rate.to_f64();
if src_fps <= 0.0 || dst_fps <= 0.0 {
return Vec::new();
}
let src_duration = (num_input_frames - 1) as f64 / src_fps;
let num_output_frames = (src_duration * dst_fps).floor() as usize + 1;
let dst_frame_dur = 1.0 / dst_fps;
let mut schedule = Vec::with_capacity(num_output_frames);
for out_idx in 0..num_output_frames {
let t_out = out_idx as f64 * dst_frame_dur;
let src_float = t_out * src_fps;
let src_floor = src_float.floor() as usize;
let frame_a = src_floor.min(num_input_frames - 2);
let frame_b = (frame_a + 1).min(num_input_frames - 1);
let blend_t = (src_float - src_floor as f64).clamp(0.0, 1.0);
schedule.push((frame_a, frame_b, blend_t));
}
schedule
}
pub fn convert(&self, input_frames: &[VideoFrame]) -> Option<Vec<Vec<u8>>> {
if input_frames.len() < 2 {
return None;
}
let w = input_frames[0].width;
let h = input_frames[0].height;
for frame in input_frames.iter().skip(1) {
if frame.width != w || frame.height != h {
return None;
}
}
let schedule = self.compute_schedule(input_frames.len());
let mut output = Vec::with_capacity(schedule.len());
for (idx_a, idx_b, t) in schedule {
let fa = &input_frames[idx_a];
let fb = &input_frames[idx_b];
let blended = match self.config.mode {
InterpolationMode::FrameBlend => blend_frames(fa, fb, 1.0 - t)?,
InterpolationMode::MotionCompensated => motion_compensated_interpolate(
fa,
fb,
t,
self.config.block_size,
self.config.search_radius,
)?,
};
output.push(blended);
}
Some(output)
}
}
#[cfg(test)]
mod tests {
use super::*;
fn make_frame(r: u8, g: u8, b: u8, w: u32, h: u32, pts: f64) -> VideoFrame {
let count = (w as usize) * (h as usize);
let mut pixels = vec![0u8; count * 3];
for i in 0..count {
pixels[i * 3] = r;
pixels[i * 3 + 1] = g;
pixels[i * 3 + 2] = b;
}
VideoFrame::new(pixels, w, h, pts).expect("frame creation should succeed")
}
#[test]
fn test_frame_rate_new() {
let fps = FrameRate::new(30, 1);
assert_eq!(fps.num, 30);
assert_eq!(fps.den, 1);
assert!((fps.to_f64() - 30.0).abs() < 1e-9);
}
#[test]
fn test_frame_rate_zero_den_clamped() {
let fps = FrameRate::new(30, 0);
assert_eq!(fps.den, 1);
}
#[test]
fn test_frame_rate_ntsc() {
let fps = FrameRate::ntsc();
assert!((fps.to_f64() - 29.97).abs() < 0.01);
}
#[test]
fn test_frame_rate_frame_duration() {
let fps = FrameRate::fps60();
assert!((fps.frame_duration_secs() - 1.0 / 60.0).abs() < 1e-9);
}
#[test]
fn test_frame_rate_display_integer() {
let fps = FrameRate::fps30();
assert_eq!(fps.to_string(), "30 fps");
}
#[test]
fn test_frame_rate_display_rational() {
let fps = FrameRate::ntsc();
let s = fps.to_string();
assert!(s.contains("30000"));
assert!(s.contains("1001"));
}
#[test]
fn test_video_frame_sample_center() {
let frame = make_frame(100, 150, 200, 4, 4, 0.0);
let p = frame.sample(2, 2);
assert_eq!(p, [100, 150, 200]);
}
#[test]
fn test_video_frame_sample_clamp_neg() {
let frame = make_frame(10, 20, 30, 4, 4, 0.0);
let p = frame.sample(-5, -5);
assert_eq!(p, [10, 20, 30]); }
#[test]
fn test_video_frame_sample_clamp_oob() {
let frame = make_frame(50, 60, 70, 4, 4, 0.0);
let p = frame.sample(100, 100);
assert_eq!(p, [50, 60, 70]); }
#[test]
fn test_video_frame_new_invalid_buffer() {
let result = VideoFrame::new(vec![0u8; 3], 4, 4, 0.0);
assert!(result.is_none());
}
#[test]
fn test_interpolation_mode_display() {
assert_eq!(InterpolationMode::FrameBlend.to_string(), "FrameBlend");
assert_eq!(
InterpolationMode::MotionCompensated.to_string(),
"MotionCompensated"
);
}
#[test]
fn test_blend_frames_alpha_zero() {
let fa = make_frame(255, 0, 0, 2, 2, 0.0);
let fb = make_frame(0, 0, 255, 2, 2, 0.0);
let result = blend_frames(&fa, &fb, 0.0).expect("blend should succeed");
for chunk in result.chunks(3) {
assert!(chunk[0] < 10, "should be mostly blue");
assert!(chunk[2] > 200, "should be mostly blue");
}
}
#[test]
fn test_blend_frames_alpha_one() {
let fa = make_frame(255, 0, 0, 2, 2, 0.0);
let fb = make_frame(0, 0, 255, 2, 2, 0.0);
let result = blend_frames(&fa, &fb, 1.0).expect("blend should succeed");
for chunk in result.chunks(3) {
assert!(chunk[0] > 200, "should be fully red");
}
}
#[test]
fn test_blend_frames_half() {
let fa = make_frame(200, 200, 200, 2, 2, 0.0);
let fb = make_frame(100, 100, 100, 2, 2, 0.0);
let result = blend_frames(&fa, &fb, 0.5).expect("blend should succeed");
for chunk in result.chunks(3) {
let v = chunk[0];
assert!((v as i32 - 150).abs() <= 1, "expected ~150, got {v}");
}
}
#[test]
fn test_blend_frames_dimension_mismatch() {
let fa = make_frame(0, 0, 0, 4, 4, 0.0);
let fb = make_frame(0, 0, 0, 2, 2, 0.0);
assert!(blend_frames(&fa, &fb, 0.5).is_none());
}
#[test]
fn test_estimate_motion_identical_frames() {
let fa = make_frame(128, 64, 32, 8, 8, 0.0);
let fb = fa.clone();
let mv = estimate_motion(&fa, &fb, 4, 2);
assert_eq!(mv.dx, 0);
assert_eq!(mv.dy, 0);
}
#[test]
fn test_motion_vector_zero() {
let mv = MotionVector::ZERO;
assert_eq!(mv.dx, 0);
assert_eq!(mv.dy, 0);
}
#[test]
fn test_estimate_motion_shifted_frame() {
let w = 8u32;
let h = 8u32;
let count = (w as usize) * (h as usize);
let mut pixels_a = vec![50u8; count * 3];
for y in 0..4usize {
for x in 0..4usize {
let base = (y * w as usize + x) * 3;
pixels_a[base] = 200;
pixels_a[base + 1] = 200;
pixels_a[base + 2] = 200;
}
}
let fa = VideoFrame::new(pixels_a.clone(), w, h, 0.0).expect("frame ok");
let warped = warp_frame(&fa, 2, 1);
let fb = VideoFrame::new(warped, w, h, 0.04).expect("frame ok");
let mv = estimate_motion(&fa, &fb, 4, 4);
assert!(
mv.dx.abs() <= 3 && mv.dy.abs() <= 2,
"motion vector ({},{}) too large",
mv.dx,
mv.dy
);
}
#[test]
fn test_motion_compensated_at_t0() {
let fa = make_frame(100, 0, 0, 4, 4, 0.0);
let fb = make_frame(200, 0, 0, 4, 4, 0.04);
let result = motion_compensated_interpolate(&fa, &fb, 0.0, 4, 2);
assert!(result.is_some());
let out = result.expect("mci should succeed");
for chunk in out.chunks(3) {
assert!(
(chunk[0] as i32 - 100).abs() <= 20,
"expected near 100, got {}",
chunk[0]
);
}
}
#[test]
fn test_motion_compensated_output_size() {
let fa = make_frame(0, 128, 0, 4, 4, 0.0);
let fb = make_frame(0, 200, 0, 4, 4, 0.04);
let result = motion_compensated_interpolate(&fa, &fb, 0.5, 4, 2);
assert!(result.is_some());
let out = result.expect("mci should succeed");
assert_eq!(out.len(), 4 * 4 * 3);
}
#[test]
fn test_motion_compensated_dimension_mismatch() {
let fa = make_frame(0, 0, 0, 4, 4, 0.0);
let fb = make_frame(0, 0, 0, 8, 4, 0.0);
assert!(motion_compensated_interpolate(&fa, &fb, 0.5, 4, 2).is_none());
}
#[test]
fn test_compute_schedule_24_to_30() {
let cfg = TemporalScalingConfig::new(FrameRate::film(), FrameRate::fps30());
let scaler = TemporalScaler::new(cfg);
let schedule = scaler.compute_schedule(5);
for &(a, b, t) in &schedule {
assert!(a < 5, "a={a} out of range");
assert!(b < 5, "b={b} out of range");
assert!(b >= a, "b must be >= a");
assert!(t >= 0.0 && t <= 1.0, "t={t} out of [0,1]");
}
assert!(!schedule.is_empty());
}
#[test]
fn test_compute_schedule_too_short() {
let cfg = TemporalScalingConfig::new(FrameRate::film(), FrameRate::fps30());
let scaler = TemporalScaler::new(cfg);
assert!(scaler.compute_schedule(0).is_empty());
assert!(scaler.compute_schedule(1).is_empty());
}
#[test]
fn test_compute_schedule_identity() {
let cfg = TemporalScalingConfig::new(FrameRate::fps30(), FrameRate::fps30());
let scaler = TemporalScaler::new(cfg);
let schedule = scaler.compute_schedule(10);
assert_eq!(schedule.len(), 10);
for &(_, _, t) in &schedule {
assert!(t < 1e-9, "t should be 0 for identity conversion, got {t}");
}
}
#[test]
fn test_convert_frame_blend() {
let frames: Vec<VideoFrame> = (0..5)
.map(|i| make_frame(i * 50, 0, 0, 4, 4, i as f64 / 24.0))
.collect();
let cfg = TemporalScalingConfig::new(FrameRate::film(), FrameRate::fps30());
let scaler = TemporalScaler::new(cfg);
let result = scaler.convert(&frames);
assert!(result.is_some());
let output = result.expect("conversion should succeed");
assert!(!output.is_empty());
for frame in &output {
assert_eq!(frame.len(), 4 * 4 * 3);
}
}
#[test]
fn test_convert_motion_compensated() {
let frames: Vec<VideoFrame> = (0..4)
.map(|i| make_frame(100 + i * 20, 0, 0, 8, 8, i as f64 / 25.0))
.collect();
let cfg = TemporalScalingConfig::new(FrameRate::pal(), FrameRate::fps30())
.with_mode(InterpolationMode::MotionCompensated)
.with_block_size(4)
.with_search_radius(2);
let scaler = TemporalScaler::new(cfg);
let result = scaler.convert(&frames);
assert!(result.is_some());
let output = result.expect("conversion should succeed");
for frame in &output {
assert_eq!(frame.len(), 8 * 8 * 3);
}
}
#[test]
fn test_convert_too_few_frames() {
let frames = vec![make_frame(0, 0, 0, 4, 4, 0.0)];
let cfg = TemporalScalingConfig::new(FrameRate::film(), FrameRate::fps30());
let scaler = TemporalScaler::new(cfg);
assert!(scaler.convert(&frames).is_none());
}
#[test]
fn test_convert_dimension_mismatch() {
let fa = make_frame(0, 0, 0, 4, 4, 0.0);
let fb = make_frame(0, 0, 0, 8, 8, 0.04);
let cfg = TemporalScalingConfig::new(FrameRate::film(), FrameRate::fps30());
let scaler = TemporalScaler::new(cfg);
assert!(scaler.convert(&[fa, fb]).is_none());
}
#[test]
fn test_config_builder() {
let cfg = TemporalScalingConfig::new(FrameRate::ntsc(), FrameRate::fps60())
.with_mode(InterpolationMode::MotionCompensated)
.with_block_size(8)
.with_search_radius(16);
assert_eq!(cfg.mode, InterpolationMode::MotionCompensated);
assert_eq!(cfg.block_size, 8);
assert_eq!(cfg.search_radius, 16);
}
#[test]
fn test_config_block_size_minimum() {
let cfg =
TemporalScalingConfig::new(FrameRate::film(), FrameRate::fps30()).with_block_size(1); assert_eq!(cfg.block_size, 4);
}
}