use crate::{
audio::utils::has_audio,
common::{errors::*, utils::*},
downloader::youtube,
pipeline::video_decoder::{self, VideoDecoder},
};
use either::Either;
use gif;
use image::{DynamicImage, ImageReader};
use libwebp_sys as webp;
use std::{
fs::File,
io::{Read, Write},
path::Path,
};
use tempfile::{tempdir, TempPath};
use url::Url;
pub enum FrameIterator {
Image(Option<DynamicImage>),
Video(VideoDecoder),
AnimatedImage {
frames: Vec<DynamicImage>,
current_frame: usize,
},
}
pub struct MediaData {
pub frame_iter: FrameIterator,
pub fps: Option<f64>,
pub audio_path: Option<Either<TempPath, String>>,
}
impl Iterator for FrameIterator {
type Item = DynamicImage;
fn next(&mut self) -> Option<Self::Item> {
match self {
FrameIterator::Image(img) => img.take(),
FrameIterator::Video(video) => video.next_frame(),
FrameIterator::AnimatedImage {
frames,
current_frame,
} => {
if *current_frame == frames.len() {
None
} else {
let frame = frames.get(*current_frame).cloned();
*current_frame += 1;
frame
}
}
}
}
}
impl FrameIterator {
pub fn skip_frames(&mut self, n: usize) {
match self {
FrameIterator::Image(_) => {
}
FrameIterator::Video(video) => {
video.skip_frames(n);
}
FrameIterator::AnimatedImage {
current_frame,
frames,
} => {
*current_frame = (*current_frame + n) % frames.len();
}
}
}
pub fn reset(&mut self) {
match self {
FrameIterator::Image(_) => {
}
FrameIterator::Video(video) => {
video.reset();
}
FrameIterator::AnimatedImage {
current_frame,
..
} => {
*current_frame = 0;
}
}
}
pub fn is_at_end(&self) -> bool {
match self {
FrameIterator::Image(img) => img.is_none(),
FrameIterator::Video(video) => video.is_at_end(),
FrameIterator::AnimatedImage {
frames,
current_frame,
} => *current_frame >= frames.len(),
}
}
pub fn seek_seconds(&mut self, seconds: f64, fps: f64) -> bool {
match self {
FrameIterator::Image(_) => false,
FrameIterator::Video(video) => video.seek_seconds(seconds),
FrameIterator::AnimatedImage {
current_frame,
frames,
} => {
if frames.is_empty() {
return false;
}
let frames_to_seek = (seconds * fps).round() as i64;
let new_frame = (*current_frame as i64 + frames_to_seek)
.rem_euclid(frames.len() as i64) as usize;
*current_frame = new_frame;
true
}
}
}
pub fn seek_to_frame(&mut self, target_frame: usize) {
match self {
FrameIterator::Image(_) => {}
FrameIterator::Video(video) => {
video.seek_to_frame(target_frame);
}
FrameIterator::AnimatedImage {
current_frame,
frames,
} => {
*current_frame = target_frame.min(frames.len().saturating_sub(1));
}
}
}
pub fn is_streaming(&self) -> bool {
matches!(self, FrameIterator::Video(video) if video.is_streaming())
}
pub fn dimensions(&self) -> Option<(u32, u32)> {
match self {
FrameIterator::Image(Some(img)) => Some((img.width(), img.height())),
FrameIterator::Image(None) => None,
FrameIterator::Video(video) => Some(video.dimensions()),
FrameIterator::AnimatedImage { frames, .. } => {
frames.first().map(|f| (f.width(), f.height()))
}
}
}
pub fn seek_to_seconds(&mut self, seconds: f64, fps: f64) -> bool {
match self {
FrameIterator::Image(_) => false,
FrameIterator::Video(video) => video.seek_to_seconds_abs(seconds),
FrameIterator::AnimatedImage {
current_frame,
frames,
} => {
if frames.is_empty() {
return false;
}
let target_frame = (seconds * fps).round() as usize;
*current_frame = target_frame.min(frames.len().saturating_sub(1));
true
}
}
}
pub fn duration_secs(&self) -> Option<f64> {
match self {
FrameIterator::Image(_) => None,
FrameIterator::Video(video) => video.duration_secs(),
FrameIterator::AnimatedImage { .. } => {
None
}
}
}
pub fn get_position_frames(&self) -> i64 {
match self {
FrameIterator::Image(_) => 0,
FrameIterator::Video(video) => video.get_position_frames(),
FrameIterator::AnimatedImage { current_frame, .. } => *current_frame as i64,
}
}
}
pub fn open_media(path: String, broswer: String) -> Result<MediaData, MyError> {
if video_decoder::is_stream_url(&path) && !path.starts_with("http") {
let frame_iter = open_video_from_url(&path)?;
let fps = extract_fps(&path);
let audio = if has_audio(&path).unwrap_or(false) {
Some(Either::Right(path))
} else {
None
};
return Ok(MediaData {
frame_iter,
fps,
audio_path: audio,
});
}
if let Ok(url) = Url::parse(path.as_str()) {
if let Some(domain) = url.domain() {
if domain.ends_with("youtube.com") || domain.ends_with("youtu.be") {
if let Ok(streaming_url) =
youtube::get_streaming_url(path.as_str(), broswer.as_str())
{
if let Ok(frame_iter) = open_video_from_url(&streaming_url) {
let fps = extract_fps(&streaming_url);
return Ok(MediaData {
frame_iter,
fps,
audio_path: Some(Either::Right(streaming_url)),
});
}
}
let video = youtube::download_video(path.as_str(), broswer.as_str())?;
let fps = extract_fps(video.as_os_str().to_str().unwrap_or(""));
let video_open = open_video(&video)?;
return Ok(MediaData {
frame_iter: video_open,
fps,
audio_path: Some(Either::Left(video)),
});
} else {
let url_str = url.as_str().to_string();
let name = url
.path_segments()
.and_then(|s| s.last())
.unwrap_or("unknown_media");
let ext = Path::new(name)
.extension()
.and_then(std::ffi::OsStr::to_str);
let is_image = matches!(
ext,
Some("png") | Some("jpg") | Some("jpeg") | Some("bmp")
| Some("gif") | Some("webp") | Some("tif") | Some("tiff")
| Some("ico")
);
let is_streaming_manifest = matches!(ext, Some("m3u8") | Some("mpd"));
if !is_image {
match open_video_from_url(&url_str) {
Ok(frame_iter) => {
let fps = extract_fps(&url_str);
let audio = if has_audio(&url_str).unwrap_or(false) {
Some(Either::Right(url_str))
} else {
None
};
return Ok(MediaData {
frame_iter,
fps,
audio_path: audio,
});
}
Err(e) if is_streaming_manifest => return Err(e),
Err(_) => {} }
}
let tmp = tempdir()?;
let p = tmp.path().join(name);
download_url_to_file(p.as_path(), url)?;
open_media_from_path(p.as_os_str().to_str().unwrap_or(""), p.as_path())
}
} else {
open_media_from_path(path.as_str(), &Path::new(path.as_str()))
}
} else {
open_media_from_path(path.as_str(), &Path::new(path.as_str()))
}
}
fn open_media_from_path(path_str: &str, path: &Path) -> Result<MediaData, MyError> {
let fps = extract_fps(path_str);
let audio = has_audio(path_str)?;
let audio_track = if audio {
Some(Either::Right(path_str.to_owned()))
} else {
None
};
let ext = path.extension().and_then(std::ffi::OsStr::to_str);
match ext {
Some("png") | Some("bmp") | Some("ico") | Some("tif") | Some("tiff") | Some("jpg")
| Some("jpeg") => Ok(MediaData {
frame_iter: open_image(path)?,
fps: None,
audio_path: None,
}),
Some("mp4") | Some("avi") | Some("webm") | Some("mkv") | Some("mov") | Some("flv")
| Some("ogg") => Ok(MediaData {
frame_iter: open_video(path)?,
fps,
audio_path: audio_track,
}),
Some("gif") => {
let (frame_iter, fps) = open_gif(path)?;
Ok(MediaData {
frame_iter,
fps: Some(fps),
audio_path: None,
})
}
Some("webp") => {
let (frame_iter, fps) = open_webp(path)?;
Ok(MediaData {
frame_iter,
fps: Some(fps),
audio_path: None,
})
}
_ => Ok(MediaData {
frame_iter: open_video(path)?,
fps,
audio_path: audio_track,
}),
}
}
fn download_url_to_file(path: &Path, url: Url) -> Result<(), MyError> {
let tmp_file = File::create(path)?;
let mut tmp_file = std::io::BufWriter::new(tmp_file);
tmp_file
.write_all(
reqwest::blocking::get(url)
.and_then(|resp| resp.bytes())
.map_err(|err| {
MyError::Application(format!(
"{error}: {err:?}",
error = ERROR_DOWNLOADING_RESOURCE
))
})?
.as_ref(),
)
.map_err(|err| {
MyError::Application(format!("{error}: {err:?}", error = ERROR_OPENING_RESOURCE))
})
}
fn open_image(path: &Path) -> Result<FrameIterator, MyError> {
let img = ImageReader::open(path)?.decode().map_err(|e| {
MyError::Application(format!("{error}: {e:?}", error = ERROR_DECODING_IMAGE))
})?;
Ok(FrameIterator::Image(Some(img)))
}
fn open_video(path: &Path) -> Result<FrameIterator, MyError> {
let path_str = path
.to_str()
.ok_or_else(|| MyError::Application(ERROR_OPENING_VIDEO.to_string()))?;
let decoder = VideoDecoder::open(path_str)?;
Ok(FrameIterator::Video(decoder))
}
fn open_video_from_url(url: &str) -> Result<FrameIterator, MyError> {
let decoder = VideoDecoder::open(url)?;
Ok(FrameIterator::Video(decoder))
}
fn open_gif(path: &Path) -> Result<(FrameIterator, f64), MyError> {
let file = File::open(path).map_err(|e| {
MyError::Application(format!("{error}: {e:?}", error = ERROR_OPENING_RESOURCE))
})?;
let mut options = gif::DecodeOptions::new();
options.set_color_output(gif::ColorOutput::Indexed);
let mut decoder = options.read_info(file).map_err(|e| {
MyError::Application(format!("{error}: {e:?}", error = ERROR_READING_GIF_HEADER))
})?;
let mut delay: u64 = 0;
let mut frames = Vec::new();
let mut screen = gif_dispose::Screen::new_decoder(&decoder);
while let Ok(Some(frame)) = decoder.read_next_frame() {
delay += frame.delay as u64;
screen.blit_frame(&frame).map_err(|e| {
MyError::Application(format!("{error}: {e:?}", error = ERROR_DECODING_IMAGE))
})?;
let (buf, width, height) = screen.pixels_rgba().to_contiguous_buf();
frames.push(DynamicImage::ImageRgba8(image::RgbaImage::from_fn(
width as u32,
height as u32,
|x, y| {
let rgba = buf.as_ref()[y as usize * width + x as usize];
image::Rgba([rgba.r, rgba.g, rgba.b, rgba.a])
},
)));
}
let fps = frames.len() as f64 / (delay.max(1) as f64 / 100.0);
Ok((
FrameIterator::AnimatedImage {
frames,
current_frame: 0,
},
fps,
))
}
fn open_webp(path: &Path) -> Result<(FrameIterator, f64), MyError> {
let mut file = File::open(path).map_err(|e| {
MyError::Application(format!("{error}: {e:?}", error = ERROR_OPENING_RESOURCE))
})?;
let mut buf = Vec::new();
file.read_to_end(&mut buf)?;
let mut frames = Vec::new();
let mut first_timestamp: i32 = i32::MAX;
let mut last_timestamp: i32 = i32::MIN;
unsafe {
let mut options = webp::WebPAnimDecoderOptions {
color_mode: webp::WEBP_CSP_MODE::MODE_RGBA,
use_threads: 0,
padding: [0, 0, 0, 0, 0, 0, 0],
};
webp::WebPAnimDecoderOptionsInit(&mut options);
let dec = webp::WebPAnimDecoderNew(
&webp::WebPData {
bytes: buf.as_ptr(),
size: buf.len(),
},
&options,
);
let mut info = webp::WebPAnimInfo::default();
webp::WebPAnimDecoderGetInfo(dec, &mut info);
let frame_sz = (info.canvas_width * info.canvas_height * 4) as usize;
for _ in 0..info.loop_count {
while webp::WebPAnimDecoderHasMoreFrames(dec) != 0 {
let mut buf: *mut u8 = std::ptr::null_mut();
let mut timestamp: i32 = 0;
webp::WebPAnimDecoderGetNext(dec, &mut buf, &mut timestamp);
first_timestamp = first_timestamp.min(timestamp);
last_timestamp = last_timestamp.max(timestamp);
if let Some(image) = image::RgbaImage::from_raw(
info.canvas_width,
info.canvas_height,
std::slice::from_raw_parts(buf, frame_sz).to_vec(),
) {
frames.push(DynamicImage::ImageRgba8(image));
} else {
}
}
webp::WebPAnimDecoderReset(dec);
}
webp::WebPAnimDecoderDelete(dec);
}
let fps = frames.len() as f64
/ ((last_timestamp.saturating_sub(first_timestamp).max(1)) as f64 / 1000.0);
Ok((
FrameIterator::AnimatedImage {
frames,
current_frame: 0,
},
fps,
))
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_animated_image_seek_forward() {
let frames: Vec<DynamicImage> = (0..100).map(|_| DynamicImage::new_rgb8(1, 1)).collect();
let mut frame_iter = FrameIterator::AnimatedImage {
frames,
current_frame: 0,
};
let fps = 30.0;
frame_iter.seek_seconds(1.0, fps);
if let FrameIterator::AnimatedImage { current_frame, .. } = frame_iter {
assert_eq!(current_frame, 30);
} else {
panic!("Expected AnimatedImage variant");
}
}
#[test]
fn test_animated_image_seek_backward() {
let frames: Vec<DynamicImage> = (0..100).map(|_| DynamicImage::new_rgb8(1, 1)).collect();
let mut frame_iter = FrameIterator::AnimatedImage {
frames,
current_frame: 50,
};
let fps = 30.0;
frame_iter.seek_seconds(-1.0, fps);
if let FrameIterator::AnimatedImage { current_frame, .. } = frame_iter {
assert_eq!(current_frame, 20);
} else {
panic!("Expected AnimatedImage variant");
}
}
#[test]
fn test_animated_image_seek_wraps_around() {
let frames: Vec<DynamicImage> = (0..100).map(|_| DynamicImage::new_rgb8(1, 1)).collect();
let mut frame_iter = FrameIterator::AnimatedImage {
frames,
current_frame: 10,
};
let fps = 30.0;
frame_iter.seek_seconds(-1.0, fps);
if let FrameIterator::AnimatedImage { current_frame, .. } = frame_iter {
assert_eq!(current_frame, 80);
} else {
panic!("Expected AnimatedImage variant");
}
}
#[test]
fn test_image_seek_is_noop() {
let img = DynamicImage::new_rgb8(1, 1);
let mut frame_iter = FrameIterator::Image(Some(img));
let result = frame_iter.seek_seconds(5.0, 30.0);
assert!(!result);
}
#[test]
fn test_animated_image_is_at_end() {
let frames: Vec<DynamicImage> = (0..10).map(|_| DynamicImage::new_rgb8(1, 1)).collect();
let frame_iter = FrameIterator::AnimatedImage {
frames: frames.clone(),
current_frame: 0,
};
assert!(!frame_iter.is_at_end());
let frame_iter = FrameIterator::AnimatedImage {
frames: frames.clone(),
current_frame: 5,
};
assert!(!frame_iter.is_at_end());
let frame_iter = FrameIterator::AnimatedImage {
frames: frames.clone(),
current_frame: 10,
};
assert!(frame_iter.is_at_end());
let frame_iter = FrameIterator::AnimatedImage {
frames,
current_frame: 15,
};
assert!(frame_iter.is_at_end());
}
#[test]
fn test_image_is_at_end() {
let frame_iter = FrameIterator::Image(Some(DynamicImage::new_rgb8(1, 1)));
assert!(!frame_iter.is_at_end());
let frame_iter = FrameIterator::Image(None);
assert!(frame_iter.is_at_end());
}
}