fframes 1.0.0-beta.7

A video generation framework
use crate::{media::GeneralVideoFileMetadata, media::ImageData, FFramesContext};
use fframes_media_loaders::VideoMedia;
use std::sync::Arc;

#[cfg(not(target_arch = "wasm32"))]
use crate::error::Result;
#[cfg(not(target_arch = "wasm32"))]
use fframes_media_loaders::ImageMetadata;
pub use fframes_media_loaders::{FrameConvertOptions, ResizeVideoFrame};
#[cfg(not(target_arch = "wasm32"))]
use std::collections::HashMap;

pub trait FFramesSyncedVideoFrame {
    /// Gets the original image from the frame
    /// This will perform color space conversion but will preserve the original frame
    /// size. If you need to render the video frame in the smaller size, use `into_resized_image`.
    ///
    /// This method take self to reuse frame data allocations, but you should not
    /// call this method more than once per the render function
    #[allow(clippy::wrong_self_convention)]
    fn into_image(&self) -> ImageData;
    /// Performs color space conversion and resizes the image to the specified size.
    /// Recommended to use this method if you render image in the smaller size than the original
    /// video e.g. as a pattern of some other shape or as a filler of standalone image.
    ///
    /// This method take self to reuse frame data allocations, but you should not
    /// call this method more than once per the render function
    #[allow(clippy::wrong_self_convention)]
    fn into_resized_image(&self, options: &FrameConvertOptions) -> Option<ImageData>;
    /// Get the native height of the frame
    fn height(&self) -> u32;
    /// Get the native size of the frame
    fn width(&self) -> u32;
    /// Get the frame index of the origin video
    fn timestamp_seconds(&self) -> f64;
    /// Returns the native frame timestamp in the decoders' timebase
    fn timestamp(&self) -> i64;
    /// Get the FPS of the origin video
    fn stream_fps(&self) -> f64;
    /// Get the duration of the origin video in frames
    fn stream_duration_in_seconds(&self) -> f64;
}

#[derive(Debug, Clone, Default)]
pub struct SyncVideoFrameInput {
    /// The frame of the fframes' video to start from.
    /// If not provided starts from the beginning of the fframes' video.
    ///
    /// It is possible to leverage the scene's offset to start video from the scene start:
    ///
    /// ```no_run
    /// use fframes::{Frame, SyncVideoFrameInput};
    ///
    /// let scene_info = ctx.get_scene_info();
    ///
    /// fnrame.get_synced_video_frame(ctx, "video.mp4", SyncVideoFrameInput {
    ///    start_from: scene_info.map(|info| info.start_frame),
    ///    ..Default::default()
    /// });
    ///
    /// ```
    pub start_from: usize,
    /// If `true` the video playback will be looped and will start from the beginning on the end.
    /// Also if `true` the `get_synced_video_frame` API should never return None, only in case of
    /// decoding errors, so in some cases it might be okay to panic if the frame is not available.
    pub looping: bool,
}

/// This is used by the editor as a fallback for the static image
#[derive(Debug, Clone)]
pub struct WasmEditorVideoFrameFallback {
    fallback_image: ImageData,
    video_metadata: GeneralVideoFileMetadata,
}

impl FFramesSyncedVideoFrame for WasmEditorVideoFrameFallback {
    fn into_image(&self) -> ImageData {
        self.fallback_image.clone()
    }

    fn into_resized_image(&self, _options: &FrameConvertOptions) -> Option<ImageData> {
        Some(self.fallback_image.clone())
    }

    fn height(&self) -> u32 {
        self.video_metadata.height
    }

    fn width(&self) -> u32 {
        self.video_metadata.width
    }

    fn timestamp_seconds(&self) -> f64 {
        unimplemented!(
            "Timestamp seconds are not supported in the editor. Please mock this data for editing."
        )
    }

    fn timestamp(&self) -> i64 {
        unimplemented!(
            "Timestamp seconds are not supported in the editor. Please mock this data for editing."
        )
    }

    fn stream_fps(&self) -> f64 {
        self.video_metadata.fps
    }

    fn stream_duration_in_seconds(&self) -> f64 {
        self.video_metadata.duration
    }
}

#[cfg(not(target_arch = "wasm32"))]
impl FFramesSyncedVideoFrame for fframes_media_loaders::FFmpegFrameBuf {
    fn into_image(&self) -> ImageData {
        let image_data = unsafe { self.scale_and_return_latest_frame() };

        let filename = image_data.id.clone();
        ImageData::new_from_raw_data(
            image_data,
            filename,
            ImageMetadata {
                width: self.get_width(),
                height: self.get_height(),
            },
        )
    }

    fn into_resized_image(&self, options: &FrameConvertOptions) -> Option<ImageData> {
        let image_data = unsafe {
            self.reinit_sws_context(options)
                .map_err(|e| {
                    crate::log!("Error while decoding video frame: {:?}", e);
                })
                .ok()?;

            self.scale_and_return_latest_frame()
        };

        let filename = image_data.id.clone();
        Some(ImageData::new_from_raw_data(
            image_data,
            filename,
            ImageMetadata {
                width: self.get_width(),
                height: self.get_height(),
            },
        ))
    }

    fn height(&self) -> u32 {
        self.get_width()
    }

    fn width(&self) -> u32 {
        self.get_height()
    }

    fn timestamp_seconds(&self) -> f64 {
        unsafe { self.timestamp_seconds() }
    }

    fn timestamp(&self) -> i64 {
        unsafe { self.get_pts() }
    }

    fn stream_duration_in_seconds(&self) -> f64 {
        unsafe { self.get_stream_duration_in_frames() }
    }

    fn stream_fps(&self) -> f64 {
        self.get_stream_fps()
    }
}

#[derive(Clone)]
#[allow(dead_code)]
pub struct VideoDecodersWorker {
    buffer_size: usize,
    #[cfg(not(target_arch = "wasm32"))]
    map: Arc<std::sync::RwLock<HashMap<String, fframes_media_loaders::FFmpegDecoder>>>,
}

impl std::fmt::Debug for VideoDecodersWorker {
    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
        f.debug_struct("WorkerLocalDecoders").finish()
    }
}

impl Default for VideoDecodersWorker {
    fn default() -> Self {
        Self::new(1)
    }
}

impl VideoDecodersWorker {
    pub fn new(buffer_size: usize) -> Self {
        VideoDecodersWorker {
            buffer_size,
            #[cfg(not(target_arch = "wasm32"))]
            map: Arc::new(std::sync::RwLock::new(HashMap::new())),
        }
    }

    #[cfg(target_arch = "wasm32")]
    pub(crate) fn get_synced_frame(
        &self,
        media_ref: &VideoMedia,
        _offset: i64,
        ctx: &FFramesContext,
        _options: &SyncVideoFrameInput,
    ) -> crate::error::Result<Option<Arc<impl FFramesSyncedVideoFrame>>> {
        let filename = media_ref.path.file_name().unwrap().to_string_lossy();
        let video_filename = filename.as_ref();

        if let Some(image) = ctx
            .get_image(format!("{video_filename}_fallback.png"))
            .or_else(|| ctx.get_image(format!("{video_filename}.fallback.jpg")))
        {
            Ok(Some(Arc::new(WasmEditorVideoFrameFallback {
                fallback_image: image.clone(),
                video_metadata: media_ref
                    .metadata
                    .expect("Failed precodition: Missing editor video file metadata"),
            })))
        } else {
            Ok(None)
        }
    }

    #[cfg(not(target_arch = "wasm32"))]
    pub fn get_buffered_frame_image(
        &self,
        pts: i64,
        filename: &str,
    ) -> Result<Option<Arc<usvgr::PreloadedImageData>>> {
        let frame = self
            .map
            .read()
            .unwrap()
            .get(filename)
            .and_then(|decoder| decoder.get_decoded_image_in_buf(pts));

        Ok(frame)
    }

    #[cfg(not(target_arch = "wasm32"))]
    pub(crate) fn get_synced_frame(
        &self,
        VideoMedia { path, .. }: &VideoMedia,
        mut offset: i64,
        ctx: &FFramesContext,
        options: &SyncVideoFrameInput,
    ) -> Result<Option<Arc<impl FFramesSyncedVideoFrame>>> {
        use crate::error::FFramesError;
        use fframes_media_loaders::FFramesMediaError;

        let resource_name = path
            .file_name()
            .ok_or_else(|| FFramesError::MediaError(FFramesMediaError::MediaDirectoryProvided))?
            .to_string_lossy()
            .to_string();

        // It is important to lock it for the whole duration of he function to
        // avoid any parallel encoder creation.
        let mut map_guard = self.map.write()?;
        let needs_new_decoder = !map_guard.contains_key(&resource_name);

        // Create new decoder if needed
        if needs_new_decoder {
            let decoder = unsafe {
                let mut decoder = fframes_media_loaders::FFmpegDecoder::new(
                    path,
                    ctx.time_base.fps,
                    self.buffer_size,
                )?;

                if offset > 0 {
                    decoder.seek_to_offset(offset)?;
                }

                decoder
            };

            map_guard.insert(resource_name.clone(), decoder);
        }

        unsafe {
            // Use write lock for modifying decoder state
            let decoder = map_guard.get_mut(&resource_name).unwrap();

            if options.looping {
                offset = decoder.adjust_offset_for_looping(offset)?;
            }

            let has_frame = decoder.decode_up_to(offset)?;
            match has_frame {
                true => Ok(Some(decoder.get_raw_frame())),
                false if options.looping => {
                    decoder.seek_to_offset(0)?;
                    let has_first_frame = decoder.decode_up_to(0)?;

                    decoder.current_loop += 1;
                    Ok(has_first_frame.then_some(decoder.get_raw_frame()))
                }
                false => Ok(None),
            }
        }
    }
}

// These implementations are already thread-safe due to RwLock
unsafe impl Send for VideoDecodersWorker {}
unsafe impl Sync for VideoDecodersWorker {}