mod audio_resampling;
mod compositor;
mod inner;
mod runner;
mod runner_layout;
mod state;
mod types;
use std::path::PathBuf;
use std::sync::atomic::{AtomicBool, AtomicU64};
use std::sync::{Arc, Mutex, mpsc};
use std::time::{Duration, Instant};
use crate::audio::{AudioMixer, AudioTrackHandle};
use crate::error::PreviewError;
use crate::event::PlayerEvent;
use crate::playback::SwsRgbaConverter;
use crate::playback::decode_buffer::DecodeBuffer;
use crate::playback::master_clock::MasterClock;
use crate::playback::player_handle::PlayerHandle;
pub use compositor::PreviewCompositor;
pub use inner::apply_xfade;
pub use runner::{Pacing, SceneRunner};
pub use types::{
Scene, SceneAudioPlacement, SceneAudioTrack, ScenePlacement, SceneSource, SceneVideoTrack,
};
use audio_resampling::spawn_audio_track_thread;
use ff_filter::{AnimatedValue, SolidSource, TextSource, XfadeTransition};
use ff_format::VideoFrame;
use state::{
AudioFadeConfig, AudioOnlyTrack, ClipState, ClipVideoSource, LavfiOverlayState, OverlayLayer,
db_to_linear,
};
fn resolve_canvas_dims(scene: &Scene) -> (u32, u32) {
if let Some(dims) = scene.canvas {
return dims;
}
for track in &scene.video_tracks {
for p in &track.placements {
if let Some(path) = p.source.as_file()
&& let Ok(info) = ff_probe::open(path)
&& let Some(v) = info.primary_video()
{
return (v.width(), v.height());
}
}
}
(0, 0)
}
fn generated_held_frame(source: &SceneSource, cw: u32, ch: u32, fps: f64) -> Option<VideoFrame> {
if cw == 0 || ch == 0 {
log::warn!("generated source has no canvas size to render into, cw={cw} ch={ch}");
return None;
}
let pulled = match source {
SceneSource::Solid(color) => {
SolidSource::new(*color, cw, ch, fps).map(|mut s| pull_first(&mut s))
}
SceneSource::Text(spec) => {
TextSource::new(spec, cw, ch, fps).map(|mut s| pull_first(&mut s))
}
SceneSource::File(_) => return None,
};
match pulled {
Ok(Some(frame)) => Some(frame),
Ok(None) => {
log::warn!("generated source produced no frame; rendering nothing");
None
}
Err(e) => {
log::warn!("generated source unavailable, rendering nothing, error={e}");
None
}
}
}
fn generated_span(out_point: Option<Duration>, in_pt: Duration) -> Duration {
if let Some(op) = out_point {
op.saturating_sub(in_pt)
} else {
log::warn!(
"generated clip has no out_point; preview shows zero duration (bound it with a trim)"
);
Duration::ZERO
}
}
fn open_clip_video_source(
source: &SceneSource,
in_pt: Duration,
cw: u32,
ch: u32,
fps: f64,
) -> Result<ClipVideoSource, PreviewError> {
match source.as_file() {
Some(path) => {
let mut buf = DecodeBuffer::open(path).build()?;
if in_pt > Duration::ZERO {
buf.seek(in_pt)?;
}
Ok(ClipVideoSource::File(buf))
}
None => Ok(ClipVideoSource::held(
generated_held_frame(source, cw, ch, fps),
in_pt,
fps,
)),
}
}
fn source_path(source: &SceneSource) -> PathBuf {
source
.as_file()
.map(std::path::Path::to_path_buf)
.unwrap_or_default()
}
fn pull_first<S: GeneratedPull>(source: &mut S) -> Option<VideoFrame> {
for _ in 0..16 {
match source.pull() {
Ok(Some(frame)) => return Some(frame),
Ok(None) => {}
Err(_) => return None,
}
}
None
}
trait GeneratedPull {
fn pull(&mut self) -> Result<Option<VideoFrame>, ff_filter::FilterError>;
}
impl GeneratedPull for SolidSource {
fn pull(&mut self) -> Result<Option<VideoFrame>, ff_filter::FilterError> {
SolidSource::pull(self)
}
}
impl GeneratedPull for TextSource {
fn pull(&mut self) -> Result<Option<VideoFrame>, ff_filter::FilterError> {
TextSource::pull(self)
}
}
const CHANNEL_CAP: usize = 64;
pub struct ScenePlayer;
impl ScenePlayer {
#[allow(clippy::too_many_lines)]
pub fn open(scene: &Scene) -> Result<(SceneRunner, PlayerHandle), PreviewError> {
struct ProbeResult {
source: SceneSource,
in_pt: Duration,
clip_dur: Duration,
offset: Duration,
out_point: Option<Duration>,
xfade_dur: Duration,
xfade_kind: Option<XfadeTransition>,
video_handle: Duration,
has_audio: bool,
video_w: u32,
video_h: u32,
speed: f64,
opacity: f32,
}
let v_tracks = &scene.video_tracks;
if v_tracks.is_empty() || v_tracks[0].placements.is_empty() {
return Err(PreviewError::Ffmpeg {
code: 0,
message: "timeline has no video clips in the primary track".into(),
});
}
let fps = scene.fps.max(1.0);
let canvas = resolve_canvas_dims(scene);
let clip_list = &v_tracks[0].placements;
let mut probes: Vec<ProbeResult> = Vec::with_capacity(clip_list.len());
let mut has_any_audio = false;
for p in clip_list {
let in_pt = p.in_point;
let speed = p.speed;
let (video_w, video_h, unscaled_dur, has_audio) = if let Some(path) = p.source.as_file()
{
let info = ff_probe::open(path)?;
let dur = p.out_point.map_or_else(
|| info.duration().saturating_sub(in_pt),
|op| op.saturating_sub(in_pt),
);
let (w, h) = info
.primary_video()
.map_or((0, 0), |v| (v.width(), v.height()));
(w, h, dur, info.has_audio())
} else {
(
canvas.0,
canvas.1,
generated_span(p.out_point, in_pt),
false,
)
};
let clip_dur = if (speed - 1.0).abs() < 1e-9 {
unscaled_dur
} else {
unscaled_dur.div_f64(speed)
};
has_any_audio |= has_audio;
probes.push(ProbeResult {
source: p.source.clone(),
in_pt,
clip_dur,
offset: p.offset,
out_point: p.out_point,
xfade_dur: p.xfade_dur,
xfade_kind: p.xfade_kind,
video_handle: p.video_handle,
has_audio,
video_w,
video_h,
speed,
opacity: p.opacity,
});
}
let (mut mixer_arc, audio_track_handles): (
Option<Arc<Mutex<AudioMixer>>>,
Vec<Option<AudioTrackHandle>>,
) = if has_any_audio {
let mut mixer = AudioMixer::new(48_000);
let handles: Vec<Option<AudioTrackHandle>> = probes
.iter()
.map(|p| {
if p.has_audio {
Some(mixer.add_track())
} else {
None
}
})
.collect();
(Some(Arc::new(Mutex::new(mixer))), handles)
} else {
(None, probes.iter().map(|_| None).collect())
};
let mut clip_states: Vec<ClipState> = Vec::with_capacity(probes.len());
for (i, p) in probes.iter().enumerate() {
let timeline_start = p.offset;
let timeline_end = timeline_start + p.clip_dur;
let decode_buf = open_clip_video_source(&p.source, p.in_pt, p.video_w, p.video_h, fps)?;
if let (Some(handle), AnimatedValue::Static(db)) =
(&audio_track_handles[i], &clip_list[i].volume)
&& *db != 0.0
{
handle.set_volume(db_to_linear(*db));
}
if let Some(handle) = &audio_track_handles[i] {
let pan0 = clip_list[i].pan.value_at(Duration::ZERO);
if pan0 != 0.0 {
#[allow(clippy::cast_possible_truncation)]
handle.set_pan(pan0 as f32);
}
}
clip_states.push(ClipState {
source: p.source.clone(),
decode_buf,
timeline_start,
timeline_end,
in_point: p.in_pt,
out_point: p.out_point,
xfade_dur: p.xfade_dur,
xfade_kind: p.xfade_kind,
video_handle: p.video_handle,
audio_track: audio_track_handles[i].clone(),
speed: p.speed,
opacity: p.opacity,
layer_desc: clip_list[i].layer.clone(),
volume: clip_list[i].volume.clone(),
fade_in: clip_list[i].fade_in,
fade_out: clip_list[i].fade_out,
pitch: clip_list[i].pitch,
});
}
let mut audio_only_tracks: Vec<AudioOnlyTrack> = Vec::new();
let mut overlay_layers: Vec<OverlayLayer> = Vec::new();
for layer in v_tracks.iter().skip(1) {
if layer.placements.is_empty() {
continue;
}
let mut layer_clips: Vec<ClipState> = Vec::new();
for p in &layer.placements {
let in_pt = p.in_point;
let (clip_dur, has_audio) = match p.source.as_file() {
Some(path) => {
let info = ff_probe::open(path)?;
let dur = p.out_point.map_or_else(
|| info.duration().saturating_sub(in_pt),
|op| op.saturating_sub(in_pt),
);
(dur, info.has_audio())
}
None => (generated_span(p.out_point, in_pt), false),
};
let timeline_start = p.offset;
let timeline_end = timeline_start + clip_dur;
let decode_buf = open_clip_video_source(&p.source, in_pt, canvas.0, canvas.1, fps)?;
if has_audio {
let mixer_ref = mixer_arc
.get_or_insert_with(|| Arc::new(Mutex::new(AudioMixer::new(48_000))));
let handle = mixer_ref
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.add_track();
if let AnimatedValue::Static(db) = &p.volume
&& *db != 0.0
{
handle.set_volume(db_to_linear(*db));
}
let pan0 = p.pan.value_at(Duration::ZERO);
if pan0 != 0.0 {
#[allow(clippy::cast_possible_truncation)]
handle.set_pan(pan0 as f32);
}
audio_only_tracks.push(AudioOnlyTrack {
source: source_path(&p.source),
timeline_start,
timeline_end,
in_point: in_pt,
fade_in: p.fade_in,
fade_out: p.fade_out,
clip_dur,
speed: p.speed,
pitch: p.pitch,
handle,
volume: p.volume.clone(),
cancel: None,
thread: None,
});
}
layer_clips.push(ClipState {
source: p.source.clone(),
decode_buf,
timeline_start,
timeline_end,
in_point: in_pt,
out_point: p.out_point,
xfade_dur: Duration::ZERO,
xfade_kind: None,
video_handle: Duration::ZERO,
audio_track: None,
speed: p.speed,
opacity: p.opacity,
layer_desc: p.layer.clone(),
volume: p.volume.clone(),
fade_in: p.fade_in,
fade_out: p.fade_out,
pitch: p.pitch,
});
}
overlay_layers.push(OverlayLayer {
clips: layer_clips,
active: 0,
sws: SwsRgbaConverter::new(),
rgba: Vec::new(),
cur_dims: None,
pending: None,
});
}
for track in &scene.audio_tracks {
for p in &track.placements {
let in_pt = p.in_point;
let info = ff_probe::open(&p.source)?;
if !info.has_audio() {
continue;
}
let clip_dur = p.out_point.map_or_else(
|| info.duration().saturating_sub(in_pt),
|op| op.saturating_sub(in_pt),
);
let timeline_start = p.offset;
let timeline_end = timeline_start + clip_dur;
let mixer_ref =
mixer_arc.get_or_insert_with(|| Arc::new(Mutex::new(AudioMixer::new(48_000))));
let handle = mixer_ref
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.add_track();
if let AnimatedValue::Static(db) = &p.volume
&& *db != 0.0
{
handle.set_volume(db_to_linear(*db));
}
let pan0 = p.pan.value_at(Duration::ZERO);
if pan0 != 0.0 {
#[allow(clippy::cast_possible_truncation)]
handle.set_pan(pan0 as f32);
}
audio_only_tracks.push(AudioOnlyTrack {
source: p.source.clone(),
timeline_start,
timeline_end,
in_point: in_pt,
fade_in: p.fade_in,
fade_out: p.fade_out,
clip_dur,
speed: p.speed,
pitch: p.pitch,
handle,
volume: p.volume.clone(),
cancel: None,
thread: None,
});
}
}
let total_dur = clip_states
.iter()
.map(|c| c.timeline_end)
.max()
.unwrap_or(Duration::ZERO);
let duration_millis = u64::try_from(total_dur.as_millis()).unwrap_or(u64::MAX);
let current_pts = Arc::new(AtomicU64::new(0));
let paused = Arc::new(AtomicBool::new(false));
let stopped = Arc::new(AtomicBool::new(false));
let (cmd_tx, cmd_rx) = mpsc::sync_channel(CHANNEL_CAP);
let (event_tx, event_rx) = mpsc::sync_channel::<PlayerEvent>(CHANNEL_CAP);
let first_clip_at_origin = clip_states
.first()
.is_some_and(|c| c.timeline_start == Duration::ZERO);
let (initial_audio_cancel, initial_audio_thread) = if first_clip_at_origin {
if let Some(handle) = clip_states.first().and_then(|c| c.audio_track.clone()) {
let source = source_path(&clip_states[0].source);
let in_pt = clip_states[0].in_point;
let clip0_speed = clip_states[0].speed;
let clip0_pitch = clip_states[0].pitch;
let cancel = Arc::new(AtomicBool::new(false));
let thread = spawn_audio_track_thread(
source,
in_pt,
handle,
Arc::clone(&cancel),
AudioFadeConfig {
speed: clip0_speed,
pitch: clip0_pitch,
..AudioFadeConfig::NONE
},
);
(Some(cancel), Some(thread))
} else {
(None, None)
}
} else {
(None, None)
};
let (initial_last_w, initial_last_h) =
probes.first().map_or((0, 0), |p| (p.video_w, p.video_h));
let runner = SceneRunner {
clips: clip_states,
overlay_layers,
audio_only_tracks,
active: 0,
transition: None,
cmd_rx,
event_tx,
sink: None,
gpu_compositor: None,
current_pts: Arc::clone(¤t_pts),
paused: Arc::clone(&paused),
stopped: Arc::clone(&stopped),
fps,
rate: 1.0,
clock: MasterClock::System {
started_at: Instant::now(),
base_pts: Duration::ZERO,
rate: 1.0,
},
resume_pts: Duration::ZERO,
sws_a: SwsRgbaConverter::new(),
sws_b: SwsRgbaConverter::new(),
rgba_a: Vec::new(),
rgba_b: Vec::new(),
blend_buf: Vec::new(),
dissolve_field: Vec::new(),
dissolve_field_dims: (0, 0),
last_frame_w: initial_last_w,
last_frame_h: initial_last_h,
gap_buf: Vec::new(),
audio_mixer: mixer_arc.clone(),
active_audio_cancel: initial_audio_cancel,
active_audio_thread: initial_audio_thread,
composer: None,
composer_key: Vec::new(),
canvas: scene.canvas,
lavfi: scene
.lavfi_overlay
.as_deref()
.and_then(LavfiOverlayState::new),
};
let handle = PlayerHandle::for_timeline(
cmd_tx,
Arc::new(Mutex::new(event_rx)),
current_pts,
paused,
stopped,
duration_millis,
mixer_arc,
);
Ok((runner, handle))
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn resolve_canvas_dims_should_prefer_explicit_canvas_then_fall_back() {
let scene = |canvas| Scene {
fps: 30.0,
canvas,
lavfi_overlay: None,
video_tracks: vec![],
audio_tracks: vec![],
};
assert_eq!(
resolve_canvas_dims(&scene(Some((1920, 1080)))),
(1920, 1080)
);
assert_eq!(resolve_canvas_dims(&scene(None)), (0, 0));
}
#[test]
#[ignore = "requires the color/drawtext filters; run with -- --include-ignored"]
fn preview_should_render_text_and_solid_sources() {
use ff_format::{Color, TextSpec};
let red = Color::rgb(200, 30, 40);
let Some(frame) = generated_held_frame(&SceneSource::Solid(red), 16, 16, 30.0) else {
println!("Skipping: color filter unavailable");
return;
};
assert_eq!((frame.width(), frame.height()), (16, 16));
let Some(plane) = frame.plane(0) else {
println!("Skipping: no rgba plane");
return;
};
let stride = frame.stride(0).unwrap_or(16 * 4);
let off = 8 * stride + 8 * 4;
let (r, g, b) = (plane[off], plane[off + 1], plane[off + 2]);
assert!(
r.abs_diff(200) <= 6 && g.abs_diff(30) <= 6 && b.abs_diff(40) <= 6,
"solid centre pixel must be ~red, got ({r}, {g}, {b})"
);
if let Some(tf) =
generated_held_frame(&SceneSource::Text(TextSpec::new("Hi")), 64, 32, 30.0)
{
assert_eq!((tf.width(), tf.height()), (64, 32));
} else {
println!("Skipping text: drawtext filter unavailable");
}
}
#[test]
fn inner_blend_rgba_at_zero_alpha_should_return_a() {
let a = vec![255u8, 0, 0, 255];
let b = vec![0u8, 0, 255, 255];
let mut dst = Vec::new();
inner::blend_rgba(&a, &b, 0.0, &mut dst);
assert_eq!(dst, a);
}
#[test]
fn timeline_player_open_should_fail_when_no_video_tracks() {
let _ = PreviewError::SeekOutOfRange {
pts: Duration::from_secs(1),
};
}
}