ff_preview/scene/runner.rs
1//! The timeline decode/present state machine.
2//!
3//! [`SceneRunner`] owns the per-track decode buffers and the audio mixer,
4//! and drives frame presentation. Construct it via
5//! [`ScenePlayer::open`](super::ScenePlayer::open).
6
7use std::sync::atomic::{AtomicBool, AtomicU64, Ordering};
8use std::sync::{Arc, Mutex, mpsc};
9use std::thread::{self, JoinHandle};
10use std::time::Duration;
11
12use ff_filter::{
13 AnimatedValue, BlendMode, CompositeOp, RealtimeComposer, RealtimeLayer, XfadeTransition,
14};
15use ff_format::{PixelFormat, Rational, Timestamp, VideoFrame};
16
17use crate::audio::AudioMixer;
18use crate::error::PreviewError;
19use crate::event::PlayerEvent;
20use crate::playback::SwsRgbaConverter;
21use crate::playback::decode_buffer::FrameResult;
22use crate::playback::master_clock::MasterClock;
23use crate::playback::player::PlayerCommand;
24use crate::playback::sink::FrameSink;
25
26use super::audio_resampling::spawn_audio_track_thread;
27use super::compositor::PreviewCompositor;
28use super::inner;
29use super::state::{
30 AudioFadeConfig, AudioOnlyTrack, ClipState, LavfiOverlayState, OverlayLayer, TransitionState,
31 db_to_linear,
32};
33
34// SceneRunner
35
36/// Exclusive owner of the timeline decode pipeline.
37///
38/// Move to a background thread and call [`run`](Self::run). Register a
39/// [`FrameSink`] with [`set_sink`](Self::set_sink) before calling `run`.
40pub struct SceneRunner {
41 pub(super) clips: Vec<ClipState>,
42 /// Secondary video overlay layers (V2, V3, …). Each is composited over V1
43 /// in order before the frame is delivered to the sink.
44 pub(super) overlay_layers: Vec<OverlayLayer>,
45 /// Dedicated audio-only clips (from A1, A2, … tracks). Each is started and
46 /// stopped as the playhead crosses its timeline window.
47 pub(super) audio_only_tracks: Vec<AudioOnlyTrack>,
48 /// Index of the clip currently being decoded and presented.
49 pub(super) active: usize,
50 /// Non-`None` while a crossfade transition is in progress.
51 pub(super) transition: Option<TransitionState>,
52 pub(super) cmd_rx: mpsc::Receiver<PlayerCommand>,
53 pub(super) event_tx: mpsc::SyncSender<PlayerEvent>,
54 pub(super) sink: Option<Box<dyn FrameSink>>,
55 /// Optional injected GPU compositor, tried before the built-in CPU compositor.
56 /// `avio` supplies one over `ff-render`; `None` (the default) uses the CPU path.
57 pub(super) gpu_compositor: Option<Box<dyn PreviewCompositor>>,
58 pub(super) current_pts: Arc<AtomicU64>,
59 pub(super) paused: Arc<AtomicBool>,
60 pub(super) stopped: Arc<AtomicBool>,
61 pub(super) fps: f64,
62 pub(super) rate: f64,
63 pub(super) clock: MasterClock,
64 /// Media PTS to re-anchor the System clock to when `PlayerCommand::Play`
65 /// is received from a paused state. Updated on every seek and after every
66 /// presented frame so that accumulated wall-clock time during pause does
67 /// not advance `current_pts()` past the last known media position.
68 pub(super) resume_pts: Duration,
69 /// Pixel-format converter for the active (outgoing) frame.
70 pub(super) sws_a: SwsRgbaConverter,
71 /// Pixel-format converter for the incoming frame during transitions.
72 pub(super) sws_b: SwsRgbaConverter,
73 pub(super) rgba_a: Vec<u8>,
74 pub(super) rgba_b: Vec<u8>,
75 pub(super) blend_buf: Vec<u8>,
76 /// `xfade`'s dissolve noise, tabulated for the current frame size. The hash depends
77 /// only on the pixel coordinates, so recomputing it per frame was costing a 4 K
78 /// dissolve more than a whole 30 fps budget (#1736). Kept beside the rgba scratch
79 /// because it has the same lifetime: rebuilt when the frame size changes, held
80 /// across transitions so a dissolve does not pay for it again.
81 pub(super) dissolve_field: Vec<f32>,
82 /// The frame size `dissolve_field` was built for. `(0, 0)` until the first dissolve,
83 /// so no field is built for a timeline that never dissolves.
84 pub(super) dissolve_field_dims: (u32, u32),
85 /// Width of the most recently presented primary-track frame; used to
86 /// synthesise fill frames during primary-track gaps.
87 pub(super) last_frame_w: u32,
88 /// Height of the most recently presented primary-track frame.
89 pub(super) last_frame_h: u32,
90 /// Scratch buffer for synthesising black fill frames during primary-track gaps.
91 pub(super) gap_buf: Vec<u8>,
92 /// Multi-track audio mixer — `None` when no clip has audio.
93 pub(super) audio_mixer: Option<Arc<Mutex<AudioMixer>>>,
94 /// Cancel flag for the currently running audio decode thread.
95 pub(super) active_audio_cancel: Option<Arc<AtomicBool>>,
96 /// Handle to the currently running audio decode thread.
97 pub(super) active_audio_thread: Option<JoinHandle<()>>,
98 /// Cached real-time compositor that applies per-clip effects + blend modes
99 /// (the same chain as export). Rebuilt only when the active clip set or frame
100 /// geometry changes; `None` until the first composite.
101 pub(super) composer: Option<RealtimeComposer>,
102 /// Identifies the composer's current configuration as
103 /// `(layer_id, active_clip_idx, width, height)` per layer. Rebuild on change.
104 pub(super) composer_key: Vec<(usize, usize, u32, u32)>,
105 /// Project output canvas. When `Some`, every layer is placed on a canvas of these
106 /// dimensions (the base included, at its native size unless scaled) and every
107 /// composited frame is canvas-sized. `None` composites at the base clip's own
108 /// size, the standalone behaviour an engine never asks for.
109 pub(super) canvas: Option<(u32, u32)>,
110 /// Timeline-global generated `lavfi` overlay, composited as the topmost layer
111 /// (above every file overlay). `None` when the timeline set no `lavfi_overlay`.
112 pub(super) lavfi: Option<LavfiOverlayState>,
113}
114
115/// Rebuilds `field` when it does not already hold [`xfade_frand_field`] for `w * h`,
116/// returning whether it did.
117///
118/// A free function rather than a method so the rule can be tested without a runner, and
119/// so the caller keeps `field` as a plain field it can lend out beside its other scratch
120/// buffers. The dimensions are tracked explicitly rather than inferred from the length:
121/// `w * h` alone cannot tell 1920x1080 from 1080x1920, and a transposed field would read
122/// the wrong pixel at every coordinate while looking the right size.
123fn ensure_dissolve_field(field: &mut Vec<f32>, dims: &mut (u32, u32), w: u32, h: u32) -> bool {
124 if *dims == (w, h) && field.len() == (w as usize) * (h as usize) {
125 return false;
126 }
127 *field = ff_filter::xfade_frand_field(w, h);
128 *dims = (w, h);
129 true
130}
131
132/// How [`SceneRunner::run`] paces frame delivery.
133#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
134#[non_exhaustive]
135pub enum Pacing {
136 /// Deliver frames against the wall clock: sleep until a frame is due and
137 /// drop one that is more than a frame period late. Playback.
138 #[default]
139 RealTime,
140 /// Deliver every frame as soon as it is decoded: the clock is the runner's
141 /// own position, moved one frame period per presented frame, so nothing is
142 /// ever late and nothing is dropped or slept for. For checks that must see
143 /// each frame (tests, thumbnail strips). Reverse playback and the pause
144 /// poll still wait on the wall clock.
145 Unpaced,
146}
147
148impl SceneRunner {
149 /// Register the frame sink. Call before [`run`](Self::run).
150 pub fn set_sink(&mut self, sink: Box<dyn FrameSink>) {
151 self.sink = Some(sink);
152 }
153
154 /// Choose how frames are paced. Call before [`run`](Self::run); the default
155 /// is [`Pacing::RealTime`]. Switching keeps the current position.
156 pub fn set_pacing(&mut self, pacing: Pacing) {
157 let current = self.clock.current_pts();
158 self.clock = match pacing {
159 Pacing::RealTime => MasterClock::System {
160 started_at: std::time::Instant::now(),
161 base_pts: current,
162 // A wall clock only runs forward; reverse playback keeps its own
163 // stepping and never reads the clock's rate.
164 rate: if self.rate > 0.0 { self.rate } else { 1.0 },
165 },
166 Pacing::Unpaced => MasterClock::Stepped { pts: current },
167 };
168 }
169
170 /// Register an external GPU compositor tried before the built-in CPU path.
171 /// Call before [`run`](Self::run). `avio` supplies one over `ff-render`.
172 pub fn set_gpu_compositor(&mut self, compositor: Box<dyn PreviewCompositor>) {
173 self.gpu_compositor = Some(compositor);
174 }
175
176 /// Whether an external GPU compositor is registered (else the CPU path is used).
177 #[must_use]
178 pub fn has_gpu_compositor(&self) -> bool {
179 self.gpu_compositor.is_some()
180 }
181
182 /// Advances every overlay layer to the frame whose presentation time has
183 /// arrived at `target_pts`, holding the current frame otherwise (so a layer
184 /// whose fps differs from the timeline plays at the right speed rather than
185 /// advancing once per present). Returns `(layer_index, width, height)` for
186 /// each layer that currently has a frame to show.
187 fn sync_overlays(&mut self, target_pts: Duration) -> Vec<(usize, u32, u32)> {
188 let mut active = Vec::new();
189 for (li, layer) in self.overlay_layers.iter_mut().enumerate() {
190 let maybe_cidx = layer
191 .clips
192 .iter()
193 .position(|c| target_pts >= c.timeline_start && target_pts < c.timeline_end);
194 let Some(cidx) = maybe_cidx else {
195 layer.rgba.clear();
196 layer.cur_dims = None;
197 layer.pending = None;
198 continue;
199 };
200 if cidx != layer.active {
201 let local = layer.clips[cidx].in_point
202 + target_pts.saturating_sub(layer.clips[cidx].timeline_start);
203 let _ = layer.clips[cidx].decode_buf.seek(local);
204 layer.active = cidx;
205 layer.cur_dims = None;
206 layer.pending = None;
207 }
208 let clip_in = layer.clips[cidx].in_point;
209 let tl_start = layer.clips[cidx].timeline_start;
210 loop {
211 let f = match layer.pending.take() {
212 Some(pf) => pf,
213 None => match layer.clips[cidx].decode_buf.pop_frame() {
214 FrameResult::Frame(f) => f,
215 _ => break,
216 },
217 };
218 let v2_pts = tl_start + f.timestamp().as_duration().saturating_sub(clip_in);
219 if v2_pts > target_pts {
220 // Not due yet — hold it for a later present.
221 layer.pending = Some(f);
222 break;
223 }
224 if layer.sws.convert(&f, &mut layer.rgba) {
225 layer.cur_dims = Some((f.width(), f.height()));
226 }
227 }
228 match layer.cur_dims {
229 Some((ow, oh)) => active.push((li, ow, oh)),
230 None => layer.rgba.clear(),
231 }
232 }
233 active
234 }
235
236 /// Advances the generated `lavfi` overlay (if any) to the frame due at
237 /// `target_pts`, holding otherwise — see [`LavfiOverlayState::advance_to`].
238 /// Returns the current frame's `(width, height)`, or `None` when there is no
239 /// lavfi overlay / no frame yet.
240 fn sync_lavfi(&mut self, target_pts: Duration) -> Option<(u32, u32)> {
241 self.lavfi.as_mut()?.advance_to(target_pts)
242 }
243
244 /// Composites `base_frame` (the bottom layer) with the given overlay layers
245 /// through the cached [`RealtimeComposer`], applying each layer's effects and
246 /// blend mode. `base_id` identifies the base for cache invalidation — the V1
247 /// clip index, or `usize::MAX` for the gap-fill black base. Returns the
248 /// composited RGBA frame together with its actual `(width, height)`, or `None`
249 /// on failure.
250 ///
251 /// The composited size can differ from `base_w`/`base_h` when the base layer's
252 /// effect chain resizes the frame (`Crop`, `Scale`, `Pad`, `FitToAspect`), so
253 /// callers must push the returned dimensions to the sink rather than the
254 /// decoded ones — otherwise the buffer length no longer matches the reported
255 /// size and the frame is dropped.
256 #[allow(clippy::too_many_arguments)]
257 fn composite_frame(
258 &mut self,
259 base_layer: RealtimeLayer,
260 base_id: usize,
261 mut base_frame: VideoFrame,
262 base_w: u32,
263 base_h: u32,
264 overlays: &[(usize, u32, u32)],
265 t: Duration,
266 ) -> Option<(Vec<u8>, u32, u32)> {
267 let mut specs = vec![base_layer];
268 let mut key: Vec<(usize, usize, u32, u32)> = vec![(0, base_id, base_w, base_h)];
269 for &(li, ow, oh) in overlays {
270 let oc = &self.overlay_layers[li];
271 specs.push(RealtimeLayer::with_dimensions(
272 oc.clips[oc.active].layer_desc.clone(),
273 ow,
274 oh,
275 PixelFormat::Rgba,
276 ));
277 key.push((li + 1, oc.active, ow, oh));
278 }
279 // Topmost timeline-global lavfi overlay (generated), when a frame is held.
280 // Fixed full-frame Normal/Over layer, matching export; its transparency comes
281 // from the lavfi content's own alpha. A sentinel layer id keys the cache.
282 let lavfi_dims = self.lavfi.as_ref().and_then(|s| s.dims);
283 if let Some((lw, lh)) = lavfi_dims {
284 specs.push(RealtimeLayer {
285 width: lw,
286 height: lh,
287 pixel_format: PixelFormat::Rgba,
288 effects: Vec::new(),
289 opacity: AnimatedValue::Static(1.0),
290 x: AnimatedValue::Static(0.0),
291 y: AnimatedValue::Static(0.0),
292 scale_x: AnimatedValue::Static(1.0),
293 scale_y: AnimatedValue::Static(1.0),
294 rotation: AnimatedValue::Static(0.0),
295 blend_mode: BlendMode::Normal,
296 composite_op: CompositeOp::Over,
297 });
298 key.push((usize::MAX - 1, 0, lw, lh));
299 }
300 // Build the decoded frame for every layer, in the same order as `specs`.
301 // Stamp each with the composite's timeline PTS so the graph's per-frame
302 // animation tick (in `push_video`) evaluates each layer's opacity track at
303 // the same time. Frames from `from_rgba` carry PTS 0 otherwise, and any
304 // registered `AnimationEntry` would be frozen at t=0.
305 let ts = Timestamp::from_duration(t, Rational::new(1, 1_000_000));
306 base_frame.set_timestamp(ts);
307 let mut frames = vec![base_frame];
308 for &(li, ow, oh) in overlays {
309 let mut vf =
310 VideoFrame::from_rgba(ow, oh, self.overlay_layers[li].rgba.clone()).ok()?;
311 vf.set_timestamp(ts);
312 frames.push(vf);
313 }
314 if let Some((lw, lh)) = lavfi_dims {
315 let rgba = self
316 .lavfi
317 .as_ref()
318 .map_or_else(Vec::new, |s| s.rgba.clone());
319 let mut vf = VideoFrame::from_rgba(lw, lh, rgba).ok()?;
320 vf.set_timestamp(ts);
321 frames.push(vf);
322 }
323
324 // Try the injected GPU compositor first; `None` falls through to the CPU
325 // compositor below (unsupported layer, no adapter, or a GPU error).
326 let gpu_canvas = self.canvas.unwrap_or((base_w, base_h));
327 if let Some(out) =
328 try_gpu_composite(self.gpu_compositor.as_mut(), &specs, &frames, gpu_canvas, t)
329 {
330 return Some(out);
331 }
332
333 // CPU compositor (cached, keyed by layer identity/size).
334 if self.composer.is_none() || self.composer_key != key {
335 let new_layer_set = self.composer_key != key;
336 self.composer = match RealtimeComposer::with_canvas(&specs, self.canvas) {
337 Ok(c) => Some(c),
338 Err(e) => {
339 // A GPU compositor is attached but declined this frame for an
340 // unrelated reason, and the CPU compositor refuses the operator
341 // outright (#1753), so the base frame is what gets shown. Said
342 // once per layer set rather than per frame. With no GPU attached
343 // at all the timeline is refused up front by the engine's open.
344 if new_layer_set
345 && matches!(e, ff_filter::FilterError::UnsupportedCompositeOp { .. })
346 {
347 log::warn!(
348 "preview: CPU compositor refused the layer set, showing the \
349 base frame only error={e}"
350 );
351 }
352 None
353 }
354 };
355 self.composer_key = key;
356 }
357 let composer = self.composer.as_mut()?;
358 for (slot, vf) in frames.iter().enumerate() {
359 if composer.push_layer(slot, vf).is_err() {
360 return None;
361 }
362 }
363 let f = composer.pull().ok().flatten()?;
364 let (w, h) = (f.width(), f.height());
365 f.to_rgba().map(|rgba| (rgba, w, h))
366 }
367
368 /// A/V sync presentation loop.
369 ///
370 /// Plays all clips in the primary video track from start to finish (or until
371 /// a [`PlayerCommand::Stop`] is received).
372 ///
373 /// Emits [`PlayerEvent::SeekCompleted`] after each successful seek,
374 /// [`PlayerEvent::PositionUpdate`] after each presented video frame,
375 /// [`PlayerEvent::Error`] on non-fatal decode errors, and
376 /// [`PlayerEvent::Eof`] before returning.
377 ///
378 /// # Errors
379 ///
380 /// Returns [`PreviewError::SeekOutOfRange`] if a seek command targets a
381 /// timestamp that falls outside all clips on the timeline.
382 #[allow(clippy::too_many_lines)]
383 pub fn run(mut self) -> Result<(), PreviewError> {
384 if self.clips.is_empty() {
385 let _ = self.event_tx.try_send(PlayerEvent::Eof);
386 return Ok(());
387 }
388
389 let fps = self.fps.max(1.0);
390 let frame_period = Duration::from_secs_f64(1.0 / fps);
391 // `Pacing::Unpaced`: the clock is moved by this loop, never by wall time,
392 // so the pacing sleep and the late-frame drop below are both skipped.
393 let stepped = self.clock.is_stepped();
394 self.clock.reset(Duration::ZERO);
395
396 loop {
397 // Drain commands
398 let mut pending_seek: Option<Duration> = None;
399 while let Ok(cmd) = self.cmd_rx.try_recv() {
400 match cmd {
401 PlayerCommand::Seek(pts) => pending_seek = Some(pts),
402 PlayerCommand::Play => {
403 // Always re-anchor the System clock on Play.
404 //
405 // PlayerHandle::play() sets the shared `paused` atomic
406 // to `false` BEFORE enqueueing PlayerCommand::Play, so
407 // paused.load() here always returns false — a guard on
408 // `if paused` would never fire. Re-anchoring
409 // unconditionally is safe: when the player was not
410 // actually paused, resume_pts equals the last presented
411 // frame PTS (or the seek target), which is already the
412 // clock's current base, so clock.reset() is a no-op
413 // in effect.
414 self.clock.reset(self.resume_pts);
415 self.stopped.store(false, Ordering::Release);
416 self.paused.store(false, Ordering::Release);
417 }
418 PlayerCommand::Pause => {
419 self.paused.store(true, Ordering::Release);
420 }
421 PlayerCommand::Stop => {
422 self.stopped.store(true, Ordering::Release);
423 }
424 PlayerCommand::SetRate(r) => {
425 if r != 0.0 {
426 let was_negative = self.rate < 0.0;
427 self.rate = r;
428 if r > 0.0 {
429 self.clock.set_rate(r);
430 if was_negative {
431 // Returning from reverse: rebase clock and
432 // restart audio from the current video position.
433 let pts = Duration::from_micros(
434 self.current_pts.load(Ordering::Relaxed),
435 );
436 self.clock.reset(pts);
437 self.resume_pts = pts;
438 if let Err(e) = self.seek_timeline_coarse(pts) {
439 log::warn!(
440 "timeline reverse→forward seek failed \
441 pts={pts:?} error={e}"
442 );
443 } else {
444 let ci = self.active;
445 let clip_local = self.clips[ci].in_point
446 + pts.saturating_sub(self.clips[ci].timeline_start);
447 if let Some(m) = &self.audio_mixer {
448 m.lock()
449 .unwrap_or_else(std::sync::PoisonError::into_inner)
450 .invalidate_all();
451 }
452 self.restart_audio_at(ci, clip_local);
453 }
454 }
455 } else {
456 // Entering reverse: silence audio.
457 if let Some(cancel) = &self.active_audio_cancel {
458 cancel.store(true, Ordering::Release);
459 }
460 if let Some(m) = &self.audio_mixer {
461 m.lock()
462 .unwrap_or_else(std::sync::PoisonError::into_inner)
463 .invalidate_all();
464 }
465 }
466 }
467 }
468 PlayerCommand::SetAvOffset(_) => {} // audio timing is system-clock driven
469 PlayerCommand::UpdateLayout(scene) => {
470 if let Err(e) = self.update_layout_in_place(&scene, self.resume_pts) {
471 log::warn!("timeline layout update ignored: {e}");
472 }
473 }
474 }
475 }
476
477 // Apply pending seek
478 let had_seek = pending_seek.is_some();
479 if let Some(target) = pending_seek {
480 self.seek_timeline(target)?;
481 self.clock.reset(target);
482 self.resume_pts = target;
483 let _ = self.event_tx.try_send(PlayerEvent::SeekCompleted(target));
484 }
485
486 // When a seek arrives while paused, present one preview frame so
487 // the sink reflects the new position without resuming playback.
488 if had_seek && self.paused.load(Ordering::Acquire) {
489 let active = self.active;
490 let deadline = std::time::Instant::now() + Duration::from_millis(300);
491 loop {
492 match self.clips[active].decode_buf.pop_frame() {
493 FrameResult::Frame(f) => {
494 let f_pts = f.timestamp().as_duration();
495 let elapsed = f_pts.saturating_sub(self.clips[active].in_point);
496 let tl_pts = self.clips[active].timeline_start
497 + if (self.clips[active].speed - 1.0).abs() < 1e-9 {
498 elapsed
499 } else {
500 elapsed.div_f64(self.clips[active].speed)
501 };
502 let w = f.width();
503 let h = f.height();
504 if self.sws_a.convert(&f, &mut self.rgba_a)
505 && let Some(sink) = self.sink.as_mut()
506 {
507 sink.push_frame(&self.rgba_a, w, h, tl_pts);
508 }
509 self.current_pts.store(
510 u64::try_from(tl_pts.as_micros()).unwrap_or(u64::MAX),
511 Ordering::Relaxed,
512 );
513 let _ = self.event_tx.try_send(PlayerEvent::PositionUpdate(tl_pts));
514 break;
515 }
516 FrameResult::Seeking(_) => {
517 if std::time::Instant::now() > deadline {
518 break;
519 }
520 thread::sleep(Duration::from_millis(2));
521 }
522 FrameResult::Eof => break,
523 }
524 }
525 }
526
527 // Error events from active clip (a generated held source has no channel).
528 {
529 let active = self.active;
530 if let Some(rx) = self.clips[active].decode_buf.error_events() {
531 while let Ok(msg) = rx.try_recv() {
532 let _ = self.event_tx.try_send(PlayerEvent::Error(msg));
533 }
534 }
535 }
536 let trans_next = self.transition.as_ref().map(|tp| tp.next_idx);
537 if let Some(next_idx) = trans_next
538 && let Some(rx) = self.clips[next_idx].decode_buf.error_events()
539 {
540 while let Ok(msg) = rx.try_recv() {
541 let _ = self.event_tx.try_send(PlayerEvent::Error(msg));
542 }
543 }
544
545 // Stopped / paused
546 if self.stopped.load(Ordering::Acquire) {
547 break;
548 }
549 if self.paused.load(Ordering::Acquire) {
550 thread::sleep(Duration::from_millis(5));
551 continue;
552 }
553
554 // Reverse playback path
555 if self.rate < 0.0 {
556 let current = Duration::from_micros(self.current_pts.load(Ordering::Relaxed));
557 let step = Duration::from_secs_f64(self.rate.abs() / fps.max(f64::MIN_POSITIVE));
558 let target = current.saturating_sub(step);
559
560 let clip_idx = self
561 .clips
562 .iter()
563 .position(|c| target >= c.timeline_start && target < c.timeline_end);
564
565 if let Some(ci) = clip_idx {
566 let elapsed_tl = target.saturating_sub(self.clips[ci].timeline_start);
567 let clip_local = self.clips[ci].in_point
568 + if (self.clips[ci].speed - 1.0).abs() < 1e-9 {
569 elapsed_tl
570 } else {
571 elapsed_tl.mul_f64(self.clips[ci].speed)
572 };
573 if self.clips[ci].decode_buf.seek_coarse(clip_local).is_ok() {
574 if ci != self.active {
575 self.active = ci;
576 self.transition = None;
577 }
578 let deadline = std::time::Instant::now() + Duration::from_millis(300);
579 let frame = loop {
580 match self.clips[ci].decode_buf.pop_frame() {
581 FrameResult::Frame(f) => break Some(f),
582 FrameResult::Seeking(_) => {
583 if std::time::Instant::now() > deadline {
584 break None;
585 }
586 thread::sleep(Duration::from_millis(2));
587 }
588 FrameResult::Eof => break None,
589 }
590 };
591 if let Some(f) = frame {
592 let f_pts = f.timestamp().as_duration();
593 let elapsed = f_pts.saturating_sub(self.clips[ci].in_point);
594 let tl_pts = self.clips[ci].timeline_start
595 + if (self.clips[ci].speed - 1.0).abs() < 1e-9 {
596 elapsed
597 } else {
598 elapsed.div_f64(self.clips[ci].speed)
599 };
600 let w = f.width();
601 let h = f.height();
602 if self.sws_a.convert(&f, &mut self.rgba_a)
603 && let Some(sink) = self.sink.as_mut()
604 {
605 sink.push_frame(&self.rgba_a, w, h, tl_pts);
606 }
607 self.current_pts.store(
608 u64::try_from(tl_pts.as_micros()).unwrap_or(u64::MAX),
609 Ordering::Relaxed,
610 );
611 self.resume_pts = tl_pts;
612 let _ = self.event_tx.try_send(PlayerEvent::PositionUpdate(tl_pts));
613 }
614 }
615 }
616
617 if self
618 .clips
619 .first()
620 .is_some_and(|c| target < c.timeline_start)
621 {
622 self.paused.store(true, Ordering::Release);
623 }
624 thread::sleep(frame_period);
625 continue;
626 }
627
628 // Pop frame from active clip
629 let active = self.active;
630 let pop_result = self.clips[active].decode_buf.pop_frame();
631
632 match pop_result {
633 FrameResult::Eof => {
634 let old_active = active;
635 if let Some(tp) = self.transition.take() {
636 self.active = tp.next_idx;
637 } else if active + 1 < self.clips.len() {
638 self.active += 1;
639 } else {
640 break;
641 }
642 if self.active != old_active {
643 // Clear the outgoing clip's pre-decoded audio so its stale
644 // samples do not continue to mix in after the transition.
645 if let Some(h) = self.clips[old_active].audio_track.clone() {
646 h.clear();
647 }
648 let in_pt = self.clips[self.active].in_point;
649 self.restart_audio_at(self.active, in_pt);
650 }
651 }
652
653 FrameResult::Seeking(last) => {
654 if let Some(ref f) = last {
655 let f_pts = f.timestamp().as_duration();
656 let in_pt = self.clips[active].in_point;
657 // Suppress pre-seek artefact frames: when a DecodeBuffer
658 // is opened and immediately seeked to in_point, the
659 // background thread may have decoded one frame from
660 // position 0 before processing the seek command. That
661 // frame ends up as `last` and must not be displayed —
662 // its content is from before the clip's in_point.
663 if f_pts >= in_pt {
664 let tl_start = self.clips[active].timeline_start;
665 let elapsed = f_pts.saturating_sub(in_pt);
666 let spd = self.clips[active].speed;
667 let tl_pts = tl_start
668 + if (spd - 1.0).abs() < 1e-9 {
669 elapsed
670 } else {
671 elapsed.div_f64(spd)
672 };
673 let w = f.width();
674 let h = f.height();
675 if self.sws_a.convert(f, &mut self.rgba_a)
676 && let Some(sink) = self.sink.as_mut()
677 {
678 sink.push_frame(&self.rgba_a, w, h, tl_pts);
679 }
680 }
681 }
682 }
683
684 FrameResult::Frame(frame) => {
685 let f_pts = frame.timestamp().as_duration();
686 let clip_in = self.clips[active].in_point;
687 let clip_out = self.clips[active].out_point;
688 let clip_tl_start = self.clips[active].timeline_start;
689 let clip_tl_end = self.clips[active].timeline_end;
690 let clip_speed = self.clips[active].speed;
691 // Frames past `out_point` that feed the crossfade into the next
692 // clip (ADR-0009). Without it this clip ends exactly where the next
693 // one starts, the branch below advances, and the transition-entry
694 // check further down is never reached — which is why an
695 // engine-derived scene never blended (#1737).
696 let handle = self.clips[active].video_handle;
697
698 // Skip frames before in_point (e.g. right after a seek).
699 if f_pts < clip_in {
700 continue;
701 }
702
703 // The handle in source time, to compare against `out_point` and
704 // `f_pts`: at speed 2.0 half a second of blend is a second of source.
705 let src_handle = if (clip_speed - 1.0).abs() < 1e-9 {
706 handle
707 } else {
708 handle.mul_f64(clip_speed)
709 };
710 // Treat frames past out_point (plus the handle) as EOF for this clip.
711 let past_out = clip_out.is_some_and(|op| f_pts >= op + src_handle);
712 let elapsed = f_pts.saturating_sub(clip_in);
713 // Remap source PTS → timeline PTS via speed factor.
714 // For speed=2.0 the clip occupies half the timeline duration;
715 // for speed=0.5 it occupies double.
716 let tl_elapsed = if (clip_speed - 1.0).abs() < 1e-9 {
717 elapsed
718 } else {
719 elapsed.div_f64(clip_speed)
720 };
721 // `handle` is already timeline time, so it adds to the timeline
722 // extent directly.
723 let past_end = clip_tl_start + tl_elapsed >= clip_tl_end + handle;
724
725 if past_out || past_end {
726 let old_active = active;
727 if let Some(tp) = self.transition.take() {
728 self.active = tp.next_idx;
729 } else if active + 1 < self.clips.len() {
730 self.active += 1;
731 } else {
732 break;
733 }
734 if self.active != old_active {
735 // Clear the outgoing clip's pre-decoded audio so its
736 // stale samples do not continue to mix in after the
737 // transition.
738 if let Some(h) = self.clips[old_active].audio_track.clone() {
739 h.clear();
740 }
741 // And the visual equivalent: a stateful effect (motion
742 // blur's exposure trail) accumulates across one clip's
743 // frames and must not bleed into the next. This rides the
744 // cut detection that is already here rather than adding a
745 // second notion of a boundary, which matters because clip
746 // progression is driven by each frame's own PTS (RK-019).
747 if let Some(c) = self.gpu_compositor.as_mut() {
748 c.reset_effects();
749 }
750 let in_pt = self.clips[self.active].in_point;
751 self.restart_audio_at(self.active, in_pt);
752 }
753 continue;
754 }
755
756 let timeline_pts = clip_tl_start + tl_elapsed;
757
758 // Manage audio-only decode threads
759 for at in &mut self.audio_only_tracks {
760 let should_run =
761 timeline_pts >= at.timeline_start && timeline_pts < at.timeline_end;
762 let is_running = at.cancel.is_some();
763 if should_run && !is_running {
764 let local =
765 at.in_point + timeline_pts.saturating_sub(at.timeline_start);
766 at.start_at(local);
767 } else if !should_run && is_running {
768 at.stop();
769 // Clear stale pre-decoded samples so the mixer does
770 // not play this track's buffered audio past clip end.
771 at.handle.clear();
772 }
773 // Per-clip volume automation: an animated gain is evaluated at
774 // the timeline PTS each tick (a static gain was set at open).
775 if should_run && let AnimatedValue::Track(track) = &at.volume {
776 at.handle
777 .set_volume(db_to_linear(track.value_at(timeline_pts)));
778 }
779 }
780
781 // Primary-track volume automation for the active clip.
782 if let Some(handle) = &self.clips[active].audio_track
783 && let AnimatedValue::Track(track) = &self.clips[active].volume
784 {
785 handle.set_volume(db_to_linear(track.value_at(timeline_pts)));
786 }
787
788 // Update shared current_pts and resume anchor.
789 self.current_pts.store(
790 u64::try_from(timeline_pts.as_micros()).unwrap_or(u64::MAX),
791 Ordering::Relaxed,
792 );
793 self.resume_pts = timeline_pts;
794
795 // Transition zone entry check
796 if self.transition.is_none() && active + 1 < self.clips.len() {
797 let next = &self.clips[active + 1];
798 if next.xfade_dur > Duration::ZERO && timeline_pts >= next.timeline_start {
799 if timeline_pts < next.timeline_start + next.xfade_dur {
800 self.transition = Some(TransitionState {
801 next_idx: active + 1,
802 start: next.timeline_start,
803 duration: next.xfade_dur,
804 kind: next.xfade_kind.unwrap_or(XfadeTransition::Fade),
805 });
806 } else {
807 // Jumped past the entire transition zone.
808 let old_active = active;
809 self.active = active + 1;
810 if self.active != old_active {
811 let in_pt = self.clips[self.active].in_point;
812 self.restart_audio_at(self.active, in_pt);
813 }
814 continue;
815 }
816 }
817 }
818
819 // A/V sync (system clock)
820 {
821 let clock_pts = self.clock.current_pts();
822 let diff = timeline_pts.as_secs_f64() - clock_pts.as_secs_f64();
823 let fp = frame_period.as_secs_f64();
824
825 // Only enter gap fill for an actual gap between clips.
826 // For slow-motion clips (speed < 1.0) the large diff is expected
827 // and should be handled by the `diff > fp` sleep below instead.
828 if diff > fp * 2.0
829 && (clip_speed - 1.0) > -1e-9
830 && self.transition.is_none()
831 && self.last_frame_w > 0
832 {
833 // Gap in the primary track: the next V1 clip starts more than
834 // 2 frame-periods ahead of the clock. Synthesise black frames
835 // composited with overlay-layer content for every missing
836 // frame period so that V2 overlays and audio-only tracks
837 // remain live during the gap.
838 // With an explicit canvas, fill gaps at the canvas size so
839 // the preview frame size stays constant across gaps.
840 let (gw, gh) = self
841 .canvas
842 .unwrap_or((self.last_frame_w, self.last_frame_h));
843 let n = (gw * gh * 4) as usize;
844 'gap: loop {
845 // Drain incoming commands.
846 while let Ok(cmd) = self.cmd_rx.try_recv() {
847 match cmd {
848 PlayerCommand::Play => {
849 self.clock.reset(self.resume_pts);
850 self.stopped.store(false, Ordering::Release);
851 self.paused.store(false, Ordering::Release);
852 }
853 PlayerCommand::Pause => {
854 self.paused.store(true, Ordering::Release);
855 }
856 PlayerCommand::Stop => {
857 self.stopped.store(true, Ordering::Release);
858 }
859 PlayerCommand::SetRate(r) if r > 0.0 => {
860 self.rate = r;
861 self.clock.set_rate(r);
862 }
863 _ => {}
864 }
865 }
866 if self.stopped.load(Ordering::Acquire) {
867 break 'gap;
868 }
869 if self.paused.load(Ordering::Acquire) {
870 thread::sleep(Duration::from_millis(5));
871 continue 'gap;
872 }
873 let gap_pts = self.clock.current_pts();
874 if gap_pts + frame_period >= timeline_pts {
875 break 'gap;
876 }
877 // Build a black base and composite the overlays onto it
878 // through the shared compositor — same held-frame timing,
879 // effects, and blend modes as the main present path.
880 self.gap_buf.resize(n, 0);
881 self.gap_buf.fill(0);
882 let gap_overlays = self.sync_overlays(gap_pts);
883 let gap_lavfi = self.sync_lavfi(gap_pts);
884 // Composite when there is a file overlay OR a lavfi
885 // overlay to draw over the gap's black base.
886 let gap_composited =
887 if gap_overlays.is_empty() && gap_lavfi.is_none() {
888 None
889 } else {
890 let base_layer = RealtimeLayer {
891 width: gw,
892 height: gh,
893 pixel_format: PixelFormat::Rgba,
894 effects: Vec::new(),
895 opacity: AnimatedValue::Static(1.0),
896 x: AnimatedValue::Static(0.0),
897 y: AnimatedValue::Static(0.0),
898 scale_x: AnimatedValue::Static(1.0),
899 scale_y: AnimatedValue::Static(1.0),
900 rotation: AnimatedValue::Static(0.0),
901 blend_mode: BlendMode::Normal,
902 composite_op: ff_filter::CompositeOp::Over,
903 };
904 match VideoFrame::from_rgba(gw, gh, self.gap_buf.clone()) {
905 Ok(bf) => self.composite_frame(
906 base_layer,
907 usize::MAX,
908 bf,
909 gw,
910 gh,
911 &gap_overlays,
912 gap_pts,
913 ),
914 Err(_) => None,
915 }
916 };
917 // Manage audio-only decode threads (A1/A2…).
918 for at in &mut self.audio_only_tracks {
919 let should_run =
920 gap_pts >= at.timeline_start && gap_pts < at.timeline_end;
921 let is_running = at.cancel.is_some();
922 if should_run && !is_running {
923 let local =
924 at.in_point + gap_pts.saturating_sub(at.timeline_start);
925 at.start_at(local);
926 } else if !should_run && is_running {
927 at.stop();
928 at.handle.clear();
929 }
930 }
931 // Manage V1 inline audio: start it the moment the
932 // gap clock reaches the active clip's timeline_start.
933 if self.active_audio_cancel.is_none()
934 && self.clips[self.active].audio_track.is_some()
935 && gap_pts >= self.clips[self.active].timeline_start
936 {
937 let tl_start = self.clips[self.active].timeline_start;
938 let in_pt = self.clips[self.active].in_point;
939 let gap_elapsed = gap_pts.saturating_sub(tl_start);
940 let spd = self.clips[self.active].speed;
941 let local = in_pt
942 + if (spd - 1.0).abs() < 1e-9 {
943 gap_elapsed
944 } else {
945 gap_elapsed.mul_f64(spd)
946 };
947 self.restart_audio_at(self.active, local);
948 }
949 self.current_pts.store(
950 u64::try_from(gap_pts.as_micros()).unwrap_or(u64::MAX),
951 Ordering::Relaxed,
952 );
953 self.resume_pts = gap_pts;
954 let _ =
955 self.event_tx.try_send(PlayerEvent::PositionUpdate(gap_pts));
956 if let Some(sink) = self.sink.as_mut() {
957 match &gap_composited {
958 Some((rgba, cw, ch)) => {
959 sink.push_frame(rgba, *cw, *ch, gap_pts);
960 }
961 None => sink.push_frame(&self.gap_buf, gw, gh, gap_pts),
962 }
963 }
964 if stepped {
965 self.clock.advance(frame_period);
966 } else {
967 thread::sleep(frame_period);
968 }
969 }
970 } else if !stepped && diff > fp {
971 let sleep_secs =
972 (diff - fp / 2.0).max(0.0) / self.rate.max(f64::MIN_POSITIVE);
973 thread::sleep(Duration::from_secs_f64(sleep_secs));
974 } else if !stepped && diff < -fp {
975 log::debug!(
976 "timeline dropped late frame timeline_pts={timeline_pts:?} \
977 clock_pts={clock_pts:?}"
978 );
979 continue;
980 }
981 }
982
983 // Start V1 inline audio on the first presented frame when a
984 // pre-roll gap prevented the thread from starting at open() time.
985 // The gap-fill loop attempts this but exits one frame-period before
986 // timeline_start, so we catch the remaining case here.
987 if self.active_audio_cancel.is_none()
988 && self.clips[active].audio_track.is_some()
989 {
990 let in_pt = self.clips[active].in_point;
991 let elapsed_tl =
992 timeline_pts.saturating_sub(self.clips[active].timeline_start);
993 let local = in_pt
994 + if (clip_speed - 1.0).abs() < 1e-9 {
995 elapsed_tl
996 } else {
997 elapsed_tl.mul_f64(clip_speed)
998 };
999 self.restart_audio_at(active, local);
1000 }
1001
1002 // Present frame
1003 let w = frame.width();
1004 let h = frame.height();
1005 self.last_frame_w = w;
1006 self.last_frame_h = h;
1007
1008 // Copy transition fields to avoid holding a borrow while
1009 // calling `pop_frame` on the next clip.
1010 let (in_trans, next_idx, trans_start, trans_dur, trans_kind) =
1011 match &self.transition {
1012 Some(tp) => (true, tp.next_idx, tp.start, tp.duration, tp.kind),
1013 None => (
1014 false,
1015 0,
1016 Duration::ZERO,
1017 Duration::ZERO,
1018 XfadeTransition::Fade,
1019 ),
1020 };
1021
1022 let a_ok = self.sws_a.convert(&frame, &mut self.rgba_a);
1023
1024 if a_ok {
1025 // V1 per-clip opacity: pre-multiply toward black (producer-side;
1026 // the composer ignores base-layer opacity). The merged opacity is
1027 // an `AnimatedValue`; a track is evaluated at the timeline PTS
1028 // (tracks are timeline-global), so base-layer opacity animates too.
1029 let v1_op = match &self.clips[active].layer_desc.opacity {
1030 // Value is clamped to [0.0, 1.0], so the f32 narrowing is safe.
1031 #[allow(clippy::cast_possible_truncation)]
1032 AnimatedValue::Track(track) => {
1033 track.value_at(timeline_pts).clamp(0.0, 1.0) as f32
1034 }
1035 AnimatedValue::Static(_) => self.clips[active].opacity,
1036 };
1037 if (v1_op - 1.0).abs() > 1e-6 {
1038 for chunk in self.rgba_a.as_chunks_mut::<4>().0 {
1039 #[allow(clippy::cast_possible_truncation, clippy::cast_sign_loss)]
1040 {
1041 chunk[0] = (f32::from(chunk[0]) * v1_op).round() as u8;
1042 chunk[1] = (f32::from(chunk[1]) * v1_op).round() as u8;
1043 chunk[2] = (f32::from(chunk[2]) * v1_op).round() as u8;
1044 }
1045 }
1046 }
1047
1048 // Transition crossfade (producer-side): blend the incoming clip
1049 // into rgba_a so the composer grades the crossfaded V1 frame.
1050 if in_trans
1051 && let FrameResult::Frame(next_frame) =
1052 self.clips[next_idx].decode_buf.pop_frame()
1053 && self.sws_b.convert(&next_frame, &mut self.rgba_b)
1054 {
1055 let alpha = (timeline_pts.saturating_sub(trans_start).as_secs_f32()
1056 / trans_dur.as_secs_f32())
1057 .clamp(0.0, 1.0);
1058 // Offer the blend to the injected GPU path first; `None`
1059 // falls through to the CPU one below, which covers an
1060 // unrendered kind, no adapter, and a GPU error alike.
1061 if let Some(blended) = try_gpu_blend(
1062 self.gpu_compositor.as_mut(),
1063 trans_kind,
1064 &self.rgba_a,
1065 &self.rgba_b,
1066 alpha,
1067 w,
1068 h,
1069 ) {
1070 // Moved in, not copied into the scratch buffer: the
1071 // readback already owns a correctly sized `Vec`, so
1072 // taking it costs nothing while routing it through
1073 // `blend_buf` would add a full-frame memcpy. The CPU
1074 // branch below swaps instead because `apply_xfade`
1075 // writes into a buffer it does not own.
1076 self.rgba_a = blended;
1077 } else {
1078 // Only `Dissolve` reads the field, and building one costs
1079 // what a whole frame of dissolve used to (47.4 ms at 4 K),
1080 // so a `Fade` must not pay for it.
1081 let field = if trans_kind == XfadeTransition::Dissolve {
1082 ensure_dissolve_field(
1083 &mut self.dissolve_field,
1084 &mut self.dissolve_field_dims,
1085 w,
1086 h,
1087 );
1088 Some(self.dissolve_field.as_slice())
1089 } else {
1090 None
1091 };
1092 inner::apply_xfade(
1093 trans_kind,
1094 &self.rgba_a,
1095 &self.rgba_b,
1096 alpha,
1097 (w, h),
1098 field,
1099 &mut self.blend_buf,
1100 );
1101 std::mem::swap(&mut self.rgba_a, &mut self.blend_buf);
1102 }
1103 }
1104
1105 // Update overlays (held-frame, advanced by PTS) and composite
1106 // the V1 base with them through the shared compositor.
1107 let active_overlays = self.sync_overlays(timeline_pts);
1108 self.sync_lavfi(timeline_pts);
1109 let base_layer = RealtimeLayer::with_dimensions(
1110 self.clips[active].layer_desc.clone(),
1111 w,
1112 h,
1113 PixelFormat::Rgba,
1114 );
1115 let composited = match VideoFrame::from_rgba(w, h, self.rgba_a.clone()) {
1116 Ok(bf) => self.composite_frame(
1117 base_layer,
1118 active,
1119 bf,
1120 w,
1121 h,
1122 &active_overlays,
1123 timeline_pts,
1124 ),
1125 Err(_) => None,
1126 };
1127
1128 // Deliver: the composited frame, or the raw V1 as a fallback.
1129 if let Some(sink) = self.sink.as_mut() {
1130 match &composited {
1131 Some((rgba, cw, ch)) => {
1132 sink.push_frame(rgba, *cw, *ch, timeline_pts);
1133 }
1134 None => sink.push_frame(&self.rgba_a, w, h, timeline_pts),
1135 }
1136 }
1137 // Unpaced: the next frame is due now, and a gap after this
1138 // one is measured from the slot following it.
1139 if stepped {
1140 self.clock.reset(timeline_pts + frame_period);
1141 }
1142
1143 // Advance past a completed transition.
1144 if in_trans && timeline_pts >= trans_start + trans_dur {
1145 let old_active = self.active;
1146 self.transition = None;
1147 self.active = next_idx;
1148 if self.active != old_active {
1149 let in_pt = self.clips[self.active].in_point;
1150 self.restart_audio_at(self.active, in_pt);
1151 }
1152 }
1153 }
1154
1155 let _ = self
1156 .event_tx
1157 .try_send(PlayerEvent::PositionUpdate(timeline_pts));
1158 }
1159 }
1160 }
1161
1162 let _ = self.event_tx.try_send(PlayerEvent::Eof);
1163 if let Some(sink) = self.sink.as_mut() {
1164 sink.flush();
1165 }
1166 Ok(())
1167 }
1168
1169 /// Seek all decode buffers so that `active` is the clip containing `target`
1170 /// and that clip's buffer is positioned at the correct source-file PTS.
1171 ///
1172 /// When `target` falls in a pre-roll or inter-clip gap the method finds the
1173 /// next clip after `target`, seeks it to its `in_point`, and returns without
1174 /// starting audio — the gap-fill loop in `run()` will start audio at the
1175 /// right time.
1176 pub(super) fn seek_timeline(&mut self, target: Duration) -> Result<(), PreviewError> {
1177 // Try to find a clip that contains `target`.
1178 let clip_in_range = self
1179 .clips
1180 .iter()
1181 .position(|c| target >= c.timeline_start && target < c.timeline_end);
1182
1183 // If target is in a gap, find the next clip after `target`.
1184 let (clip_idx, clip_local_pts, is_gap_seek) = if let Some(ci) = clip_in_range {
1185 let elapsed_tl = target.saturating_sub(self.clips[ci].timeline_start);
1186 let local = self.clips[ci].in_point
1187 + if (self.clips[ci].speed - 1.0).abs() < 1e-9 {
1188 elapsed_tl
1189 } else {
1190 elapsed_tl.mul_f64(self.clips[ci].speed)
1191 };
1192 (ci, local, false)
1193 } else if let Some(ci) = self.clips.iter().position(|c| c.timeline_start > target) {
1194 // Seek the clip to its in_point; gap-fill loop will tick until it starts.
1195 (ci, self.clips[ci].in_point, true)
1196 } else {
1197 return Err(PreviewError::SeekOutOfRange { pts: target });
1198 };
1199
1200 self.clips[clip_idx].decode_buf.seek(clip_local_pts)?;
1201 self.active = clip_idx;
1202 self.transition = None;
1203
1204 // Discard stale audio and restart from the seek position.
1205 if let Some(mixer_arc) = &self.audio_mixer {
1206 mixer_arc
1207 .lock()
1208 .unwrap_or_else(std::sync::PoisonError::into_inner)
1209 .invalidate_all();
1210 }
1211 // The visual equivalent of that invalidation. A stateful effect (motion
1212 // blur's exposure trail) accumulated from the frames that preceded the *old*
1213 // position, which after a seek are not the frames preceding the new one — so
1214 // it is stale whether or not the seek crossed a clip boundary (#1705).
1215 if let Some(c) = self.gpu_compositor.as_mut() {
1216 c.reset_effects();
1217 }
1218 if is_gap_seek {
1219 // Cancel any running V1 audio thread; the gap loop will restart it
1220 // once the clock reaches the clip's timeline_start.
1221 if let Some(cancel) = self.active_audio_cancel.take() {
1222 cancel.store(true, Ordering::Release);
1223 }
1224 drop(self.active_audio_thread.take());
1225 } else {
1226 self.restart_audio_at(clip_idx, clip_local_pts);
1227 }
1228
1229 // Seek overlay layers to the new target position.
1230 for layer in &mut self.overlay_layers {
1231 let cidx = layer
1232 .clips
1233 .iter()
1234 .position(|c| target >= c.timeline_start && target < c.timeline_end);
1235 if let Some(cidx) = cidx {
1236 let local = layer.clips[cidx].in_point
1237 + target.saturating_sub(layer.clips[cidx].timeline_start);
1238 let _ = layer.clips[cidx].decode_buf.seek(local);
1239 layer.active = cidx;
1240 }
1241 }
1242
1243 // The lavfi overlay source exposes no seek — rebuild it so it restarts from
1244 // t=0. Static overlays are unaffected; a time-varying lavfi restarts (a
1245 // documented limitation).
1246 if let Some(st) = &mut self.lavfi {
1247 st.rebuild();
1248 }
1249
1250 // Stop all audio-only threads; they restart on the next frame tick.
1251 for at in &mut self.audio_only_tracks {
1252 at.stop();
1253 }
1254
1255 Ok(())
1256 }
1257
1258 /// Coarse (I-frame only) seek variant of [`seek_timeline`].
1259 ///
1260 /// Does not restart audio or invalidate the mixer — caller is responsible.
1261 /// Used for the reverse→forward recovery path where latency matters more
1262 /// than frame-accurate positioning.
1263 fn seek_timeline_coarse(&mut self, target: Duration) -> Result<(), PreviewError> {
1264 let clip_idx = self
1265 .clips
1266 .iter()
1267 .position(|c| target >= c.timeline_start && target < c.timeline_end)
1268 .ok_or(PreviewError::SeekOutOfRange { pts: target })?;
1269 let elapsed_tl = target.saturating_sub(self.clips[clip_idx].timeline_start);
1270 let clip_local_pts = self.clips[clip_idx].in_point
1271 + if (self.clips[clip_idx].speed - 1.0).abs() < 1e-9 {
1272 elapsed_tl
1273 } else {
1274 elapsed_tl.mul_f64(self.clips[clip_idx].speed)
1275 };
1276 self.clips[clip_idx]
1277 .decode_buf
1278 .seek_coarse(clip_local_pts)?;
1279 self.active = clip_idx;
1280 self.transition = None;
1281 // Keep the lavfi overlay consistent with the main seek path (no source seek).
1282 if let Some(st) = &mut self.lavfi {
1283 st.rebuild();
1284 }
1285 Ok(())
1286 }
1287
1288 /// Cancel the current audio decode thread (if any) and start a new one
1289 /// for `clip_idx` beginning at `start_pts`.
1290 fn restart_audio_at(&mut self, clip_idx: usize, start_pts: Duration) {
1291 // Cancel and drop the previous thread.
1292 if let Some(cancel) = &self.active_audio_cancel {
1293 cancel.store(true, Ordering::Release);
1294 }
1295 drop(self.active_audio_thread.take());
1296 self.active_audio_cancel = None;
1297
1298 let Some(handle) = self.clips.get(clip_idx).and_then(|c| c.audio_track.clone()) else {
1299 return;
1300 };
1301 handle.clear(); // discard stale samples
1302
1303 let c = &self.clips[clip_idx];
1304 // Only a file source reaches here (a generated clip has no `audio_track`,
1305 // so the guard above returns early); derive its path for the audio thread.
1306 let source = c
1307 .source
1308 .as_file()
1309 .map(std::path::Path::to_path_buf)
1310 .unwrap_or_default();
1311 // V1 clip audio honours its own fades + speed (the A-track path already does).
1312 // `clip_dur` must be the SOURCE-time span (the resampler multiplies it by
1313 // `1/speed` to get timeline time, as the A-tracks feed `out-in` source span);
1314 // the timeline span is `source/speed`, so scale it back by `speed`.
1315 let fades = AudioFadeConfig {
1316 fade_in: c.fade_in,
1317 fade_out: c.fade_out,
1318 clip_dur: c
1319 .timeline_end
1320 .saturating_sub(c.timeline_start)
1321 .mul_f64(c.speed),
1322 in_point: c.in_point,
1323 speed: c.speed,
1324 pitch: c.pitch,
1325 };
1326 let cancel = Arc::new(AtomicBool::new(false));
1327 let thread =
1328 spawn_audio_track_thread(source, start_pts, handle, Arc::clone(&cancel), fades);
1329 self.active_audio_cancel = Some(cancel);
1330 self.active_audio_thread = Some(thread);
1331 }
1332}
1333
1334impl Drop for SceneRunner {
1335 fn drop(&mut self) {
1336 if let Some(cancel) = &self.active_audio_cancel {
1337 cancel.store(true, Ordering::Release);
1338 }
1339 if let Some(h) = self.active_audio_thread.take() {
1340 let _ = h.join();
1341 }
1342 }
1343}
1344
1345/// Pairs each layer spec with its decoded frame and asks the injected GPU
1346/// compositor to composite them, or returns `None` (no compositor, or the
1347/// compositor declined) so the caller uses the CPU path. Split out of
1348/// `composite_frame` so the seam is unit-testable without a full runner.
1349fn try_gpu_composite(
1350 gpu: Option<&mut Box<dyn PreviewCompositor>>,
1351 specs: &[RealtimeLayer],
1352 frames: &[VideoFrame],
1353 canvas: (u32, u32),
1354 t: Duration,
1355) -> Option<(Vec<u8>, u32, u32)> {
1356 let gpu = gpu?;
1357 let pairs: Vec<(&RealtimeLayer, &VideoFrame)> = specs.iter().zip(frames.iter()).collect();
1358 gpu.composite(&pairs, canvas, t)
1359}
1360
1361/// Offer a transition blend to the injected compositor, or `None` when there is none
1362/// registered or it declines.
1363///
1364/// Mirrors [`try_gpu_composite`]: the runner keeps one `if let Some` shape for "the GPU
1365/// answered" and treats every other case, including no injection at all, as the CPU path.
1366fn try_gpu_blend(
1367 gpu: Option<&mut Box<dyn PreviewCompositor>>,
1368 kind: XfadeTransition,
1369 a: &[u8],
1370 b: &[u8],
1371 progress: f32,
1372 w: u32,
1373 h: u32,
1374) -> Option<Vec<u8>> {
1375 let blended = gpu?.blend(kind, a, b, progress, w, h)?;
1376 // A short buffer would be written straight into `rgba_a` and read as a frame, so
1377 // check the length here rather than trusting the implementor (the trait is public).
1378 if blended.len() == (w as usize) * (h as usize) * 4 {
1379 Some(blended)
1380 } else {
1381 log::warn!(
1382 "preview: GPU blend returned {} bytes, expected {}; falling back to the CPU path",
1383 blended.len(),
1384 (w as usize) * (h as usize) * 4
1385 );
1386 None
1387 }
1388}
1389
1390#[cfg(test)]
1391mod tests {
1392 use super::*;
1393
1394 /// A `PreviewCompositor` that returns a fixed result, to drive the seam.
1395 struct MockCompositor {
1396 result: Option<(Vec<u8>, u32, u32)>,
1397 calls: std::cell::Cell<u32>,
1398 }
1399
1400 impl PreviewCompositor for MockCompositor {
1401 fn composite(
1402 &mut self,
1403 _layers: &[(&RealtimeLayer, &VideoFrame)],
1404 _canvas: (u32, u32),
1405 _t: Duration,
1406 ) -> Option<(Vec<u8>, u32, u32)> {
1407 self.calls.set(self.calls.get() + 1);
1408 self.result.clone()
1409 }
1410 }
1411
1412 /// A `PreviewCompositor` whose `blend` returns a fixed buffer, to drive the
1413 /// transition seam. `composite` is never the subject here.
1414 struct MockBlender {
1415 result: Option<Vec<u8>>,
1416 calls: std::cell::Cell<u32>,
1417 }
1418
1419 impl PreviewCompositor for MockBlender {
1420 fn composite(
1421 &mut self,
1422 _layers: &[(&RealtimeLayer, &VideoFrame)],
1423 _canvas: (u32, u32),
1424 _t: Duration,
1425 ) -> Option<(Vec<u8>, u32, u32)> {
1426 None
1427 }
1428
1429 fn blend(
1430 &mut self,
1431 _kind: XfadeTransition,
1432 _a: &[u8],
1433 _b: &[u8],
1434 _progress: f32,
1435 _w: u32,
1436 _h: u32,
1437 ) -> Option<Vec<u8>> {
1438 self.calls.set(self.calls.get() + 1);
1439 self.result.clone()
1440 }
1441 }
1442
1443 #[test]
1444 fn try_gpu_blend_should_return_none_without_a_compositor() {
1445 let (a, b) = (vec![0u8; 2 * 2 * 4], vec![255u8; 2 * 2 * 4]);
1446 assert!(try_gpu_blend(None, XfadeTransition::Fade, &a, &b, 0.5, 2, 2).is_none());
1447 }
1448
1449 #[test]
1450 fn try_gpu_blend_should_use_the_compositor_result_when_some() {
1451 let (a, b) = (vec![0u8; 2 * 2 * 4], vec![255u8; 2 * 2 * 4]);
1452 let want = vec![7u8; 2 * 2 * 4];
1453 let mut gpu: Box<dyn PreviewCompositor> = Box::new(MockBlender {
1454 result: Some(want.clone()),
1455 calls: std::cell::Cell::new(0),
1456 });
1457 let out = try_gpu_blend(Some(&mut gpu), XfadeTransition::Fade, &a, &b, 0.5, 2, 2);
1458 assert_eq!(out, Some(want));
1459 }
1460
1461 #[test]
1462 fn try_gpu_blend_should_return_none_when_the_compositor_declines() {
1463 let (a, b) = (vec![0u8; 2 * 2 * 4], vec![255u8; 2 * 2 * 4]);
1464 let mut gpu: Box<dyn PreviewCompositor> = Box::new(MockBlender {
1465 result: None,
1466 calls: std::cell::Cell::new(0),
1467 });
1468 assert!(try_gpu_blend(Some(&mut gpu), XfadeTransition::Fade, &a, &b, 0.5, 2, 2).is_none());
1469 }
1470
1471 #[test]
1472 fn try_gpu_blend_should_reject_a_wrongly_sized_buffer() {
1473 // The result is written straight into `rgba_a` and read back as a frame, so a
1474 // short buffer has to fall back rather than corrupt the next composite. The
1475 // trait is public, so this is not a should-not-happen.
1476 let (a, b) = (vec![0u8; 2 * 2 * 4], vec![255u8; 2 * 2 * 4]);
1477 let mut gpu: Box<dyn PreviewCompositor> = Box::new(MockBlender {
1478 result: Some(vec![7u8; 3]),
1479 calls: std::cell::Cell::new(0),
1480 });
1481 assert!(try_gpu_blend(Some(&mut gpu), XfadeTransition::Fade, &a, &b, 0.5, 2, 2).is_none());
1482 }
1483
1484 #[test]
1485 fn try_gpu_blend_should_default_to_none_for_a_composite_only_implementor() {
1486 // The trait's default: an existing `PreviewCompositor` that predates this seam
1487 // keeps working and simply never takes the GPU blend.
1488 let (a, b) = (vec![0u8; 2 * 2 * 4], vec![255u8; 2 * 2 * 4]);
1489 let mut gpu: Box<dyn PreviewCompositor> = Box::new(MockCompositor {
1490 result: None,
1491 calls: std::cell::Cell::new(0),
1492 });
1493 assert!(try_gpu_blend(Some(&mut gpu), XfadeTransition::Fade, &a, &b, 0.5, 2, 2).is_none());
1494 }
1495
1496 fn one_spec_and_frame() -> (Vec<RealtimeLayer>, Vec<VideoFrame>) {
1497 let desc = ff_filter::RealtimeLayerDescriptor {
1498 effects: Vec::new(),
1499 opacity: AnimatedValue::Static(1.0),
1500 x: AnimatedValue::Static(0.0),
1501 y: AnimatedValue::Static(0.0),
1502 scale_x: AnimatedValue::Static(1.0),
1503 scale_y: AnimatedValue::Static(1.0),
1504 rotation: AnimatedValue::Static(0.0),
1505 blend_mode: BlendMode::Normal,
1506 composite_op: CompositeOp::Over,
1507 };
1508 let spec = RealtimeLayer::with_dimensions(desc, 2, 2, PixelFormat::Rgba);
1509 let frame = VideoFrame::from_rgba(2, 2, vec![0u8; 2 * 2 * 4]).expect("frame");
1510 (vec![spec], vec![frame])
1511 }
1512
1513 #[test]
1514 fn try_gpu_composite_should_return_none_without_a_compositor() {
1515 let (specs, frames) = one_spec_and_frame();
1516 assert!(try_gpu_composite(None, &specs, &frames, (2, 2), Duration::ZERO).is_none());
1517 }
1518
1519 #[test]
1520 fn try_gpu_composite_should_use_the_compositor_result_when_some() {
1521 let (specs, frames) = one_spec_and_frame();
1522 let mut gpu: Box<dyn PreviewCompositor> = Box::new(MockCompositor {
1523 result: Some((vec![1, 2, 3, 4], 1, 1)),
1524 calls: std::cell::Cell::new(0),
1525 });
1526 let out = try_gpu_composite(Some(&mut gpu), &specs, &frames, (2, 2), Duration::ZERO);
1527 assert_eq!(out, Some((vec![1, 2, 3, 4], 1, 1)));
1528 }
1529
1530 #[test]
1531 fn try_gpu_composite_should_return_none_when_the_compositor_declines() {
1532 let (specs, frames) = one_spec_and_frame();
1533 let mut gpu: Box<dyn PreviewCompositor> = Box::new(MockCompositor {
1534 result: None,
1535 calls: std::cell::Cell::new(0),
1536 });
1537 // A declining compositor yields None so the caller falls back to CPU.
1538 assert!(
1539 try_gpu_composite(Some(&mut gpu), &specs, &frames, (2, 2), Duration::ZERO).is_none()
1540 );
1541 }
1542
1543 #[test]
1544 fn ensure_dissolve_field_should_build_once_and_rebuild_on_a_size_change() {
1545 // The acceptance criterion directly: a dissolve of n frames builds the field
1546 // once, not n times, and a change of frame size does rebuild it.
1547 let mut field = Vec::new();
1548 let mut dims = (0, 0);
1549
1550 assert!(
1551 ensure_dissolve_field(&mut field, &mut dims, 7, 5),
1552 "the first frame of a dissolve has to build the field"
1553 );
1554 assert_eq!(field.len(), 35);
1555 for _ in 0..10 {
1556 assert!(
1557 !ensure_dissolve_field(&mut field, &mut dims, 7, 5),
1558 "every later frame at the same size must reuse it"
1559 );
1560 }
1561
1562 assert!(
1563 ensure_dissolve_field(&mut field, &mut dims, 9, 4),
1564 "a change of frame size has to rebuild"
1565 );
1566 assert_eq!(field.len(), 36);
1567
1568 // 5x7 has the same pixel count as 7x5, so a length check alone would reuse a
1569 // transposed field and read the wrong pixel at every coordinate.
1570 assert!(
1571 ensure_dissolve_field(&mut field, &mut dims, 4, 9),
1572 "a transposed frame is a different field, not the same one"
1573 );
1574 assert_eq!(field, ff_filter::xfade_frand_field(4, 9));
1575 }
1576}