1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
1196
1197
1198
1199
1200
1201
1202
1203
1204
1205
1206
1207
1208
1209
1210
1211
1212
1213
1214
1215
1216
1217
1218
1219
1220
1221
1222
1223
1224
1225
1226
1227
1228
1229
1230
1231
1232
1233
1234
1235
1236
1237
1238
1239
1240
1241
1242
1243
1244
1245
1246
1247
1248
1249
1250
1251
1252
1253
1254
1255
1256
1257
1258
1259
1260
1261
1262
1263
1264
1265
1266
1267
1268
1269
1270
1271
1272
1273
1274
1275
1276
1277
1278
1279
1280
1281
1282
1283
1284
1285
1286
1287
1288
1289
1290
1291
1292
1293
1294
1295
1296
1297
1298
1299
1300
1301
1302
1303
1304
1305
1306
1307
1308
1309
1310
1311
1312
1313
1314
1315
1316
1317
1318
1319
1320
1321
1322
1323
1324
1325
1326
1327
1328
1329
1330
1331
1332
1333
1334
1335
1336
1337
1338
1339
1340
1341
1342
1343
1344
1345
1346
1347
1348
1349
1350
1351
1352
1353
1354
1355
1356
1357
1358
1359
1360
1361
1362
1363
1364
1365
1366
1367
1368
1369
1370
1371
1372
1373
1374
1375
1376
1377
1378
1379
1380
1381
1382
1383
1384
1385
1386
1387
1388
1389
1390
1391
1392
1393
1394
1395
1396
1397
1398
1399
1400
1401
1402
1403
1404
1405
1406
1407
1408
1409
1410
1411
1412
1413
1414
1415
1416
1417
1418
1419
1420
1421
1422
1423
1424
1425
1426
1427
1428
1429
1430
1431
1432
1433
1434
1435
1436
1437
1438
1439
1440
1441
1442
1443
1444
1445
1446
1447
1448
1449
1450
1451
1452
1453
1454
1455
1456
1457
1458
1459
1460
1461
1462
1463
1464
1465
1466
1467
1468
1469
1470
1471
1472
1473
1474
1475
1476
1477
1478
1479
1480
1481
1482
1483
1484
1485
1486
1487
1488
1489
1490
1491
1492
1493
1494
1495
1496
1497
1498
1499
1500
1501
1502
1503
1504
1505
1506
1507
1508
1509
1510
1511
1512
1513
1514
1515
1516
1517
1518
1519
1520
1521
1522
1523
1524
1525
1526
1527
1528
1529
1530
1531
1532
1533
1534
1535
1536
1537
1538
1539
1540
1541
1542
1543
1544
1545
1546
1547
1548
1549
1550
1551
1552
1553
1554
1555
1556
1557
1558
1559
1560
1561
1562
1563
1564
1565
1566
1567
1568
1569
1570
1571
1572
1573
1574
1575
1576
//! The timeline decode/present state machine.
//!
//! [`SceneRunner`] owns the per-track decode buffers and the audio mixer,
//! and drives frame presentation. Construct it via
//! [`ScenePlayer::open`](super::ScenePlayer::open).
use std::sync::atomic::{AtomicBool, AtomicU64, Ordering};
use std::sync::{Arc, Mutex, mpsc};
use std::thread::{self, JoinHandle};
use std::time::Duration;
use ff_filter::{
AnimatedValue, BlendMode, CompositeOp, RealtimeComposer, RealtimeLayer, XfadeTransition,
};
use ff_format::{PixelFormat, Rational, Timestamp, VideoFrame};
use crate::audio::AudioMixer;
use crate::error::PreviewError;
use crate::event::PlayerEvent;
use crate::playback::SwsRgbaConverter;
use crate::playback::decode_buffer::FrameResult;
use crate::playback::master_clock::MasterClock;
use crate::playback::player::PlayerCommand;
use crate::playback::sink::FrameSink;
use super::audio_resampling::spawn_audio_track_thread;
use super::compositor::PreviewCompositor;
use super::inner;
use super::state::{
AudioFadeConfig, AudioOnlyTrack, ClipState, LavfiOverlayState, OverlayLayer, TransitionState,
db_to_linear,
};
// SceneRunner
/// Exclusive owner of the timeline decode pipeline.
///
/// Move to a background thread and call [`run`](Self::run). Register a
/// [`FrameSink`] with [`set_sink`](Self::set_sink) before calling `run`.
pub struct SceneRunner {
pub(super) clips: Vec<ClipState>,
/// Secondary video overlay layers (V2, V3, …). Each is composited over V1
/// in order before the frame is delivered to the sink.
pub(super) overlay_layers: Vec<OverlayLayer>,
/// Dedicated audio-only clips (from A1, A2, … tracks). Each is started and
/// stopped as the playhead crosses its timeline window.
pub(super) audio_only_tracks: Vec<AudioOnlyTrack>,
/// Index of the clip currently being decoded and presented.
pub(super) active: usize,
/// Non-`None` while a crossfade transition is in progress.
pub(super) transition: Option<TransitionState>,
pub(super) cmd_rx: mpsc::Receiver<PlayerCommand>,
pub(super) event_tx: mpsc::SyncSender<PlayerEvent>,
pub(super) sink: Option<Box<dyn FrameSink>>,
/// Optional injected GPU compositor, tried before the built-in CPU compositor.
/// `avio` supplies one over `ff-render`; `None` (the default) uses the CPU path.
pub(super) gpu_compositor: Option<Box<dyn PreviewCompositor>>,
pub(super) current_pts: Arc<AtomicU64>,
pub(super) paused: Arc<AtomicBool>,
pub(super) stopped: Arc<AtomicBool>,
pub(super) fps: f64,
pub(super) rate: f64,
pub(super) clock: MasterClock,
/// Media PTS to re-anchor the System clock to when `PlayerCommand::Play`
/// is received from a paused state. Updated on every seek and after every
/// presented frame so that accumulated wall-clock time during pause does
/// not advance `current_pts()` past the last known media position.
pub(super) resume_pts: Duration,
/// Pixel-format converter for the active (outgoing) frame.
pub(super) sws_a: SwsRgbaConverter,
/// Pixel-format converter for the incoming frame during transitions.
pub(super) sws_b: SwsRgbaConverter,
pub(super) rgba_a: Vec<u8>,
pub(super) rgba_b: Vec<u8>,
pub(super) blend_buf: Vec<u8>,
/// `xfade`'s dissolve noise, tabulated for the current frame size. The hash depends
/// only on the pixel coordinates, so recomputing it per frame was costing a 4 K
/// dissolve more than a whole 30 fps budget (#1736). Kept beside the rgba scratch
/// because it has the same lifetime: rebuilt when the frame size changes, held
/// across transitions so a dissolve does not pay for it again.
pub(super) dissolve_field: Vec<f32>,
/// The frame size `dissolve_field` was built for. `(0, 0)` until the first dissolve,
/// so no field is built for a timeline that never dissolves.
pub(super) dissolve_field_dims: (u32, u32),
/// Width of the most recently presented primary-track frame; used to
/// synthesise fill frames during primary-track gaps.
pub(super) last_frame_w: u32,
/// Height of the most recently presented primary-track frame.
pub(super) last_frame_h: u32,
/// Scratch buffer for synthesising black fill frames during primary-track gaps.
pub(super) gap_buf: Vec<u8>,
/// Multi-track audio mixer — `None` when no clip has audio.
pub(super) audio_mixer: Option<Arc<Mutex<AudioMixer>>>,
/// Cancel flag for the currently running audio decode thread.
pub(super) active_audio_cancel: Option<Arc<AtomicBool>>,
/// Handle to the currently running audio decode thread.
pub(super) active_audio_thread: Option<JoinHandle<()>>,
/// Cached real-time compositor that applies per-clip effects + blend modes
/// (the same chain as export). Rebuilt only when the active clip set or frame
/// geometry changes; `None` until the first composite.
pub(super) composer: Option<RealtimeComposer>,
/// Identifies the composer's current configuration as
/// `(layer_id, active_clip_idx, width, height)` per layer. Rebuild on change.
pub(super) composer_key: Vec<(usize, usize, u32, u32)>,
/// Project output canvas. When `Some`, every layer is placed on a canvas of these
/// dimensions (the base included, at its native size unless scaled) and every
/// composited frame is canvas-sized. `None` composites at the base clip's own
/// size, the standalone behaviour an engine never asks for.
pub(super) canvas: Option<(u32, u32)>,
/// Timeline-global generated `lavfi` overlay, composited as the topmost layer
/// (above every file overlay). `None` when the timeline set no `lavfi_overlay`.
pub(super) lavfi: Option<LavfiOverlayState>,
}
/// Rebuilds `field` when it does not already hold [`xfade_frand_field`] for `w * h`,
/// returning whether it did.
///
/// A free function rather than a method so the rule can be tested without a runner, and
/// so the caller keeps `field` as a plain field it can lend out beside its other scratch
/// buffers. The dimensions are tracked explicitly rather than inferred from the length:
/// `w * h` alone cannot tell 1920x1080 from 1080x1920, and a transposed field would read
/// the wrong pixel at every coordinate while looking the right size.
fn ensure_dissolve_field(field: &mut Vec<f32>, dims: &mut (u32, u32), w: u32, h: u32) -> bool {
if *dims == (w, h) && field.len() == (w as usize) * (h as usize) {
return false;
}
*field = ff_filter::xfade_frand_field(w, h);
*dims = (w, h);
true
}
/// How [`SceneRunner::run`] paces frame delivery.
#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
#[non_exhaustive]
pub enum Pacing {
/// Deliver frames against the wall clock: sleep until a frame is due and
/// drop one that is more than a frame period late. Playback.
#[default]
RealTime,
/// Deliver every frame as soon as it is decoded: the clock is the runner's
/// own position, moved one frame period per presented frame, so nothing is
/// ever late and nothing is dropped or slept for. For checks that must see
/// each frame (tests, thumbnail strips). Reverse playback and the pause
/// poll still wait on the wall clock.
Unpaced,
}
impl SceneRunner {
/// Register the frame sink. Call before [`run`](Self::run).
pub fn set_sink(&mut self, sink: Box<dyn FrameSink>) {
self.sink = Some(sink);
}
/// Choose how frames are paced. Call before [`run`](Self::run); the default
/// is [`Pacing::RealTime`]. Switching keeps the current position.
pub fn set_pacing(&mut self, pacing: Pacing) {
let current = self.clock.current_pts();
self.clock = match pacing {
Pacing::RealTime => MasterClock::System {
started_at: std::time::Instant::now(),
base_pts: current,
// A wall clock only runs forward; reverse playback keeps its own
// stepping and never reads the clock's rate.
rate: if self.rate > 0.0 { self.rate } else { 1.0 },
},
Pacing::Unpaced => MasterClock::Stepped { pts: current },
};
}
/// Register an external GPU compositor tried before the built-in CPU path.
/// Call before [`run`](Self::run). `avio` supplies one over `ff-render`.
pub fn set_gpu_compositor(&mut self, compositor: Box<dyn PreviewCompositor>) {
self.gpu_compositor = Some(compositor);
}
/// Whether an external GPU compositor is registered (else the CPU path is used).
#[must_use]
pub fn has_gpu_compositor(&self) -> bool {
self.gpu_compositor.is_some()
}
/// Advances every overlay layer to the frame whose presentation time has
/// arrived at `target_pts`, holding the current frame otherwise (so a layer
/// whose fps differs from the timeline plays at the right speed rather than
/// advancing once per present). Returns `(layer_index, width, height)` for
/// each layer that currently has a frame to show.
fn sync_overlays(&mut self, target_pts: Duration) -> Vec<(usize, u32, u32)> {
let mut active = Vec::new();
for (li, layer) in self.overlay_layers.iter_mut().enumerate() {
let maybe_cidx = layer
.clips
.iter()
.position(|c| target_pts >= c.timeline_start && target_pts < c.timeline_end);
let Some(cidx) = maybe_cidx else {
layer.rgba.clear();
layer.cur_dims = None;
layer.pending = None;
continue;
};
if cidx != layer.active {
let local = layer.clips[cidx].in_point
+ target_pts.saturating_sub(layer.clips[cidx].timeline_start);
let _ = layer.clips[cidx].decode_buf.seek(local);
layer.active = cidx;
layer.cur_dims = None;
layer.pending = None;
}
let clip_in = layer.clips[cidx].in_point;
let tl_start = layer.clips[cidx].timeline_start;
loop {
let f = match layer.pending.take() {
Some(pf) => pf,
None => match layer.clips[cidx].decode_buf.pop_frame() {
FrameResult::Frame(f) => f,
_ => break,
},
};
let v2_pts = tl_start + f.timestamp().as_duration().saturating_sub(clip_in);
if v2_pts > target_pts {
// Not due yet — hold it for a later present.
layer.pending = Some(f);
break;
}
if layer.sws.convert(&f, &mut layer.rgba) {
layer.cur_dims = Some((f.width(), f.height()));
}
}
match layer.cur_dims {
Some((ow, oh)) => active.push((li, ow, oh)),
None => layer.rgba.clear(),
}
}
active
}
/// Advances the generated `lavfi` overlay (if any) to the frame due at
/// `target_pts`, holding otherwise — see [`LavfiOverlayState::advance_to`].
/// Returns the current frame's `(width, height)`, or `None` when there is no
/// lavfi overlay / no frame yet.
fn sync_lavfi(&mut self, target_pts: Duration) -> Option<(u32, u32)> {
self.lavfi.as_mut()?.advance_to(target_pts)
}
/// Composites `base_frame` (the bottom layer) with the given overlay layers
/// through the cached [`RealtimeComposer`], applying each layer's effects and
/// blend mode. `base_id` identifies the base for cache invalidation — the V1
/// clip index, or `usize::MAX` for the gap-fill black base. Returns the
/// composited RGBA frame together with its actual `(width, height)`, or `None`
/// on failure.
///
/// The composited size can differ from `base_w`/`base_h` when the base layer's
/// effect chain resizes the frame (`Crop`, `Scale`, `Pad`, `FitToAspect`), so
/// callers must push the returned dimensions to the sink rather than the
/// decoded ones — otherwise the buffer length no longer matches the reported
/// size and the frame is dropped.
#[allow(clippy::too_many_arguments)]
fn composite_frame(
&mut self,
base_layer: RealtimeLayer,
base_id: usize,
mut base_frame: VideoFrame,
base_w: u32,
base_h: u32,
overlays: &[(usize, u32, u32)],
t: Duration,
) -> Option<(Vec<u8>, u32, u32)> {
let mut specs = vec![base_layer];
let mut key: Vec<(usize, usize, u32, u32)> = vec![(0, base_id, base_w, base_h)];
for &(li, ow, oh) in overlays {
let oc = &self.overlay_layers[li];
specs.push(RealtimeLayer::with_dimensions(
oc.clips[oc.active].layer_desc.clone(),
ow,
oh,
PixelFormat::Rgba,
));
key.push((li + 1, oc.active, ow, oh));
}
// Topmost timeline-global lavfi overlay (generated), when a frame is held.
// Fixed full-frame Normal/Over layer, matching export; its transparency comes
// from the lavfi content's own alpha. A sentinel layer id keys the cache.
let lavfi_dims = self.lavfi.as_ref().and_then(|s| s.dims);
if let Some((lw, lh)) = lavfi_dims {
specs.push(RealtimeLayer {
width: lw,
height: lh,
pixel_format: PixelFormat::Rgba,
effects: Vec::new(),
opacity: AnimatedValue::Static(1.0),
x: AnimatedValue::Static(0.0),
y: AnimatedValue::Static(0.0),
scale_x: AnimatedValue::Static(1.0),
scale_y: AnimatedValue::Static(1.0),
rotation: AnimatedValue::Static(0.0),
blend_mode: BlendMode::Normal,
composite_op: CompositeOp::Over,
});
key.push((usize::MAX - 1, 0, lw, lh));
}
// Build the decoded frame for every layer, in the same order as `specs`.
// Stamp each with the composite's timeline PTS so the graph's per-frame
// animation tick (in `push_video`) evaluates each layer's opacity track at
// the same time. Frames from `from_rgba` carry PTS 0 otherwise, and any
// registered `AnimationEntry` would be frozen at t=0.
let ts = Timestamp::from_duration(t, Rational::new(1, 1_000_000));
base_frame.set_timestamp(ts);
let mut frames = vec![base_frame];
for &(li, ow, oh) in overlays {
let mut vf =
VideoFrame::from_rgba(ow, oh, self.overlay_layers[li].rgba.clone()).ok()?;
vf.set_timestamp(ts);
frames.push(vf);
}
if let Some((lw, lh)) = lavfi_dims {
let rgba = self
.lavfi
.as_ref()
.map_or_else(Vec::new, |s| s.rgba.clone());
let mut vf = VideoFrame::from_rgba(lw, lh, rgba).ok()?;
vf.set_timestamp(ts);
frames.push(vf);
}
// Try the injected GPU compositor first; `None` falls through to the CPU
// compositor below (unsupported layer, no adapter, or a GPU error).
let gpu_canvas = self.canvas.unwrap_or((base_w, base_h));
if let Some(out) =
try_gpu_composite(self.gpu_compositor.as_mut(), &specs, &frames, gpu_canvas, t)
{
return Some(out);
}
// CPU compositor (cached, keyed by layer identity/size).
if self.composer.is_none() || self.composer_key != key {
let new_layer_set = self.composer_key != key;
self.composer = match RealtimeComposer::with_canvas(&specs, self.canvas) {
Ok(c) => Some(c),
Err(e) => {
// A GPU compositor is attached but declined this frame for an
// unrelated reason, and the CPU compositor refuses the operator
// outright (#1753), so the base frame is what gets shown. Said
// once per layer set rather than per frame. With no GPU attached
// at all the timeline is refused up front by the engine's open.
if new_layer_set
&& matches!(e, ff_filter::FilterError::UnsupportedCompositeOp { .. })
{
log::warn!(
"preview: CPU compositor refused the layer set, showing the \
base frame only error={e}"
);
}
None
}
};
self.composer_key = key;
}
let composer = self.composer.as_mut()?;
for (slot, vf) in frames.iter().enumerate() {
if composer.push_layer(slot, vf).is_err() {
return None;
}
}
let f = composer.pull().ok().flatten()?;
let (w, h) = (f.width(), f.height());
f.to_rgba().map(|rgba| (rgba, w, h))
}
/// A/V sync presentation loop.
///
/// Plays all clips in the primary video track from start to finish (or until
/// a [`PlayerCommand::Stop`] is received).
///
/// Emits [`PlayerEvent::SeekCompleted`] after each successful seek,
/// [`PlayerEvent::PositionUpdate`] after each presented video frame,
/// [`PlayerEvent::Error`] on non-fatal decode errors, and
/// [`PlayerEvent::Eof`] before returning.
///
/// # Errors
///
/// Returns [`PreviewError::SeekOutOfRange`] if a seek command targets a
/// timestamp that falls outside all clips on the timeline.
#[allow(clippy::too_many_lines)]
pub fn run(mut self) -> Result<(), PreviewError> {
if self.clips.is_empty() {
let _ = self.event_tx.try_send(PlayerEvent::Eof);
return Ok(());
}
let fps = self.fps.max(1.0);
let frame_period = Duration::from_secs_f64(1.0 / fps);
// `Pacing::Unpaced`: the clock is moved by this loop, never by wall time,
// so the pacing sleep and the late-frame drop below are both skipped.
let stepped = self.clock.is_stepped();
self.clock.reset(Duration::ZERO);
loop {
// Drain commands
let mut pending_seek: Option<Duration> = None;
while let Ok(cmd) = self.cmd_rx.try_recv() {
match cmd {
PlayerCommand::Seek(pts) => pending_seek = Some(pts),
PlayerCommand::Play => {
// Always re-anchor the System clock on Play.
//
// PlayerHandle::play() sets the shared `paused` atomic
// to `false` BEFORE enqueueing PlayerCommand::Play, so
// paused.load() here always returns false — a guard on
// `if paused` would never fire. Re-anchoring
// unconditionally is safe: when the player was not
// actually paused, resume_pts equals the last presented
// frame PTS (or the seek target), which is already the
// clock's current base, so clock.reset() is a no-op
// in effect.
self.clock.reset(self.resume_pts);
self.stopped.store(false, Ordering::Release);
self.paused.store(false, Ordering::Release);
}
PlayerCommand::Pause => {
self.paused.store(true, Ordering::Release);
}
PlayerCommand::Stop => {
self.stopped.store(true, Ordering::Release);
}
PlayerCommand::SetRate(r) => {
if r != 0.0 {
let was_negative = self.rate < 0.0;
self.rate = r;
if r > 0.0 {
self.clock.set_rate(r);
if was_negative {
// Returning from reverse: rebase clock and
// restart audio from the current video position.
let pts = Duration::from_micros(
self.current_pts.load(Ordering::Relaxed),
);
self.clock.reset(pts);
self.resume_pts = pts;
if let Err(e) = self.seek_timeline_coarse(pts) {
log::warn!(
"timeline reverse→forward seek failed \
pts={pts:?} error={e}"
);
} else {
let ci = self.active;
let clip_local = self.clips[ci].in_point
+ pts.saturating_sub(self.clips[ci].timeline_start);
if let Some(m) = &self.audio_mixer {
m.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.invalidate_all();
}
self.restart_audio_at(ci, clip_local);
}
}
} else {
// Entering reverse: silence audio.
if let Some(cancel) = &self.active_audio_cancel {
cancel.store(true, Ordering::Release);
}
if let Some(m) = &self.audio_mixer {
m.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.invalidate_all();
}
}
}
}
PlayerCommand::SetAvOffset(_) => {} // audio timing is system-clock driven
PlayerCommand::UpdateLayout(scene) => {
if let Err(e) = self.update_layout_in_place(&scene, self.resume_pts) {
log::warn!("timeline layout update ignored: {e}");
}
}
}
}
// Apply pending seek
let had_seek = pending_seek.is_some();
if let Some(target) = pending_seek {
self.seek_timeline(target)?;
self.clock.reset(target);
self.resume_pts = target;
let _ = self.event_tx.try_send(PlayerEvent::SeekCompleted(target));
}
// When a seek arrives while paused, present one preview frame so
// the sink reflects the new position without resuming playback.
if had_seek && self.paused.load(Ordering::Acquire) {
let active = self.active;
let deadline = std::time::Instant::now() + Duration::from_millis(300);
loop {
match self.clips[active].decode_buf.pop_frame() {
FrameResult::Frame(f) => {
let f_pts = f.timestamp().as_duration();
let elapsed = f_pts.saturating_sub(self.clips[active].in_point);
let tl_pts = self.clips[active].timeline_start
+ if (self.clips[active].speed - 1.0).abs() < 1e-9 {
elapsed
} else {
elapsed.div_f64(self.clips[active].speed)
};
let w = f.width();
let h = f.height();
if self.sws_a.convert(&f, &mut self.rgba_a)
&& let Some(sink) = self.sink.as_mut()
{
sink.push_frame(&self.rgba_a, w, h, tl_pts);
}
self.current_pts.store(
u64::try_from(tl_pts.as_micros()).unwrap_or(u64::MAX),
Ordering::Relaxed,
);
let _ = self.event_tx.try_send(PlayerEvent::PositionUpdate(tl_pts));
break;
}
FrameResult::Seeking(_) => {
if std::time::Instant::now() > deadline {
break;
}
thread::sleep(Duration::from_millis(2));
}
FrameResult::Eof => break,
}
}
}
// Error events from active clip (a generated held source has no channel).
{
let active = self.active;
if let Some(rx) = self.clips[active].decode_buf.error_events() {
while let Ok(msg) = rx.try_recv() {
let _ = self.event_tx.try_send(PlayerEvent::Error(msg));
}
}
}
let trans_next = self.transition.as_ref().map(|tp| tp.next_idx);
if let Some(next_idx) = trans_next
&& let Some(rx) = self.clips[next_idx].decode_buf.error_events()
{
while let Ok(msg) = rx.try_recv() {
let _ = self.event_tx.try_send(PlayerEvent::Error(msg));
}
}
// Stopped / paused
if self.stopped.load(Ordering::Acquire) {
break;
}
if self.paused.load(Ordering::Acquire) {
thread::sleep(Duration::from_millis(5));
continue;
}
// Reverse playback path
if self.rate < 0.0 {
let current = Duration::from_micros(self.current_pts.load(Ordering::Relaxed));
let step = Duration::from_secs_f64(self.rate.abs() / fps.max(f64::MIN_POSITIVE));
let target = current.saturating_sub(step);
let clip_idx = self
.clips
.iter()
.position(|c| target >= c.timeline_start && target < c.timeline_end);
if let Some(ci) = clip_idx {
let elapsed_tl = target.saturating_sub(self.clips[ci].timeline_start);
let clip_local = self.clips[ci].in_point
+ if (self.clips[ci].speed - 1.0).abs() < 1e-9 {
elapsed_tl
} else {
elapsed_tl.mul_f64(self.clips[ci].speed)
};
if self.clips[ci].decode_buf.seek_coarse(clip_local).is_ok() {
if ci != self.active {
self.active = ci;
self.transition = None;
}
let deadline = std::time::Instant::now() + Duration::from_millis(300);
let frame = loop {
match self.clips[ci].decode_buf.pop_frame() {
FrameResult::Frame(f) => break Some(f),
FrameResult::Seeking(_) => {
if std::time::Instant::now() > deadline {
break None;
}
thread::sleep(Duration::from_millis(2));
}
FrameResult::Eof => break None,
}
};
if let Some(f) = frame {
let f_pts = f.timestamp().as_duration();
let elapsed = f_pts.saturating_sub(self.clips[ci].in_point);
let tl_pts = self.clips[ci].timeline_start
+ if (self.clips[ci].speed - 1.0).abs() < 1e-9 {
elapsed
} else {
elapsed.div_f64(self.clips[ci].speed)
};
let w = f.width();
let h = f.height();
if self.sws_a.convert(&f, &mut self.rgba_a)
&& let Some(sink) = self.sink.as_mut()
{
sink.push_frame(&self.rgba_a, w, h, tl_pts);
}
self.current_pts.store(
u64::try_from(tl_pts.as_micros()).unwrap_or(u64::MAX),
Ordering::Relaxed,
);
self.resume_pts = tl_pts;
let _ = self.event_tx.try_send(PlayerEvent::PositionUpdate(tl_pts));
}
}
}
if self
.clips
.first()
.is_some_and(|c| target < c.timeline_start)
{
self.paused.store(true, Ordering::Release);
}
thread::sleep(frame_period);
continue;
}
// Pop frame from active clip
let active = self.active;
let pop_result = self.clips[active].decode_buf.pop_frame();
match pop_result {
FrameResult::Eof => {
let old_active = active;
if let Some(tp) = self.transition.take() {
self.active = tp.next_idx;
} else if active + 1 < self.clips.len() {
self.active += 1;
} else {
break;
}
if self.active != old_active {
// Clear the outgoing clip's pre-decoded audio so its stale
// samples do not continue to mix in after the transition.
if let Some(h) = self.clips[old_active].audio_track.clone() {
h.clear();
}
let in_pt = self.clips[self.active].in_point;
self.restart_audio_at(self.active, in_pt);
}
}
FrameResult::Seeking(last) => {
if let Some(ref f) = last {
let f_pts = f.timestamp().as_duration();
let in_pt = self.clips[active].in_point;
// Suppress pre-seek artefact frames: when a DecodeBuffer
// is opened and immediately seeked to in_point, the
// background thread may have decoded one frame from
// position 0 before processing the seek command. That
// frame ends up as `last` and must not be displayed —
// its content is from before the clip's in_point.
if f_pts >= in_pt {
let tl_start = self.clips[active].timeline_start;
let elapsed = f_pts.saturating_sub(in_pt);
let spd = self.clips[active].speed;
let tl_pts = tl_start
+ if (spd - 1.0).abs() < 1e-9 {
elapsed
} else {
elapsed.div_f64(spd)
};
let w = f.width();
let h = f.height();
if self.sws_a.convert(f, &mut self.rgba_a)
&& let Some(sink) = self.sink.as_mut()
{
sink.push_frame(&self.rgba_a, w, h, tl_pts);
}
}
}
}
FrameResult::Frame(frame) => {
let f_pts = frame.timestamp().as_duration();
let clip_in = self.clips[active].in_point;
let clip_out = self.clips[active].out_point;
let clip_tl_start = self.clips[active].timeline_start;
let clip_tl_end = self.clips[active].timeline_end;
let clip_speed = self.clips[active].speed;
// Frames past `out_point` that feed the crossfade into the next
// clip (ADR-0009). Without it this clip ends exactly where the next
// one starts, the branch below advances, and the transition-entry
// check further down is never reached — which is why an
// engine-derived scene never blended (#1737).
let handle = self.clips[active].video_handle;
// Skip frames before in_point (e.g. right after a seek).
if f_pts < clip_in {
continue;
}
// The handle in source time, to compare against `out_point` and
// `f_pts`: at speed 2.0 half a second of blend is a second of source.
let src_handle = if (clip_speed - 1.0).abs() < 1e-9 {
handle
} else {
handle.mul_f64(clip_speed)
};
// Treat frames past out_point (plus the handle) as EOF for this clip.
let past_out = clip_out.is_some_and(|op| f_pts >= op + src_handle);
let elapsed = f_pts.saturating_sub(clip_in);
// Remap source PTS → timeline PTS via speed factor.
// For speed=2.0 the clip occupies half the timeline duration;
// for speed=0.5 it occupies double.
let tl_elapsed = if (clip_speed - 1.0).abs() < 1e-9 {
elapsed
} else {
elapsed.div_f64(clip_speed)
};
// `handle` is already timeline time, so it adds to the timeline
// extent directly.
let past_end = clip_tl_start + tl_elapsed >= clip_tl_end + handle;
if past_out || past_end {
let old_active = active;
if let Some(tp) = self.transition.take() {
self.active = tp.next_idx;
} else if active + 1 < self.clips.len() {
self.active += 1;
} else {
break;
}
if self.active != old_active {
// Clear the outgoing clip's pre-decoded audio so its
// stale samples do not continue to mix in after the
// transition.
if let Some(h) = self.clips[old_active].audio_track.clone() {
h.clear();
}
// And the visual equivalent: a stateful effect (motion
// blur's exposure trail) accumulates across one clip's
// frames and must not bleed into the next. This rides the
// cut detection that is already here rather than adding a
// second notion of a boundary, which matters because clip
// progression is driven by each frame's own PTS (RK-019).
if let Some(c) = self.gpu_compositor.as_mut() {
c.reset_effects();
}
let in_pt = self.clips[self.active].in_point;
self.restart_audio_at(self.active, in_pt);
}
continue;
}
let timeline_pts = clip_tl_start + tl_elapsed;
// Manage audio-only decode threads
for at in &mut self.audio_only_tracks {
let should_run =
timeline_pts >= at.timeline_start && timeline_pts < at.timeline_end;
let is_running = at.cancel.is_some();
if should_run && !is_running {
let local =
at.in_point + timeline_pts.saturating_sub(at.timeline_start);
at.start_at(local);
} else if !should_run && is_running {
at.stop();
// Clear stale pre-decoded samples so the mixer does
// not play this track's buffered audio past clip end.
at.handle.clear();
}
// Per-clip volume automation: an animated gain is evaluated at
// the timeline PTS each tick (a static gain was set at open).
if should_run && let AnimatedValue::Track(track) = &at.volume {
at.handle
.set_volume(db_to_linear(track.value_at(timeline_pts)));
}
}
// Primary-track volume automation for the active clip.
if let Some(handle) = &self.clips[active].audio_track
&& let AnimatedValue::Track(track) = &self.clips[active].volume
{
handle.set_volume(db_to_linear(track.value_at(timeline_pts)));
}
// Update shared current_pts and resume anchor.
self.current_pts.store(
u64::try_from(timeline_pts.as_micros()).unwrap_or(u64::MAX),
Ordering::Relaxed,
);
self.resume_pts = timeline_pts;
// Transition zone entry check
if self.transition.is_none() && active + 1 < self.clips.len() {
let next = &self.clips[active + 1];
if next.xfade_dur > Duration::ZERO && timeline_pts >= next.timeline_start {
if timeline_pts < next.timeline_start + next.xfade_dur {
self.transition = Some(TransitionState {
next_idx: active + 1,
start: next.timeline_start,
duration: next.xfade_dur,
kind: next.xfade_kind.unwrap_or(XfadeTransition::Fade),
});
} else {
// Jumped past the entire transition zone.
let old_active = active;
self.active = active + 1;
if self.active != old_active {
let in_pt = self.clips[self.active].in_point;
self.restart_audio_at(self.active, in_pt);
}
continue;
}
}
}
// A/V sync (system clock)
{
let clock_pts = self.clock.current_pts();
let diff = timeline_pts.as_secs_f64() - clock_pts.as_secs_f64();
let fp = frame_period.as_secs_f64();
// Only enter gap fill for an actual gap between clips.
// For slow-motion clips (speed < 1.0) the large diff is expected
// and should be handled by the `diff > fp` sleep below instead.
if diff > fp * 2.0
&& (clip_speed - 1.0) > -1e-9
&& self.transition.is_none()
&& self.last_frame_w > 0
{
// Gap in the primary track: the next V1 clip starts more than
// 2 frame-periods ahead of the clock. Synthesise black frames
// composited with overlay-layer content for every missing
// frame period so that V2 overlays and audio-only tracks
// remain live during the gap.
// With an explicit canvas, fill gaps at the canvas size so
// the preview frame size stays constant across gaps.
let (gw, gh) = self
.canvas
.unwrap_or((self.last_frame_w, self.last_frame_h));
let n = (gw * gh * 4) as usize;
'gap: loop {
// Drain incoming commands.
while let Ok(cmd) = self.cmd_rx.try_recv() {
match cmd {
PlayerCommand::Play => {
self.clock.reset(self.resume_pts);
self.stopped.store(false, Ordering::Release);
self.paused.store(false, Ordering::Release);
}
PlayerCommand::Pause => {
self.paused.store(true, Ordering::Release);
}
PlayerCommand::Stop => {
self.stopped.store(true, Ordering::Release);
}
PlayerCommand::SetRate(r) if r > 0.0 => {
self.rate = r;
self.clock.set_rate(r);
}
_ => {}
}
}
if self.stopped.load(Ordering::Acquire) {
break 'gap;
}
if self.paused.load(Ordering::Acquire) {
thread::sleep(Duration::from_millis(5));
continue 'gap;
}
let gap_pts = self.clock.current_pts();
if gap_pts + frame_period >= timeline_pts {
break 'gap;
}
// Build a black base and composite the overlays onto it
// through the shared compositor — same held-frame timing,
// effects, and blend modes as the main present path.
self.gap_buf.resize(n, 0);
self.gap_buf.fill(0);
let gap_overlays = self.sync_overlays(gap_pts);
let gap_lavfi = self.sync_lavfi(gap_pts);
// Composite when there is a file overlay OR a lavfi
// overlay to draw over the gap's black base.
let gap_composited =
if gap_overlays.is_empty() && gap_lavfi.is_none() {
None
} else {
let base_layer = RealtimeLayer {
width: gw,
height: gh,
pixel_format: PixelFormat::Rgba,
effects: Vec::new(),
opacity: AnimatedValue::Static(1.0),
x: AnimatedValue::Static(0.0),
y: AnimatedValue::Static(0.0),
scale_x: AnimatedValue::Static(1.0),
scale_y: AnimatedValue::Static(1.0),
rotation: AnimatedValue::Static(0.0),
blend_mode: BlendMode::Normal,
composite_op: ff_filter::CompositeOp::Over,
};
match VideoFrame::from_rgba(gw, gh, self.gap_buf.clone()) {
Ok(bf) => self.composite_frame(
base_layer,
usize::MAX,
bf,
gw,
gh,
&gap_overlays,
gap_pts,
),
Err(_) => None,
}
};
// Manage audio-only decode threads (A1/A2…).
for at in &mut self.audio_only_tracks {
let should_run =
gap_pts >= at.timeline_start && gap_pts < at.timeline_end;
let is_running = at.cancel.is_some();
if should_run && !is_running {
let local =
at.in_point + gap_pts.saturating_sub(at.timeline_start);
at.start_at(local);
} else if !should_run && is_running {
at.stop();
at.handle.clear();
}
}
// Manage V1 inline audio: start it the moment the
// gap clock reaches the active clip's timeline_start.
if self.active_audio_cancel.is_none()
&& self.clips[self.active].audio_track.is_some()
&& gap_pts >= self.clips[self.active].timeline_start
{
let tl_start = self.clips[self.active].timeline_start;
let in_pt = self.clips[self.active].in_point;
let gap_elapsed = gap_pts.saturating_sub(tl_start);
let spd = self.clips[self.active].speed;
let local = in_pt
+ if (spd - 1.0).abs() < 1e-9 {
gap_elapsed
} else {
gap_elapsed.mul_f64(spd)
};
self.restart_audio_at(self.active, local);
}
self.current_pts.store(
u64::try_from(gap_pts.as_micros()).unwrap_or(u64::MAX),
Ordering::Relaxed,
);
self.resume_pts = gap_pts;
let _ =
self.event_tx.try_send(PlayerEvent::PositionUpdate(gap_pts));
if let Some(sink) = self.sink.as_mut() {
match &gap_composited {
Some((rgba, cw, ch)) => {
sink.push_frame(rgba, *cw, *ch, gap_pts);
}
None => sink.push_frame(&self.gap_buf, gw, gh, gap_pts),
}
}
if stepped {
self.clock.advance(frame_period);
} else {
thread::sleep(frame_period);
}
}
} else if !stepped && diff > fp {
let sleep_secs =
(diff - fp / 2.0).max(0.0) / self.rate.max(f64::MIN_POSITIVE);
thread::sleep(Duration::from_secs_f64(sleep_secs));
} else if !stepped && diff < -fp {
log::debug!(
"timeline dropped late frame timeline_pts={timeline_pts:?} \
clock_pts={clock_pts:?}"
);
continue;
}
}
// Start V1 inline audio on the first presented frame when a
// pre-roll gap prevented the thread from starting at open() time.
// The gap-fill loop attempts this but exits one frame-period before
// timeline_start, so we catch the remaining case here.
if self.active_audio_cancel.is_none()
&& self.clips[active].audio_track.is_some()
{
let in_pt = self.clips[active].in_point;
let elapsed_tl =
timeline_pts.saturating_sub(self.clips[active].timeline_start);
let local = in_pt
+ if (clip_speed - 1.0).abs() < 1e-9 {
elapsed_tl
} else {
elapsed_tl.mul_f64(clip_speed)
};
self.restart_audio_at(active, local);
}
// Present frame
let w = frame.width();
let h = frame.height();
self.last_frame_w = w;
self.last_frame_h = h;
// Copy transition fields to avoid holding a borrow while
// calling `pop_frame` on the next clip.
let (in_trans, next_idx, trans_start, trans_dur, trans_kind) =
match &self.transition {
Some(tp) => (true, tp.next_idx, tp.start, tp.duration, tp.kind),
None => (
false,
0,
Duration::ZERO,
Duration::ZERO,
XfadeTransition::Fade,
),
};
let a_ok = self.sws_a.convert(&frame, &mut self.rgba_a);
if a_ok {
// V1 per-clip opacity: pre-multiply toward black (producer-side;
// the composer ignores base-layer opacity). The merged opacity is
// an `AnimatedValue`; a track is evaluated at the timeline PTS
// (tracks are timeline-global), so base-layer opacity animates too.
let v1_op = match &self.clips[active].layer_desc.opacity {
// Value is clamped to [0.0, 1.0], so the f32 narrowing is safe.
#[allow(clippy::cast_possible_truncation)]
AnimatedValue::Track(track) => {
track.value_at(timeline_pts).clamp(0.0, 1.0) as f32
}
AnimatedValue::Static(_) => self.clips[active].opacity,
};
if (v1_op - 1.0).abs() > 1e-6 {
for chunk in self.rgba_a.as_chunks_mut::<4>().0 {
#[allow(clippy::cast_possible_truncation, clippy::cast_sign_loss)]
{
chunk[0] = (f32::from(chunk[0]) * v1_op).round() as u8;
chunk[1] = (f32::from(chunk[1]) * v1_op).round() as u8;
chunk[2] = (f32::from(chunk[2]) * v1_op).round() as u8;
}
}
}
// Transition crossfade (producer-side): blend the incoming clip
// into rgba_a so the composer grades the crossfaded V1 frame.
if in_trans
&& let FrameResult::Frame(next_frame) =
self.clips[next_idx].decode_buf.pop_frame()
&& self.sws_b.convert(&next_frame, &mut self.rgba_b)
{
let alpha = (timeline_pts.saturating_sub(trans_start).as_secs_f32()
/ trans_dur.as_secs_f32())
.clamp(0.0, 1.0);
// Offer the blend to the injected GPU path first; `None`
// falls through to the CPU one below, which covers an
// unrendered kind, no adapter, and a GPU error alike.
if let Some(blended) = try_gpu_blend(
self.gpu_compositor.as_mut(),
trans_kind,
&self.rgba_a,
&self.rgba_b,
alpha,
w,
h,
) {
// Moved in, not copied into the scratch buffer: the
// readback already owns a correctly sized `Vec`, so
// taking it costs nothing while routing it through
// `blend_buf` would add a full-frame memcpy. The CPU
// branch below swaps instead because `apply_xfade`
// writes into a buffer it does not own.
self.rgba_a = blended;
} else {
// Only `Dissolve` reads the field, and building one costs
// what a whole frame of dissolve used to (47.4 ms at 4 K),
// so a `Fade` must not pay for it.
let field = if trans_kind == XfadeTransition::Dissolve {
ensure_dissolve_field(
&mut self.dissolve_field,
&mut self.dissolve_field_dims,
w,
h,
);
Some(self.dissolve_field.as_slice())
} else {
None
};
inner::apply_xfade(
trans_kind,
&self.rgba_a,
&self.rgba_b,
alpha,
(w, h),
field,
&mut self.blend_buf,
);
std::mem::swap(&mut self.rgba_a, &mut self.blend_buf);
}
}
// Update overlays (held-frame, advanced by PTS) and composite
// the V1 base with them through the shared compositor.
let active_overlays = self.sync_overlays(timeline_pts);
self.sync_lavfi(timeline_pts);
let base_layer = RealtimeLayer::with_dimensions(
self.clips[active].layer_desc.clone(),
w,
h,
PixelFormat::Rgba,
);
let composited = match VideoFrame::from_rgba(w, h, self.rgba_a.clone()) {
Ok(bf) => self.composite_frame(
base_layer,
active,
bf,
w,
h,
&active_overlays,
timeline_pts,
),
Err(_) => None,
};
// Deliver: the composited frame, or the raw V1 as a fallback.
if let Some(sink) = self.sink.as_mut() {
match &composited {
Some((rgba, cw, ch)) => {
sink.push_frame(rgba, *cw, *ch, timeline_pts);
}
None => sink.push_frame(&self.rgba_a, w, h, timeline_pts),
}
}
// Unpaced: the next frame is due now, and a gap after this
// one is measured from the slot following it.
if stepped {
self.clock.reset(timeline_pts + frame_period);
}
// Advance past a completed transition.
if in_trans && timeline_pts >= trans_start + trans_dur {
let old_active = self.active;
self.transition = None;
self.active = next_idx;
if self.active != old_active {
let in_pt = self.clips[self.active].in_point;
self.restart_audio_at(self.active, in_pt);
}
}
}
let _ = self
.event_tx
.try_send(PlayerEvent::PositionUpdate(timeline_pts));
}
}
}
let _ = self.event_tx.try_send(PlayerEvent::Eof);
if let Some(sink) = self.sink.as_mut() {
sink.flush();
}
Ok(())
}
/// Seek all decode buffers so that `active` is the clip containing `target`
/// and that clip's buffer is positioned at the correct source-file PTS.
///
/// When `target` falls in a pre-roll or inter-clip gap the method finds the
/// next clip after `target`, seeks it to its `in_point`, and returns without
/// starting audio — the gap-fill loop in `run()` will start audio at the
/// right time.
pub(super) fn seek_timeline(&mut self, target: Duration) -> Result<(), PreviewError> {
// Try to find a clip that contains `target`.
let clip_in_range = self
.clips
.iter()
.position(|c| target >= c.timeline_start && target < c.timeline_end);
// If target is in a gap, find the next clip after `target`.
let (clip_idx, clip_local_pts, is_gap_seek) = if let Some(ci) = clip_in_range {
let elapsed_tl = target.saturating_sub(self.clips[ci].timeline_start);
let local = self.clips[ci].in_point
+ if (self.clips[ci].speed - 1.0).abs() < 1e-9 {
elapsed_tl
} else {
elapsed_tl.mul_f64(self.clips[ci].speed)
};
(ci, local, false)
} else if let Some(ci) = self.clips.iter().position(|c| c.timeline_start > target) {
// Seek the clip to its in_point; gap-fill loop will tick until it starts.
(ci, self.clips[ci].in_point, true)
} else {
return Err(PreviewError::SeekOutOfRange { pts: target });
};
self.clips[clip_idx].decode_buf.seek(clip_local_pts)?;
self.active = clip_idx;
self.transition = None;
// Discard stale audio and restart from the seek position.
if let Some(mixer_arc) = &self.audio_mixer {
mixer_arc
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.invalidate_all();
}
// The visual equivalent of that invalidation. A stateful effect (motion
// blur's exposure trail) accumulated from the frames that preceded the *old*
// position, which after a seek are not the frames preceding the new one — so
// it is stale whether or not the seek crossed a clip boundary (#1705).
if let Some(c) = self.gpu_compositor.as_mut() {
c.reset_effects();
}
if is_gap_seek {
// Cancel any running V1 audio thread; the gap loop will restart it
// once the clock reaches the clip's timeline_start.
if let Some(cancel) = self.active_audio_cancel.take() {
cancel.store(true, Ordering::Release);
}
drop(self.active_audio_thread.take());
} else {
self.restart_audio_at(clip_idx, clip_local_pts);
}
// Seek overlay layers to the new target position.
for layer in &mut self.overlay_layers {
let cidx = layer
.clips
.iter()
.position(|c| target >= c.timeline_start && target < c.timeline_end);
if let Some(cidx) = cidx {
let local = layer.clips[cidx].in_point
+ target.saturating_sub(layer.clips[cidx].timeline_start);
let _ = layer.clips[cidx].decode_buf.seek(local);
layer.active = cidx;
}
}
// The lavfi overlay source exposes no seek — rebuild it so it restarts from
// t=0. Static overlays are unaffected; a time-varying lavfi restarts (a
// documented limitation).
if let Some(st) = &mut self.lavfi {
st.rebuild();
}
// Stop all audio-only threads; they restart on the next frame tick.
for at in &mut self.audio_only_tracks {
at.stop();
}
Ok(())
}
/// Coarse (I-frame only) seek variant of [`seek_timeline`].
///
/// Does not restart audio or invalidate the mixer — caller is responsible.
/// Used for the reverse→forward recovery path where latency matters more
/// than frame-accurate positioning.
fn seek_timeline_coarse(&mut self, target: Duration) -> Result<(), PreviewError> {
let clip_idx = self
.clips
.iter()
.position(|c| target >= c.timeline_start && target < c.timeline_end)
.ok_or(PreviewError::SeekOutOfRange { pts: target })?;
let elapsed_tl = target.saturating_sub(self.clips[clip_idx].timeline_start);
let clip_local_pts = self.clips[clip_idx].in_point
+ if (self.clips[clip_idx].speed - 1.0).abs() < 1e-9 {
elapsed_tl
} else {
elapsed_tl.mul_f64(self.clips[clip_idx].speed)
};
self.clips[clip_idx]
.decode_buf
.seek_coarse(clip_local_pts)?;
self.active = clip_idx;
self.transition = None;
// Keep the lavfi overlay consistent with the main seek path (no source seek).
if let Some(st) = &mut self.lavfi {
st.rebuild();
}
Ok(())
}
/// Cancel the current audio decode thread (if any) and start a new one
/// for `clip_idx` beginning at `start_pts`.
fn restart_audio_at(&mut self, clip_idx: usize, start_pts: Duration) {
// Cancel and drop the previous thread.
if let Some(cancel) = &self.active_audio_cancel {
cancel.store(true, Ordering::Release);
}
drop(self.active_audio_thread.take());
self.active_audio_cancel = None;
let Some(handle) = self.clips.get(clip_idx).and_then(|c| c.audio_track.clone()) else {
return;
};
handle.clear(); // discard stale samples
let c = &self.clips[clip_idx];
// Only a file source reaches here (a generated clip has no `audio_track`,
// so the guard above returns early); derive its path for the audio thread.
let source = c
.source
.as_file()
.map(std::path::Path::to_path_buf)
.unwrap_or_default();
// V1 clip audio honours its own fades + speed (the A-track path already does).
// `clip_dur` must be the SOURCE-time span (the resampler multiplies it by
// `1/speed` to get timeline time, as the A-tracks feed `out-in` source span);
// the timeline span is `source/speed`, so scale it back by `speed`.
let fades = AudioFadeConfig {
fade_in: c.fade_in,
fade_out: c.fade_out,
clip_dur: c
.timeline_end
.saturating_sub(c.timeline_start)
.mul_f64(c.speed),
in_point: c.in_point,
speed: c.speed,
pitch: c.pitch,
};
let cancel = Arc::new(AtomicBool::new(false));
let thread =
spawn_audio_track_thread(source, start_pts, handle, Arc::clone(&cancel), fades);
self.active_audio_cancel = Some(cancel);
self.active_audio_thread = Some(thread);
}
}
impl Drop for SceneRunner {
fn drop(&mut self) {
if let Some(cancel) = &self.active_audio_cancel {
cancel.store(true, Ordering::Release);
}
if let Some(h) = self.active_audio_thread.take() {
let _ = h.join();
}
}
}
/// Pairs each layer spec with its decoded frame and asks the injected GPU
/// compositor to composite them, or returns `None` (no compositor, or the
/// compositor declined) so the caller uses the CPU path. Split out of
/// `composite_frame` so the seam is unit-testable without a full runner.
fn try_gpu_composite(
gpu: Option<&mut Box<dyn PreviewCompositor>>,
specs: &[RealtimeLayer],
frames: &[VideoFrame],
canvas: (u32, u32),
t: Duration,
) -> Option<(Vec<u8>, u32, u32)> {
let gpu = gpu?;
let pairs: Vec<(&RealtimeLayer, &VideoFrame)> = specs.iter().zip(frames.iter()).collect();
gpu.composite(&pairs, canvas, t)
}
/// Offer a transition blend to the injected compositor, or `None` when there is none
/// registered or it declines.
///
/// Mirrors [`try_gpu_composite`]: the runner keeps one `if let Some` shape for "the GPU
/// answered" and treats every other case, including no injection at all, as the CPU path.
fn try_gpu_blend(
gpu: Option<&mut Box<dyn PreviewCompositor>>,
kind: XfadeTransition,
a: &[u8],
b: &[u8],
progress: f32,
w: u32,
h: u32,
) -> Option<Vec<u8>> {
let blended = gpu?.blend(kind, a, b, progress, w, h)?;
// A short buffer would be written straight into `rgba_a` and read as a frame, so
// check the length here rather than trusting the implementor (the trait is public).
if blended.len() == (w as usize) * (h as usize) * 4 {
Some(blended)
} else {
log::warn!(
"preview: GPU blend returned {} bytes, expected {}; falling back to the CPU path",
blended.len(),
(w as usize) * (h as usize) * 4
);
None
}
}
#[cfg(test)]
mod tests {
use super::*;
/// A `PreviewCompositor` that returns a fixed result, to drive the seam.
struct MockCompositor {
result: Option<(Vec<u8>, u32, u32)>,
calls: std::cell::Cell<u32>,
}
impl PreviewCompositor for MockCompositor {
fn composite(
&mut self,
_layers: &[(&RealtimeLayer, &VideoFrame)],
_canvas: (u32, u32),
_t: Duration,
) -> Option<(Vec<u8>, u32, u32)> {
self.calls.set(self.calls.get() + 1);
self.result.clone()
}
}
/// A `PreviewCompositor` whose `blend` returns a fixed buffer, to drive the
/// transition seam. `composite` is never the subject here.
struct MockBlender {
result: Option<Vec<u8>>,
calls: std::cell::Cell<u32>,
}
impl PreviewCompositor for MockBlender {
fn composite(
&mut self,
_layers: &[(&RealtimeLayer, &VideoFrame)],
_canvas: (u32, u32),
_t: Duration,
) -> Option<(Vec<u8>, u32, u32)> {
None
}
fn blend(
&mut self,
_kind: XfadeTransition,
_a: &[u8],
_b: &[u8],
_progress: f32,
_w: u32,
_h: u32,
) -> Option<Vec<u8>> {
self.calls.set(self.calls.get() + 1);
self.result.clone()
}
}
#[test]
fn try_gpu_blend_should_return_none_without_a_compositor() {
let (a, b) = (vec![0u8; 2 * 2 * 4], vec![255u8; 2 * 2 * 4]);
assert!(try_gpu_blend(None, XfadeTransition::Fade, &a, &b, 0.5, 2, 2).is_none());
}
#[test]
fn try_gpu_blend_should_use_the_compositor_result_when_some() {
let (a, b) = (vec![0u8; 2 * 2 * 4], vec![255u8; 2 * 2 * 4]);
let want = vec![7u8; 2 * 2 * 4];
let mut gpu: Box<dyn PreviewCompositor> = Box::new(MockBlender {
result: Some(want.clone()),
calls: std::cell::Cell::new(0),
});
let out = try_gpu_blend(Some(&mut gpu), XfadeTransition::Fade, &a, &b, 0.5, 2, 2);
assert_eq!(out, Some(want));
}
#[test]
fn try_gpu_blend_should_return_none_when_the_compositor_declines() {
let (a, b) = (vec![0u8; 2 * 2 * 4], vec![255u8; 2 * 2 * 4]);
let mut gpu: Box<dyn PreviewCompositor> = Box::new(MockBlender {
result: None,
calls: std::cell::Cell::new(0),
});
assert!(try_gpu_blend(Some(&mut gpu), XfadeTransition::Fade, &a, &b, 0.5, 2, 2).is_none());
}
#[test]
fn try_gpu_blend_should_reject_a_wrongly_sized_buffer() {
// The result is written straight into `rgba_a` and read back as a frame, so a
// short buffer has to fall back rather than corrupt the next composite. The
// trait is public, so this is not a should-not-happen.
let (a, b) = (vec![0u8; 2 * 2 * 4], vec![255u8; 2 * 2 * 4]);
let mut gpu: Box<dyn PreviewCompositor> = Box::new(MockBlender {
result: Some(vec![7u8; 3]),
calls: std::cell::Cell::new(0),
});
assert!(try_gpu_blend(Some(&mut gpu), XfadeTransition::Fade, &a, &b, 0.5, 2, 2).is_none());
}
#[test]
fn try_gpu_blend_should_default_to_none_for_a_composite_only_implementor() {
// The trait's default: an existing `PreviewCompositor` that predates this seam
// keeps working and simply never takes the GPU blend.
let (a, b) = (vec![0u8; 2 * 2 * 4], vec![255u8; 2 * 2 * 4]);
let mut gpu: Box<dyn PreviewCompositor> = Box::new(MockCompositor {
result: None,
calls: std::cell::Cell::new(0),
});
assert!(try_gpu_blend(Some(&mut gpu), XfadeTransition::Fade, &a, &b, 0.5, 2, 2).is_none());
}
fn one_spec_and_frame() -> (Vec<RealtimeLayer>, Vec<VideoFrame>) {
let desc = ff_filter::RealtimeLayerDescriptor {
effects: Vec::new(),
opacity: AnimatedValue::Static(1.0),
x: AnimatedValue::Static(0.0),
y: AnimatedValue::Static(0.0),
scale_x: AnimatedValue::Static(1.0),
scale_y: AnimatedValue::Static(1.0),
rotation: AnimatedValue::Static(0.0),
blend_mode: BlendMode::Normal,
composite_op: CompositeOp::Over,
};
let spec = RealtimeLayer::with_dimensions(desc, 2, 2, PixelFormat::Rgba);
let frame = VideoFrame::from_rgba(2, 2, vec![0u8; 2 * 2 * 4]).expect("frame");
(vec![spec], vec![frame])
}
#[test]
fn try_gpu_composite_should_return_none_without_a_compositor() {
let (specs, frames) = one_spec_and_frame();
assert!(try_gpu_composite(None, &specs, &frames, (2, 2), Duration::ZERO).is_none());
}
#[test]
fn try_gpu_composite_should_use_the_compositor_result_when_some() {
let (specs, frames) = one_spec_and_frame();
let mut gpu: Box<dyn PreviewCompositor> = Box::new(MockCompositor {
result: Some((vec![1, 2, 3, 4], 1, 1)),
calls: std::cell::Cell::new(0),
});
let out = try_gpu_composite(Some(&mut gpu), &specs, &frames, (2, 2), Duration::ZERO);
assert_eq!(out, Some((vec![1, 2, 3, 4], 1, 1)));
}
#[test]
fn try_gpu_composite_should_return_none_when_the_compositor_declines() {
let (specs, frames) = one_spec_and_frame();
let mut gpu: Box<dyn PreviewCompositor> = Box::new(MockCompositor {
result: None,
calls: std::cell::Cell::new(0),
});
// A declining compositor yields None so the caller falls back to CPU.
assert!(
try_gpu_composite(Some(&mut gpu), &specs, &frames, (2, 2), Duration::ZERO).is_none()
);
}
#[test]
fn ensure_dissolve_field_should_build_once_and_rebuild_on_a_size_change() {
// The acceptance criterion directly: a dissolve of n frames builds the field
// once, not n times, and a change of frame size does rebuild it.
let mut field = Vec::new();
let mut dims = (0, 0);
assert!(
ensure_dissolve_field(&mut field, &mut dims, 7, 5),
"the first frame of a dissolve has to build the field"
);
assert_eq!(field.len(), 35);
for _ in 0..10 {
assert!(
!ensure_dissolve_field(&mut field, &mut dims, 7, 5),
"every later frame at the same size must reuse it"
);
}
assert!(
ensure_dissolve_field(&mut field, &mut dims, 9, 4),
"a change of frame size has to rebuild"
);
assert_eq!(field.len(), 36);
// 5x7 has the same pixel count as 7x5, so a length check alone would reuse a
// transposed field and read the wrong pixel at every coordinate.
assert!(
ensure_dissolve_field(&mut field, &mut dims, 4, 9),
"a transposed frame is a different field, not the same one"
);
assert_eq!(field, ff_filter::xfade_frand_field(4, 9));
}
}