alpha2_engine/
playback.rs

1//! How real-time playback trades pictures against sound, and the clock that enforces it.
2//!
3//! A [`crate::QualityProfile`] decides how a frame is drawn; a [`PlaybackPolicy`] decides which
4//! frame is shown when.
5
6use alpha2_time::{FrameRate, TICKS_PER_SECOND};
7use serde::{Deserialize, Serialize};
8
9use crate::MissingFramePolicy;
10
11/// How early a paced frame may advance, in milliseconds. Display refresh timestamps are rounded
12/// (browsers to 0.1 ms or coarser), so a due refresh can read slightly early.
13const PACING_SLACK_MS: f64 = 0.5;
14
15/// Which frame real-time playback shows, and whether sound plays.
16#[derive(Copy, Clone, Debug, Default, Deserialize, Eq, PartialEq, Serialize)]
17#[serde(rename_all = "snake_case")]
18pub enum PlaybackPolicy {
19    /// Sound leads at 1x; the picture follows the audio clock, dropping late frames and holding
20    /// the last one while inputs are missing. Other rates follow the wall clock silently.
21    #[default]
22    DropVideoKeepAudio,
23    /// Every frame in order once its inputs are ready, paced no faster than the rate. Slows
24    /// rather than skips. Silent.
25    PreserveEveryVideoFrame,
26    /// The picture follows the wall clock at any rate, holding the last frame while inputs are
27    /// missing. Silent; for shuttling and scrubbing.
28    SilentScrub,
29    /// Every frame in order as fast as it renders, like an export. Silent.
30    OfflineExact,
31}
32
33impl PlaybackPolicy {
34    /// Every policy, in the order a host lists them.
35    pub const ALL: [Self; 4] = [
36        Self::DropVideoKeepAudio,
37        Self::PreserveEveryVideoFrame,
38        Self::SilentScrub,
39        Self::OfflineExact,
40    ];
41
42    /// What a frame whose inputs are missing shows.
43    #[must_use]
44    pub const fn missing_frames(self) -> MissingFramePolicy {
45        match self {
46            Self::DropVideoKeepAudio | Self::SilentScrub => MissingFramePolicy::HoldPrevious,
47            Self::PreserveEveryVideoFrame | Self::OfflineExact => MissingFramePolicy::Wait,
48        }
49    }
50
51    /// Whether sound plays at `rate` (only when it leads, at 1x).
52    #[must_use]
53    pub fn audible(self, rate: f64) -> bool {
54        self == Self::DropVideoKeepAudio && (rate - 1.0).abs() < f64::EPSILON
55    }
56
57    /// Whether the clock waits for each frame to be shown before the next.
58    #[must_use]
59    pub const fn waits_for_frames(self) -> bool {
60        matches!(self, Self::PreserveEveryVideoFrame | Self::OfflineExact)
61    }
62}
63
64/// What the clock asks for at one moment.
65#[derive(Copy, Clone, Debug, Eq, PartialEq)]
66pub struct ClockStep {
67    /// The start of the frame to show, in ticks.
68    pub tick: i64,
69    /// Whether sound should be playing.
70    pub audible: bool,
71    /// Whether this frame must be shown before the clock moves on.
72    pub wait_for_frame: bool,
73}
74
75/// Milliseconds the audio clock may stall before the picture continues on the wall clock.
76const AUDIO_STALL_MS: f64 = 250.0;
77
78/// The clock for one playback run, asked each display refresh which time to show.
79#[derive(Clone, Debug)]
80pub struct PlaybackClock {
81    policy: PlaybackPolicy,
82    rate: f64,
83    frame_rate: FrameRate,
84    /// Wall time and tick the wall clock counts from.
85    origin_ms: f64,
86    origin_tick: i64,
87    /// The tick playback started at; neither clock runs before it.
88    start_tick: i64,
89    /// Whether the audio clock leads.
90    following_audio: bool,
91    last_audio: Option<i64>,
92    last_audio_ms: f64,
93    /// For frame-by-frame policies: the next frame and when the current one began showing.
94    frame: i64,
95    frame_ms: f64,
96}
97
98impl PlaybackClock {
99    /// A clock starting at `tick` at wall time `now_ms`, playing at `rate` (negative is backwards).
100    #[must_use]
101    pub fn new(
102        policy: PlaybackPolicy,
103        frame_rate: FrameRate,
104        rate: f64,
105        tick: i64,
106        now_ms: f64,
107    ) -> Self {
108        let frame = frame_rate.frame_at_or_before(alpha2_time::MediaTime(tick.max(0)));
109        Self {
110            policy,
111            rate,
112            frame_rate,
113            origin_ms: now_ms,
114            origin_tick: tick,
115            start_tick: tick,
116            following_audio: true,
117            last_audio: None,
118            last_audio_ms: now_ms,
119            frame: frame.unwrap_or(0),
120            frame_ms: now_ms,
121        }
122    }
123
124    /// The policy it enforces.
125    #[must_use]
126    pub const fn policy(&self) -> PlaybackPolicy {
127        self.policy
128    }
129
130    /// The time to show at wall time `now_ms`. `heard` is the audio output position in ticks
131    /// while it runs; `presented` says whether the last requested frame has been shown.
132    pub fn step(&mut self, now_ms: f64, heard: Option<i64>, presented: bool) -> ClockStep {
133        let audible = self.policy.audible(self.rate);
134        match self.policy {
135            PlaybackPolicy::DropVideoKeepAudio | PlaybackPolicy::SilentScrub => {
136                let time = if audible {
137                    self.follow_audio(now_ms, heard)
138                } else {
139                    self.wall_tick(now_ms)
140                };
141                ClockStep {
142                    tick: self.frame_start_at(time),
143                    audible,
144                    wait_for_frame: false,
145                }
146            }
147            PlaybackPolicy::PreserveEveryVideoFrame | PlaybackPolicy::OfflineExact => {
148                let paced = self.policy == PlaybackPolicy::PreserveEveryVideoFrame;
149                let period = self.frame_ms_at_rate();
150                // Carry overshoot to the next deadline so frames average the rate on any display.
151                let due = !paced || now_ms - self.frame_ms >= period - PACING_SLACK_MS;
152                if presented && due {
153                    self.frame = (self.frame + if self.rate < 0.0 { -1 } else { 1 }).max(0);
154                    self.frame_ms = if paced {
155                        (self.frame_ms + period).max(now_ms - period)
156                    } else {
157                        now_ms
158                    };
159                }
160                ClockStep {
161                    tick: self.frame_tick(),
162                    audible: false,
163                    wait_for_frame: true,
164                }
165            }
166        }
167    }
168
169    /// The audio clock while it advances, else the wall clock from where the audio stopped.
170    fn follow_audio(&mut self, now_ms: f64, heard: Option<i64>) -> i64 {
171        // Audio a tick before the start (host sample rounding) still shows the start frame.
172        let heard = heard.map(|position| position.max(self.start_tick));
173        match heard {
174            Some(position) if Some(position) != self.last_audio => {
175                self.last_audio = Some(position);
176                self.last_audio_ms = now_ms;
177                self.following_audio = true;
178            }
179            Some(position)
180                if self.following_audio && now_ms - self.last_audio_ms > AUDIO_STALL_MS =>
181            {
182                self.following_audio = false;
183                self.origin_ms = now_ms;
184                self.origin_tick = position;
185            }
186            _ => {}
187        }
188        match heard {
189            Some(position) if self.following_audio => position,
190            _ => self.wall_tick(now_ms),
191        }
192    }
193
194    fn wall_tick(&self, now_ms: f64) -> i64 {
195        // 120,000 is exact in f64.
196        #[allow(clippy::cast_precision_loss)]
197        let ticks_per_ms = TICKS_PER_SECOND as f64 / 1000.0;
198        // `requestAnimationFrame` times can precede the clock's start.
199        let elapsed = (now_ms - self.origin_ms).max(0.0) * ticks_per_ms;
200        // Within any timeline length; the host stops at the end.
201        #[allow(clippy::cast_possible_truncation)]
202        let advanced = (elapsed * self.rate).floor() as i64;
203        self.origin_tick.saturating_add(advanced)
204    }
205
206    /// The start of the frame containing `tick`, in either direction.
207    fn frame_start_at(&self, tick: i64) -> i64 {
208        self.frame_rate
209            .frame_at_or_before(alpha2_time::MediaTime(tick.max(0)))
210            .and_then(|frame| self.frame_rate.frame_start(frame))
211            .map_or(0, alpha2_time::MediaTime::ticks)
212    }
213
214    fn frame_tick(&self) -> i64 {
215        self.frame_rate
216            .frame_start(self.frame)
217            .map_or(0, alpha2_time::MediaTime::ticks)
218    }
219
220    /// How long one frame shows at the clock's rate, in milliseconds.
221    fn frame_ms_at_rate(&self) -> f64 {
222        let frame_ms = 1000.0 * f64::from(self.frame_rate.denominator())
223            / f64::from(self.frame_rate.numerator());
224        frame_ms / self.rate.abs().max(f64::MIN_POSITIVE)
225    }
226}
227
228#[cfg(test)]
229mod tests {
230    use super::*;
231
232    fn rate() -> FrameRate {
233        FrameRate::new(25, 1).expect("rate")
234    }
235
236    #[test]
237    fn keeping_audio_follows_it_and_falls_back_to_the_wall_clock_when_it_stalls() {
238        let mut clock = PlaybackClock::new(PlaybackPolicy::DropVideoKeepAudio, rate(), 1.0, 0, 0.0);
239        // One frame at 25 fps is 4,800 ticks: the clock asks for the frame the audio is in.
240        let step = clock.step(10.0, Some(1_200), false);
241        assert_eq!(
242            (step.tick, step.audible, step.wait_for_frame),
243            (0, true, false)
244        );
245        clock.step(100.0, Some(1_200), true);
246        let stalled = clock.step(400.0, Some(1_200), true);
247        assert_eq!(stalled.tick, 0);
248        // 1,200 + 12,000 ticks is inside frame 2.
249        let later = clock.step(500.0, Some(1_200), true);
250        assert_eq!(later.tick, 2 * 4_800);
251        assert_eq!(clock.step(510.0, Some(50_000), true).tick, 10 * 4_800);
252        let mut fast = PlaybackClock::new(PlaybackPolicy::DropVideoKeepAudio, rate(), 2.0, 0, 0.0);
253        let step = fast.step(1_000.0, Some(7), true);
254        assert_eq!((step.tick, step.audible), (240_000, false));
255    }
256
257    #[test]
258    fn audio_heard_before_the_start_shows_the_start() {
259        // 29.97 fps from frame 2 (tick 8008). 48 kHz output heard at tick 8007 (frame 1) still
260        // shows frame 2.
261        let ntsc = FrameRate::new(30_000, 1_001).expect("rate");
262        let mut clock =
263            PlaybackClock::new(PlaybackPolicy::DropVideoKeepAudio, ntsc, 1.0, 8_008, 0.0);
264        assert_eq!(clock.step(5.0, Some(8_007), false).tick, 8_008);
265        assert_eq!(clock.step(10.0, Some(8_007), true).tick, 8_008);
266        assert_eq!(clock.step(20.0, Some(9_000), true).tick, 8_008);
267        let mut stalled =
268            PlaybackClock::new(PlaybackPolicy::DropVideoKeepAudio, ntsc, 1.0, 8_008, 0.0);
269        stalled.step(1.0, Some(8_007), false);
270        assert_eq!(stalled.step(300.0, Some(8_007), true).tick, 8_008);
271        // 8,008 + 12,000 ticks is inside frame 4, which starts at 16,016.
272        assert_eq!(stalled.step(400.0, Some(8_007), true).tick, 16_016);
273    }
274
275    #[test]
276    fn preserving_every_frame_advances_one_frame_at_a_time_and_waits() {
277        let mut clock =
278            PlaybackClock::new(PlaybackPolicy::PreserveEveryVideoFrame, rate(), 1.0, 0, 0.0);
279        let frame = 4_800; // one frame at 25 fps
280        assert_eq!(clock.step(0.0, None, false).tick, 0);
281        // Shown, but its 40 ms are not up.
282        assert_eq!(clock.step(20.0, None, true).tick, 0);
283        let next = clock.step(41.0, None, true);
284        assert_eq!(
285            (next.tick, next.audible, next.wait_for_frame),
286            (frame, false, true)
287        );
288        assert_eq!(clock.step(500.0, None, false).tick, frame);
289        assert_eq!(clock.step(501.0, None, true).tick, 2 * frame);
290    }
291
292    #[test]
293    fn preserving_every_frame_keeps_the_rate_on_any_display() {
294        // Without carrying overshoot, 29.97 fps would play at 20 fps on a 60 Hz display.
295        for (num, den) in [
296            (24_000, 1_001),
297            (25, 1),
298            (30_000, 1_001),
299            (30, 1),
300            (60_000, 1_001),
301            (60, 1),
302        ] {
303            let rate = FrameRate::new(num, den).expect("rate");
304            let fps = f64::from(num) / f64::from(den);
305            for hz in [59.94, 60.0, 120.0, 144.0] {
306                for grain in [0.0, 0.1, 1.0] {
307                    if fps > hz {
308                        continue;
309                    }
310                    let mut clock = PlaybackClock::new(
311                        PlaybackPolicy::PreserveEveryVideoFrame,
312                        rate,
313                        1.0,
314                        0,
315                        0.0,
316                    );
317                    let seconds = 60.0;
318                    #[allow(clippy::cast_possible_truncation, clippy::cast_sign_loss)]
319                    let refreshes = (seconds * hz) as u64;
320                    let mut tick = 0;
321                    for refresh in 0..=refreshes {
322                        #[allow(clippy::cast_precision_loss)]
323                        let exact = refresh as f64 * 1_000.0 / hz;
324                        let now = if grain > 0.0 {
325                            (exact / grain).round() * grain
326                        } else {
327                            exact
328                        };
329                        tick = clock.step(now, None, true).tick;
330                    }
331                    let shown = clock.frame;
332                    let expected = seconds * fps;
333                    #[allow(clippy::cast_precision_loss)]
334                    let short = expected - shown as f64;
335                    assert!(
336                        short.abs() <= 1.0,
337                        "{fps} fps at {hz} Hz, {grain} ms: {shown} frames in {seconds} s, not {expected} (tick {tick})"
338                    );
339                }
340            }
341        }
342    }
343
344    #[test]
345    fn offline_exact_moves_as_soon_as_each_frame_is_shown() {
346        let mut clock = PlaybackClock::new(PlaybackPolicy::OfflineExact, rate(), 1.0, 9_600, 0.0);
347        assert_eq!(clock.step(0.0, None, false).tick, 9_600);
348        assert_eq!(clock.step(0.0, None, true).tick, 14_400);
349        assert_eq!(
350            PlaybackPolicy::OfflineExact.missing_frames(),
351            MissingFramePolicy::Wait
352        );
353        assert_eq!(
354            PlaybackPolicy::SilentScrub.missing_frames(),
355            MissingFramePolicy::HoldPrevious
356        );
357    }
358
359    #[test]
360    fn the_wall_clock_holds_at_its_start_until_its_start_time() {
361        let mut clock = PlaybackClock::new(PlaybackPolicy::SilentScrub, rate(), 1.0, 0, 1_000.0);
362        assert_eq!(clock.step(992.0, None, false).tick, 0);
363        // 60,000 ticks is inside frame 12.
364        assert_eq!(clock.step(1_500.0, None, true).tick, 12 * 4_800);
365        let mut silent = PlaybackClock::new(
366            PlaybackPolicy::DropVideoKeepAudio,
367            rate(),
368            1.0,
369            9_600,
370            1_000.0,
371        );
372        assert_eq!(silent.step(990.0, None, false).tick, 9_600);
373    }
374
375    #[test]
376    fn scrubbing_is_silent_and_follows_the_wall_clock_backwards_too() {
377        let mut clock = PlaybackClock::new(PlaybackPolicy::SilentScrub, rate(), -1.0, 120_000, 0.0);
378        let step = clock.step(500.0, Some(0), true);
379        assert_eq!((step.tick, step.audible), (12 * 4_800, false));
380    }
381
382    #[test]
383    fn audio_positions_inside_one_frame_ask_for_that_frame_once() {
384        let mut clock = PlaybackClock::new(PlaybackPolicy::DropVideoKeepAudio, rate(), 1.0, 0, 0.0);
385        // Three display refreshes inside frame 1 (4,800..9,600) ask for the same tick.
386        let ticks: Vec<i64> = [(16.0, 4_800), (32.0, 6_400), (48.0, 9_599)]
387            .into_iter()
388            .map(|(now, heard)| clock.step(now, Some(heard), true).tick)
389            .collect();
390        assert_eq!(ticks, [4_800; 3]);
391        // Crossing the boundary asks for the next frame's start.
392        assert_eq!(clock.step(64.0, Some(9_600), true).tick, 9_600);
393    }
394
395    #[test]
396    fn the_wall_clock_after_an_audio_stall_is_quantized_too() {
397        let mut clock = PlaybackClock::new(PlaybackPolicy::DropVideoKeepAudio, rate(), 1.0, 0, 0.0);
398        clock.step(0.0, Some(100), false);
399        // Stalled past `AUDIO_STALL_MS`: the wall clock continues from 100 ticks.
400        assert_eq!(clock.step(300.0, Some(100), true).tick, 0);
401        // 30 ms later is 3,700 ticks, still frame 0; 40 ms later is 4,900, frame 1.
402        assert_eq!(clock.step(330.0, Some(100), true).tick, 0);
403        assert_eq!(clock.step(340.0, Some(100), true).tick, 4_800);
404    }
405
406    #[test]
407    fn reverse_play_asks_for_the_frame_containing_the_time() {
408        // From inside frame 25 (120,000..124,800) backwards: 10 ms back is 121,000, then 30 ms
409        // more is frame 24.
410        let mut clock = PlaybackClock::new(PlaybackPolicy::SilentScrub, rate(), -1.0, 122_200, 0.0);
411        assert_eq!(clock.step(10.0, None, true).tick, 120_000);
412        assert_eq!(clock.step(40.0, None, true).tick, 115_200);
413        // Past the start it holds frame 0.
414        assert_eq!(clock.step(10_000.0, None, true).tick, 0);
415    }
416}
alpha2_engine/playback.rs

alpha2_engine/
playback.rs
