alpha2_engine/
playback.rs

1//! How real-time playback trades pictures against sound, and the clock that
2//! enforces it.
3//!
4//! Quality (how a frame is drawn) and the clock (which frame is shown when)
5//! are separate choices: a [`crate::QualityProfile`] decides the first, a
6//! [`PlaybackPolicy`] the second.
7
8use alpha2_time::{FrameRate, TICKS_PER_SECOND};
9use serde::{Deserialize, Serialize};
10
11use crate::MissingFramePolicy;
12
13/// How early a paced frame may advance, in milliseconds: a display's
14/// refresh timestamps are rounded (browsers to 0.1 ms or coarser), so a
15/// refresh that is due can read a fraction early.
16const PACING_SLACK_MS: f64 = 0.5;
17
18/// Which frame real-time playback shows, and whether sound plays.
19#[derive(Copy, Clone, Debug, Default, Deserialize, Eq, PartialEq, Serialize)]
20#[serde(rename_all = "snake_case")]
21pub enum PlaybackPolicy {
22    /// Sound leads at 1x: the picture shows the frame at the audio clock's
23    /// time, dropping frames that are late and holding the last one while
24    /// inputs are missing. At any other rate the picture follows the wall clock and
25    /// sound is silent.
26    #[default]
27    DropVideoKeepAudio,
28    /// Every frame is shown, in order, each once its inputs are ready, paced
29    /// no faster than the rate asks: playback slows rather than skips, and is
30    /// silent.
31    PreserveEveryVideoFrame,
32    /// The picture follows the wall clock at any rate, holding the last frame
33    /// while inputs are missing, and is silent: for shuttling and scrubbing.
34    SilentScrub,
35    /// Every frame, in order, each after its inputs are ready, as fast as it
36    /// renders: what an export does. Silent.
37    OfflineExact,
38}
39
40impl PlaybackPolicy {
41    /// Every policy, in the order a host lists them.
42    pub const ALL: [Self; 4] = [
43        Self::DropVideoKeepAudio,
44        Self::PreserveEveryVideoFrame,
45        Self::SilentScrub,
46        Self::OfflineExact,
47    ];
48
49    /// What a frame whose inputs are missing shows.
50    #[must_use]
51    pub const fn missing_frames(self) -> MissingFramePolicy {
52        match self {
53            Self::DropVideoKeepAudio | Self::SilentScrub => MissingFramePolicy::HoldPrevious,
54            Self::PreserveEveryVideoFrame | Self::OfflineExact => MissingFramePolicy::Wait,
55        }
56    }
57
58    /// Whether sound plays at `rate`: only when it leads, at 1x.
59    #[must_use]
60    pub fn audible(self, rate: f64) -> bool {
61        self == Self::DropVideoKeepAudio && (rate - 1.0).abs() < f64::EPSILON
62    }
63
64    /// Whether the clock waits for each frame to be shown before the next.
65    #[must_use]
66    pub const fn waits_for_frames(self) -> bool {
67        matches!(self, Self::PreserveEveryVideoFrame | Self::OfflineExact)
68    }
69}
70
71/// What the clock asks for at one moment.
72#[derive(Copy, Clone, Debug, Eq, PartialEq)]
73pub struct ClockStep {
74    /// The timeline time to show, in ticks.
75    pub tick: i64,
76    /// Whether sound should be playing.
77    pub audible: bool,
78    /// Whether this frame must be shown before the clock moves on.
79    pub wait_for_frame: bool,
80}
81
82/// How long the audio clock may stand still before the picture continues on
83/// the wall clock from where it stood, in milliseconds.
84const AUDIO_STALL_MS: f64 = 250.0;
85
86/// The playback clock of one run of playback: asked each display refresh
87/// which timeline time to show.
88#[derive(Clone, Debug)]
89pub struct PlaybackClock {
90    policy: PlaybackPolicy,
91    rate: f64,
92    frame_rate: FrameRate,
93    /// Wall time and tick the wall clock counts from.
94    origin_ms: f64,
95    origin_tick: i64,
96    /// The tick playback started at: neither clock runs before it.
97    start_tick: i64,
98    /// Whether the audio clock leads.
99    following_audio: bool,
100    last_audio: Option<i64>,
101    last_audio_ms: f64,
102    /// For a frame-by-frame policy: the frame shown next, and when the
103    /// current one began showing.
104    frame: i64,
105    frame_ms: f64,
106}
107
108impl PlaybackClock {
109    /// A clock starting at `tick` at wall time `now_ms`, playing at `rate`
110    /// (negative plays backwards) under `policy`.
111    #[must_use]
112    pub fn new(
113        policy: PlaybackPolicy,
114        frame_rate: FrameRate,
115        rate: f64,
116        tick: i64,
117        now_ms: f64,
118    ) -> Self {
119        let frame = frame_rate.frame_at_or_before(alpha2_time::MediaTime(tick.max(0)));
120        Self {
121            policy,
122            rate,
123            frame_rate,
124            origin_ms: now_ms,
125            origin_tick: tick,
126            start_tick: tick,
127            following_audio: true,
128            last_audio: None,
129            last_audio_ms: now_ms,
130            frame: frame.unwrap_or(0),
131            frame_ms: now_ms,
132        }
133    }
134
135    /// The policy it enforces.
136    #[must_use]
137    pub const fn policy(&self) -> PlaybackPolicy {
138        self.policy
139    }
140
141    /// The time to show at wall time `now_ms`. `heard` is the audio output's
142    /// position in ticks while it runs (a position before the clock's start
143    /// is its start); `presented` says whether the frame the clock last asked
144    /// for has been shown.
145    pub fn step(&mut self, now_ms: f64, heard: Option<i64>, presented: bool) -> ClockStep {
146        let audible = self.policy.audible(self.rate);
147        match self.policy {
148            PlaybackPolicy::DropVideoKeepAudio | PlaybackPolicy::SilentScrub => {
149                let tick = if audible {
150                    self.follow_audio(now_ms, heard)
151                } else {
152                    self.wall_tick(now_ms)
153                };
154                ClockStep {
155                    tick,
156                    audible,
157                    wait_for_frame: false,
158                }
159            }
160            PlaybackPolicy::PreserveEveryVideoFrame | PlaybackPolicy::OfflineExact => {
161                let paced = self.policy == PlaybackPolicy::PreserveEveryVideoFrame;
162                let period = self.frame_ms_at_rate();
163                // Carry each refresh's overshoot to the next deadline so frames average the rate on
164                // any display; a late frame is followed by at most one more. The slack absorbs
165                // rounded timestamps.
166                let due = !paced || now_ms - self.frame_ms >= period - PACING_SLACK_MS;
167                if presented && due {
168                    self.frame = (self.frame + if self.rate < 0.0 { -1 } else { 1 }).max(0);
169                    self.frame_ms = if paced {
170                        (self.frame_ms + period).max(now_ms - period)
171                    } else {
172                        now_ms
173                    };
174                }
175                ClockStep {
176                    tick: self.frame_tick(),
177                    audible: false,
178                    wait_for_frame: true,
179                }
180            }
181        }
182    }
183
184    /// The audio clock while it advances; the wall clock, from where the
185    /// audio stood, while it does not.
186    fn follow_audio(&mut self, now_ms: f64, heard: Option<i64>) -> i64 {
187        // Sound leads only at 1x, forwards. Output that starts a tick before the start (a host
188        // rounding to its nearest sample) still shows the start frame.
189        let heard = heard.map(|position| position.max(self.start_tick));
190        match heard {
191            Some(position) if Some(position) != self.last_audio => {
192                self.last_audio = Some(position);
193                self.last_audio_ms = now_ms;
194                self.following_audio = true;
195            }
196            Some(position)
197                if self.following_audio && now_ms - self.last_audio_ms > AUDIO_STALL_MS =>
198            {
199                self.following_audio = false;
200                self.origin_ms = now_ms;
201                self.origin_tick = position;
202            }
203            _ => {}
204        }
205        match heard {
206            Some(position) if self.following_audio => position,
207            _ => self.wall_tick(now_ms),
208        }
209    }
210
211    fn wall_tick(&self, now_ms: f64) -> i64 {
212        // 120,000 is exact in f64.
213        #[allow(clippy::cast_precision_loss)]
214        let ticks_per_ms = TICKS_PER_SECOND as f64 / 1000.0;
215        // A refresh can be stamped before the clock started (a browser's `requestAnimationFrame`
216        // time is the frame's start).
217        let elapsed = (now_ms - self.origin_ms).max(0.0) * ticks_per_ms;
218        // Within any timeline length; the host stops at the end.
219        #[allow(clippy::cast_possible_truncation)]
220        let advanced = (elapsed * self.rate).floor() as i64;
221        self.origin_tick.saturating_add(advanced)
222    }
223
224    fn frame_tick(&self) -> i64 {
225        self.frame_rate
226            .frame_start(self.frame)
227            .map_or(0, alpha2_time::MediaTime::ticks)
228    }
229
230    /// How long one frame shows at the clock's rate, in milliseconds.
231    fn frame_ms_at_rate(&self) -> f64 {
232        let frame_ms = 1000.0 * f64::from(self.frame_rate.denominator())
233            / f64::from(self.frame_rate.numerator());
234        frame_ms / self.rate.abs().max(f64::MIN_POSITIVE)
235    }
236}
237
238#[cfg(test)]
239mod tests {
240    use super::*;
241
242    fn rate() -> FrameRate {
243        FrameRate::new(25, 1).expect("rate")
244    }
245
246    #[test]
247    fn keeping_audio_follows_it_and_falls_back_to_the_wall_clock_when_it_stalls() {
248        let mut clock = PlaybackClock::new(PlaybackPolicy::DropVideoKeepAudio, rate(), 1.0, 0, 0.0);
249        let step = clock.step(10.0, Some(1_200), false);
250        assert_eq!(
251            (step.tick, step.audible, step.wait_for_frame),
252            (1_200, true, false)
253        );
254        // The audio stands still past the stall: the picture continues on the wall clock from
255        // there.
256        clock.step(100.0, Some(1_200), true);
257        let stalled = clock.step(400.0, Some(1_200), true);
258        assert_eq!(stalled.tick, 1_200);
259        let later = clock.step(500.0, Some(1_200), true);
260        assert_eq!(later.tick, 1_200 + 12_000);
261        assert_eq!(clock.step(510.0, Some(50_000), true).tick, 50_000);
262        // At any rate other than 1x it is silent and on the wall clock.
263        let mut fast = PlaybackClock::new(PlaybackPolicy::DropVideoKeepAudio, rate(), 2.0, 0, 0.0);
264        let step = fast.step(1_000.0, Some(7), true);
265        assert_eq!((step.tick, step.audible), (240_000, false));
266    }
267
268    #[test]
269    fn audio_heard_before_the_start_shows_the_start() {
270        // 29.97 fps, playback from frame 2 (tick 8008). A host that starts its output on the
271        // nearest sample at 48 kHz (3203, tick 8007.5) hears tick 8007 first: frame 1.
272        let ntsc = FrameRate::new(30_000, 1_001).expect("rate");
273        let mut clock =
274            PlaybackClock::new(PlaybackPolicy::DropVideoKeepAudio, ntsc, 1.0, 8_008, 0.0);
275        assert_eq!(clock.step(5.0, Some(8_007), false).tick, 8_008);
276        assert_eq!(clock.step(10.0, Some(8_007), true).tick, 8_008);
277        assert_eq!(clock.step(20.0, Some(9_000), true).tick, 9_000);
278        // A stall continues on the wall clock from where it stood, never before the start.
279        let mut stalled =
280            PlaybackClock::new(PlaybackPolicy::DropVideoKeepAudio, ntsc, 1.0, 8_008, 0.0);
281        stalled.step(1.0, Some(8_007), false);
282        assert_eq!(stalled.step(300.0, Some(8_007), true).tick, 8_008);
283        assert_eq!(stalled.step(400.0, Some(8_007), true).tick, 8_008 + 12_000);
284    }
285
286    #[test]
287    fn preserving_every_frame_advances_one_frame_at_a_time_and_waits() {
288        let mut clock =
289            PlaybackClock::new(PlaybackPolicy::PreserveEveryVideoFrame, rate(), 1.0, 0, 0.0);
290        let frame = 4_800; // one frame at 25 fps
291        assert_eq!(clock.step(0.0, None, false).tick, 0);
292        // Shown, but its 40 ms are not up.
293        assert_eq!(clock.step(20.0, None, true).tick, 0);
294        let next = clock.step(41.0, None, true);
295        assert_eq!(
296            (next.tick, next.audible, next.wait_for_frame),
297            (frame, false, true)
298        );
299        assert_eq!(clock.step(500.0, None, false).tick, frame);
300        assert_eq!(clock.step(501.0, None, true).tick, 2 * frame);
301    }
302
303    #[test]
304    fn preserving_every_frame_keeps_the_rate_on_any_display() {
305        // Restarting each frame's wait at the refresh it advanced on would hold every frame a
306        // whole number of refreshes, so 29.97 fps would play at 20 fps on a 60 Hz display.
307        for (num, den) in [
308            (24_000, 1_001),
309            (25, 1),
310            (30_000, 1_001),
311            (30, 1),
312            (60_000, 1_001),
313            (60, 1),
314        ] {
315            let rate = FrameRate::new(num, den).expect("rate");
316            let fps = f64::from(num) / f64::from(den);
317            for hz in [59.94, 60.0, 120.0, 144.0] {
318                for grain in [0.0, 0.1, 1.0] {
319                    if fps > hz {
320                        continue;
321                    }
322                    let mut clock = PlaybackClock::new(
323                        PlaybackPolicy::PreserveEveryVideoFrame,
324                        rate,
325                        1.0,
326                        0,
327                        0.0,
328                    );
329                    let seconds = 60.0;
330                    #[allow(clippy::cast_possible_truncation, clippy::cast_sign_loss)]
331                    let refreshes = (seconds * hz) as u64;
332                    let mut tick = 0;
333                    for refresh in 0..=refreshes {
334                        #[allow(clippy::cast_precision_loss)]
335                        let exact = refresh as f64 * 1_000.0 / hz;
336                        let now = if grain > 0.0 {
337                            (exact / grain).round() * grain
338                        } else {
339                            exact
340                        };
341                        tick = clock.step(now, None, true).tick;
342                    }
343                    let shown = clock.frame;
344                    let expected = seconds * fps;
345                    #[allow(clippy::cast_precision_loss)]
346                    let short = expected - shown as f64;
347                    assert!(
348                        short.abs() <= 1.0,
349                        "{fps} fps at {hz} Hz, {grain} ms: {shown} frames in {seconds} s, not {expected} (tick {tick})"
350                    );
351                }
352            }
353        }
354    }
355
356    #[test]
357    fn offline_exact_moves_as_soon_as_each_frame_is_shown() {
358        let mut clock = PlaybackClock::new(PlaybackPolicy::OfflineExact, rate(), 1.0, 9_600, 0.0);
359        assert_eq!(clock.step(0.0, None, false).tick, 9_600);
360        assert_eq!(clock.step(0.0, None, true).tick, 14_400);
361        assert_eq!(
362            PlaybackPolicy::OfflineExact.missing_frames(),
363            MissingFramePolicy::Wait
364        );
365        assert_eq!(
366            PlaybackPolicy::SilentScrub.missing_frames(),
367            MissingFramePolicy::HoldPrevious
368        );
369    }
370
371    #[test]
372    fn the_wall_clock_holds_at_its_start_until_its_start_time() {
373        let mut clock = PlaybackClock::new(PlaybackPolicy::SilentScrub, rate(), 1.0, 0, 1_000.0);
374        assert_eq!(clock.step(992.0, None, false).tick, 0);
375        assert_eq!(clock.step(1_500.0, None, true).tick, 60_000);
376        let mut silent = PlaybackClock::new(
377            PlaybackPolicy::DropVideoKeepAudio,
378            rate(),
379            1.0,
380            9_600,
381            1_000.0,
382        );
383        assert_eq!(silent.step(990.0, None, false).tick, 9_600);
384    }
385
386    #[test]
387    fn scrubbing_is_silent_and_follows_the_wall_clock_backwards_too() {
388        let mut clock = PlaybackClock::new(PlaybackPolicy::SilentScrub, rate(), -1.0, 120_000, 0.0);
389        let step = clock.step(500.0, Some(0), true);
390        assert_eq!((step.tick, step.audible), (60_000, false));
391    }
392}
alpha2_engine/playback.rs

alpha2_engine/
playback.rs
