long_screen/
source.rs

1//! Where frames come from. A run reads its source once per pass (scan, solve, render, and the reveal's sweeps), in
2//! order, as the web page re-decodes its file for every pass; [`FrameSource::frames`] starts one such read.
3//!
4//! `VideoSource` (feature `decode`) decodes a file with the `decode` module (AVFoundation, Media Foundation or
5//! ffmpeg) and turns each frame into RGBA with the kernels' converter, as the page does: 10-bit PQ/HLG frames are
6//! rounded to 8 bits first and converted through the kernels' HDR path. [`RawSource`] replays RGBA frames already in
7//! memory or on disk (the parity harness feeds the web page's exact frames this way).
8
9#[cfg(feature = "decode")]
10use crate::decode::{self as decode, PixelFormat};
11#[cfg(feature = "decode")]
12use crate::kernels::yuv::{self, Format, Signal};
13use crate::value::{Js, Obj};
14use std::fmt;
15use std::fs::File;
16use std::io::{Read, Seek, SeekFrom};
17#[cfg(feature = "decode")]
18use std::path::Path;
19use std::path::PathBuf;
20
21/// Per-channel decode noise a decoded video declares (`DECODED_VIDEO_NOISE` in src/media/source.ts).
22pub const DECODED_VIDEO_NOISE: f64 = 10.0;
23
24/// One decoded frame, as RGBA.
25pub struct Frame {
26    pub index: u32,
27    /// Seconds.
28    pub time: f64,
29    pub duration: f64,
30    pub width: usize,
31    pub height: usize,
32    pub rgba: Vec<u8>,
33}
34
35/// A decoder notice the scan pass reports (`MediaNotice`).
36#[derive(Clone, Debug)]
37pub struct Notice {
38    pub code: String,
39    pub message: String,
40    pub count: u64,
41}
42
43/// What a source says about itself (the page's `MediaInfo`). `js` is the object persisted as the project's
44/// `media` field, the other fields the values the run reads from it.
45#[derive(Clone, Debug)]
46pub struct MediaInfo {
47    pub name: String,
48    pub width: usize,
49    pub height: usize,
50    pub duration: f64,
51    pub frame_count: Option<u64>,
52    pub codec: String,
53    pub mode: String,
54    pub noise: f64,
55    pub warnings: Vec<String>,
56    pub notices: Vec<Notice>,
57    pub js: Obj,
58}
59
60impl MediaInfo {
61    /// Reads a `MediaInfo` object as the page builds it.
62    pub fn from_js(js: Obj) -> MediaInfo {
63        let strings = |v: Option<&Js>| -> Vec<String> {
64            v.and_then(Js::as_arr)
65                .map(|a| {
66                    a.iter()
67                        .filter_map(|s| s.as_str().map(String::from))
68                        .collect()
69                })
70                .unwrap_or_default()
71        };
72        let notices = js
73            .get("notices")
74            .and_then(Js::as_arr)
75            .map(|a| {
76                a.iter()
77                    .filter_map(Js::as_obj)
78                    .map(|n| Notice {
79                        code: n.str("code").unwrap_or_default().to_string(),
80                        message: n.str("message").unwrap_or_default().to_string(),
81                        count: n.num("count") as u64,
82                    })
83                    .collect()
84            })
85            .unwrap_or_default();
86        MediaInfo {
87            name: js.str("name").unwrap_or("video").to_string(),
88            width: js.num("width") as usize,
89            height: js.num("height") as usize,
90            duration: js.num("duration"),
91            frame_count: js
92                .get("frameCount")
93                .map(Js::num)
94                .filter(|n| n.is_finite())
95                .map(|n| n as u64),
96            codec: js.str("codec").unwrap_or_default().to_string(),
97            mode: js.str("mode").unwrap_or_default().to_string(),
98            noise: js.get("noise").map_or(DECODED_VIDEO_NOISE, Js::num),
99            warnings: strings(js.get("warnings")),
100            notices,
101            js,
102        }
103    }
104}
105
106/// A failure to read the next frame.
107#[derive(Clone, Debug, PartialEq)]
108pub struct SourceError(pub String);
109
110impl fmt::Display for SourceError {
111    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
112        f.write_str(&self.0)
113    }
114}
115
116impl std::error::Error for SourceError {}
117
118/// One read of the source, frame by frame.
119pub type Frames<'a> = Box<dyn Iterator<Item = Result<Frame, SourceError>> + 'a>;
120
121/// A recording a run reads, once per pass.
122pub trait FrameSource: Send {
123    fn info(&self) -> &MediaInfo;
124    /// Starts a read from the first frame.
125    fn frames(&mut self) -> Result<Frames<'_>, SourceError>;
126
127    /// Like [`FrameSource::frames`], but a frame whose entry in `wanted` is false (or past its end) may come without
128    /// pixels (an empty `rgba`), which saves converting it. The default converts every frame.
129    fn sparse_frames(&mut self, wanted: &[bool]) -> Result<Frames<'_>, SourceError> {
130        let _ = wanted;
131        self.frames()
132    }
133}
134
135// --- Decoded video (feature `decode`) ---
136
137/// A video file decoded natively.
138#[cfg(feature = "decode")]
139pub struct VideoSource {
140    path: PathBuf,
141    options: decode::Options,
142    video: decode::VideoInfo,
143    info: MediaInfo,
144}
145
146#[cfg(feature = "decode")]
147impl VideoSource {
148    /// Opens `path` with `options` (probing it once); errors carry the decoder's reason, including how to install
149    /// ffmpeg when it is needed and missing.
150    pub fn open(path: &Path, options: decode::Options) -> Result<VideoSource, decode::Error> {
151        let video = decode::probe(path, &options)?;
152        let (width, height) = video.display_size();
153        let name = path
154            .file_name()
155            .map(|n| n.to_string_lossy().into_owned())
156            .unwrap_or_else(|| "video".into());
157        let size = std::fs::metadata(path).map(|m| m.len()).unwrap_or(0);
158        let mode = format!("Native {} decoder", video.backend.name());
159        // The page's MediaInfo field order (src/media/source.ts PreciseSource).
160        let mut js = Obj::new()
161            .with("name", name.as_str())
162            .with("size", size)
163            .with("width", width)
164            .with("height", height)
165            .with("codedWidth", video.width)
166            .with("codedHeight", video.height)
167            .with("rotation", video.rotation as u32)
168            .with("duration", video.duration.unwrap_or(0.0))
169            .with("codec", video.codec.as_str());
170        if let Some(n) = video.frame_count {
171            js.set("frameCount", n);
172        }
173        js.set("mode", mode.as_str());
174        js.set("noise", DECODED_VIDEO_NOISE);
175        js.set("warnings", Js::Arr(Vec::new()));
176        js.set("notices", Js::Arr(Vec::new()));
177        let info = MediaInfo::from_js(js);
178        Ok(VideoSource {
179            path: path.to_path_buf(),
180            options,
181            video,
182            info,
183        })
184    }
185
186    /// The decoder's view of the file.
187    pub fn video(&self) -> &decode::VideoInfo {
188        &self.video
189    }
190}
191
192#[cfg(feature = "decode")]
193impl FrameSource for VideoSource {
194    fn info(&self) -> &MediaInfo {
195        &self.info
196    }
197
198    fn frames(&mut self) -> Result<Frames<'_>, SourceError> {
199        self.read(None)
200    }
201
202    fn sparse_frames(&mut self, wanted: &[bool]) -> Result<Frames<'_>, SourceError> {
203        self.read(Some(wanted.to_vec()))
204    }
205}
206
207#[cfg(feature = "decode")]
208impl VideoSource {
209    fn read(&mut self, wanted: Option<Vec<bool>>) -> Result<Frames<'_>, SourceError> {
210        let mut decoder =
211            decode::open(&self.path, &self.options).map_err(|e| SourceError(e.to_string()))?;
212        let video = self.video.clone();
213        let signal = Signal::from_code(video.colour.core_code(video.height))
214            .ok_or_else(|| SourceError("unsupported colour signal".into()))?;
215        let duration = video.duration;
216        let mut pending: Option<decode::Frame> = None;
217        let mut done = false;
218        let mut index = 0u32;
219        let mut last_step = 1.0 / 60.0;
220        Ok(Box::new(std::iter::from_fn(move || {
221            if done && pending.is_none() {
222                return None;
223            }
224            // One frame of lookahead gives each frame its duration (the next timestamp minus its own).
225            {
226                if pending.is_none() {
227                    match decoder.next_frame() {
228                        Ok(Some(f)) => pending = Some(f),
229                        Ok(None) => {
230                            done = true;
231                            return None;
232                        }
233                        Err(e) => {
234                            done = true;
235                            return Some(Err(SourceError(e.to_string())));
236                        }
237                    }
238                }
239                let current = pending.take().unwrap();
240                let next = if done {
241                    None
242                } else {
243                    match decoder.next_frame() {
244                        Ok(f) => f,
245                        Err(e) => {
246                            done = true;
247                            pending = None;
248                            return Some(Err(SourceError(e.to_string())));
249                        }
250                    }
251                };
252                let step = match &next {
253                    Some(n) if n.timestamp > current.timestamp => n.timestamp - current.timestamp,
254                    Some(_) => last_step,
255                    None => duration
256                        .map(|d| d - current.timestamp)
257                        .filter(|d| *d > 0.0)
258                        .unwrap_or(last_step),
259                };
260                last_step = step;
261                if next.is_none() {
262                    done = true;
263                }
264                pending = next;
265                let skip = wanted
266                    .as_ref()
267                    .is_some_and(|w| !w.get(index as usize).copied().unwrap_or(false));
268                let rgba = if skip {
269                    let (w, h) = video.display_size();
270                    (w, h, Vec::new())
271                } else {
272                    match to_rgba(&current, signal, video.rotation) {
273                        Ok(f) => f,
274                        Err(e) => {
275                            done = true;
276                            pending = None;
277                            return Some(Err(e));
278                        }
279                    }
280                };
281                let frame = Frame {
282                    index,
283                    time: current.timestamp,
284                    duration: step,
285                    width: rgba.0,
286                    height: rgba.1,
287                    rgba: rgba.2,
288                };
289                index += 1;
290                Some(Ok(frame))
291            }
292        })))
293    }
294}
295
296/// The core's YUV → RGBA conversion of one decoded frame (10-bit frames rounded to 8 bits first, as the page
297/// copies them), turned for display.
298#[cfg(feature = "decode")]
299fn to_rgba(
300    frame: &decode::Frame,
301    signal: Signal,
302    rotation: u16,
303) -> Result<(usize, usize, Vec<u8>), SourceError> {
304    let eight;
305    let frame = match frame.format {
306        Some(PixelFormat::I420P10) | Some(PixelFormat::P010) => {
307            eight = frame.to_8bit();
308            &eight
309        }
310        Some(_) => frame,
311        None => return Err(SourceError("frame without a pixel format".into())),
312    };
313    let format = match frame.format {
314        Some(PixelFormat::I420) => Format::I420,
315        Some(PixelFormat::Nv12) => Format::Nv12,
316        _ => return Err(SourceError("unsupported pixel format".into())),
317    };
318    let (w, h) = (frame.width, frame.height);
319    let planes: Vec<yuv::Plane> = frame
320        .planes
321        .iter()
322        .map(|p| yuv::Plane {
323            offset: p.offset,
324            stride: p.stride,
325        })
326        .collect();
327    let mut rgba = vec![0u8; w * h * 4];
328    yuv::convert(&frame.data, format, &planes, w, h, signal, &mut rgba);
329    Ok(rotate(w, h, rgba, rotation))
330}
331
332/// Turns an RGBA frame clockwise by `rotation` degrees (0, 90, 180, 270).
333#[cfg(feature = "decode")]
334fn rotate(w: usize, h: usize, rgba: Vec<u8>, rotation: u16) -> (usize, usize, Vec<u8>) {
335    if rotation == 0 {
336        return (w, h, rgba);
337    }
338    let (ow, oh) = if rotation % 180 == 90 { (h, w) } else { (w, h) };
339    let mut out = vec![0u8; rgba.len()];
340    for y in 0..h {
341        for x in 0..w {
342            let (ox, oy) = match rotation {
343                90 => (h - 1 - y, x),
344                180 => (w - 1 - x, h - 1 - y),
345                _ => (y, w - 1 - x),
346            };
347            let s = (y * w + x) * 4;
348            let d = (oy * ow + ox) * 4;
349            out[d..d + 4].copy_from_slice(&rgba[s..s + 4]);
350        }
351    }
352    (ow, oh, out)
353}
354
355// --- Raw RGBA frames ---
356
357/// One frame of a [`RawSource`]: its time, duration and size.
358#[derive(Clone, Copy, Debug, PartialEq)]
359pub struct RawFrame {
360    pub time: f64,
361    pub duration: f64,
362    pub width: usize,
363    pub height: usize,
364}
365
366/// RGBA frames in a file, back to back, with each frame's time, duration and size.
367pub struct RawSource {
368    path: PathBuf,
369    frames: Vec<RawFrame>,
370    info: MediaInfo,
371}
372
373impl RawSource {
374    /// `path` holds the frames `frames` describes, in order.
375    pub fn new(path: PathBuf, frames: Vec<RawFrame>, info: MediaInfo) -> RawSource {
376        RawSource { path, frames, info }
377    }
378}
379
380impl FrameSource for RawSource {
381    fn info(&self) -> &MediaInfo {
382        &self.info
383    }
384
385    fn frames(&mut self) -> Result<Frames<'_>, SourceError> {
386        let mut file = File::open(&self.path).map_err(|e| SourceError(e.to_string()))?;
387        let frames = self.frames.clone();
388        let mut index = 0usize;
389        let mut offset = 0u64;
390        Ok(Box::new(std::iter::from_fn(move || {
391            let &RawFrame {
392                time,
393                duration,
394                width: w,
395                height: h,
396            } = frames.get(index)?;
397            let bytes = w * h * 4;
398            let mut rgba = vec![0u8; bytes];
399            let read = file
400                .seek(SeekFrom::Start(offset))
401                .and_then(|_| file.read_exact(&mut rgba));
402            offset += bytes as u64;
403            if let Err(e) = read {
404                return Some(Err(SourceError(e.to_string())));
405            }
406            let frame = Frame {
407                index: index as u32,
408                time,
409                duration,
410                width: w,
411                height: h,
412                rgba,
413            };
414            index += 1;
415            Some(Ok(frame))
416        })))
417    }
418}
long_screen/source.rs

long_screen/
source.rs
