alpha2_media/io.rs
1//! Platform-neutral media I/O contracts.
2//!
3//! Adapters translate `FFmpeg`, `WebCodecs`, or other host objects at this boundary;
4//! none of those platform types are part of these contracts.
5
6use std::sync::Arc;
7use std::sync::atomic::{AtomicBool, Ordering};
8use std::{error::Error, fmt};
9
10use alpha2_time::{FrameRate, MediaTime};
11
12use crate::{ColorMetadata, SourceId};
13
14/// Rectangle inside a coded video frame.
15#[derive(Copy, Clone, Debug, Eq, PartialEq)]
16pub struct Rect {
17 /// Horizontal origin in coded pixels.
18 pub x: u32,
19 /// Vertical origin in coded pixels.
20 pub y: u32,
21 /// Visible width.
22 pub width: u32,
23 /// Visible height.
24 pub height: u32,
25}
26
27/// Clockwise rotation to apply when displaying a frame.
28///
29/// This is the convention of the `WebCodecs` `VideoFrame.rotation`. A binding
30/// whose source states rotation another way converts it: `FFmpeg`'s display
31/// matrix angle is counter-clockwise, so a binding reports `(-angle) mod 360`.
32#[derive(Copy, Clone, Debug, Default, Eq, Hash, PartialEq)]
33pub enum Rotation {
34 /// No rotation.
35 #[default]
36 Deg0,
37 /// Clockwise quarter turn.
38 Deg90,
39 /// Half turn.
40 Deg180,
41 /// Clockwise three-quarter turn, i.e. a counter-clockwise quarter turn.
42 Deg270,
43}
44
45impl Rotation {
46 /// Clockwise quarter turns, `0..4`.
47 #[must_use]
48 pub const fn quarter_turns(self) -> u32 {
49 match self {
50 Self::Deg0 => 0,
51 Self::Deg90 => 1,
52 Self::Deg180 => 2,
53 Self::Deg270 => 3,
54 }
55 }
56
57 /// The rotation of `quarter_turns` clockwise quarter turns, of any sign.
58 #[must_use]
59 pub const fn from_quarter_turns(quarter_turns: i32) -> Self {
60 match quarter_turns.rem_euclid(4) {
61 0 => Self::Deg0,
62 1 => Self::Deg90,
63 2 => Self::Deg180,
64 _ => Self::Deg270,
65 }
66 }
67
68 /// Whether the rotation exchanges width and height.
69 #[must_use]
70 pub const fn swaps_axes(self) -> bool {
71 matches!(self, Self::Deg90 | Self::Deg270)
72 }
73}
74
75/// How to show a decoded picture upright: mirror it left to right when
76/// `mirror` is set, then turn it clockwise by `rotation`.
77///
78/// The two together reach every orientation a container or an image can
79/// declare: the eight of the Exif `Orientation` tag, every display matrix
80/// whose angle is a multiple of 90 degrees, a Matroska projection's pose. A binding
81/// converts its source's own statement to this one; `FFmpeg`'s display matrix
82/// mirrors after it rotates, and counter-clockwise.
83#[derive(Copy, Clone, Debug, Default, Eq, Hash, PartialEq)]
84pub struct Orientation {
85 /// Clockwise rotation, applied after the mirror.
86 pub rotation: Rotation,
87 /// Whether the stored picture is mirrored left to right before it turns.
88 pub mirror: bool,
89}
90
91impl Orientation {
92 /// Shown as stored.
93 pub const UPRIGHT: Self = Self {
94 rotation: Rotation::Deg0,
95 mirror: false,
96 };
97
98 /// A rotation without a mirror.
99 #[must_use]
100 pub const fn rotated(rotation: Rotation) -> Self {
101 Self {
102 rotation,
103 mirror: false,
104 }
105 }
106
107 /// The orientation an Exif `Orientation` value (1 to 8) declares; `None`
108 /// for any other value. 2, 4, 5, and 7 mirror.
109 #[must_use]
110 pub const fn from_exif(value: u16) -> Option<Self> {
111 let (rotation, mirror) = match value {
112 1 => (Rotation::Deg0, false),
113 2 => (Rotation::Deg0, true),
114 3 => (Rotation::Deg180, false),
115 4 => (Rotation::Deg180, true),
116 5 => (Rotation::Deg270, true),
117 6 => (Rotation::Deg90, false),
118 7 => (Rotation::Deg90, true),
119 8 => (Rotation::Deg270, false),
120 _ => return None,
121 };
122 Some(Self { rotation, mirror })
123 }
124
125 /// Whether the picture is shown as stored.
126 #[must_use]
127 pub const fn is_upright(self) -> bool {
128 matches!(self.rotation, Rotation::Deg0) && !self.mirror
129 }
130
131 /// The displayed size of a stored `width` x `height` picture.
132 #[must_use]
133 pub const fn displayed_size(self, width: u32, height: u32) -> (u32, u32) {
134 if self.rotation.swaps_axes() {
135 (height, width)
136 } else {
137 (width, height)
138 }
139 }
140}
141
142/// Reduced pixel-width to pixel-height ratio.
143#[derive(Copy, Clone, Debug, Eq, PartialEq)]
144pub struct SampleAspectRatio {
145 /// Ratio numerator.
146 pub num: u32,
147 /// Ratio denominator, never zero.
148 pub den: u32,
149}
150
151impl Default for SampleAspectRatio {
152 fn default() -> Self {
153 Self { num: 1, den: 1 }
154 }
155}
156
157impl SampleAspectRatio {
158 /// The ratio `num`:`den` in lowest terms, or `None` for a ratio with a
159 /// zero term, which declares nothing (`FFmpeg` writes an unknown ratio
160 /// as 0/1) or does not fit.
161 #[must_use]
162 pub fn new(num: u64, den: u64) -> Option<Self> {
163 if num == 0 || den == 0 {
164 return None;
165 }
166 let divisor = greatest_common_divisor(num, den);
167 Some(Self {
168 num: u32::try_from(num / divisor).ok()?,
169 den: u32::try_from(den / divisor).ok()?,
170 })
171 }
172
173 /// The ratio of the pixels of a `visible` picture shown at `displayed`
174 /// size (width, height), as a `WebCodecs` frame's display size says it.
175 ///
176 /// A display size is the stored size with one side stretched by the
177 /// ratio and rounded to whole pixels, so it cannot say the ratio exactly
178 /// (128x96 of 4:3 pixels shows 171x96, not 170 2/3). The ratio is the
179 /// simplest fraction that rounds to that size, which is the declared one
180 /// wherever a declared ratio's terms are small (4:3, 64:45, 10:11), and
181 /// otherwise within half a displayed pixel of the size.
182 #[must_use]
183 pub fn of_display(visible: (u32, u32), displayed: (u32, u32)) -> Option<Self> {
184 let [
185 visible_width,
186 visible_height,
187 displayed_width,
188 displayed_height,
189 ] = [visible.0, visible.1, displayed.0, displayed.1].map(u128::from);
190 if visible_width == 0
191 || visible_height == 0
192 || displayed_width == 0
193 || displayed_height == 0
194 {
195 return None;
196 }
197 // A side the display stretched is known to half a pixel; one it kept is exact.
198 let slack = |shown: u128, stored: u128| u128::from(shown != stored);
199 let width_slack = slack(displayed_width, visible_width);
200 let height_slack = slack(displayed_height, visible_height);
201 // ratio = (displayed_width / visible_width) / (displayed_height / visible_height), in
202 // halves of a pixel.
203 let low = (
204 (2 * displayed_width - width_slack) * visible_height,
205 (2 * displayed_height + height_slack) * visible_width,
206 );
207 let high = (
208 (2 * displayed_width + width_slack) * visible_height,
209 (2 * displayed_height - height_slack) * visible_width,
210 );
211 let (num, den) = simplest_fraction_between(low, high)?;
212 Self::new(u64::try_from(num).ok()?, u64::try_from(den).ok()?)
213 }
214
215 /// The size a stored `width` x `height` picture of these pixels is shown
216 /// at in square pixels, as a browser demuxer reports it: the side the
217 /// pixels are longer along stretched, rounded half up.
218 #[must_use]
219 pub fn square_pixel_size(self, width: u32, height: u32) -> (u32, u32) {
220 let stretch = |side: u32, by: u32, of: u32| {
221 let scaled =
222 (u64::from(side) * u64::from(by) * 2 + u64::from(of)) / (2 * u64::from(of));
223 u32::try_from(scaled).unwrap_or(u32::MAX)
224 };
225 if self.num == 0 || self.den == 0 {
226 return (width, height);
227 }
228 match self.num.cmp(&self.den) {
229 std::cmp::Ordering::Greater => (stretch(width, self.num, self.den), height),
230 std::cmp::Ordering::Less => (width, stretch(height, self.den, self.num)),
231 std::cmp::Ordering::Equal => (width, height),
232 }
233 }
234}
235
236/// The fraction with the smallest denominator in the closed interval
237/// `low`..=`high` of positive fractions (numerator, denominator), by the
238/// continued fractions of its ends.
239fn simplest_fraction_between(low: (u128, u128), high: (u128, u128)) -> Option<(u128, u128)> {
240 let ((low_num, low_den), (high_num, high_den)) = (low, high);
241 if low_den == 0 || high_den == 0 || low_num * high_den > high_num * low_den {
242 return None;
243 }
244 let whole = low_num / low_den;
245 if whole * low_den == low_num {
246 return Some((whole, 1));
247 }
248 if (whole + 1) * high_den <= high_num {
249 return Some((whole + 1, 1));
250 }
251 // Both ends lie strictly between `whole` and `whole + 1`: x = whole + 1/y, with y between the
252 // reciprocals of the fractional parts.
253 let (num, den) = simplest_fraction_between(
254 (high_den, high_num - whole * high_den),
255 (low_den, low_num - whole * low_den),
256 )?;
257 Some((whole * num + den, num))
258}
259
260fn greatest_common_divisor(mut a: u64, mut b: u64) -> u64 {
261 while b != 0 {
262 (a, b) = (b, a % b);
263 }
264 a
265}
266
267/// Plane organization delivered by a decoder.
268///
269/// A [`Decoder`] delivers only the layouts the render stage consumes directly:
270/// planar or semi-planar YCbCr of any chroma subsampling and bit depth that
271/// [`YuvFormat`] describes, 8-bit RGBA, and linear-light half-float RGBA for
272/// sources with more range than 8 bits hold. Every other decoded layout, such as
273/// packed RGB orders, palettes, or YCbCr with alpha, must be converted by the
274/// decoder before it is handed over, and is then reported as
275/// [`PixelFormat::Rgba8`]. The bundled native decoder does this with
276/// `libswscale`; a custom decoder is responsible for the same conversion.
277#[derive(Copy, Clone, Debug, Eq, PartialEq)]
278#[non_exhaustive]
279pub enum PixelFormat {
280 /// Planar or semi-planar YCbCr, one [`Plane`] per stored plane.
281 Yuv(YuvFormat),
282 /// Straight-alpha 8-bit red, green, blue, alpha in one plane.
283 Rgba8,
284 /// Premultiplied linear-light red, green, blue, alpha in one plane, each
285 /// a little-endian IEEE 754 half float, in BT.709 primaries with `1.0` at
286 /// SDR white. Values above `1.0` and below `0.0` are allowed; the frame's
287 /// color metadata does not apply.
288 LinearRgbaF16,
289}
290
291/// How a planar or semi-planar YCbCr frame stores its samples.
292///
293/// Luma is always the first plane. Chroma follows as separate U and V planes,
294/// or as one plane of interleaved U, V pairs. Each chroma plane covers
295/// `ceil(width / 2^chroma_shift_x)` by `ceil(height / 2^chroma_shift_y)`
296/// samples. A sample of more than eight bits is a little-endian `u16`. A
297/// format with [`alpha`](Self::alpha) stores one more plane after the
298/// chroma: straight, full-range alpha at the luma's size and depth.
299#[derive(Copy, Clone, Debug, Eq, Hash, PartialEq)]
300pub struct YuvFormat {
301 /// Log2 of the horizontal chroma subsampling: 0 for 4:4:4, 1 for 4:2:x.
302 pub chroma_shift_x: u8,
303 /// Log2 of the vertical chroma subsampling: 1 for 4:2:0, 0 otherwise.
304 pub chroma_shift_y: u8,
305 /// Significant bits per sample, 8 through 16.
306 pub bit_depth: u8,
307 /// Whether U and V interleave in one plane, as NV12 and P010 do.
308 pub interleaved_chroma: bool,
309 /// Whether a sample of fewer than 16 bits sits in the high bits of its
310 /// `u16`, as P010's do, rather than the low bits, as `yuv420p10le`'s do.
311 pub msb_aligned: bool,
312 /// Whether an alpha plane follows the chroma, as `yuva444p16le`'s does.
313 pub alpha: bool,
314}
315
316impl YuvFormat {
317 /// 8-bit planar 4:2:0 (`yuv420p`).
318 pub const I420: Self = Self::planar(1, 1, 8);
319 /// 8-bit 4:2:0 with interleaved chroma (`nv12`).
320 pub const NV12: Self = Self {
321 interleaved_chroma: true,
322 ..Self::I420
323 };
324 /// 10-bit 4:2:0 with interleaved, high-aligned chroma (`p010le`).
325 pub const P010: Self = Self {
326 bit_depth: 10,
327 msb_aligned: true,
328 ..Self::NV12
329 };
330
331 /// Planar YCbCr with low-aligned samples.
332 #[must_use]
333 pub const fn planar(chroma_shift_x: u8, chroma_shift_y: u8, bit_depth: u8) -> Self {
334 Self {
335 chroma_shift_x,
336 chroma_shift_y,
337 bit_depth,
338 interleaved_chroma: false,
339 msb_aligned: false,
340 alpha: false,
341 }
342 }
343
344 /// The same format with an alpha plane after the chroma.
345 #[must_use]
346 pub const fn with_alpha(self) -> Self {
347 Self {
348 alpha: true,
349 ..self
350 }
351 }
352
353 /// Whether the renderer can sample this format: 8 to 16 bits, chroma
354 /// subsampled by at most four on either axis, and high alignment only
355 /// where there are spare bits to align in.
356 #[must_use]
357 pub const fn is_supported(self) -> bool {
358 self.bit_depth >= 8
359 && self.bit_depth <= 16
360 && self.chroma_shift_x <= 2
361 && self.chroma_shift_y <= 2
362 && (!self.msb_aligned || self.bit_depth > 8)
363 }
364
365 /// Bytes each stored sample takes: one up to 8 bits, two above.
366 #[must_use]
367 pub const fn bytes_per_sample(self) -> u32 {
368 if self.bit_depth > 8 { 2 } else { 1 }
369 }
370
371 /// Number of stored planes: luma and chroma (three, or two with
372 /// interleaved chroma), then alpha when the format has it.
373 #[must_use]
374 pub const fn plane_count(self) -> usize {
375 self.color_plane_count() + if self.alpha { 1 } else { 0 }
376 }
377
378 /// Number of planes that hold luma and chroma: three, or two with
379 /// interleaved chroma. The alpha plane, if any, is the one after them.
380 #[must_use]
381 pub const fn color_plane_count(self) -> usize {
382 if self.interleaved_chroma { 2 } else { 3 }
383 }
384
385 /// Chroma plane dimensions, in sample positions, for a coded frame size.
386 #[must_use]
387 pub const fn chroma_size(self, width: u32, height: u32) -> (u32, u32) {
388 (
389 width.div_ceil(1 << self.chroma_shift_x),
390 height.div_ceil(1 << self.chroma_shift_y),
391 )
392 }
393}
394
395/// Chroma sample location for subsampled YUV.
396#[derive(Copy, Clone, Debug, Default, Eq, PartialEq)]
397pub enum ChromaLocation {
398 /// Adapter could not determine a location.
399 #[default]
400 Unspecified,
401 /// MPEG-2/most video: horizontally left, vertically centered.
402 Left,
403 /// JPEG-style centered chroma.
404 Center,
405 /// Top-left co-sited chroma.
406 TopLeft,
407}
408
409/// One decoded image plane.
410///
411/// The bytes are shared rather than owned outright, so a decoder can hand
412/// over its own reference-counted frame memory without copying it, and a
413/// cloned frame shares its planes.
414#[derive(Clone, Debug, Eq, PartialEq)]
415pub struct Plane {
416 /// Plane bytes, including row padding: `stride * height` of them.
417 pub bytes: bytes::Bytes,
418 /// Bytes between adjacent rows.
419 pub stride: u32,
420 /// Plane width in sample positions; an interleaved chroma plane holds a
421 /// U, V pair at each.
422 pub width: u32,
423 /// Plane height in samples.
424 pub height: u32,
425}
426
427/// Largest frame edge, in pixels, anything in alpha2 decodes, renders, or
428/// composes a project at.
429pub const MAX_FRAME_DIMENSION: u32 = 16_384;
430/// Largest frame area, in pixels, anything in alpha2 decodes, renders, or
431/// composes a project at: a decoder refuses a larger frame before it
432/// allocates one.
433pub const MAX_FRAME_PIXELS: u64 = 67_108_864;
434
435/// Whether a `width` x `height` frame is within [`MAX_FRAME_DIMENSION`] and
436/// [`MAX_FRAME_PIXELS`].
437#[must_use]
438pub const fn frame_size_supported(width: u32, height: u32) -> bool {
439 width <= MAX_FRAME_DIMENSION
440 && height <= MAX_FRAME_DIMENSION
441 && (width as u64) * (height as u64) <= MAX_FRAME_PIXELS
442}
443
444/// Host-independent decoded video frame.
445#[derive(Clone, Debug, PartialEq)]
446pub struct DecodedVideoFrame {
447 /// Presentation timestamp.
448 pub pts: MediaTime,
449 /// Optional decode timestamp.
450 pub dts: Option<MediaTime>,
451 /// Nominal frame duration when known.
452 pub duration: Option<MediaTime>,
453 /// The visible crop within the coded picture, in coded pixels.
454 pub visible_rect: Rect,
455 /// Pixel aspect ratio.
456 pub sample_aspect_ratio: SampleAspectRatio,
457 /// How the picture is shown upright.
458 pub orientation: Orientation,
459 /// Plane layout.
460 pub pixel_format: PixelFormat,
461 /// Chroma siting.
462 pub chroma_location: ChromaLocation,
463 /// Declared color metadata: the project's declaration, else the frame's
464 /// or stream's tags. `None` when nothing declares one: the renderer then
465 /// applies [`crate::ColorMetadata::UNDECLARED`] and reports it.
466 pub color: Option<ColorMetadata>,
467 /// Image planes in the order [`Self::pixel_format`] stores them.
468 pub planes: Vec<Plane>,
469}
470
471/// Most planes an [`AudioBlock`] carries: a 22.2 layout's 24, the most of
472/// any layout `FFmpeg` names.
473///
474/// Neither binding downmixes a source with more: both refuse it, so no
475/// source is mixed one way natively and another in the browser.
476pub const MAX_AUDIO_CHANNELS: usize = 24;
477
478/// The loudspeaker one plane of an [`AudioBlock`] feeds.
479///
480/// The positions are `FFmpeg`'s named channels (`AVChannel`), which take in
481/// the WAVE / `WAVEFORMATEXTENSIBLE` mask positions, Apple's channel labels,
482/// and ISO/IEC 23001-8's. [`Speaker::name`] is `FFmpeg`'s abbreviation.
483#[derive(Copy, Clone, Debug, Eq, Hash, PartialEq)]
484pub enum Speaker {
485 /// Front left (`FL`).
486 FrontLeft,
487 /// Front right (`FR`).
488 FrontRight,
489 /// Front center (`FC`): a mono source's one channel, and most films' dialogue.
490 FrontCenter,
491 /// Low-frequency effects (`LFE`).
492 LowFrequency,
493 /// Back left (`BL`).
494 BackLeft,
495 /// Back right (`BR`).
496 BackRight,
497 /// Front left of center (`FLC`).
498 FrontLeftOfCenter,
499 /// Front right of center (`FRC`).
500 FrontRightOfCenter,
501 /// Back center (`BC`).
502 BackCenter,
503 /// Side left (`SL`).
504 SideLeft,
505 /// Side right (`SR`).
506 SideRight,
507 /// Top center (`TC`).
508 TopCenter,
509 /// Top front left (`TFL`).
510 TopFrontLeft,
511 /// Top front center (`TFC`).
512 TopFrontCenter,
513 /// Top front right (`TFR`).
514 TopFrontRight,
515 /// Top back left (`TBL`).
516 TopBackLeft,
517 /// Top back center (`TBC`).
518 TopBackCenter,
519 /// Top back right (`TBR`).
520 TopBackRight,
521 /// Left of a matrix-encoded stereo downmix (`DL`, Lt).
522 DownmixLeft,
523 /// Right of a matrix-encoded stereo downmix (`DR`, Rt).
524 DownmixRight,
525 /// Wide left (`WL`).
526 WideLeft,
527 /// Wide right (`WR`).
528 WideRight,
529 /// Surround direct left (`SDL`).
530 SurroundDirectLeft,
531 /// Surround direct right (`SDR`).
532 SurroundDirectRight,
533 /// Second low-frequency effects (`LFE2`).
534 LowFrequency2,
535 /// Top side left (`TSL`).
536 TopSideLeft,
537 /// Top side right (`TSR`).
538 TopSideRight,
539 /// Bottom front center (`BFC`).
540 BottomFrontCenter,
541 /// Bottom front left (`BFL`).
542 BottomFrontLeft,
543 /// Bottom front right (`BFR`).
544 BottomFrontRight,
545 /// Side surround left (`SSL`).
546 SideSurroundLeft,
547 /// Side surround right (`SSR`).
548 SideSurroundRight,
549 /// Top surround left (`TTL`).
550 TopSurroundLeft,
551 /// Top surround right (`TTR`).
552 TopSurroundRight,
553 /// Binaural left (`BIL`).
554 BinauralLeft,
555 /// Binaural right (`BIR`).
556 BinauralRight,
557 /// A channel no loudspeaker position names (`UNK`): unknown, unused, or
558 /// one ambisonic component. The fold leaves it out.
559 Unnamed,
560}
561
562impl Speaker {
563 /// Every speaker with its `FFmpeg` name and channel number (`AVChannel`);
564 /// [`Speaker::Unnamed`] is `FFmpeg`'s `AV_CHAN_UNKNOWN`.
565 const TABLE: [(Self, &'static str, u32); 37] = [
566 (Self::FrontLeft, "FL", 0),
567 (Self::FrontRight, "FR", 1),
568 (Self::FrontCenter, "FC", 2),
569 (Self::LowFrequency, "LFE", 3),
570 (Self::BackLeft, "BL", 4),
571 (Self::BackRight, "BR", 5),
572 (Self::FrontLeftOfCenter, "FLC", 6),
573 (Self::FrontRightOfCenter, "FRC", 7),
574 (Self::BackCenter, "BC", 8),
575 (Self::SideLeft, "SL", 9),
576 (Self::SideRight, "SR", 10),
577 (Self::TopCenter, "TC", 11),
578 (Self::TopFrontLeft, "TFL", 12),
579 (Self::TopFrontCenter, "TFC", 13),
580 (Self::TopFrontRight, "TFR", 14),
581 (Self::TopBackLeft, "TBL", 15),
582 (Self::TopBackCenter, "TBC", 16),
583 (Self::TopBackRight, "TBR", 17),
584 (Self::DownmixLeft, "DL", 29),
585 (Self::DownmixRight, "DR", 30),
586 (Self::WideLeft, "WL", 31),
587 (Self::WideRight, "WR", 32),
588 (Self::SurroundDirectLeft, "SDL", 33),
589 (Self::SurroundDirectRight, "SDR", 34),
590 (Self::LowFrequency2, "LFE2", 35),
591 (Self::TopSideLeft, "TSL", 36),
592 (Self::TopSideRight, "TSR", 37),
593 (Self::BottomFrontCenter, "BFC", 38),
594 (Self::BottomFrontLeft, "BFL", 39),
595 (Self::BottomFrontRight, "BFR", 40),
596 (Self::SideSurroundLeft, "SSL", 41),
597 (Self::SideSurroundRight, "SSR", 42),
598 (Self::TopSurroundLeft, "TTL", 43),
599 (Self::TopSurroundRight, "TTR", 44),
600 (Self::BinauralLeft, "BIL", 61),
601 (Self::BinauralRight, "BIR", 62),
602 (Self::Unnamed, "UNK", 0x300),
603 ];
604
605 /// `FFmpeg`'s abbreviation (`FL`, `FR`, `FC`, `LFE`, ...); `UNK` for
606 /// [`Speaker::Unnamed`].
607 #[must_use]
608 pub fn name(self) -> &'static str {
609 Self::TABLE
610 .iter()
611 .find(|(speaker, _, _)| *speaker == self)
612 .map_or("UNK", |(_, name, _)| name)
613 }
614
615 /// The speaker `FFmpeg` names `name`, or `None` for a name it does not
616 /// give a loudspeaker position (`UNK` is [`Speaker::Unnamed`]).
617 #[must_use]
618 pub fn from_name(name: &str) -> Option<Self> {
619 Self::TABLE
620 .iter()
621 .find(|(_, known, _)| *known == name)
622 .map(|(speaker, _, _)| *speaker)
623 }
624
625 /// `FFmpeg`'s channel number (`enum AVChannel`) for this speaker.
626 #[must_use]
627 pub fn ffmpeg_channel(self) -> u32 {
628 Self::TABLE
629 .iter()
630 .find(|(speaker, _, _)| *speaker == self)
631 .map_or(0x300, |(_, _, channel)| *channel)
632 }
633
634 /// The speaker of `FFmpeg` channel number `channel`; any channel without
635 /// a loudspeaker position (unknown, unused, ambisonic, user) is
636 /// [`Speaker::Unnamed`].
637 #[must_use]
638 pub fn from_ffmpeg_channel(channel: i64) -> Self {
639 Self::TABLE
640 .iter()
641 .find(|(_, _, number)| i64::from(*number) == channel)
642 .map_or(Self::Unnamed, |(speaker, _, _)| *speaker)
643 }
644}
645
646/// The speakers an [`AudioBlock`]'s planes feed, in plane order: 1 to
647/// [`MAX_AUDIO_CHANNELS`] of them.
648///
649/// A provider reports the layout its source declares (a WAVE channel mask, a
650/// `QuickTime` `chan` box, a codec's channel configuration), in the order its
651/// planes arrive; neither binding rematrixes a source into another layout.
652/// A source that declares none, or only channels no speaker names, has
653/// [`SpeakerLayout::default_for`] its channel count.
654#[derive(Copy, Clone, Eq, Hash, PartialEq)]
655pub struct SpeakerLayout {
656 speakers: [Speaker; MAX_AUDIO_CHANNELS],
657 count: u8,
658}
659
660impl std::fmt::Debug for SpeakerLayout {
661 fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
662 formatter
663 .debug_list()
664 .entries(self.speakers().iter().map(|speaker| speaker.name()))
665 .finish()
666 }
667}
668
669impl SpeakerLayout {
670 /// One front-center plane: a mono source, and a mono project's mix.
671 pub const MONO: Self = Self::fixed(&[Speaker::FrontCenter]);
672 /// Front left and right: a stereo source, and a stereo project's mix.
673 pub const STEREO: Self = Self::fixed(&[Speaker::FrontLeft, Speaker::FrontRight]);
674
675 const fn fixed(speakers: &[Speaker]) -> Self {
676 let mut layout = [Speaker::Unnamed; MAX_AUDIO_CHANNELS];
677 let mut index = 0;
678 while index < speakers.len() {
679 layout[index] = speakers[index];
680 index += 1;
681 }
682 #[allow(clippy::cast_possible_truncation)]
683 Self {
684 speakers: layout,
685 count: speakers.len() as u8,
686 }
687 }
688
689 /// The layout of these speakers in this plane order, or `None` for none
690 /// or more than [`MAX_AUDIO_CHANNELS`].
691 #[must_use]
692 pub fn new(speakers: &[Speaker]) -> Option<Self> {
693 if speakers.is_empty() || speakers.len() > MAX_AUDIO_CHANNELS {
694 return None;
695 }
696 let mut layout = Self {
697 speakers: [Speaker::Unnamed; MAX_AUDIO_CHANNELS],
698 count: u8::try_from(speakers.len()).ok()?,
699 };
700 layout.speakers[..speakers.len()].copy_from_slice(speakers);
701 Some(layout)
702 }
703
704 /// The layout a source of `channels` planes that declares none has:
705 /// `FFmpeg`'s default for the count (`av_channel_layout_default`), in the
706 /// WAVE order. Mono, stereo, 2.1, 4.0, 5.0, 5.1, 6.1 and 7.1 up to 8;
707 /// 5.1.4, 7.1.4, 9.1.4, 9.1.6 and 22.2 for 10, 12, 14, 16 and 24. A count
708 /// `FFmpeg` names no layout for is a 7.1 with its further channels
709 /// unnamed. `None` for no channels or more than [`MAX_AUDIO_CHANNELS`].
710 #[must_use]
711 pub fn default_for(channels: usize) -> Option<Self> {
712 /// `FFmpeg`'s native-order channel masks (`AV_CH_LAYOUT_*`).
713 const SEVEN_ONE: u64 = 0x63f;
714 let mask: u64 = match channels {
715 1 => 0x4,
716 2 => 0x3,
717 3 => 0xb,
718 4 => 0x107,
719 5 => 0x37,
720 6 => 0x3f,
721 7 => 0x70f,
722 10 => 0x2_d60f,
723 12 => 0x2_d63f,
724 14 => 0x2_d6ff,
725 16 => 0x30_0002_d6ff,
726 24 => 0x1f8_0003_ffff,
727 8..=MAX_AUDIO_CHANNELS => SEVEN_ONE,
728 _ => return None,
729 };
730 let mut layout = Self::of_mask(mask)?;
731 // A count without a layout of its own: the 7.1's further channels name no speaker.
732 layout.count = u8::try_from(channels).ok()?;
733 Some(layout)
734 }
735
736 /// The speakers of a native-order channel mask, in bit order.
737 fn of_mask(mask: u64) -> Option<Self> {
738 let speakers = (0..64_i64)
739 .filter(|bit| mask & (1 << bit) != 0)
740 .map(Speaker::from_ffmpeg_channel)
741 .collect::<Vec<_>>();
742 Self::new(&speakers)
743 }
744
745 /// The layout a source declares, or [`SpeakerLayout::default_for`] its
746 /// channel count when it names no speaker at all. The flag says the
747 /// default was assumed.
748 #[must_use]
749 pub fn declared_or_default(speakers: &[Speaker]) -> Option<(Self, bool)> {
750 if speakers.iter().all(|speaker| *speaker == Speaker::Unnamed) {
751 return Self::default_for(speakers.len()).map(|layout| (layout, true));
752 }
753 Self::new(speakers).map(|layout| (layout, false))
754 }
755
756 /// The speakers, in plane order.
757 #[must_use]
758 pub fn speakers(&self) -> &[Speaker] {
759 &self.speakers[..usize::from(self.count)]
760 }
761
762 /// How many planes the layout has.
763 #[must_use]
764 pub fn channels(&self) -> usize {
765 usize::from(self.count)
766 }
767
768 /// The speakers' `FFmpeg` names, joined by `+` (`FL+FR+FC`), as
769 /// `av_channel_layout_from_string` reads them.
770 #[must_use]
771 pub fn describe(&self) -> String {
772 self.speakers()
773 .iter()
774 .map(|speaker| speaker.name())
775 .collect::<Vec<_>>()
776 .join("+")
777 }
778}
779
780/// Planar floating-point audio at the project sample rate.
781///
782/// Positions are sample indices at `sample_rate`, never ticks: a tick is not a
783/// whole number of samples at 44.1 kHz, so a start in ticks cannot say which
784/// sample a block begins with, and every conversion between the two would need
785/// a rounding rule both sides agree on. Sample `n` is at `n / sample_rate`
786/// seconds on the block's timeline, the source's for a decoder and the
787/// project's for a mixer.
788///
789/// A block from a decoder or an [`AudioBlockProvider`] keeps its source's
790/// channels, 1 to [`MAX_AUDIO_CHANNELS`] planes, and says which speaker each
791/// feeds; the mixer folds them into the project's layout by those speakers.
792/// A mixer's own blocks have the project's channels.
793#[derive(Clone, Debug, PartialEq)]
794pub struct AudioBlock {
795 /// Index of the first sample, at `sample_rate`.
796 pub start_sample: i64,
797 /// Project sample rate used by every returned block.
798 pub sample_rate: u32,
799 /// The speaker each of `channels` feeds, one per plane.
800 pub speakers: SpeakerLayout,
801 /// One vector per channel, all with equal lengths, in `speakers`' order.
802 pub channels: Vec<Vec<f32>>,
803}
804
805impl AudioBlock {
806 /// Samples per channel.
807 #[must_use]
808 pub fn frames(&self) -> usize {
809 self.channels.first().map_or(0, Vec::len)
810 }
811
812 /// Index one past the last sample.
813 #[must_use]
814 pub fn end_sample(&self) -> i64 {
815 self.start_sample
816 .saturating_add(i64::try_from(self.frames()).unwrap_or(i64::MAX))
817 }
818
819 /// The first sample's time, at or before it on the tick grid.
820 ///
821 /// For display and for APIs that take a [`MediaTime`]; positions within
822 /// audio stay in samples.
823 #[must_use]
824 pub fn start_time(&self) -> MediaTime {
825 MediaTime::from_ticks(sample_ticks_floor(self.start_sample, self.sample_rate))
826 }
827}
828
829/// The tick at or before sample `sample` at `sample_rate`.
830#[must_use]
831pub fn sample_ticks_floor(sample: i64, sample_rate: u32) -> i64 {
832 let ticks = (i128::from(sample) * i128::from(alpha2_time::TICKS_PER_SECOND))
833 .div_euclid(i128::from(sample_rate.max(1)));
834 i64::try_from(ticks).unwrap_or(if ticks < 0 { i64::MIN } else { i64::MAX })
835}
836
837/// Generic color declaration for a host-rasterized texture.
838#[derive(Copy, Clone, Debug, Eq, PartialEq)]
839pub enum RasterColorSpace {
840 /// IEC sRGB encoded RGB.
841 Srgb,
842 /// Linear-light RGB with BT.709/sRGB primaries.
843 LinearSrgb,
844}
845
846/// Alpha representation of host pixels.
847#[derive(Copy, Clone, Debug, Eq, PartialEq)]
848pub enum Premultiplication {
849 /// RGB has already been multiplied by alpha.
850 Premultiplied,
851 /// RGB is straight and must be premultiplied at upload.
852 Straight,
853 /// Pixels are opaque.
854 Opaque,
855}
856
857/// Generic image/video/text texture descriptor owned by a host.
858#[derive(Clone, Debug, Eq, PartialEq)]
859pub struct HostTexture {
860 /// Host-defined cache identity.
861 pub content_version: u64,
862 /// Width in pixels.
863 pub width: u32,
864 /// Height in pixels.
865 pub height: u32,
866 /// Alpha representation.
867 pub premultiplication: Premultiplication,
868 /// RGB encoding.
869 pub color_space: RasterColorSpace,
870}
871
872/// Horizontal alignment applied by a host text layout engine.
873#[derive(Copy, Clone, Debug, Default, Eq, PartialEq)]
874pub enum TextAlign {
875 /// Align each line to its paragraph's leading edge: the left of a
876 /// left-to-right paragraph, the right of a right-to-left one.
877 #[default]
878 Start,
879 /// Center each line in the requested layout width.
880 Center,
881 /// Align each line to its paragraph's trailing edge.
882 End,
883}
884
885/// Policy for glyphs whose only available representation is a color emoji.
886#[derive(Copy, Clone, Debug, Default, Eq, PartialEq)]
887pub enum EmojiPolicy {
888 /// Rasterize a monochrome outline when the selected host stack provides one.
889 #[default]
890 Monochrome,
891 /// Draw each grapheme shown as an emoji (UTS #51: one with an
892 /// `Emoji_Presentation` character, or U+FE0F with a pictograph or a
893 /// keycap, and no U+FE0E), sequence and modifiers included, as one
894 /// U+FFFD REPLACEMENT CHARACTER. A symbol shown as text by default,
895 /// such as (c) or a sun, stays.
896 Replace,
897}
898
899/// Layout facts attached to a host-rasterized text texture.
900///
901/// The compositor consumes [`Self::texture`] exactly like an image or decoded
902/// video texture. These values are retained for placement, cache validation,
903/// and cross-host contract tests; they never select a text-specific GPU path.
904#[derive(Clone, Debug, PartialEq)]
905pub struct TextTexture {
906 /// Generic texture identity and pixel declaration.
907 pub texture: HostTexture,
908 /// Where the layout box's left edge lies in the texture, in pixels. The
909 /// texture adds the same margin on both sides of the box for ink and
910 /// decorations that reach past it.
911 pub layout_x: f32,
912 /// Where the layout box's top edge lies in the texture, in pixels, with
913 /// the same margin below the box.
914 pub layout_y: f32,
915 /// Logical layout width before integer pixel rounding.
916 pub layout_width: f32,
917 /// Logical layout height before integer pixel rounding.
918 pub layout_height: f32,
919 /// Distance from the layout box's top edge to the first alphabetic
920 /// baseline.
921 pub first_baseline: f32,
922 /// Highest any line's ink reaches above its baseline, and at least the
923 /// nominal ascent the lines are spaced by.
924 pub ascent: f32,
925 /// Lowest any line's ink reaches below its baseline, and at least the
926 /// nominal descent.
927 pub descent: f32,
928 /// Number of laid-out lines.
929 pub line_count: u32,
930}
931
932impl TextTexture {
933 /// Checks dimensions and finite layout metrics at the public boundary.
934 #[must_use]
935 pub fn is_valid(&self) -> bool {
936 self.texture.width > 0
937 && self.texture.height > 0
938 && self.line_count > 0
939 && [
940 self.layout_x,
941 self.layout_y,
942 self.layout_width,
943 self.layout_height,
944 self.first_baseline,
945 self.ascent,
946 self.descent,
947 ]
948 .into_iter()
949 .all(|value| value.is_finite() && value >= 0.0)
950 && self.first_baseline <= self.layout_height
951 && f64::from(self.layout_x) + f64::from(self.layout_width)
952 <= f64::from(self.texture.width)
953 && f64::from(self.layout_y) + f64::from(self.layout_height)
954 <= f64::from(self.texture.height)
955 }
956}
957
958/// RGBA8 pixels returned by an image, video, or text rasterization host.
959#[derive(Clone, Debug, PartialEq)]
960pub struct HostTexturePixels {
961 /// Shared texture descriptor. Text hosts additionally return [`TextTexture`].
962 pub texture: HostTexture,
963 /// Tightly packed top-to-bottom RGBA8 rows.
964 pub rgba8: Vec<u8>,
965}
966
967impl HostTexturePixels {
968 /// Checks that the owned bytes exactly cover the declared texture.
969 #[must_use]
970 pub fn is_valid(&self) -> bool {
971 usize::try_from(self.texture.width)
972 .ok()
973 .and_then(|width| {
974 usize::try_from(self.texture.height)
975 .ok()
976 .and_then(|height| width.checked_mul(height))
977 })
978 .and_then(|pixels| pixels.checked_mul(4))
979 == Some(self.rgba8.len())
980 }
981}
982
983/// Media stream domain.
984#[derive(Copy, Clone, Debug, Eq, PartialEq)]
985pub enum StreamKind {
986 /// Video stream.
987 Video,
988 /// Audio stream.
989 Audio,
990}
991
992/// Platform-neutral probe result for one stream.
993#[derive(Clone, Debug, PartialEq)]
994pub struct StreamInfo {
995 /// Demuxer stream index.
996 pub index: usize,
997 /// Stream domain.
998 pub kind: StreamKind,
999 /// Codec short name when available.
1000 pub codec: String,
1001 /// Stream duration when known.
1002 pub duration: Option<MediaTime>,
1003 /// Video dimensions.
1004 pub dimensions: Option<(u32, u32)>,
1005 /// Video frame rate.
1006 pub frame_rate: Option<FrameRate>,
1007 /// Audio sample rate.
1008 pub sample_rate: Option<u32>,
1009 /// Audio channel count.
1010 pub channels: Option<u32>,
1011 /// How the stream's pictures are shown upright.
1012 pub orientation: Orientation,
1013 /// Sample aspect ratio.
1014 pub sample_aspect_ratio: SampleAspectRatio,
1015 /// Declared video color.
1016 pub color: Option<ColorMetadata>,
1017}
1018
1019/// Kind of decoder cache/refill operation.
1020#[derive(Copy, Clone, Debug, Eq, PartialEq)]
1021pub enum DecodeRequestKind {
1022 /// Normal look-ahead fill.
1023 Prefetch,
1024 /// Exact seek: flush decoder state and begin at/before the requested time.
1025 Seek,
1026 /// Refill after cache eviction.
1027 Evicted,
1028 /// Refill every referenced input after GPU device recovery.
1029 DeviceRecovery,
1030}
1031
1032/// One idempotent decoder request. Hosts discard results with an old generation.
1033#[derive(Clone, Debug, Eq, PartialEq)]
1034pub struct DecodeRequest {
1035 /// Source identity.
1036 pub source: SourceId,
1037 /// Requested source timestamp.
1038 pub time: MediaTime,
1039 /// Why the engine needs the input.
1040 pub kind: DecodeRequestKind,
1041 /// Monotonically increasing seek/recovery generation.
1042 pub generation: u64,
1043}
1044
1045/// Result correlated to a decoder request, safe for out-of-order completion.
1046#[derive(Clone, Debug, PartialEq)]
1047pub struct DecodeResult {
1048 /// Original request.
1049 pub request: DecodeRequest,
1050 /// Decoder outcome.
1051 pub event: DecodeEvent,
1052}
1053
1054/// Decoder output and terminal states.
1055#[derive(Clone, Debug, PartialEq)]
1056pub enum DecodeEvent {
1057 /// Decoded video became available.
1058 Video(DecodedVideoFrame),
1059 /// Decoded audio became available.
1060 Audio(AudioBlock),
1061 /// No exact frame is available yet; retry after feeding more input.
1062 Retry,
1063 /// The selected stream has ended.
1064 EndOfStream,
1065}
1066
1067/// Cancellation flag shared across adapter operations.
1068#[derive(Clone, Debug, Default)]
1069pub struct CancelToken(Arc<AtomicBool>);
1070
1071impl CancelToken {
1072 /// Requests cancellation. The operation must stop at its next adapter boundary.
1073 pub fn cancel(&self) {
1074 self.0.store(true, Ordering::Release);
1075 }
1076
1077 /// Returns whether cancellation was requested.
1078 #[must_use]
1079 pub fn is_cancelled(&self) -> bool {
1080 self.0.load(Ordering::Acquire)
1081 }
1082}
1083
1084/// The sentence every wait on GPU work that timed out carries, wherever it
1085/// was waited on (the renderer, a native interop helper, an encoder input),
1086/// so a host that sees only error text tells a hung GPU from a generic
1087/// failure. Hosts classify it as `GPU_TIMEOUT`.
1088pub const GPU_TIMEOUT_MESSAGE: &str = "GPU work did not complete within the wait timeout";
1089
1090/// What a failed GPU call says about the device it ran on, read from the
1091/// platform's own status for that call (a `VkResult`, `HRESULT`, or
1092/// `CVReturn`) where it failed, not from any message.
1093#[derive(Copy, Clone, Debug, Eq, PartialEq)]
1094#[non_exhaustive]
1095pub enum GpuFault {
1096 /// The device was lost or removed; nothing more runs on it.
1097 DeviceLost,
1098 /// The device could not allocate memory of its own.
1099 DeviceMemory,
1100 /// The driver could not allocate host (CPU) memory for the call.
1101 HostMemory,
1102 /// GPU work the call waited on did not complete within the wait timeout.
1103 Timeout,
1104}
1105
1106impl GpuFault {
1107 /// A stable name for logs and reports.
1108 #[must_use]
1109 pub const fn as_str(self) -> &'static str {
1110 match self {
1111 Self::DeviceLost => "device-lost",
1112 Self::DeviceMemory => "device-memory",
1113 Self::HostMemory => "host-memory",
1114 Self::Timeout => "timeout",
1115 }
1116 }
1117}
1118
1119/// Typed decoder failure.
1120///
1121/// Damaged input a decoder's policy lets it skip is not one: the decoder
1122/// goes on to the next frame and reports the damage through its adapter's
1123/// diagnostics, so a caller never retries around it. Damage it does not skip
1124/// is an adapter failure.
1125#[derive(Clone, Debug, Eq, PartialEq)]
1126#[non_exhaustive]
1127pub enum DecodeError {
1128 /// The operation was cancelled.
1129 Cancelled,
1130 /// The requested stream or timestamp is outside the resource.
1131 OutOfRange,
1132 /// The adapter does not support this media representation.
1133 Unsupported(String),
1134 /// Adapter-specific failure, represented as stable text rather than a platform type.
1135 Adapter(String),
1136 /// A GPU call the adapter made failed, and its platform status says what
1137 /// that was evidence of. `message` names the call and the status.
1138 Gpu {
1139 /// What the status says about the device.
1140 fault: GpuFault,
1141 /// The failed call and its platform status.
1142 message: String,
1143 },
1144}
1145
1146impl DecodeError {
1147 /// Returns whether the decoder must be discarded instead of retried.
1148 #[must_use]
1149 pub const fn is_fatal(&self) -> bool {
1150 matches!(
1151 self,
1152 Self::Unsupported(_) | Self::Adapter(_) | Self::Gpu { .. }
1153 )
1154 }
1155}
1156
1157impl fmt::Display for DecodeError {
1158 fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
1159 match self {
1160 Self::Cancelled => formatter.write_str("decode was cancelled"),
1161 Self::OutOfRange => formatter.write_str("decode request is out of range"),
1162 Self::Unsupported(message) => write!(formatter, "unsupported media: {message}"),
1163 Self::Adapter(message) | Self::Gpu { message, .. } => {
1164 write!(formatter, "decoder adapter failed: {message}")
1165 }
1166 }
1167 }
1168}
1169
1170impl Error for DecodeError {}
1171
1172/// Stateful media decoder implemented by a platform binding.
1173pub trait Decoder {
1174 /// Returns deterministic stream metadata and stream selection.
1175 fn streams(&self) -> &[StreamInfo];
1176 /// Flushes queued packets and frames without changing stream selection.
1177 ///
1178 /// # Errors
1179 /// Returns a platform-neutral adapter failure when flushing fails.
1180 fn flush(&mut self) -> Result<(), DecodeError>;
1181 /// Seeks and returns what is presented at the request's time.
1182 ///
1183 /// For video that is the frame whose presentation interval contains the
1184 /// time: the last frame whose timestamp is at or before the time's
1185 /// presentation deadline, the time plus the stream's presentation slack
1186 /// ([`alpha2_time::presentation_slack`]), which absorbs the rounding a
1187 /// stored timestamp went through. A seek starts from the keyframe at or
1188 /// before that deadline. Deciding so takes the frame after, which the
1189 /// next [`Self::decode_next`] returns. A stream whose first frame is later
1190 /// answers with that frame, and past the last frame the last one answers,
1191 /// however far past: a stream shows its last picture after its end as it
1192 /// shows its first before its start. A stream with no frame at all is
1193 /// [`DecodeError::OutOfRange`]. For audio it is the block holding the
1194 /// time's sample.
1195 ///
1196 /// # Errors
1197 /// Returns cancellation, range, unsupported-media, or adapter failures.
1198 fn seek(
1199 &mut self,
1200 request: DecodeRequest,
1201 cancel: &CancelToken,
1202 ) -> Result<DecodeResult, DecodeError>;
1203 /// Advances packet draining by one observable event.
1204 ///
1205 /// # Errors
1206 /// Returns cancellation, unsupported-media, or adapter failures.
1207 fn decode_next(&mut self, cancel: &CancelToken) -> Result<DecodeEvent, DecodeError>;
1208}
1209
1210/// A provider's answer to one request: source samples, and what the source
1211/// holds on either side of them.
1212#[derive(Clone, Copy, Debug, PartialEq)]
1213pub struct AudioSpan<'a> {
1214 /// Source samples at the project rate, read by their own `start_sample`.
1215 pub block: &'a AudioBlock,
1216 /// Whether the source has no samples before `block.start_sample`: the
1217 /// block begins where the source's audio does, such as a stream that
1218 /// starts 50 ms into its container.
1219 pub starts_source: bool,
1220 /// Whether the source has no samples from `block.end_sample()` on: the
1221 /// block ends where the source's audio does.
1222 pub ends_source: bool,
1223}
1224
1225impl<'a> AudioSpan<'a> {
1226 /// A block that holds its source's entire audio, as a source decoded
1227 /// once from end to end is.
1228 #[must_use]
1229 pub const fn whole(block: &'a AudioBlock) -> Self {
1230 Self {
1231 block,
1232 starts_source: true,
1233 ends_source: true,
1234 }
1235 }
1236
1237 /// Whether the span accounts for every sample of `[start, end)`: each is
1238 /// in the block, or before it with [`Self::starts_source`], or after it
1239 /// with [`Self::ends_source`], where the source is silent.
1240 #[must_use]
1241 pub fn covers(&self, start: i64, end: i64) -> bool {
1242 (self.starts_source || self.block.start_sample <= start)
1243 && (self.ends_source || self.block.end_sample() >= end)
1244 }
1245}
1246
1247/// Retrieves project-rate planar audio without exposing its backing cache.
1248pub trait AudioBlockProvider {
1249 /// Returns source samples `[start_sample, start_sample + frames)` at the
1250 /// project rate, as a span that [covers](AudioSpan::covers) all of them.
1251 ///
1252 /// The block may start before `start_sample` and end after the request.
1253 /// Where the source has no audio, before its first sample or after its
1254 /// last, the provider either holds zeros there or says so with
1255 /// [`AudioSpan::starts_source`] and [`AudioSpan::ends_source`], which
1256 /// spares allocating silence. A provider that cannot yet account for the
1257 /// whole request returns `None`, or a span that does not cover it: the
1258 /// mixer then reports the source missing instead of mixing the gap as
1259 /// silence and calling the block complete.
1260 fn block(&self, source: &SourceId, start_sample: i64, frames: usize) -> Option<AudioSpan<'_>>;
1261}
1262
1263/// Receives project-rate planar audio from offline or realtime graph hosts.
1264pub trait AudioBlockSink {
1265 /// Writes one monotonic audio block.
1266 ///
1267 /// # Errors
1268 /// Returns a platform-neutral encode error on cancellation, invalid timing,
1269 /// or host failure.
1270 fn write_audio(&mut self, block: &AudioBlock, cancel: &CancelToken) -> Result<(), EncodeError>;
1271
1272 /// Drains and commits the sink.
1273 ///
1274 /// # Errors
1275 /// Returns cancellation or host failures; success means all delayed
1276 /// samples were committed.
1277 fn finish_audio(&mut self, cancel: &CancelToken) -> Result<(), EncodeError>;
1278
1279 /// Discards partial output.
1280 ///
1281 /// # Errors
1282 /// Returns an error when cleanup fails.
1283 fn abort_audio(&mut self) -> Result<(), EncodeError>;
1284}
1285
1286/// One input accepted by an encoding/mux adapter.
1287#[derive(Clone, Debug, PartialEq)]
1288pub enum EncodeEvent {
1289 /// Premultiplied rendered video.
1290 Video(DecodedVideoFrame),
1291 /// Project-rate planar audio.
1292 Audio(AudioBlock),
1293}
1294
1295/// Typed encoding failure.
1296#[derive(Clone, Debug, Eq, PartialEq)]
1297#[non_exhaustive]
1298pub enum EncodeError {
1299 /// The operation was cancelled; partial output must be removed by policy.
1300 Cancelled,
1301 /// Input timestamps violated monotonic/interleave requirements.
1302 InvalidTimestamp,
1303 /// Adapter failure without leaking a platform error type.
1304 Adapter(String),
1305 /// The output file could not be created or claimed: it already exists,
1306 /// its path is unusable, or the file system refused. Nothing was encoded.
1307 Output(String),
1308 /// Another export, live in this process or another, is writing the same
1309 /// output: it holds the output's partial file. Nothing was encoded, and
1310 /// the other export's file is untouched.
1311 OutputInUse(String),
1312 /// A GPU call the adapter made failed, and its platform status says what
1313 /// that was evidence of. `message` names the call and the status.
1314 Gpu {
1315 /// What the status says about the device.
1316 fault: GpuFault,
1317 /// The failed call and its platform status.
1318 message: String,
1319 },
1320}
1321
1322impl EncodeError {
1323 /// A wait on GPU work an encode needs, for `what`, that timed out: a
1324 /// [`GpuFault::Timeout`] whose message carries [`GPU_TIMEOUT_MESSAGE`].
1325 #[must_use]
1326 pub fn gpu_timeout(what: impl fmt::Display) -> Self {
1327 Self::Gpu {
1328 fault: GpuFault::Timeout,
1329 message: format!("{what}: {GPU_TIMEOUT_MESSAGE}"),
1330 }
1331 }
1332}
1333
1334impl fmt::Display for EncodeError {
1335 fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
1336 match self {
1337 Self::Cancelled => formatter.write_str("encode was cancelled"),
1338 Self::InvalidTimestamp => formatter.write_str("encode timestamp is invalid"),
1339 Self::Adapter(message) | Self::Gpu { message, .. } => {
1340 write!(formatter, "encoder adapter failed: {message}")
1341 }
1342 Self::Output(message) | Self::OutputInUse(message) => {
1343 write!(formatter, "cannot create the output: {message}")
1344 }
1345 }
1346 }
1347}
1348
1349impl Error for EncodeError {}
1350
1351/// Host encoding and muxing boundary.
1352pub trait EncodeSink {
1353 /// Queues one timestamped media item.
1354 ///
1355 /// # Errors
1356 /// Returns cancellation, invalid timestamp, or adapter failures.
1357 fn write(&mut self, event: EncodeEvent, cancel: &CancelToken) -> Result<(), EncodeError>;
1358 /// Drains encoder delay, interleaves remaining packets, and writes the trailer.
1359 ///
1360 /// # Errors
1361 /// Returns cancellation or adapter failures; success means the trailer exists.
1362 fn finish(&mut self, cancel: &CancelToken) -> Result<(), EncodeError>;
1363 /// Aborts and applies the adapter's partial-output cleanup policy.
1364 ///
1365 /// # Errors
1366 /// Returns an adapter failure if partial output could not be cleaned up.
1367 fn abort(&mut self) -> Result<(), EncodeError>;
1368}
1369
1370#[cfg(test)]
1371mod tests {
1372 use super::SampleAspectRatio;
1373
1374 #[test]
1375 fn a_pixel_aspect_ratio_is_kept_in_lowest_terms_and_shows_as_a_browser_shows_it() {
1376 let ratio = |num, den| SampleAspectRatio { num, den };
1377 assert_eq!(SampleAspectRatio::new(8, 6), Some(ratio(4, 3)));
1378 assert_eq!(SampleAspectRatio::new(0, 1), None);
1379 assert_eq!(SampleAspectRatio::new(1, 0), None);
1380 assert_eq!(SampleAspectRatio::new(u64::from(u32::MAX) + 1, 1), None);
1381 // A display size rounds the stretched side: the declared ratio is the simplest that rounds
1382 // to it.
1383 let shown = |visible, displayed| SampleAspectRatio::of_display(visible, displayed);
1384 assert_eq!(shown((128, 96), (171, 96)), Some(ratio(4, 3)));
1385 assert_eq!(shown((96, 128), (96, 171)), Some(ratio(3, 4)));
1386 assert_eq!(shown((720, 576), (1024, 576)), Some(ratio(64, 45)));
1387 assert_eq!(shown((720, 480), (655, 480)), Some(ratio(10, 11)));
1388 assert_eq!(shown((720, 480), (720, 524)), Some(ratio(11, 12)));
1389 assert_eq!(shown((1440, 1080), (1920, 1080)), Some(ratio(4, 3)));
1390 assert_eq!(shown((128, 96), (128, 96)), Some(ratio(1, 1)));
1391 // Whatever it finds rounds to the size shown.
1392 for (visible, displayed) in [
1393 ((128, 96), (170, 96)),
1394 ((1000, 999), (1777, 999)),
1395 ((3, 2), (5, 2)),
1396 ] {
1397 let found = shown(visible, displayed).expect("a ratio");
1398 assert_eq!(
1399 found.square_pixel_size(visible.0, visible.1),
1400 displayed,
1401 "{visible:?} {displayed:?}"
1402 );
1403 }
1404 assert_eq!(shown((0, 96), (171, 96)), None);
1405 // mediabunny: Math.round(128 * 4 / 3) = 171, and a narrow pixel stretches the height.
1406 assert_eq!(ratio(4, 3).square_pixel_size(128, 96), (171, 96));
1407 assert_eq!(ratio(64, 45).square_pixel_size(720, 576), (1024, 576));
1408 assert_eq!(ratio(3, 4).square_pixel_size(96, 128), (96, 171));
1409 assert_eq!(ratio(1, 1).square_pixel_size(65, 49), (65, 49));
1410 // Half a pixel rounds up: 3 * 3 / 2 = 4.5.
1411 assert_eq!(ratio(3, 2).square_pixel_size(3, 7), (5, 7));
1412 }
1413
1414 use super::*;
1415
1416 #[test]
1417 fn a_frame_size_is_bounded_on_each_side_and_in_area() {
1418 assert!(frame_size_supported(16_384, 4_096));
1419 assert!(frame_size_supported(8_192, 8_192));
1420 assert!(!frame_size_supported(8_193, 8_192));
1421 assert!(!frame_size_supported(16_385, 1));
1422 assert!(!frame_size_supported(1, 16_385));
1423 assert!(!frame_size_supported(u32::MAX, u32::MAX));
1424 }
1425
1426 #[test]
1427 fn cancellation_is_shared_and_monotonic() {
1428 let token = CancelToken::default();
1429 let other = token.clone();
1430 assert!(!other.is_cancelled());
1431 token.cancel();
1432 assert!(other.is_cancelled());
1433 }
1434
1435 #[test]
1436 fn a_layout_names_each_plane_by_ffmpegs_channel_names() {
1437 let lcr = SpeakerLayout::new(&[
1438 Speaker::FrontCenter,
1439 Speaker::FrontLeft,
1440 Speaker::FrontRight,
1441 ])
1442 .expect("layout");
1443 assert_eq!(lcr.channels(), 3);
1444 assert_eq!(lcr.describe(), "FC+FL+FR");
1445 assert_eq!(format!("{lcr:?}"), r#"["FC", "FL", "FR"]"#);
1446 for (speaker, name, channel) in Speaker::TABLE {
1447 assert_eq!(speaker.name(), name);
1448 assert_eq!(Speaker::from_name(name), Some(speaker));
1449 assert_eq!(speaker.ffmpeg_channel(), channel);
1450 assert_eq!(Speaker::from_ffmpeg_channel(i64::from(channel)), speaker);
1451 }
1452 // Unused (0x200), ambisonic (0x400 and up) and user channels name no speaker.
1453 for channel in [-1, 18, 0x200, 0x400, 0x401, 0x7ff, 1_000] {
1454 assert_eq!(Speaker::from_ffmpeg_channel(channel), Speaker::Unnamed);
1455 }
1456 assert_eq!(Speaker::from_name("front left"), None);
1457 assert!(SpeakerLayout::new(&[]).is_none());
1458 assert!(SpeakerLayout::new(&[Speaker::FrontLeft; MAX_AUDIO_CHANNELS + 1]).is_none());
1459 }
1460
1461 #[test]
1462 fn a_source_that_declares_no_speaker_has_ffmpegs_default_for_its_count() {
1463 let names = |channels| {
1464 SpeakerLayout::default_for(channels)
1465 .expect("a default")
1466 .describe()
1467 };
1468 assert_eq!(names(1), "FC");
1469 assert_eq!(names(3), "FL+FR+LFE");
1470 assert_eq!(names(4), "FL+FR+FC+BC");
1471 assert_eq!(names(8), "FL+FR+FC+LFE+BL+BR+SL+SR");
1472 assert_eq!(names(9), "FL+FR+FC+LFE+BL+BR+SL+SR+UNK");
1473 assert_eq!(names(10), "FL+FR+FC+LFE+SL+SR+TFL+TFR+TBL+TBR");
1474 assert_eq!(SpeakerLayout::default_for(24).expect("22.2").channels(), 24);
1475 assert!(SpeakerLayout::default_for(0).is_none());
1476 assert!(SpeakerLayout::default_for(MAX_AUDIO_CHANNELS + 1).is_none());
1477 let unnamed = [Speaker::Unnamed; 2];
1478 assert_eq!(
1479 SpeakerLayout::declared_or_default(&unnamed),
1480 Some((SpeakerLayout::default_for(2).expect("stereo"), true))
1481 );
1482 let partly = [Speaker::FrontCenter, Speaker::Unnamed];
1483 assert_eq!(
1484 SpeakerLayout::declared_or_default(&partly),
1485 Some((SpeakerLayout::new(&partly).expect("layout"), false))
1486 );
1487 }
1488
1489 #[test]
1490 fn a_span_covers_a_request_only_where_it_holds_samples_or_knows_silence() {
1491 let block = AudioBlock {
1492 start_sample: 100,
1493 sample_rate: 48_000,
1494 speakers: SpeakerLayout::default_for(1).expect("mono"),
1495 channels: vec![vec![0.5; 50]],
1496 };
1497 let partial = AudioSpan {
1498 block: &block,
1499 starts_source: false,
1500 ends_source: false,
1501 };
1502 assert!(partial.covers(100, 150));
1503 assert!(partial.covers(120, 130));
1504 assert!(!partial.covers(99, 150), "a sample before the block");
1505 assert!(!partial.covers(100, 151), "a sample after the block");
1506 let starts = AudioSpan {
1507 starts_source: true,
1508 ..partial
1509 };
1510 assert!(starts.covers(0, 150));
1511 assert!(!starts.covers(0, 151));
1512 assert!(AudioSpan::whole(&block).covers(-5, 1_000));
1513 }
1514
1515 #[test]
1516 fn text_descriptor_and_generic_pixels_validate_without_compositor_types() {
1517 let texture = HostTexture {
1518 content_version: 2,
1519 width: 4,
1520 height: 3,
1521 premultiplication: Premultiplication::Premultiplied,
1522 color_space: RasterColorSpace::Srgb,
1523 };
1524 let text = TextTexture {
1525 texture: texture.clone(),
1526 layout_x: 0.0,
1527 layout_y: 0.0,
1528 layout_width: 4.0,
1529 layout_height: 3.0,
1530 first_baseline: 2.0,
1531 ascent: 2.0,
1532 descent: 1.0,
1533 line_count: 1,
1534 };
1535 assert!(text.is_valid());
1536 assert!(
1537 HostTexturePixels {
1538 texture,
1539 rgba8: vec![0; 4 * 3 * 4],
1540 }
1541 .is_valid()
1542 );
1543 }
1544}