player_plugin/
native_frame.rs

1use serde::{Deserialize, Serialize};
2
3use crate::{DecoderFrameFormat, DecoderMediaKind};
4
5/// Opaque host-side identity for one plugin-owned native-frame lease.
6///
7/// The token is separate from the platform native handle and media frame IDs.
8/// Plugin authors should leave it unset; checked host loaders attach it to
9/// frames that require an explicit release through the producing session.
10#[doc(hidden)]
11#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
12pub struct NativeFrameLeaseToken {
13    interface_token: u64,
14    session_token: u64,
15    lease_token: u64,
16}
17
18impl NativeFrameLeaseToken {
19    #[doc(hidden)]
20    pub fn from_host_lease(interface_token: u64, session_token: u64, lease_token: u64) -> Self {
21        Self {
22            interface_token,
23            session_token,
24            lease_token,
25        }
26    }
27
28    #[doc(hidden)]
29    pub fn host_lease_parts(self) -> (u64, u64, u64) {
30        (self.interface_token, self.session_token, self.lease_token)
31    }
32}
33
34/// Native frame handle kinds shared by decoder, frame processor, and presenter paths.
35#[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)]
36pub enum NativeHandleKind {
37    CvPixelBuffer,
38    IoSurface,
39    MetalTexture,
40    DmaBuf,
41    VaapiSurface,
42    D3D11Texture2D,
43    DxgiSurface,
44    VulkanImage,
45    MediaCodecHardwareBuffer,
46    MediaCodecSurfaceTexture,
47    Unknown(String),
48}
49
50/// Cross-component native-frame pipeline profiles used for decoder/processor/presenter matching.
51#[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)]
52pub enum NativeFramePipelineProfile {
53    VideoToolboxCvPixelBuffer,
54    MetalTexture,
55    D3D11Texture2D,
56    MediaCodecHardwareBuffer,
57    MediaCodecSurfaceTexture,
58    Unknown(String),
59}
60
61impl NativeFramePipelineProfile {
62    /// Returns the best-known pipeline profile implied by a native handle kind.
63    pub fn from_handle_kind(handle_kind: &NativeHandleKind) -> Self {
64        match handle_kind {
65            NativeHandleKind::CvPixelBuffer => Self::VideoToolboxCvPixelBuffer,
66            NativeHandleKind::MetalTexture => Self::MetalTexture,
67            NativeHandleKind::D3D11Texture2D => Self::D3D11Texture2D,
68            NativeHandleKind::MediaCodecHardwareBuffer => Self::MediaCodecHardwareBuffer,
69            NativeHandleKind::MediaCodecSurfaceTexture => Self::MediaCodecSurfaceTexture,
70            NativeHandleKind::IoSurface => Self::Unknown("io_surface".to_owned()),
71            NativeHandleKind::DmaBuf => Self::Unknown("dma_buf".to_owned()),
72            NativeHandleKind::VaapiSurface => Self::Unknown("vaapi_surface".to_owned()),
73            NativeHandleKind::DxgiSurface => Self::Unknown("dxgi_surface".to_owned()),
74            NativeHandleKind::VulkanImage => Self::Unknown("vulkan_image".to_owned()),
75            NativeHandleKind::Unknown(name) => Self::Unknown(name.clone()),
76        }
77    }
78
79    /// Returns the stable diagnostics label used by runtime and platform bridges.
80    pub fn label(&self) -> String {
81        match self {
82            Self::VideoToolboxCvPixelBuffer => "video_toolbox_cv_pixel_buffer".to_owned(),
83            Self::MetalTexture => "metal_texture".to_owned(),
84            Self::D3D11Texture2D => "d3d11_texture_2d".to_owned(),
85            Self::MediaCodecHardwareBuffer => "media_codec_hardware_buffer".to_owned(),
86            Self::MediaCodecSurfaceTexture => "media_codec_surface_texture".to_owned(),
87            Self::Unknown(name) => name.clone(),
88        }
89    }
90}
91
92/// Visible content rectangle within a coded native frame.
93#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
94pub struct VisibleRect {
95    pub x: u32,
96    pub y: u32,
97    pub width: u32,
98    pub height: u32,
99}
100
101/// Release tracking diagnostics attached to a native frame.
102#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
103pub struct NativeFrameReleaseTracking {
104    pub frame_id: Option<u64>,
105    pub requires_release: bool,
106}
107
108/// Platform synchronization information associated with a native frame.
109#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
110pub struct NativeFrameSyncInfo {
111    pub kind: String,
112    #[serde(default)]
113    pub handle: Option<u64>,
114    #[serde(default)]
115    pub value: Option<u64>,
116}
117
118/// Display transform metadata that must be preserved across native-frame stages.
119#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
120pub struct NativeFrameTransform {
121    pub rotation_degrees: u16,
122    #[serde(default)]
123    pub mirrored_horizontal: bool,
124    #[serde(default)]
125    pub mirrored_vertical: bool,
126}
127
128/// Color characteristics that must be preserved for HDR native-frame playback.
129#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
130pub struct NativeFrameColorMetadata {
131    #[serde(default)]
132    pub primaries: Option<String>,
133    #[serde(default)]
134    pub transfer: Option<String>,
135    #[serde(default)]
136    pub matrix: Option<String>,
137    #[serde(default)]
138    pub range: Option<String>,
139    #[serde(default)]
140    pub bit_depth: Option<u8>,
141}
142
143impl NativeFrameColorMetadata {
144    /// Returns whether this metadata describes a known HDR transfer function.
145    pub fn is_hdr_transfer(&self) -> bool {
146        self.transfer
147            .as_deref()
148            .map(|transfer| {
149                let transfer = transfer.to_ascii_lowercase();
150                transfer.contains("pq")
151                    || transfer.contains("st2084")
152                    || transfer.contains("smpte2084")
153                    || transfer.contains("hlg")
154                    || transfer.contains("arib-std-b67")
155                    || transfer.contains("arib_std_b67")
156            })
157            .unwrap_or(false)
158    }
159
160    /// Returns whether this color metadata requires explicit preservation.
161    pub fn requires_preservation(&self) -> bool {
162        self.bit_depth.is_some_and(|bit_depth| bit_depth > 8)
163            || self.is_hdr_transfer()
164            || self.primaries.as_deref().is_some_and(is_wide_color_label)
165            || self.matrix.as_deref().is_some_and(is_wide_color_label)
166            || self
167                .transfer
168                .as_deref()
169                .is_some_and(is_wide_color_transfer_label)
170    }
171}
172
173fn is_wide_color_label(label: &str) -> bool {
174    let normalized = normalize_color_label(label);
175    matches!(
176        normalized.as_str(),
177        "bt2020"
178            | "rec2020"
179            | "bt2020nc"
180            | "bt2020ncl"
181            | "bt2020c"
182            | "bt2020cl"
183            | "smpte431"
184            | "smpte431p3"
185            | "smpte432"
186            | "smpte432p3"
187            | "displayp3"
188            | "displayp3d65"
189            | "p3"
190            | "dcip3"
191            | "ictcp"
192    )
193}
194
195fn is_wide_color_transfer_label(label: &str) -> bool {
196    let normalized = normalize_color_label(label);
197    matches!(
198        normalized.as_str(),
199        "bt2020" | "bt202010" | "bt202012" | "smpte2084" | "st2084" | "pq" | "hlg" | "aribstdb67"
200    )
201}
202
203fn normalize_color_label(label: &str) -> String {
204    label
205        .trim()
206        .to_ascii_lowercase()
207        .chars()
208        .filter(|character| character.is_ascii_alphanumeric())
209        .collect()
210}
211
212/// Mastering display metadata carried by HDR10-style streams.
213#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
214pub struct NativeFrameMasteringDisplayMetadata {
215    #[serde(default)]
216    pub display_primaries: Option<String>,
217    #[serde(default)]
218    pub white_point: Option<String>,
219    #[serde(default)]
220    pub max_luminance_nits: Option<u32>,
221    #[serde(default)]
222    pub min_luminance_nits: Option<u32>,
223}
224
225/// Content light metadata carried by HDR streams.
226#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
227pub struct NativeFrameContentLightMetadata {
228    #[serde(default)]
229    pub max_content_light_level: Option<u32>,
230    #[serde(default)]
231    pub max_frame_average_light_level: Option<u32>,
232}
233
234/// Dolby Vision stream metadata used for diagnostics and route selection.
235#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
236pub struct NativeFrameDolbyVisionMetadata {
237    #[serde(default)]
238    pub profile: Option<u8>,
239    #[serde(default)]
240    pub level: Option<u8>,
241    #[serde(default)]
242    pub compatibility_id: Option<u8>,
243    #[serde(default)]
244    pub has_rpu: bool,
245    #[serde(default)]
246    pub has_el: bool,
247    #[serde(default)]
248    pub has_bl: bool,
249}
250
251/// Structured HDR metadata attached to tracks and native frames.
252#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
253pub struct NativeFrameHdrMetadata {
254    pub kind: String,
255    #[serde(default)]
256    pub mastering_display: Option<NativeFrameMasteringDisplayMetadata>,
257    #[serde(default)]
258    pub content_light: Option<NativeFrameContentLightMetadata>,
259    #[serde(default)]
260    pub dolby_vision: Option<NativeFrameDolbyVisionMetadata>,
261}
262
263impl NativeFrameHdrMetadata {
264    /// Returns whether the metadata describes Dolby Vision.
265    pub fn is_dolby_vision(&self) -> bool {
266        self.kind.eq_ignore_ascii_case("dolbyVision") || self.dolby_vision.is_some()
267    }
268}
269
270/// Metadata shared by native frame producers, processors, and consumers.
271#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
272pub struct NativeFrameMetadata {
273    pub media_kind: DecoderMediaKind,
274    pub format: DecoderFrameFormat,
275    pub codec: String,
276    pub pts_us: Option<i64>,
277    pub duration_us: Option<i64>,
278    pub width: u32,
279    pub height: u32,
280    #[serde(default)]
281    pub coded_width: Option<u32>,
282    #[serde(default)]
283    pub coded_height: Option<u32>,
284    #[serde(default)]
285    pub visible_rect: Option<VisibleRect>,
286    pub handle_kind: NativeHandleKind,
287    #[serde(default)]
288    pub pipeline_profile: Option<NativeFramePipelineProfile>,
289    #[serde(default)]
290    pub color_space: Option<String>,
291    #[serde(default)]
292    pub hdr_metadata: Option<String>,
293    #[serde(default)]
294    pub color: Option<NativeFrameColorMetadata>,
295    #[serde(default)]
296    pub hdr: Option<NativeFrameHdrMetadata>,
297    #[serde(default)]
298    pub sync_info: Option<NativeFrameSyncInfo>,
299    #[serde(default)]
300    pub transform: Option<NativeFrameTransform>,
301    #[serde(default)]
302    pub frame_id: Option<u64>,
303    #[serde(default)]
304    pub release_tracking: Option<NativeFrameReleaseTracking>,
305}
306
307impl NativeFrameMetadata {
308    /// Returns the explicit pipeline profile or derives one from the handle kind.
309    pub fn effective_pipeline_profile(&self) -> NativeFramePipelineProfile {
310        self.pipeline_profile
311            .clone()
312            .unwrap_or_else(|| NativeFramePipelineProfile::from_handle_kind(&self.handle_kind))
313    }
314
315    /// Returns whether this frame should be treated as HDR by native-frame gates.
316    pub fn requires_hdr_preservation(&self) -> bool {
317        self.hdr.is_some()
318            || self
319                .hdr_metadata
320                .as_deref()
321                .map(|metadata| !metadata.trim().is_empty())
322                .unwrap_or(false)
323            || self
324                .color
325                .as_ref()
326                .map(NativeFrameColorMetadata::is_hdr_transfer)
327                .unwrap_or(false)
328    }
329
330    /// Returns whether this frame carries color metadata that should be preserved.
331    pub fn requires_color_preservation(&self) -> bool {
332        self.color
333            .as_ref()
334            .is_some_and(NativeFrameColorMetadata::requires_preservation)
335            || self.color_space.as_deref().is_some_and(is_wide_color_label)
336            || self.requires_hdr_preservation()
337    }
338}
339
340/// A native frame handle plus metadata.
341#[must_use = "native frames may own externally retained resources and must be released through the producing session"]
342#[derive(Debug, Clone, PartialEq, Eq)]
343pub struct NativeFrame {
344    pub metadata: NativeFrameMetadata,
345    pub handle: usize,
346    #[doc(hidden)]
347    pub lease_token: Option<NativeFrameLeaseToken>,
348}
349
350impl NativeFrame {
351    pub fn new(metadata: NativeFrameMetadata, handle: usize) -> Self {
352        Self {
353            metadata,
354            handle,
355            lease_token: None,
356        }
357    }
358
359    #[doc(hidden)]
360    pub fn with_plugin_lease(
361        metadata: NativeFrameMetadata,
362        handle: usize,
363        lease_token: NativeFrameLeaseToken,
364    ) -> Self {
365        Self {
366            metadata,
367            handle,
368            lease_token: Some(lease_token),
369        }
370    }
371}
372
373#[cfg(test)]
374mod tests {
375    use super::{
376        NativeFrameColorMetadata, NativeFrameContentLightMetadata, NativeFrameDolbyVisionMetadata,
377        NativeFrameHdrMetadata, NativeFrameMasteringDisplayMetadata, NativeFrameMetadata,
378        NativeFramePipelineProfile, NativeFrameReleaseTracking, NativeFrameSyncInfo,
379        NativeFrameTransform, NativeHandleKind, VisibleRect,
380    };
381    use crate::{DecoderFrameFormat, DecoderMediaKind};
382
383    fn test_metadata() -> NativeFrameMetadata {
384        NativeFrameMetadata {
385            media_kind: DecoderMediaKind::Video,
386            format: DecoderFrameFormat::Nv12,
387            codec: "h264".to_owned(),
388            pts_us: Some(42_000),
389            duration_us: Some(16_667),
390            width: 1_920,
391            height: 1_080,
392            coded_width: Some(1_920),
393            coded_height: Some(1_088),
394            visible_rect: Some(VisibleRect {
395                x: 0,
396                y: 0,
397                width: 1_920,
398                height: 1_080,
399            }),
400            handle_kind: NativeHandleKind::CvPixelBuffer,
401            pipeline_profile: Some(NativeFramePipelineProfile::VideoToolboxCvPixelBuffer),
402            color_space: Some("bt709".to_owned()),
403            hdr_metadata: Some("hdr10".to_owned()),
404            color: Some(NativeFrameColorMetadata {
405                primaries: Some("bt2020".to_owned()),
406                transfer: Some("smpte2084".to_owned()),
407                matrix: Some("bt2020-ncl".to_owned()),
408                range: Some("limited".to_owned()),
409                bit_depth: Some(10),
410            }),
411            hdr: Some(NativeFrameHdrMetadata {
412                kind: "hdr10".to_owned(),
413                mastering_display: Some(NativeFrameMasteringDisplayMetadata {
414                    display_primaries: Some("bt2020".to_owned()),
415                    white_point: Some("d65".to_owned()),
416                    max_luminance_nits: Some(1_000),
417                    min_luminance_nits: Some(0),
418                }),
419                content_light: Some(NativeFrameContentLightMetadata {
420                    max_content_light_level: Some(1_000),
421                    max_frame_average_light_level: Some(400),
422                }),
423                dolby_vision: None,
424            }),
425            sync_info: Some(NativeFrameSyncInfo {
426                kind: "test_fence".to_owned(),
427                handle: Some(12),
428                value: Some(34),
429            }),
430            transform: Some(NativeFrameTransform {
431                rotation_degrees: 90,
432                mirrored_horizontal: false,
433                mirrored_vertical: true,
434            }),
435            frame_id: Some(7),
436            release_tracking: Some(NativeFrameReleaseTracking {
437                frame_id: Some(7),
438                requires_release: true,
439            }),
440        }
441    }
442
443    #[test]
444    fn native_frame_metadata_round_trips_through_json() {
445        let metadata = test_metadata();
446
447        let encoded = serde_json::to_string(&metadata).expect("serialize metadata");
448        let decoded: NativeFrameMetadata =
449            serde_json::from_str(&encoded).expect("deserialize metadata");
450
451        assert_eq!(decoded, metadata);
452    }
453
454    #[test]
455    fn native_frame_metadata_detects_hdr_preservation_requirement() {
456        let mut metadata = test_metadata();
457
458        assert!(metadata.requires_color_preservation());
459        assert!(metadata.requires_hdr_preservation());
460
461        metadata.color = Some(NativeFrameColorMetadata {
462            primaries: Some("bt2020".to_owned()),
463            transfer: Some("arib-std-b67".to_owned()),
464            matrix: Some("bt2020-ncl".to_owned()),
465            range: Some("limited".to_owned()),
466            bit_depth: Some(10),
467        });
468        metadata.hdr = None;
469        metadata.hdr_metadata = None;
470
471        assert!(metadata.requires_hdr_preservation());
472
473        metadata.color = None;
474        metadata.color_space = None;
475
476        assert!(!metadata.requires_color_preservation());
477        assert!(!metadata.requires_hdr_preservation());
478    }
479
480    #[test]
481    fn native_frame_metadata_does_not_require_color_preservation_for_ordinary_sdr() {
482        let mut metadata = test_metadata();
483        metadata.color_space = Some("bt709".to_owned());
484        metadata.color = Some(NativeFrameColorMetadata {
485            primaries: Some("bt709".to_owned()),
486            transfer: Some("sdr-video".to_owned()),
487            matrix: Some("bt709".to_owned()),
488            range: Some("limited".to_owned()),
489            bit_depth: Some(8),
490        });
491        metadata.hdr = None;
492        metadata.hdr_metadata = None;
493
494        assert!(!metadata.requires_color_preservation());
495        assert!(!metadata.requires_hdr_preservation());
496
497        metadata.color_space = Some("bt2020".to_owned());
498        assert!(metadata.requires_color_preservation());
499    }
500
501    #[test]
502    fn native_frame_metadata_accepts_ffmpeg_sdr_color_spellings() {
503        let mut metadata = test_metadata();
504        metadata.color_space = None;
505        metadata.hdr = None;
506        metadata.hdr_metadata = None;
507
508        for label in ["bt470bg", "fcc", "smpte240m"] {
509            metadata.color = Some(NativeFrameColorMetadata {
510                primaries: Some(label.to_owned()),
511                transfer: Some("bt709".to_owned()),
512                matrix: Some(label.to_owned()),
513                range: Some("limited".to_owned()),
514                bit_depth: Some(8),
515            });
516
517            assert!(
518                !metadata.requires_color_preservation(),
519                "{label} should be treated as ordinary SDR"
520            );
521            assert!(!metadata.requires_hdr_preservation());
522        }
523    }
524
525    #[test]
526    fn native_frame_metadata_requires_preservation_for_wide_color_labels() {
527        let mut metadata = test_metadata();
528        metadata.color_space = None;
529        metadata.hdr = None;
530        metadata.hdr_metadata = None;
531
532        for label in ["bt2020", "display-p3", "ictcp"] {
533            metadata.color = Some(NativeFrameColorMetadata {
534                primaries: Some(label.to_owned()),
535                transfer: Some("sdr-video".to_owned()),
536                matrix: Some(label.to_owned()),
537                range: Some("limited".to_owned()),
538                bit_depth: Some(8),
539            });
540
541            assert!(
542                metadata.requires_color_preservation(),
543                "{label} should require preservation"
544            );
545        }
546    }
547
548    #[test]
549    fn native_frame_color_metadata_recognizes_android_hdr_transfer_labels() {
550        let mut color = NativeFrameColorMetadata {
551            primaries: Some("bt2020".to_owned()),
552            transfer: Some("st2084".to_owned()),
553            matrix: Some("bt2020-ncl".to_owned()),
554            range: Some("limited".to_owned()),
555            bit_depth: Some(10),
556        };
557
558        assert!(color.is_hdr_transfer());
559        assert!(color.requires_preservation());
560
561        color.transfer = Some("hlg".to_owned());
562        assert!(color.is_hdr_transfer());
563        assert!(color.requires_preservation());
564    }
565
566    #[test]
567    fn native_frame_hdr_metadata_identifies_dolby_vision() {
568        let metadata = NativeFrameHdrMetadata {
569            kind: "dolbyVision".to_owned(),
570            mastering_display: None,
571            content_light: None,
572            dolby_vision: Some(NativeFrameDolbyVisionMetadata {
573                profile: Some(8),
574                level: Some(6),
575                compatibility_id: Some(1),
576                has_rpu: true,
577                has_el: false,
578                has_bl: true,
579            }),
580        };
581
582        assert!(metadata.is_dolby_vision());
583    }
584
585    #[test]
586    fn native_frame_metadata_derives_pipeline_profile_from_handle_kind() {
587        let mut metadata = test_metadata();
588        metadata.pipeline_profile = None;
589
590        assert_eq!(
591            metadata.effective_pipeline_profile(),
592            NativeFramePipelineProfile::VideoToolboxCvPixelBuffer
593        );
594
595        metadata.handle_kind = NativeHandleKind::MediaCodecHardwareBuffer;
596        assert_eq!(
597            metadata.effective_pipeline_profile(),
598            NativeFramePipelineProfile::MediaCodecHardwareBuffer
599        );
600
601        metadata.handle_kind = NativeHandleKind::MediaCodecSurfaceTexture;
602        assert_eq!(
603            metadata.effective_pipeline_profile(),
604            NativeFramePipelineProfile::MediaCodecSurfaceTexture
605        );
606    }
607
608    #[test]
609    fn native_frame_pipeline_profile_has_stable_diagnostic_label() {
610        assert_eq!(
611            NativeFramePipelineProfile::MediaCodecHardwareBuffer.label(),
612            "media_codec_hardware_buffer"
613        );
614        assert_eq!(
615            NativeFramePipelineProfile::MediaCodecSurfaceTexture.label(),
616            "media_codec_surface_texture"
617        );
618        assert_eq!(
619            NativeFramePipelineProfile::Unknown("fixture".to_owned()).label(),
620            "fixture"
621        );
622    }
623}
player_plugin/native_frame.rs

player_plugin/
native_frame.rs
