openipc_video/backends/macos/
decoder.rs

1use crate::{
2    runtime::{LatestFrameMailbox, StatsHandle},
3    CodecCapability, CodecConfig, CodecConfigTracker, ConfigUpdate, DecodedFrame,
4    DecoderCapabilities, DecoderOptions, DecoderStats, EncodedAccessUnit, SubmitOutcome,
5    VideoCodec, VideoDecoder, VideoError,
6};
7
8use super::{callback::CallbackState, session::VideoToolboxSession, MacOsVideoFrame};
9
10/// Low-latency H.264/H.265 decoder backed by macOS VideoToolbox.
11pub struct MacOsDecoder {
12    options: DecoderOptions,
13    tracker: CodecConfigTracker,
14    session: Option<VideoToolboxSession>,
15    active_config: Option<CodecConfig>,
16    waiting_for_keyframe: bool,
17    frames: LatestFrameMailbox<DecodedFrame<MacOsVideoFrame>>,
18    stats: StatsHandle,
19    callback: CallbackState,
20}
21
22impl MacOsDecoder {
23    /// Create a VideoToolbox decoder using the supplied latency policy.
24    pub fn new(options: DecoderOptions) -> Result<Self, VideoError> {
25        if options.max_frames_in_flight == 0 {
26            return Err(VideoError::InvalidOption(
27                "max_frames_in_flight must be greater than zero",
28            ));
29        }
30        let frames = LatestFrameMailbox::default();
31        let stats = StatsHandle::default();
32        let callback = CallbackState::new(frames.clone(), stats.clone());
33        Ok(Self {
34            options,
35            tracker: CodecConfigTracker::default(),
36            session: None,
37            active_config: None,
38            waiting_for_keyframe: true,
39            frames,
40            stats,
41            callback,
42        })
43    }
44
45    /// Query VideoToolbox H.264 and H.265 hardware support.
46    pub fn probe_capabilities() -> DecoderCapabilities {
47        use objc2_core_media::{kCMVideoCodecType_H264, kCMVideoCodecType_HEVC};
48        use objc2_video_toolbox::VTIsHardwareDecodeSupported;
49
50        // SAFETY: These codec constants are valid inputs and the function is
51        // available on every macOS version supported by this crate.
52        let h264 = unsafe { VTIsHardwareDecodeSupported(kCMVideoCodecType_H264) };
53        // SAFETY: See the H.264 call above.
54        let h265 = unsafe { VTIsHardwareDecodeSupported(kCMVideoCodecType_HEVC) };
55        DecoderCapabilities {
56            backend: "videotoolbox",
57            codecs: vec![
58                CodecCapability {
59                    codec: VideoCodec::H264,
60                    supported: true,
61                    hardware_accelerated: h264,
62                    hardware_acceleration_known: true,
63                },
64                CodecCapability {
65                    codec: VideoCodec::H265,
66                    supported: true,
67                    hardware_accelerated: h265,
68                    hardware_acceleration_known: true,
69                },
70            ],
71            native_surfaces: true,
72        }
73    }
74
75    fn supports_requested_codec(&self, codec: VideoCodec) -> Result<(), VideoError> {
76        let capability = Self::probe_capabilities().codec(codec);
77        let Some(capability) = capability.filter(|entry| entry.supported) else {
78            return Err(VideoError::UnsupportedCodec {
79                codec,
80                backend: "videotoolbox",
81            });
82        };
83        if self.options.require_hardware && !capability.hardware_accelerated {
84            return Err(VideoError::HardwareDecoderUnavailable {
85                codec,
86                backend: "videotoolbox",
87            });
88        }
89        Ok(())
90    }
91
92    fn replace_session(&mut self, config: CodecConfig) -> Result<(), VideoError> {
93        log::info!(target: "openipc_video::videotoolbox", "configuring decoder codec={}", config.codec());
94        self.supports_requested_codec(config.codec())?;
95        if let Some(session) = self.session.take() {
96            session.finish()?;
97        }
98        self.frames.clear();
99        self.session = Some(VideoToolboxSession::new(
100            &config,
101            self.options,
102            self.callback.clone(),
103        )?);
104        self.active_config = Some(config);
105        self.waiting_for_keyframe = true;
106        self.stats.update(|stats| stats.reconfigurations += 1);
107        Ok(())
108    }
109}
110
111impl VideoDecoder for MacOsDecoder {
112    type Surface = MacOsVideoFrame;
113
114    fn capabilities(&self) -> DecoderCapabilities {
115        Self::probe_capabilities()
116    }
117
118    fn configure(&mut self, config: CodecConfig) -> Result<(), VideoError> {
119        self.replace_session(config)
120    }
121
122    fn submit(&mut self, mut frame: EncodedAccessUnit) -> Result<SubmitOutcome, VideoError> {
123        self.stats.update(|stats| stats.access_units_received += 1);
124        let (update, observed_keyframe) = self.tracker.inspect(frame.codec, &frame.data)?;
125        frame.keyframe |= observed_keyframe;
126        let mut reconfigured = false;
127        if let ConfigUpdate::Changed(config) = update {
128            self.replace_session(config)?;
129            reconfigured = true;
130        } else if self.session.is_none() {
131            let Some(config) = self.tracker.config(frame.codec).cloned() else {
132                self.stats.update(|stats| stats.waiting_drops += 1);
133                return Ok(SubmitOutcome::WaitingForConfiguration);
134            };
135            self.replace_session(config)?;
136            reconfigured = true;
137        }
138
139        let configured = self
140            .active_config
141            .as_ref()
142            .map(CodecConfig::codec)
143            .expect("configured VideoToolbox session must have an active codec");
144        if configured != frame.codec {
145            return Err(VideoError::CodecMismatch {
146                configured,
147                received: frame.codec,
148            });
149        }
150        if self.waiting_for_keyframe && !frame.can_resynchronize() {
151            self.stats.update(|stats| stats.waiting_drops += 1);
152            return Ok(SubmitOutcome::WaitingForKeyframe);
153        }
154        if self.stats.snapshot().frames_in_flight >= self.options.max_frames_in_flight {
155            log::warn!(target: "openipc_video::videotoolbox", "decoder backpressure; waiting for the next keyframe");
156            self.frames.clear();
157            self.waiting_for_keyframe = true;
158            self.stats.update(|stats| stats.backpressure_drops += 1);
159            return Ok(SubmitOutcome::DroppedForBackpressure);
160        }
161
162        self.session
163            .as_mut()
164            .expect("configured VideoToolbox session must exist")
165            .submit(&mut frame)?;
166        if frame.keyframe {
167            self.waiting_for_keyframe = false;
168        }
169        Ok(if reconfigured {
170            SubmitOutcome::Reconfigured
171        } else {
172            SubmitOutcome::Submitted
173        })
174    }
175
176    fn latest_frame(&mut self) -> Option<DecodedFrame<Self::Surface>> {
177        self.frames.take()
178    }
179
180    fn flush(&mut self) -> Result<(), VideoError> {
181        if let Some(session) = self.session.take() {
182            session.finish()?;
183        }
184        self.frames.clear();
185        self.tracker.reset();
186        self.active_config = None;
187        self.waiting_for_keyframe = true;
188        Ok(())
189    }
190
191    fn stats(&self) -> DecoderStats {
192        self.stats.snapshot()
193    }
194}
195
196impl Drop for MacOsDecoder {
197    fn drop(&mut self) {
198        if let Some(session) = self.session.take() {
199            let _ = session.finish();
200        }
201    }
202}
203
204#[cfg(test)]
205mod tests {
206    use super::MacOsDecoder;
207
208    #[test]
209    fn decoder_can_move_to_a_worker_thread() {
210        fn assert_send_sync<T: Send + Sync>() {}
211        assert_send_sync::<MacOsDecoder>();
212    }
213}
openipc_video/backends/macos/decoder.rs

openipc_video/backends/macos/
decoder.rs
