Skip to main content

ferroui_media_windows/
lib.rs

1//! Media Foundation native surfaces on the compositor's compatible D3D11 device.
2#![cfg(windows)]
3#![deny(unsafe_code)]
4#[allow(unsafe_code)]
5mod native;
6mod frame_producer;
7pub use frame_producer::WindowsFrameProducer;
8use frame_producer::ReadyWindowsFrame;
9use ferroui_base::{
10    reactive::Subscription,
11    threading::{DispatcherPriority, DispatcherTimer},
12};
13use ferroui_media::{frames::*, *};
14#[cfg(feature = "controls")]
15pub use ferroui_media_opengl::NativeGlTextureImporter;
16pub use native::{
17    EncodedRecording, NativeDevice, NativeFrame, NativeRecording, NativeSession, NativeState,
18};
19
20use std::{cell::RefCell, rc::Rc, task::Poll, time::Duration};
21
22fn unsupported(message: impl Into<String>) -> MediaError {
23    MediaError::new(MediaErrorKind::Unsupported, message)
24}
25pub fn poll_native<T: 'static>(
26    mut poll: impl FnMut() -> Poll<Result<T, MediaError>> + 'static,
27) -> MediaFuture<T> {
28    let mut timer: Option<Subscription> = None;
29    Box::pin(std::future::poll_fn(move |cx| {
30        timer.take();
31        match poll() {
32            Poll::Ready(v) => Poll::Ready(v),
33            Poll::Pending => {
34                let w = cx.waker().clone();
35                timer = Some(DispatcherTimer::run_once(
36                    move || w.wake_by_ref(),
37                    Duration::from_millis(16),
38                    DispatcherPriority::NORMAL,
39                ));
40                Poll::Pending
41            }
42        }
43    }))
44}
45/// At most one queued import. The native worker separately bounds three pending
46/// video samples, one latest sample and one read callback; driver pools are opaque.
47pub struct WindowsFrameStream {
48    pub native: NativeSession,
49    producer: WindowsFrameProducer,
50    source: u64,
51    generation: u64,
52    epoch: Rc<std::cell::Cell<u64>>,
53    pending: RefCell<Option<MediaFuture<Option<ReadyWindowsFrame>>>>,
54    ready: RefCell<Option<ReadyWindowsFrame>>,
55    retry: RefCell<Option<NativeFrame>>,
56}
57
58// Drain immediately completed stages in this tick. A real asynchronous import
59// or producer fence can defer delivery; merely moving between Rust states must
60// not consume another presentation interval.
61fn poll_frame_import(
62    pending: &mut Option<MediaFuture<Option<ReadyWindowsFrame>>>,
63    ready: &mut Option<ReadyWindowsFrame>,
64    admitted: impl FnOnce(),
65) -> Result<Option<FrameLease>, MediaError> {
66    if let Some(job) = pending.as_mut() {
67        let mut cx = std::task::Context::from_waker(std::task::Waker::noop());
68        match job.as_mut().poll(&mut cx) {
69            Poll::Pending => return Ok(None),
70            Poll::Ready(result) => {
71                pending.take();
72                *ready = result?;
73                if ready.is_some() { admitted(); }
74            }
75        }
76    }
77    let frame = match ready.as_ref() {
78        Some(frame) => frame.take_if_ready()?,
79        None => None,
80    };
81    if frame.is_some() { ready.take(); }
82    Ok(frame)
83}
84
85impl WindowsFrameStream {
86    pub fn new(
87        native: NativeSession,
88        producer: impl Into<WindowsFrameProducer>,
89        source: u64,
90        generation: u64,
91    ) -> Self {
92        Self {
93            native,
94            producer: producer.into(),
95            source,
96            generation,
97            epoch: Rc::new(std::cell::Cell::new(1)),
98            pending: RefCell::new(None),
99            ready: RefCell::new(None),
100            retry: RefCell::new(None),
101        }
102    }
103    pub fn retire(&self, epoch: u64) {
104        self.epoch.set(epoch);
105        self.pending.borrow_mut().take();
106        self.ready.borrow_mut().take();
107        self.retry.borrow_mut().take();
108    }
109    pub fn has_pending_frame_work(&self) -> bool {
110        self.pending.borrow().is_some()
111            || self.ready.borrow().is_some()
112            || self.retry.borrow().is_some()
113    }
114    fn poll_import(&self) -> Result<Option<FrameLease>, MediaError> {
115        let result = poll_frame_import(
116            &mut self.pending.borrow_mut(),
117            &mut self.ready.borrow_mut(),
118            || {
119                self.producer.budget().counters()
120                    .gpu_conversion(GpuTransferReason::PixelFormatConversion);
121                self.retry.borrow_mut().take();
122            },
123        );
124        if result.is_err() { self.retry.borrow_mut().take(); }
125        result
126    }
127    pub fn take_frame(&self) -> Result<Option<FrameLease>, MediaError> {
128        // Capture/device errors must interrupt even a saturated or pending
129        // import; waiting for another native take can hide a disconnection.
130        self.native.state()?;
131        if self.producer.budget().quarantined_reads() != 0 {
132            return Err(unsupported(
133                "GPU completion is indeterminate; retained storage is quarantined",
134            ));
135        }
136        // Deliver a completed import on this tick and start its successor without waiting for
137        // another tick. A pending import or producer fence still owns the sole import slot until
138        // it completes.
139        let frame = self.poll_import()?;
140        if self.pending.borrow().is_some() || self.ready.borrow().is_some() {
141            return Ok(frame);
142        }
143        let job = self.queue_import()?;
144        *self.pending.borrow_mut() = job;
145        if frame.is_none() {
146            // An import that completes synchronously is delivered on the tick that queued it.
147            return self.poll_import();
148        }
149        Ok(frame)
150    }
151
152    fn queue_import(&self) -> Result<Option<MediaFuture<Option<ReadyWindowsFrame>>>, MediaError> {
153        if let Some(frame) = self.native.take_frame()? {
154            if frame.info().epoch == self.epoch.get() {
155                self.retry.replace(Some(frame));
156            }
157        }
158        let Some(frame) = self.retry.borrow().clone() else {
159            return Ok(None);
160        };
161        let info = frame.info();
162        if info.epoch != self.epoch.get() {
163            return Ok(None);
164        }
165        let output_bytes = u64::from(info.width) * u64::from(info.height) * 4;
166        let metadata = FrameMetadata {
167            identity: FrameIdentity {
168                source: self.source,
169                generation: self.generation,
170                sequence: info.sequence,
171            },
172            width: info.width,
173            height: info.height,
174            visible: FrameRegion {
175                x: info.x,
176                y: info.y,
177                width: info.visible_width,
178                height: info.visible_height,
179            },
180            format: self.producer.format(),
181            color: if info.color == 1 {
182                FrameColor::Srgb
183            } else {
184                FrameColor::Rec709Full
185            },
186            alpha: FrameAlpha::Opaque,
187            rotation: match info.rotation {
188                0 => FrameRotation::None,
189                90 => FrameRotation::Clockwise90,
190                180 => FrameRotation::Clockwise180,
191                270 => FrameRotation::Clockwise270,
192                _ => return Err(unsupported("Unknown native frame rotation")),
193            },
194            mirrored: info.mirrored != 0,
195            top_left_origin: true,
196            timestamp: Duration::from_nanos(info.timestamp_ns),
197            device: Some(self.producer.device()),
198            device_generation: self.producer.device_generation(),
199            planes: vec![FramePlane {
200                offset: 0,
201                stride: u64::from(info.width) * 4,
202                length: output_bytes,
203            }],
204            retained_bytes: info.bytes,
205        };
206        // The requested native MF video processor supplies RGB DXGI storage.
207        // No framework video copy/readback is used by this import.
208        Ok(Some(self.producer.import(metadata, frame)))
209    }
210}
211
212#[cfg(all(test, feature = "direct2d"))]
213mod import_pacing_tests {
214    use super::*;
215    use std::cell::Cell;
216
217    fn frame(budget: &FrameBudget) -> ReadyWindowsFrame {
218        let metadata = FrameMetadata {
219            identity: FrameIdentity { source: 1, generation: 1, sequence: 1 },
220            width: 2, height: 2,
221            visible: FrameRegion { x: 0, y: 0, width: 2, height: 2 },
222            format: FrameFormat::Bgra8, color: FrameColor::Srgb, alpha: FrameAlpha::Opaque,
223            rotation: FrameRotation::None, mirrored: false, top_left_origin: true,
224            timestamp: Duration::ZERO, device: Some(DeviceIdentity::Context(1)), device_generation: 1,
225            planes: vec![FramePlane { offset: 0, stride: 8, length: 16 }], retained_bytes: 16,
226        };
227        ReadyWindowsFrame::Direct3D11(budget.lease(metadata, ()).unwrap())
228    }
229
230    #[test]
231    fn synchronous_import_is_delivered_in_the_same_tick_and_keeps_its_lease() {
232        let budget = FrameBudget::new(Default::default());
233        let frame = frame(&budget);
234        let mut pending: Option<MediaFuture<_>> = Some(Box::pin(async move { Ok(Some(frame)) }));
235        let mut ready = None;
236        let admissions = Cell::new(0);
237        let delivered = poll_frame_import(&mut pending, &mut ready, || admissions.set(admissions.get() + 1)).unwrap().unwrap();
238        assert_eq!(admissions.get(), 1);
239        assert!(pending.is_none() && ready.is_none());
240        assert_eq!(budget.usage().frames, 1);
241        drop(delivered);
242        assert_eq!(budget.usage().frames, 0);
243        assert!(poll_frame_import(&mut pending, &mut ready, || panic!("duplicate admission")).unwrap().is_none());
244    }
245
246    #[test]
247    fn pending_import_delivers_on_its_first_ready_poll_without_an_extra_tick() {
248        let budget = FrameBudget::new(Default::default());
249        let available = Rc::new(Cell::new(false));
250        let signal = available.clone();
251        let mut frame = Some(frame(&budget));
252        let mut pending: Option<MediaFuture<_>> = Some(Box::pin(std::future::poll_fn(move |_| {
253            if signal.get() { Poll::Ready(Ok(frame.take())) } else { Poll::Pending }
254        })));
255        let mut ready = None;
256        assert!(poll_frame_import(&mut pending, &mut ready, || panic!("not admitted yet")).unwrap().is_none());
257        assert!(pending.is_some());
258        assert_eq!(budget.usage().frames, 1);
259        available.set(true);
260        let delivered = poll_frame_import(&mut pending, &mut ready, || {}).unwrap().unwrap();
261        assert!(pending.is_none() && ready.is_none());
262        drop(delivered);
263        assert_eq!(budget.usage().frames, 0);
264    }
265
266    #[test]
267    fn budget_backpressure_does_not_report_a_successful_import() {
268        let mut pending: Option<MediaFuture<Option<ReadyWindowsFrame>>> = Some(Box::pin(async { Ok(None) }));
269        let mut ready = None;
270        assert!(poll_frame_import(&mut pending, &mut ready, || panic!("budget rejection is not admission")).unwrap().is_none());
271        assert!(pending.is_none() && ready.is_none());
272    }
273
274    #[test]
275    #[ignore = "Requires hardware D3D11, Media Foundation and an audio endpoint"]
276    fn native_video_stream_delivery_cadence() {
277        use std::time::Instant;
278        use ferroui_base::platform::*;
279        use ferroui_base::{PixelSize, Rect, Vector};
280        let renderer = ferroui_direct2d::Direct2DRenderInterface;
281        let context = renderer.create_backend_context(None);
282        let feature = context.try_get_feature(std::any::TypeId::of::<dyn IExternalObjectsRenderInterfaceContextFeature>()).unwrap();
283        let feature = feature.downcast_ref::<Rc<dyn IExternalObjectsRenderInterfaceContextFeature>>().unwrap().clone();
284        let target = renderer.create_render_target_bitmap(PixelSize::new(32, 32), Vector::new(96., 96.));
285        let device = NativeDevice::from_direct2d().unwrap();
286        let path = std::path::Path::new(env!("CARGO_MANIFEST_DIR"))
287            .join("../../samples/ControlCatalog/Media/sintel-sample.mp4").canonicalize().unwrap();
288        let path = path.to_string_lossy();
289        let native = NativeSession::open(&device, path.trim_start_matches("\\\\?\\"), false, false, None).unwrap();
290        let budget = FrameBudget::new(Default::default());
291        let producer = WindowsFrameProducer::Direct3D11 {
292            budget: budget.clone(), device: DeviceIdentity::Luid(ferroui_direct2d::device_luid()),
293            generation: ferroui_direct2d::device_generation(),
294        };
295        let stream = WindowsFrameStream::new(native, producer, 1, 1);
296        let preparing = Instant::now();
297        while stream.native.state().unwrap().state != 1 {
298            assert!(preparing.elapsed() < Duration::from_secs(10));
299            std::thread::sleep(Duration::from_millis(5));
300        }
301        stream.native.intent(1);
302        let start = Instant::now();
303        let mut count = 0;
304        let mut first_pts = None;
305        let mut last_pts = Duration::ZERO;
306        while start.elapsed() < Duration::from_secs(5) {
307            if let Some(frame) = stream.take_frame().unwrap() {
308                count += 1;
309                first_pts.get_or_insert(frame.metadata().timestamp);
310                last_pts = frame.metadata().timestamp;
311                let m = frame.metadata();
312                let size = (m.width, m.height);
313                let handle = LeasedPlatformHandle::new(0, Some("SharedFrameD3D11Texture2D"), frame.clone());
314                let image = feature.import_image(Rc::new(handle), PlatformGraphicsExternalImageProperties {
315                    width: m.width as i32, height: m.height as i32,
316                    format: PlatformGraphicsExternalImageFormat::B8G8R8A8UNorm,
317                    top_left_origin: true, memory_size: m.retained_bytes, ..Default::default()
318                });
319                let bitmap = image.acquire_shared_bitmap(frame.clone()).unwrap();
320                // Match preview ownership: the acquired bitmap must survive
321                // disposal of the temporary import and the caller's lease.
322                image.dispose();
323                drop(image);
324                drop(frame);
325                let mut drawing = target.create_drawing_context();
326                drawing.draw_bitmap(&*bitmap, 1., Rect::new(0., 0., size.0 as f64, size.1 as f64), Rect::new(0., 0., 32., 32.));
327                drawing.dispose();
328            }
329            std::thread::sleep(Duration::from_millis(5));
330        }
331        let fps = count as f64 / start.elapsed().as_secs_f64();
332        println!("NATIVE_STREAM_DELIVERY frames={count} fps={fps:.2} pts_span_ms={}",
333            (last_pts - first_pts.unwrap()).as_millis());
334        assert!(fps >= 21.6, "24 fps source was undersampled: {fps:.2}");
335        stream.native.intent(0);
336        drop(stream);
337        let retiring = Instant::now();
338        while budget.usage().frames != 0 && retiring.elapsed() < Duration::from_secs(2) {
339            std::thread::sleep(Duration::from_millis(1));
340        }
341        assert_eq!(budget.usage().frames, 0);
342    }
343}