1#![cfg(windows)]
3#![deny(unsafe_code)]
4#[allow(unsafe_code)]
5mod native;
6mod frame_producer;
7pub use frame_producer::WindowsFrameProducer;
8use frame_producer::ReadyWindowsFrame;
9use ferroui_base::{
10 reactive::Subscription,
11 threading::{DispatcherPriority, DispatcherTimer},
12};
13use ferroui_media::{frames::*, *};
14#[cfg(feature = "controls")]
15pub use ferroui_media_opengl::NativeGlTextureImporter;
16pub use native::{
17 EncodedRecording, NativeDevice, NativeFrame, NativeRecording, NativeSession, NativeState,
18};
19
20use std::{cell::RefCell, rc::Rc, task::Poll, time::Duration};
21
22fn unsupported(message: impl Into<String>) -> MediaError {
23 MediaError::new(MediaErrorKind::Unsupported, message)
24}
25pub fn poll_native<T: 'static>(
26 mut poll: impl FnMut() -> Poll<Result<T, MediaError>> + 'static,
27) -> MediaFuture<T> {
28 let mut timer: Option<Subscription> = None;
29 Box::pin(std::future::poll_fn(move |cx| {
30 timer.take();
31 match poll() {
32 Poll::Ready(v) => Poll::Ready(v),
33 Poll::Pending => {
34 let w = cx.waker().clone();
35 timer = Some(DispatcherTimer::run_once(
36 move || w.wake_by_ref(),
37 Duration::from_millis(16),
38 DispatcherPriority::NORMAL,
39 ));
40 Poll::Pending
41 }
42 }
43 }))
44}
45pub struct WindowsFrameStream {
48 pub native: NativeSession,
49 producer: WindowsFrameProducer,
50 source: u64,
51 generation: u64,
52 epoch: Rc<std::cell::Cell<u64>>,
53 pending: RefCell<Option<MediaFuture<Option<ReadyWindowsFrame>>>>,
54 ready: RefCell<Option<ReadyWindowsFrame>>,
55 retry: RefCell<Option<NativeFrame>>,
56}
57
58fn poll_frame_import(
62 pending: &mut Option<MediaFuture<Option<ReadyWindowsFrame>>>,
63 ready: &mut Option<ReadyWindowsFrame>,
64 admitted: impl FnOnce(),
65) -> Result<Option<FrameLease>, MediaError> {
66 if let Some(job) = pending.as_mut() {
67 let mut cx = std::task::Context::from_waker(std::task::Waker::noop());
68 match job.as_mut().poll(&mut cx) {
69 Poll::Pending => return Ok(None),
70 Poll::Ready(result) => {
71 pending.take();
72 *ready = result?;
73 if ready.is_some() { admitted(); }
74 }
75 }
76 }
77 let frame = match ready.as_ref() {
78 Some(frame) => frame.take_if_ready()?,
79 None => None,
80 };
81 if frame.is_some() { ready.take(); }
82 Ok(frame)
83}
84
85impl WindowsFrameStream {
86 pub fn new(
87 native: NativeSession,
88 producer: impl Into<WindowsFrameProducer>,
89 source: u64,
90 generation: u64,
91 ) -> Self {
92 Self {
93 native,
94 producer: producer.into(),
95 source,
96 generation,
97 epoch: Rc::new(std::cell::Cell::new(1)),
98 pending: RefCell::new(None),
99 ready: RefCell::new(None),
100 retry: RefCell::new(None),
101 }
102 }
103 pub fn retire(&self, epoch: u64) {
104 self.epoch.set(epoch);
105 self.pending.borrow_mut().take();
106 self.ready.borrow_mut().take();
107 self.retry.borrow_mut().take();
108 }
109 pub fn has_pending_frame_work(&self) -> bool {
110 self.pending.borrow().is_some()
111 || self.ready.borrow().is_some()
112 || self.retry.borrow().is_some()
113 }
114 fn poll_import(&self) -> Result<Option<FrameLease>, MediaError> {
115 let result = poll_frame_import(
116 &mut self.pending.borrow_mut(),
117 &mut self.ready.borrow_mut(),
118 || {
119 self.producer.budget().counters()
120 .gpu_conversion(GpuTransferReason::PixelFormatConversion);
121 self.retry.borrow_mut().take();
122 },
123 );
124 if result.is_err() { self.retry.borrow_mut().take(); }
125 result
126 }
127 pub fn take_frame(&self) -> Result<Option<FrameLease>, MediaError> {
128 self.native.state()?;
131 if self.producer.budget().quarantined_reads() != 0 {
132 return Err(unsupported(
133 "GPU completion is indeterminate; retained storage is quarantined",
134 ));
135 }
136 let frame = self.poll_import()?;
140 if self.pending.borrow().is_some() || self.ready.borrow().is_some() {
141 return Ok(frame);
142 }
143 let job = self.queue_import()?;
144 *self.pending.borrow_mut() = job;
145 if frame.is_none() {
146 return self.poll_import();
148 }
149 Ok(frame)
150 }
151
152 fn queue_import(&self) -> Result<Option<MediaFuture<Option<ReadyWindowsFrame>>>, MediaError> {
153 if let Some(frame) = self.native.take_frame()? {
154 if frame.info().epoch == self.epoch.get() {
155 self.retry.replace(Some(frame));
156 }
157 }
158 let Some(frame) = self.retry.borrow().clone() else {
159 return Ok(None);
160 };
161 let info = frame.info();
162 if info.epoch != self.epoch.get() {
163 return Ok(None);
164 }
165 let output_bytes = u64::from(info.width) * u64::from(info.height) * 4;
166 let metadata = FrameMetadata {
167 identity: FrameIdentity {
168 source: self.source,
169 generation: self.generation,
170 sequence: info.sequence,
171 },
172 width: info.width,
173 height: info.height,
174 visible: FrameRegion {
175 x: info.x,
176 y: info.y,
177 width: info.visible_width,
178 height: info.visible_height,
179 },
180 format: self.producer.format(),
181 color: if info.color == 1 {
182 FrameColor::Srgb
183 } else {
184 FrameColor::Rec709Full
185 },
186 alpha: FrameAlpha::Opaque,
187 rotation: match info.rotation {
188 0 => FrameRotation::None,
189 90 => FrameRotation::Clockwise90,
190 180 => FrameRotation::Clockwise180,
191 270 => FrameRotation::Clockwise270,
192 _ => return Err(unsupported("Unknown native frame rotation")),
193 },
194 mirrored: info.mirrored != 0,
195 top_left_origin: true,
196 timestamp: Duration::from_nanos(info.timestamp_ns),
197 device: Some(self.producer.device()),
198 device_generation: self.producer.device_generation(),
199 planes: vec![FramePlane {
200 offset: 0,
201 stride: u64::from(info.width) * 4,
202 length: output_bytes,
203 }],
204 retained_bytes: info.bytes,
205 };
206 Ok(Some(self.producer.import(metadata, frame)))
209 }
210}
211
212#[cfg(all(test, feature = "direct2d"))]
213mod import_pacing_tests {
214 use super::*;
215 use std::cell::Cell;
216
217 fn frame(budget: &FrameBudget) -> ReadyWindowsFrame {
218 let metadata = FrameMetadata {
219 identity: FrameIdentity { source: 1, generation: 1, sequence: 1 },
220 width: 2, height: 2,
221 visible: FrameRegion { x: 0, y: 0, width: 2, height: 2 },
222 format: FrameFormat::Bgra8, color: FrameColor::Srgb, alpha: FrameAlpha::Opaque,
223 rotation: FrameRotation::None, mirrored: false, top_left_origin: true,
224 timestamp: Duration::ZERO, device: Some(DeviceIdentity::Context(1)), device_generation: 1,
225 planes: vec![FramePlane { offset: 0, stride: 8, length: 16 }], retained_bytes: 16,
226 };
227 ReadyWindowsFrame::Direct3D11(budget.lease(metadata, ()).unwrap())
228 }
229
230 #[test]
231 fn synchronous_import_is_delivered_in_the_same_tick_and_keeps_its_lease() {
232 let budget = FrameBudget::new(Default::default());
233 let frame = frame(&budget);
234 let mut pending: Option<MediaFuture<_>> = Some(Box::pin(async move { Ok(Some(frame)) }));
235 let mut ready = None;
236 let admissions = Cell::new(0);
237 let delivered = poll_frame_import(&mut pending, &mut ready, || admissions.set(admissions.get() + 1)).unwrap().unwrap();
238 assert_eq!(admissions.get(), 1);
239 assert!(pending.is_none() && ready.is_none());
240 assert_eq!(budget.usage().frames, 1);
241 drop(delivered);
242 assert_eq!(budget.usage().frames, 0);
243 assert!(poll_frame_import(&mut pending, &mut ready, || panic!("duplicate admission")).unwrap().is_none());
244 }
245
246 #[test]
247 fn pending_import_delivers_on_its_first_ready_poll_without_an_extra_tick() {
248 let budget = FrameBudget::new(Default::default());
249 let available = Rc::new(Cell::new(false));
250 let signal = available.clone();
251 let mut frame = Some(frame(&budget));
252 let mut pending: Option<MediaFuture<_>> = Some(Box::pin(std::future::poll_fn(move |_| {
253 if signal.get() { Poll::Ready(Ok(frame.take())) } else { Poll::Pending }
254 })));
255 let mut ready = None;
256 assert!(poll_frame_import(&mut pending, &mut ready, || panic!("not admitted yet")).unwrap().is_none());
257 assert!(pending.is_some());
258 assert_eq!(budget.usage().frames, 1);
259 available.set(true);
260 let delivered = poll_frame_import(&mut pending, &mut ready, || {}).unwrap().unwrap();
261 assert!(pending.is_none() && ready.is_none());
262 drop(delivered);
263 assert_eq!(budget.usage().frames, 0);
264 }
265
266 #[test]
267 fn budget_backpressure_does_not_report_a_successful_import() {
268 let mut pending: Option<MediaFuture<Option<ReadyWindowsFrame>>> = Some(Box::pin(async { Ok(None) }));
269 let mut ready = None;
270 assert!(poll_frame_import(&mut pending, &mut ready, || panic!("budget rejection is not admission")).unwrap().is_none());
271 assert!(pending.is_none() && ready.is_none());
272 }
273
274 #[test]
275 #[ignore = "Requires hardware D3D11, Media Foundation and an audio endpoint"]
276 fn native_video_stream_delivery_cadence() {
277 use std::time::Instant;
278 use ferroui_base::platform::*;
279 use ferroui_base::{PixelSize, Rect, Vector};
280 let renderer = ferroui_direct2d::Direct2DRenderInterface;
281 let context = renderer.create_backend_context(None);
282 let feature = context.try_get_feature(std::any::TypeId::of::<dyn IExternalObjectsRenderInterfaceContextFeature>()).unwrap();
283 let feature = feature.downcast_ref::<Rc<dyn IExternalObjectsRenderInterfaceContextFeature>>().unwrap().clone();
284 let target = renderer.create_render_target_bitmap(PixelSize::new(32, 32), Vector::new(96., 96.));
285 let device = NativeDevice::from_direct2d().unwrap();
286 let path = std::path::Path::new(env!("CARGO_MANIFEST_DIR"))
287 .join("../../samples/ControlCatalog/Media/sintel-sample.mp4").canonicalize().unwrap();
288 let path = path.to_string_lossy();
289 let native = NativeSession::open(&device, path.trim_start_matches("\\\\?\\"), false, false, None).unwrap();
290 let budget = FrameBudget::new(Default::default());
291 let producer = WindowsFrameProducer::Direct3D11 {
292 budget: budget.clone(), device: DeviceIdentity::Luid(ferroui_direct2d::device_luid()),
293 generation: ferroui_direct2d::device_generation(),
294 };
295 let stream = WindowsFrameStream::new(native, producer, 1, 1);
296 let preparing = Instant::now();
297 while stream.native.state().unwrap().state != 1 {
298 assert!(preparing.elapsed() < Duration::from_secs(10));
299 std::thread::sleep(Duration::from_millis(5));
300 }
301 stream.native.intent(1);
302 let start = Instant::now();
303 let mut count = 0;
304 let mut first_pts = None;
305 let mut last_pts = Duration::ZERO;
306 while start.elapsed() < Duration::from_secs(5) {
307 if let Some(frame) = stream.take_frame().unwrap() {
308 count += 1;
309 first_pts.get_or_insert(frame.metadata().timestamp);
310 last_pts = frame.metadata().timestamp;
311 let m = frame.metadata();
312 let size = (m.width, m.height);
313 let handle = LeasedPlatformHandle::new(0, Some("SharedFrameD3D11Texture2D"), frame.clone());
314 let image = feature.import_image(Rc::new(handle), PlatformGraphicsExternalImageProperties {
315 width: m.width as i32, height: m.height as i32,
316 format: PlatformGraphicsExternalImageFormat::B8G8R8A8UNorm,
317 top_left_origin: true, memory_size: m.retained_bytes, ..Default::default()
318 });
319 let bitmap = image.acquire_shared_bitmap(frame.clone()).unwrap();
320 image.dispose();
323 drop(image);
324 drop(frame);
325 let mut drawing = target.create_drawing_context();
326 drawing.draw_bitmap(&*bitmap, 1., Rect::new(0., 0., size.0 as f64, size.1 as f64), Rect::new(0., 0., 32., 32.));
327 drawing.dispose();
328 }
329 std::thread::sleep(Duration::from_millis(5));
330 }
331 let fps = count as f64 / start.elapsed().as_secs_f64();
332 println!("NATIVE_STREAM_DELIVERY frames={count} fps={fps:.2} pts_span_ms={}",
333 (last_pts - first_pts.unwrap()).as_millis());
334 assert!(fps >= 21.6, "24 fps source was undersampled: {fps:.2}");
335 stream.native.intent(0);
336 drop(stream);
337 let retiring = Instant::now();
338 while budget.usage().frames != 0 && retiring.elapsed() < Duration::from_secs(2) {
339 std::thread::sleep(Duration::from_millis(1));
340 }
341 assert_eq!(budget.usage().frames, 0);
342 }
343}