mirror_resample/
lib.rs

1use rubato::{FastFixedIn, PolynomialDegree, Resampler};
2
3pub use rubato::{ResampleError, ResampleResult, ResamplerConstructionError};
4
5/// Audio resampler, quickly resample input to a single channel count and
6/// different sampling rates.
7///
8/// Note that due to the fast sampling, the quality may be reduced.
9pub struct AudioResampler {
10    sampler: Option<FastFixedIn<f32>>,
11    input_buffer: Vec<f32>,
12    output_buffer: Vec<f32>,
13    samples: Vec<i16>,
14}
15
16impl AudioResampler {
17    pub fn new(input: f64, output: f64, frames: usize) -> Result<Self, ResamplerConstructionError> {
18        Ok(Self {
19            samples: Vec::with_capacity(frames),
20            input_buffer: Vec::with_capacity(48000),
21            output_buffer: vec![0.0; 48000],
22            sampler: if input != output {
23                Some(FastFixedIn::new(
24                    output / input,
25                    2.0,
26                    PolynomialDegree::Linear,
27                    frames,
28                    1,
29                )?)
30            } else {
31                None
32            },
33        })
34    }
35
36    pub fn resample<'a>(
37        &'a mut self,
38        buffer: &'a [i16],
39        channels: usize,
40    ) -> ResampleResult<&'a [i16]> {
41        if channels == 1 && self.sampler.is_none() {
42            Ok(buffer)
43        } else {
44            self.samples.clear();
45            self.input_buffer.clear();
46
47            for item in buffer.iter().step_by(channels) {
48                if self.sampler.is_none() {
49                    self.samples.push(*item);
50                } else {
51                    // need resample
52                    self.input_buffer.push(*item as f32);
53                }
54            }
55
56            if let Some(sampler) = &mut self.sampler {
57                let (_, size) = sampler.process_into_buffer(
58                    &[&self.input_buffer[..]],
59                    &mut [&mut self.output_buffer],
60                    None,
61                )?;
62
63                for item in &self.output_buffer[..size] {
64                    self.samples.push(*item as i16);
65                }
66            }
67
68            Ok(&self.samples[..])
69        }
70    }
71}
72
73#[cfg(target_os = "windows")]
74pub mod win32 {
75    use std::mem::ManuallyDrop;
76
77    use mirror_common::{
78        win32::{
79            windows::{
80                core::{Error, Interface},
81                Win32::{
82                    Foundation::RECT,
83                    Graphics::{
84                        Direct3D11::{
85                            ID3D11Device, ID3D11DeviceContext, ID3D11Texture2D, ID3D11VideoContext,
86                            ID3D11VideoDevice, ID3D11VideoProcessor,
87                            ID3D11VideoProcessorEnumerator, ID3D11VideoProcessorInputView,
88                            ID3D11VideoProcessorOutputView, D3D11_BIND_RENDER_TARGET,
89                            D3D11_CPU_ACCESS_READ, D3D11_MAPPED_SUBRESOURCE, D3D11_MAP_READ,
90                            D3D11_RESOURCE_MISC_SHARED, D3D11_TEXTURE2D_DESC, D3D11_USAGE_DEFAULT,
91                            D3D11_USAGE_STAGING, D3D11_VIDEO_FRAME_FORMAT_PROGRESSIVE,
92                            D3D11_VIDEO_PROCESSOR_COLOR_SPACE, D3D11_VIDEO_PROCESSOR_CONTENT_DESC,
93                            D3D11_VIDEO_PROCESSOR_INPUT_VIEW_DESC,
94                            D3D11_VIDEO_PROCESSOR_OUTPUT_VIEW_DESC, D3D11_VIDEO_PROCESSOR_STREAM,
95                            D3D11_VIDEO_USAGE_PLAYBACK_NORMAL, D3D11_VPIV_DIMENSION_TEXTURE2D,
96                            D3D11_VPOV_DIMENSION_TEXTURE2D,
97                        },
98                        Dxgi::Common::DXGI_FORMAT,
99                    },
100                },
101            },
102            Direct3DDevice,
103        },
104        Size,
105    };
106
107    pub enum Resource {
108        Default(DXGI_FORMAT, Size),
109        Texture(ID3D11Texture2D),
110    }
111
112    pub struct VideoResamplerDescriptor {
113        pub direct3d: Direct3DDevice,
114        pub input: Resource,
115        pub output: Resource,
116    }
117
118    /// Used to convert video frames using hardware accelerators, including
119    /// color space conversion and scaling. Note that the output is fixed to
120    /// NV12, but the input is optional and is RGBA by default. However, if
121    /// you use the `process` method, you can let the external texture
122    /// decide what format to use, because this method does not copy the
123    /// texture.
124    #[allow(unused)]
125    pub struct VideoResampler {
126        d3d_device: ID3D11Device,
127        d3d_context: ID3D11DeviceContext,
128        video_device: ID3D11VideoDevice,
129        video_context: ID3D11VideoContext,
130        input_texture: ID3D11Texture2D,
131        output_texture: ID3D11Texture2D,
132        video_enumerator: ID3D11VideoProcessorEnumerator,
133        video_processor: ID3D11VideoProcessor,
134        input_view: ID3D11VideoProcessorInputView,
135        output_view: ID3D11VideoProcessorOutputView,
136    }
137
138    unsafe impl Send for VideoResampler {}
139    unsafe impl Sync for VideoResampler {}
140
141    impl VideoResampler {
142        /// Create `VideoResampler`, the default_device parameter is used to
143        /// directly use the device when it has been created externally, so
144        /// there is no need to copy across devices, which improves
145        /// processing performance.
146        pub fn new(options: VideoResamplerDescriptor) -> Result<Self, Error> {
147            let (d3d_device, d3d_context) = (options.direct3d.device, options.direct3d.context);
148            let video_device = d3d_device.cast::<ID3D11VideoDevice>()?;
149            let video_context = d3d_context.cast::<ID3D11VideoContext>()?;
150
151            let input_texture = match options.input {
152                Resource::Texture(texture) => texture,
153                Resource::Default(format, size) => unsafe {
154                    let mut desc = D3D11_TEXTURE2D_DESC::default();
155                    desc.Width = size.width;
156                    desc.Height = size.height;
157                    desc.MipLevels = 1;
158                    desc.ArraySize = 1;
159                    desc.Format = format.into();
160                    desc.SampleDesc.Count = 1;
161                    desc.SampleDesc.Quality = 0;
162                    desc.Usage = D3D11_USAGE_DEFAULT;
163                    desc.BindFlags = D3D11_BIND_RENDER_TARGET.0 as u32;
164                    desc.CPUAccessFlags = 0;
165                    desc.MiscFlags = 0;
166
167                    let mut texture = None;
168                    d3d_device.CreateTexture2D(&desc, None, Some(&mut texture))?;
169                    texture.unwrap()
170                },
171            };
172
173            let output_texture = match options.output {
174                Resource::Texture(texture) => texture,
175                Resource::Default(format, size) => unsafe {
176                    let mut desc = D3D11_TEXTURE2D_DESC::default();
177                    desc.Width = size.width;
178                    desc.Height = size.height;
179                    desc.MipLevels = 1;
180                    desc.ArraySize = 1;
181                    desc.Format = format.into();
182                    desc.SampleDesc.Count = 1;
183                    desc.SampleDesc.Quality = 0;
184                    desc.Usage = D3D11_USAGE_DEFAULT;
185                    desc.BindFlags = D3D11_BIND_RENDER_TARGET.0 as u32;
186                    desc.CPUAccessFlags = 0;
187                    desc.MiscFlags = D3D11_RESOURCE_MISC_SHARED.0 as u32;
188
189                    let mut texture = None;
190                    d3d_device.CreateTexture2D(&desc, None, Some(&mut texture))?;
191                    texture.unwrap()
192                },
193            };
194
195            let mut input_desc = D3D11_TEXTURE2D_DESC::default();
196            unsafe {
197                input_texture.GetDesc(&mut input_desc);
198            }
199
200            let mut output_desc = D3D11_TEXTURE2D_DESC::default();
201            unsafe {
202                output_texture.GetDesc(&mut output_desc);
203            }
204
205            let (video_enumerator, video_processor) = unsafe {
206                let mut desc = D3D11_VIDEO_PROCESSOR_CONTENT_DESC::default();
207                desc.InputFrameFormat = D3D11_VIDEO_FRAME_FORMAT_PROGRESSIVE;
208                desc.InputWidth = input_desc.Width;
209                desc.InputHeight = input_desc.Height;
210                desc.OutputWidth = output_desc.Width;
211                desc.OutputHeight = output_desc.Height;
212                desc.Usage = D3D11_VIDEO_USAGE_PLAYBACK_NORMAL;
213
214                let enumerator = video_device.CreateVideoProcessorEnumerator(&desc)?;
215                let processor = video_device.CreateVideoProcessor(&enumerator, 0)?;
216                (enumerator, processor)
217            };
218
219            let input_view = unsafe {
220                let mut desc = D3D11_VIDEO_PROCESSOR_INPUT_VIEW_DESC::default();
221                desc.FourCC = 0;
222                desc.ViewDimension = D3D11_VPIV_DIMENSION_TEXTURE2D;
223                desc.Anonymous.Texture2D.MipSlice = 0;
224
225                let mut view = None;
226                video_device.CreateVideoProcessorInputView(
227                    &input_texture,
228                    &video_enumerator,
229                    &desc,
230                    Some(&mut view),
231                )?;
232
233                view.unwrap()
234            };
235
236            let output_view = unsafe {
237                let mut desc = D3D11_VIDEO_PROCESSOR_OUTPUT_VIEW_DESC::default();
238                desc.ViewDimension = D3D11_VPOV_DIMENSION_TEXTURE2D;
239
240                let mut view = None;
241                video_device.CreateVideoProcessorOutputView(
242                    &output_texture,
243                    &video_enumerator,
244                    &desc,
245                    Some(&mut view),
246                )?;
247
248                view.unwrap()
249            };
250
251            unsafe {
252                video_context.VideoProcessorSetStreamSourceRect(
253                    &video_processor,
254                    0,
255                    true,
256                    Some(&RECT {
257                        left: 0,
258                        top: 0,
259                        right: input_desc.Width as i32,
260                        bottom: input_desc.Height as i32,
261                    }),
262                );
263            }
264
265            unsafe {
266                video_context.VideoProcessorSetStreamDestRect(
267                    &video_processor,
268                    0,
269                    true,
270                    Some(&RECT {
271                        left: 0,
272                        top: 0,
273                        right: output_desc.Width as i32,
274                        bottom: output_desc.Height as i32,
275                    }),
276                );
277            }
278
279            unsafe {
280                let color_space = D3D11_VIDEO_PROCESSOR_COLOR_SPACE::default();
281                video_context.VideoProcessorSetStreamColorSpace(&video_processor, 0, &color_space);
282            }
283
284            Ok(Self {
285                d3d_device,
286                d3d_context,
287                video_device,
288                video_context,
289                video_enumerator,
290                video_processor,
291                input_texture,
292                output_texture,
293                input_view,
294                output_view,
295            })
296        }
297
298        /// To update the internal texture, simply copy it to the internal
299        /// texture.
300        pub fn update_input(&mut self, texture: &ID3D11Texture2D) {
301            unsafe {
302                self.d3d_context.CopyResource(&self.input_texture, texture);
303            }
304        }
305
306        /// Perform the conversion. This method will copy the texture array to
307        /// the internal texture, so there are restrictions on the
308        /// format of the incoming texture. Because the internal one is
309        /// fixed to RGBA, the external texture can only be RGBA.
310        pub fn update_input_from_buffer(
311            &mut self,
312            buf: *const u8,
313            stride: u32,
314        ) -> Result<(), Error> {
315            unsafe {
316                self.d3d_context.UpdateSubresource(
317                    &self.input_texture,
318                    0,
319                    None,
320                    buf as *const _,
321                    stride,
322                    0,
323                );
324            }
325
326            Ok(())
327        }
328
329        /// Perform the conversion. This method will not copy the passed
330        /// texture, but will use the texture directly, which can save a
331        /// copy step and improve performance.
332        pub fn create_input_view(
333            &mut self,
334            texture: &ID3D11Texture2D,
335            index: u32,
336        ) -> Result<ID3D11VideoProcessorInputView, Error> {
337            let input_view = unsafe {
338                let mut desc = D3D11_VIDEO_PROCESSOR_INPUT_VIEW_DESC::default();
339                desc.FourCC = 0;
340                desc.ViewDimension = D3D11_VPIV_DIMENSION_TEXTURE2D;
341                desc.Anonymous.Texture2D.MipSlice = 0;
342                desc.Anonymous.Texture2D.ArraySlice = index;
343
344                let mut view = None;
345                self.video_device.CreateVideoProcessorInputView(
346                    texture,
347                    &self.video_enumerator,
348                    &desc,
349                    Some(&mut view),
350                )?;
351
352                view.unwrap()
353            };
354
355            Ok(input_view)
356        }
357
358        pub fn get_output(&self) -> &ID3D11Texture2D {
359            &self.output_texture
360        }
361
362        pub fn get_output_buffer(&mut self) -> Result<TextureBuffer, Error> {
363            Ok(TextureBuffer::new(
364                &self.d3d_device,
365                &self.d3d_context,
366                &self.output_texture,
367            )?)
368        }
369
370        pub fn process(
371            &mut self,
372            input_view: Option<ID3D11VideoProcessorInputView>,
373        ) -> Result<(), Error> {
374            unsafe {
375                let mut streams = [D3D11_VIDEO_PROCESSOR_STREAM::default()];
376                streams[0].Enable = true.into();
377                streams[0].OutputIndex = 0;
378                streams[0].InputFrameOrField = 0;
379                streams[0].pInputSurface =
380                    ManuallyDrop::new(Some(input_view.unwrap_or_else(|| self.input_view.clone())));
381
382                self.video_context.VideoProcessorBlt(
383                    &self.video_processor,
384                    &self.output_view,
385                    0,
386                    &streams,
387                )?;
388
389                ManuallyDrop::drop(&mut streams[0].pInputSurface);
390            }
391
392            Ok(())
393        }
394    }
395
396    pub struct TextureBuffer<'a> {
397        d3d_context: &'a ID3D11DeviceContext,
398        texture: ID3D11Texture2D,
399        resource: D3D11_MAPPED_SUBRESOURCE,
400    }
401
402    unsafe impl Send for TextureBuffer<'_> {}
403    unsafe impl Sync for TextureBuffer<'_> {}
404
405    impl<'a> TextureBuffer<'a> {
406        pub fn new(
407            d3d_device: &ID3D11Device,
408            d3d_context: &'a ID3D11DeviceContext,
409            source_texture: &ID3D11Texture2D,
410        ) -> Result<Self, Error> {
411            let texture = unsafe {
412                let mut desc = D3D11_TEXTURE2D_DESC::default();
413                source_texture.GetDesc(&mut desc);
414
415                desc.CPUAccessFlags = D3D11_CPU_ACCESS_READ.0 as u32;
416                desc.Usage = D3D11_USAGE_STAGING;
417                desc.BindFlags = 0;
418                desc.MiscFlags = 0;
419
420                let mut texture = None;
421                d3d_device.CreateTexture2D(&desc, None, Some(&mut texture))?;
422                texture.unwrap()
423            };
424
425            unsafe {
426                d3d_context.CopyResource(&texture, source_texture);
427            }
428
429            let mut resource = D3D11_MAPPED_SUBRESOURCE::default();
430            unsafe {
431                d3d_context.Map(&texture, 0, D3D11_MAP_READ, 0, Some(&mut resource))?;
432            }
433
434            Ok(Self {
435                d3d_context,
436                resource,
437                texture,
438            })
439        }
440
441        /// Represents a pointer to texture data. Internally, the texture is
442        /// copied to the CPU first, and then the internal data is
443        /// mapped.
444        pub fn buffer(&self) -> *const u8 {
445            self.resource.pData as *const _
446        }
447
448        /// The stride of the texture data
449        pub fn stride(&self) -> usize {
450            self.resource.RowPitch as usize
451        }
452    }
453
454    impl Drop for TextureBuffer<'_> {
455        fn drop(&mut self) {
456            unsafe {
457                self.d3d_context.Unmap(&self.texture, 0);
458            }
459        }
460    }
461}
mirror_resample/lib.rs

mirror_resample/
lib.rs
