windows_capture/
encoder.rs

1use std::{
2    fs::{self, File},
3    path::Path,
4    slice,
5    sync::{
6        atomic::{self, AtomicBool},
7        mpsc, Arc,
8    },
9    thread::{self, JoinHandle},
10};
11
12use parking_lot::{Condvar, Mutex};
13use windows::{
14    core::HSTRING,
15    Foundation::{EventRegistrationToken, TimeSpan, TypedEventHandler},
16    Graphics::{
17        DirectX::Direct3D11::IDirect3DSurface,
18        Imaging::{BitmapAlphaMode, BitmapEncoder, BitmapPixelFormat},
19    },
20    Media::{
21        Core::{
22            AudioStreamDescriptor, MediaStreamSample, MediaStreamSource,
23            MediaStreamSourceSampleRequestedEventArgs, MediaStreamSourceStartingEventArgs,
24            VideoStreamDescriptor,
25        },
26        MediaProperties::{
27            AudioEncodingProperties, ContainerEncodingProperties, MediaEncodingProfile,
28            MediaEncodingSubtypes, VideoEncodingProperties,
29        },
30        Transcoding::MediaTranscoder,
31    },
32    Security::Cryptography::CryptographicBuffer,
33    Storage::{
34        FileAccessMode, StorageFile,
35        Streams::{
36            Buffer, DataReader, IRandomAccessStream, InMemoryRandomAccessStream, InputStreamOptions,
37        },
38    },
39};
40
41use crate::{
42    d3d11::SendDirectX,
43    frame::{Frame, ImageFormat},
44    settings::ColorFormat,
45};
46
47#[derive(thiserror::Error, Eq, PartialEq, Clone, Debug)]
48pub enum ImageEncoderError {
49    #[error("This color format is not supported for saving as image")]
50    UnsupportedFormat,
51    #[error("Windows API Error: {0}")]
52    WindowsError(#[from] windows::core::Error),
53}
54
55/// The `ImageEncoder` struct represents an image encoder that can be used to encode image buffers to image bytes with a specified format and color format.
56pub struct ImageEncoder {
57    format: ImageFormat,
58    color_format: ColorFormat,
59}
60
61impl ImageEncoder {
62    /// Create a new ImageEncoder with the specified format and color format.
63    ///
64    /// # Arguments
65    ///
66    /// * `format` - The desired image format.
67    /// * `color_format` - The desired color format.
68    ///
69    /// # Returns
70    ///
71    /// A new `ImageEncoder` instance.
72    #[must_use]
73    #[inline]
74    pub const fn new(format: ImageFormat, color_format: ColorFormat) -> Self {
75        Self {
76            format,
77            color_format,
78        }
79    }
80
81    /// Encode the image buffer to image bytes with the specified format.
82    ///
83    /// # Arguments
84    ///
85    /// * `image_buffer` - The image buffer to encode.
86    /// * `width` - The width of the image.
87    /// * `height` - The height of the image.
88    ///
89    /// # Returns
90    ///
91    /// The encoded image bytes as a `Vec<u8>`.
92    ///
93    /// # Errors
94    ///
95    /// Returns an `Error` if the encoding fails or if the color format is unsupported.
96    #[inline]
97    pub fn encode(
98        &self,
99        image_buffer: &[u8],
100        width: u32,
101        height: u32,
102    ) -> Result<Vec<u8>, ImageEncoderError> {
103        let encoder = match self.format {
104            ImageFormat::Jpeg => BitmapEncoder::JpegEncoderId()?,
105            ImageFormat::Png => BitmapEncoder::PngEncoderId()?,
106            ImageFormat::Gif => BitmapEncoder::GifEncoderId()?,
107            ImageFormat::Tiff => BitmapEncoder::TiffEncoderId()?,
108            ImageFormat::Bmp => BitmapEncoder::BmpEncoderId()?,
109            ImageFormat::JpegXr => BitmapEncoder::JpegXREncoderId()?,
110        };
111
112        let stream = InMemoryRandomAccessStream::new()?;
113        let encoder = BitmapEncoder::CreateAsync(encoder, &stream)?.get()?;
114
115        let pixelformat = match self.color_format {
116            ColorFormat::Bgra8 => BitmapPixelFormat::Bgra8,
117            ColorFormat::Rgba8 => BitmapPixelFormat::Rgba8,
118            ColorFormat::Rgba16F => return Err(ImageEncoderError::UnsupportedFormat),
119        };
120
121        encoder.SetPixelData(
122            pixelformat,
123            BitmapAlphaMode::Premultiplied,
124            width,
125            height,
126            1.0,
127            1.0,
128            image_buffer,
129        )?;
130
131        encoder.FlushAsync()?.get()?;
132
133        let buffer = Buffer::Create(u32::try_from(stream.Size()?).unwrap())?;
134        stream
135            .ReadAsync(&buffer, buffer.Capacity()?, InputStreamOptions::None)?
136            .get()?;
137
138        let data_reader = DataReader::FromBuffer(&buffer)?;
139        let length = data_reader.UnconsumedBufferLength()?;
140        let mut bytes = vec![0u8; length as usize];
141        data_reader.ReadBytes(&mut bytes)?;
142
143        Ok(bytes)
144    }
145}
146
147#[derive(thiserror::Error, Debug)]
148pub enum VideoEncoderError {
149    #[error("Windows API Error: {0}")]
150    WindowsError(#[from] windows::core::Error),
151    #[error("Frame send error: {0}")]
152    FrameSendError(#[from] mpsc::SendError<Option<(VideoEncoderSource, TimeSpan)>>),
153    #[error("Audio send error: {0}")]
154    AudioSendError(#[from] mpsc::SendError<Option<(AudioEncoderSource, TimeSpan)>>),
155    #[error("Video is disabled")]
156    VideoDisabled,
157    #[error("Audio is disabled")]
158    AudioDisabled,
159    #[error("IO Error: {0}")]
160    IoError(#[from] std::io::Error),
161}
162
163unsafe impl Send for VideoEncoderError {}
164unsafe impl Sync for VideoEncoderError {}
165
166/// The `VideoEncoderSource` struct represents all the types that can be send to the encoder.
167pub enum VideoEncoderSource {
168    DirectX(SendDirectX<IDirect3DSurface>),
169    Buffer((SendDirectX<*const u8>, usize)),
170}
171
172/// The `AudioEncoderSource` struct represents all the types that can be send to the encoder.
173pub enum AudioEncoderSource {
174    Buffer((SendDirectX<*const u8>, usize)),
175}
176
177/// The `VideoSettings` struct represents the settings for the video encoder.
178pub struct VideoSettingsBuilder {
179    sub_type: VideoSettingsSubType,
180    bitrate: u32,
181    width: u32,
182    height: u32,
183    frame_rate: u32,
184    pixel_aspect_ratio: (u32, u32),
185    disabled: bool,
186}
187
188impl VideoSettingsBuilder {
189    pub const fn new(width: u32, height: u32) -> Self {
190        Self {
191            bitrate: 15000000,
192            frame_rate: 60,
193            pixel_aspect_ratio: (1, 1),
194            sub_type: VideoSettingsSubType::HEVC,
195            width,
196            height,
197            disabled: false,
198        }
199    }
200
201    pub const fn sub_type(mut self, sub_type: VideoSettingsSubType) -> Self {
202        self.sub_type = sub_type;
203        self
204    }
205
206    pub const fn bitrate(mut self, bitrate: u32) -> Self {
207        self.bitrate = bitrate;
208        self
209    }
210
211    pub const fn width(mut self, width: u32) -> Self {
212        self.width = width;
213        self
214    }
215
216    pub const fn height(mut self, height: u32) -> Self {
217        self.height = height;
218        self
219    }
220
221    pub const fn frame_rate(mut self, frame_rate: u32) -> Self {
222        self.frame_rate = frame_rate;
223        self
224    }
225
226    pub const fn pixel_aspect_ratio(mut self, pixel_aspect_ratio: (u32, u32)) -> Self {
227        self.pixel_aspect_ratio = pixel_aspect_ratio;
228        self
229    }
230
231    pub const fn disabled(mut self, disabled: bool) -> Self {
232        self.disabled = disabled;
233        self
234    }
235
236    fn build(self) -> Result<(VideoEncodingProperties, bool), VideoEncoderError> {
237        let properties = VideoEncodingProperties::new()?;
238
239        properties.SetSubtype(&self.sub_type.to_hstring())?;
240        properties.SetBitrate(self.bitrate)?;
241        properties.SetWidth(self.width)?;
242        properties.SetHeight(self.height)?;
243        properties.FrameRate()?.SetNumerator(self.frame_rate)?;
244        properties.FrameRate()?.SetDenominator(1)?;
245        properties
246            .PixelAspectRatio()?
247            .SetNumerator(self.pixel_aspect_ratio.0)?;
248        properties
249            .PixelAspectRatio()?
250            .SetDenominator(self.pixel_aspect_ratio.1)?;
251
252        Ok((properties, self.disabled))
253    }
254}
255
256/// The `AudioSettingsSubType` enum represents the settings for the audio encoder.
257pub struct AudioSettingsBuilder {
258    bitrate: u32,
259    channel_count: u32,
260    sample_rate: u32,
261    bit_per_sample: u32,
262    sub_type: AudioSettingsSubType,
263    disabled: bool,
264}
265
266impl AudioSettingsBuilder {
267    pub const fn new() -> Self {
268        Self {
269            bitrate: 192000,
270            channel_count: 2,
271            sample_rate: 48000,
272            bit_per_sample: 16,
273            sub_type: AudioSettingsSubType::AAC,
274            disabled: false,
275        }
276    }
277    pub const fn bitrate(mut self, bitrate: u32) -> Self {
278        self.bitrate = bitrate;
279        self
280    }
281
282    pub const fn channel_count(mut self, channel_count: u32) -> Self {
283        self.channel_count = channel_count;
284        self
285    }
286
287    pub const fn sample_rate(mut self, sample_rate: u32) -> Self {
288        self.sample_rate = sample_rate;
289        self
290    }
291
292    pub const fn bit_per_sample(mut self, bit_per_sample: u32) -> Self {
293        self.bit_per_sample = bit_per_sample;
294        self
295    }
296
297    pub const fn sub_type(mut self, sub_type: AudioSettingsSubType) -> Self {
298        self.sub_type = sub_type;
299        self
300    }
301
302    pub const fn disabled(mut self, disabled: bool) -> Self {
303        self.disabled = disabled;
304        self
305    }
306
307    fn build(self) -> Result<(AudioEncodingProperties, bool), VideoEncoderError> {
308        let properties = AudioEncodingProperties::new()?;
309        properties.SetBitrate(self.bitrate)?;
310        properties.SetChannelCount(self.channel_count)?;
311        properties.SetSampleRate(self.sample_rate)?;
312        properties.SetBitsPerSample(self.bit_per_sample)?;
313        properties.SetSubtype(&self.sub_type.to_hstring())?;
314
315        Ok((properties, self.disabled))
316    }
317}
318
319impl Default for AudioSettingsBuilder {
320    fn default() -> Self {
321        Self::new()
322    }
323}
324
325/// The `ContainerSettingsSubType` enum represents the settings for the container encoder.
326pub struct ContainerSettingsBuilder {
327    sub_type: ContainerSettingsSubType,
328}
329
330impl ContainerSettingsBuilder {
331    pub const fn new() -> Self {
332        Self {
333            sub_type: ContainerSettingsSubType::MPEG4,
334        }
335    }
336
337    pub const fn sub_type(mut self, sub_type: ContainerSettingsSubType) -> Self {
338        self.sub_type = sub_type;
339        self
340    }
341
342    fn build(self) -> Result<ContainerEncodingProperties, VideoEncoderError> {
343        let properties = ContainerEncodingProperties::new()?;
344        properties.SetSubtype(&self.sub_type.to_hstring())?;
345        Ok(properties)
346    }
347}
348
349impl Default for ContainerSettingsBuilder {
350    fn default() -> Self {
351        Self::new()
352    }
353}
354
355/// The `VideoSettingsSubType` enum represents the subtypes for the video encoder.
356#[derive(Eq, PartialEq, Clone, Copy, Debug)]
357pub enum VideoSettingsSubType {
358    ARGB32,
359    BGRA8,
360    D16,
361    H263,
362    H264,
363    H264ES,
364    HEVC,
365    HEVCES,
366    IYUV,
367    L8,
368    L16,
369    MJPG,
370    NV12,
371    MPEG1,
372    MPEG2,
373    RGB24,
374    RGB32,
375    WMV3,
376    WVC1,
377    VP9,
378    YUY2,
379    YV12,
380}
381
382impl VideoSettingsSubType {
383    pub fn to_hstring(&self) -> HSTRING {
384        let s = match self {
385            Self::ARGB32 => "ARGB32",
386            Self::BGRA8 => "BGRA8",
387            Self::D16 => "D16",
388            Self::H263 => "H263",
389            Self::H264 => "H264",
390            Self::H264ES => "H264ES",
391            Self::HEVC => "HEVC",
392            Self::HEVCES => "HEVCES",
393            Self::IYUV => "IYUV",
394            Self::L8 => "L8",
395            Self::L16 => "L16",
396            Self::MJPG => "MJPG",
397            Self::NV12 => "NV12",
398            Self::MPEG1 => "MPEG1",
399            Self::MPEG2 => "MPEG2",
400            Self::RGB24 => "RGB24",
401            Self::RGB32 => "RGB32",
402            Self::WMV3 => "WMV3",
403            Self::WVC1 => "WVC1",
404            Self::VP9 => "VP9",
405            Self::YUY2 => "YUY2",
406            Self::YV12 => "YV12",
407        };
408
409        HSTRING::from(s)
410    }
411}
412
413/// The `AudioSettingsSubType` enum represents the subtypes for the audio encoder.
414#[derive(Eq, PartialEq, Clone, Copy, Debug)]
415pub enum AudioSettingsSubType {
416    AAC,
417    AC3,
418    AACADTS,
419    AACHDCP,
420    AC3SPDIF,
421    AC3HDCP,
422    ADTS,
423    ALAC,
424    AMRNB,
425    AWRWB,
426    DTS,
427    EAC3,
428    FLAC,
429    Float,
430    MP3,
431    MPEG,
432    OPUS,
433    PCM,
434    WMA8,
435    WMA9,
436    Vorbis,
437}
438
439impl AudioSettingsSubType {
440    pub fn to_hstring(&self) -> HSTRING {
441        let s = match self {
442            Self::AAC => "AAC",
443            Self::AC3 => "AC3",
444            Self::AACADTS => "AACADTS",
445            Self::AACHDCP => "AACHDCP",
446            Self::AC3SPDIF => "AC3SPDIF",
447            Self::AC3HDCP => "AC3HDCP",
448            Self::ADTS => "ADTS",
449            Self::ALAC => "ALAC",
450            Self::AMRNB => "AMRNB",
451            Self::AWRWB => "AWRWB",
452            Self::DTS => "DTS",
453            Self::EAC3 => "EAC3",
454            Self::FLAC => "FLAC",
455            Self::Float => "Float",
456            Self::MP3 => "MP3",
457            Self::MPEG => "MPEG",
458            Self::OPUS => "OPUS",
459            Self::PCM => "PCM",
460            Self::WMA8 => "WMA8",
461            Self::WMA9 => "WMA9",
462            Self::Vorbis => "Vorbis",
463        };
464
465        HSTRING::from(s)
466    }
467}
468
469/// The `Subtype` enum represents the subtypes for the video encoder.
470#[derive(Eq, PartialEq, Clone, Copy, Debug)]
471pub enum ContainerSettingsSubType {
472    ASF,
473    MP3,
474    MPEG4,
475    AVI,
476    MPEG2,
477    WAVE,
478    AACADTS,
479    ADTS,
480    GP3,
481    AMR,
482    FLAC,
483}
484
485impl ContainerSettingsSubType {
486    pub fn to_hstring(&self) -> HSTRING {
487        match self {
488            Self::ASF => HSTRING::from("ASF"),
489            Self::MP3 => HSTRING::from("MP3"),
490            Self::MPEG4 => HSTRING::from("MPEG4"),
491            Self::AVI => HSTRING::from("AVI"),
492            Self::MPEG2 => HSTRING::from("MPEG2"),
493            Self::WAVE => HSTRING::from("WAVE"),
494            Self::AACADTS => HSTRING::from("AACADTS"),
495            Self::ADTS => HSTRING::from("ADTS"),
496            Self::GP3 => HSTRING::from("3GP"),
497            Self::AMR => HSTRING::from("AMR"),
498            Self::FLAC => HSTRING::from("FLAC"),
499        }
500    }
501}
502
503/// The `VideoEncoder` struct represents a video encoder that can be used to encode video frames and save them to a specified file path.
504pub struct VideoEncoder {
505    first_timespan: Option<TimeSpan>,
506    frame_sender: mpsc::Sender<Option<(VideoEncoderSource, TimeSpan)>>,
507    audio_sender: mpsc::Sender<Option<(AudioEncoderSource, TimeSpan)>>,
508    sample_requested: EventRegistrationToken,
509    media_stream_source: MediaStreamSource,
510    starting: EventRegistrationToken,
511    transcode_thread: Option<JoinHandle<Result<(), VideoEncoderError>>>,
512    frame_notify: Arc<(Mutex<bool>, Condvar)>,
513    audio_notify: Arc<(Mutex<bool>, Condvar)>,
514    error_notify: Arc<AtomicBool>,
515    is_video_disabled: bool,
516    is_audio_disabled: bool,
517}
518
519impl VideoEncoder {
520    /// Creates a new `VideoEncoder` instance with the specified parameters.
521    ///
522    /// # Arguments
523    ///
524    /// * `encoder_type` - The type of video encoder to use.
525    /// * `encoder_quality` - The quality of the video encoder.
526    /// * `width` - The width of the video frames.
527    /// * `height` - The height of the video frames.
528    /// * `path` - The file path where the encoded video will be saved.
529    ///
530    /// # Returns
531    ///
532    /// Returns a `Result` containing the `VideoEncoder` instance if successful, or a
533    /// `VideoEncoderError` if an error occurs.
534    #[inline]
535    pub fn new<P: AsRef<Path>>(
536        video_settings: VideoSettingsBuilder,
537        audio_settings: AudioSettingsBuilder,
538        container_settings: ContainerSettingsBuilder,
539        path: P,
540    ) -> Result<Self, VideoEncoderError> {
541        let path = path.as_ref();
542        let media_encoding_profile = MediaEncodingProfile::new()?;
543
544        let (video_encoding_properties, is_video_disabled) = video_settings.build()?;
545        media_encoding_profile.SetVideo(&video_encoding_properties)?;
546        let (audio_encoding_properties, is_audio_disabled) = audio_settings.build()?;
547        media_encoding_profile.SetAudio(&audio_encoding_properties)?;
548        let container_encoding_properties = container_settings.build()?;
549        media_encoding_profile.SetContainer(&container_encoding_properties)?;
550
551        let video_encoding_properties = VideoEncodingProperties::CreateUncompressed(
552            &MediaEncodingSubtypes::Bgra8()?,
553            video_encoding_properties.Width()?,
554            video_encoding_properties.Height()?,
555        )?;
556        let video_stream_descriptor = VideoStreamDescriptor::Create(&video_encoding_properties)?;
557
558        let audio_encoding_properties = AudioEncodingProperties::CreateAac(
559            audio_encoding_properties.SampleRate()?,
560            audio_encoding_properties.ChannelCount()?,
561            audio_encoding_properties.Bitrate()?,
562        )?;
563        let audio_stream_descriptor = AudioStreamDescriptor::Create(&audio_encoding_properties)?;
564
565        let media_stream_source = MediaStreamSource::CreateFromDescriptors(
566            &video_stream_descriptor,
567            &audio_stream_descriptor,
568        )?;
569        media_stream_source.SetBufferTime(TimeSpan::default())?;
570
571        let starting = media_stream_source.Starting(&TypedEventHandler::<
572            MediaStreamSource,
573            MediaStreamSourceStartingEventArgs,
574        >::new(move |_, stream_start| {
575            let stream_start = stream_start
576                .as_ref()
577                .expect("MediaStreamSource Starting parameter was None This Should Not Happen.");
578
579            stream_start
580                .Request()?
581                .SetActualStartPosition(TimeSpan { Duration: 0 })?;
582            Ok(())
583        }))?;
584
585        let (frame_sender, frame_receiver) =
586            mpsc::channel::<Option<(VideoEncoderSource, TimeSpan)>>();
587
588        let (audio_sender, audio_receiver) =
589            mpsc::channel::<Option<(AudioEncoderSource, TimeSpan)>>();
590
591        let frame_notify = Arc::new((Mutex::new(false), Condvar::new()));
592        let audio_notify = Arc::new((Mutex::new(false), Condvar::new()));
593
594        let sample_requested = media_stream_source.SampleRequested(&TypedEventHandler::<
595            MediaStreamSource,
596            MediaStreamSourceSampleRequestedEventArgs,
597        >::new({
598            let frame_receiver = frame_receiver;
599            let frame_notify = frame_notify.clone();
600
601            let audio_receiver = audio_receiver;
602            let audio_notify = audio_notify.clone();
603
604            move |_, sample_requested| {
605                let sample_requested = sample_requested.as_ref().expect(
606                    "MediaStreamSource SampleRequested parameter was None This Should Not Happen.",
607                );
608
609                if sample_requested
610                    .Request()?
611                    .StreamDescriptor()?
612                    .GetRuntimeClassName()?
613                    == "Windows.Media.Core.AudioStreamDescriptor"
614                {
615                    if is_audio_disabled {
616                        sample_requested.Request()?.SetSample(None)?;
617
618                        return Ok(());
619                    }
620
621                    let audio = match audio_receiver.recv() {
622                        Ok(audio) => audio,
623                        Err(e) => panic!("Failed to receive audio from audio sender: {e}"),
624                    };
625
626                    match audio {
627                        Some((source, timespan)) => {
628                            let sample = match source {
629                                AudioEncoderSource::Buffer(buffer_data) => {
630                                    let buffer = buffer_data.0;
631                                    let buffer =
632                                        unsafe { slice::from_raw_parts(buffer.0, buffer_data.1) };
633                                    let buffer = CryptographicBuffer::CreateFromByteArray(buffer)?;
634                                    MediaStreamSample::CreateFromBuffer(&buffer, timespan)?
635                                }
636                            };
637
638                            sample_requested.Request()?.SetSample(&sample)?;
639                        }
640                        None => {
641                            sample_requested.Request()?.SetSample(None)?;
642                        }
643                    }
644
645                    let (lock, cvar) = &*audio_notify;
646                    *lock.lock() = true;
647                    cvar.notify_one();
648                } else {
649                    if is_video_disabled {
650                        sample_requested.Request()?.SetSample(None)?;
651
652                        return Ok(());
653                    }
654
655                    let frame = match frame_receiver.recv() {
656                        Ok(frame) => frame,
657                        Err(e) => panic!("Failed to receive frame from frame sender: {e}"),
658                    };
659
660                    match frame {
661                        Some((source, timespan)) => {
662                            let sample = match source {
663                                VideoEncoderSource::DirectX(surface) => {
664                                    MediaStreamSample::CreateFromDirect3D11Surface(
665                                        &surface.0, timespan,
666                                    )?
667                                }
668                                VideoEncoderSource::Buffer(buffer_data) => {
669                                    let buffer = buffer_data.0;
670                                    let buffer =
671                                        unsafe { slice::from_raw_parts(buffer.0, buffer_data.1) };
672                                    let buffer = CryptographicBuffer::CreateFromByteArray(buffer)?;
673                                    MediaStreamSample::CreateFromBuffer(&buffer, timespan)?
674                                }
675                            };
676
677                            sample_requested.Request()?.SetSample(&sample)?;
678                        }
679                        None => {
680                            sample_requested.Request()?.SetSample(None)?;
681                        }
682                    }
683
684                    let (lock, cvar) = &*frame_notify;
685                    *lock.lock() = true;
686                    cvar.notify_one();
687                }
688
689                Ok(())
690            }
691        }))?;
692
693        let media_transcoder = MediaTranscoder::new()?;
694        media_transcoder.SetHardwareAccelerationEnabled(true)?;
695
696        File::create(path)?;
697        let path = fs::canonicalize(path).unwrap().to_string_lossy()[4..].to_string();
698        let path = Path::new(&path);
699
700        let path = &HSTRING::from(path.as_os_str().to_os_string());
701
702        let file = StorageFile::GetFileFromPathAsync(path)?.get()?;
703        let media_stream_output = file.OpenAsync(FileAccessMode::ReadWrite)?.get()?;
704
705        let transcode = media_transcoder
706            .PrepareMediaStreamSourceTranscodeAsync(
707                &media_stream_source,
708                &media_stream_output,
709                &media_encoding_profile,
710            )?
711            .get()?;
712
713        let error_notify = Arc::new(AtomicBool::new(false));
714        let transcode_thread = thread::spawn({
715            let error_notify = error_notify.clone();
716
717            move || -> Result<(), VideoEncoderError> {
718                let result = transcode.TranscodeAsync();
719
720                if result.is_err() {
721                    error_notify.store(true, atomic::Ordering::Relaxed);
722                }
723
724                result?.get()?;
725
726                drop(media_transcoder);
727
728                Ok(())
729            }
730        });
731
732        Ok(Self {
733            first_timespan: None,
734            frame_sender,
735            audio_sender,
736            sample_requested,
737            media_stream_source,
738            starting,
739            transcode_thread: Some(transcode_thread),
740            frame_notify,
741            audio_notify,
742            error_notify,
743            is_video_disabled,
744            is_audio_disabled,
745        })
746    }
747
748    /// Creates a new `VideoEncoder` instance with the specified parameters.
749    ///
750    /// # Arguments
751    ///
752    /// * `encoder_type` - The type of video encoder to use.
753    /// * `encoder_quality` - The quality of the video encoder.
754    /// * `width` - The width of the video frames.
755    /// * `height` - The height of the video frames.
756    /// * `stream` - The stream where the encoded video will be saved.
757    ///
758    /// # Returns
759    ///
760    /// Returns a `Result` containing the `VideoEncoder` instance if successful, or a
761    /// `VideoEncoderError` if an error occurs.
762    #[inline]
763    pub fn new_from_stream(
764        video_settings: VideoSettingsBuilder,
765        audio_settings: AudioSettingsBuilder,
766        container_settings: ContainerSettingsBuilder,
767        stream: IRandomAccessStream,
768    ) -> Result<Self, VideoEncoderError> {
769        let media_encoding_profile = MediaEncodingProfile::new()?;
770
771        let (video_encoding_properties, is_video_disabled) = video_settings.build()?;
772        media_encoding_profile.SetVideo(&video_encoding_properties)?;
773        let (audio_encoding_properties, is_audio_disabled) = audio_settings.build()?;
774        media_encoding_profile.SetAudio(&audio_encoding_properties)?;
775        let container_encoding_properties = container_settings.build()?;
776        media_encoding_profile.SetContainer(&container_encoding_properties)?;
777
778        let video_encoding_properties = VideoEncodingProperties::CreateUncompressed(
779            &MediaEncodingSubtypes::Bgra8()?,
780            video_encoding_properties.Width()?,
781            video_encoding_properties.Height()?,
782        )?;
783        let video_stream_descriptor = VideoStreamDescriptor::Create(&video_encoding_properties)?;
784
785        let audio_encoding_properties = AudioEncodingProperties::CreateAac(
786            audio_encoding_properties.SampleRate()?,
787            audio_encoding_properties.ChannelCount()?,
788            audio_encoding_properties.Bitrate()?,
789        )?;
790        let audio_stream_descriptor = AudioStreamDescriptor::Create(&audio_encoding_properties)?;
791
792        let media_stream_source = MediaStreamSource::CreateFromDescriptors(
793            &video_stream_descriptor,
794            &audio_stream_descriptor,
795        )?;
796        media_stream_source.SetBufferTime(TimeSpan::default())?;
797
798        let starting = media_stream_source.Starting(&TypedEventHandler::<
799            MediaStreamSource,
800            MediaStreamSourceStartingEventArgs,
801        >::new(move |_, stream_start| {
802            let stream_start = stream_start
803                .as_ref()
804                .expect("MediaStreamSource Starting parameter was None This Should Not Happen.");
805
806            stream_start
807                .Request()?
808                .SetActualStartPosition(TimeSpan { Duration: 0 })?;
809            Ok(())
810        }))?;
811
812        let (frame_sender, frame_receiver) =
813            mpsc::channel::<Option<(VideoEncoderSource, TimeSpan)>>();
814
815        let (audio_sender, audio_receiver) =
816            mpsc::channel::<Option<(AudioEncoderSource, TimeSpan)>>();
817
818        let frame_notify = Arc::new((Mutex::new(false), Condvar::new()));
819        let audio_notify = Arc::new((Mutex::new(false), Condvar::new()));
820
821        let sample_requested = media_stream_source.SampleRequested(&TypedEventHandler::<
822            MediaStreamSource,
823            MediaStreamSourceSampleRequestedEventArgs,
824        >::new({
825            let frame_receiver = frame_receiver;
826            let frame_notify = frame_notify.clone();
827
828            let audio_receiver = audio_receiver;
829            let audio_notify = audio_notify.clone();
830
831            move |_, sample_requested| {
832                let sample_requested = sample_requested.as_ref().expect(
833                    "MediaStreamSource SampleRequested parameter was None This Should Not Happen.",
834                );
835
836                if sample_requested
837                    .Request()?
838                    .StreamDescriptor()?
839                    .GetRuntimeClassName()?
840                    == "Windows.Media.Core.AudioStreamDescriptor"
841                {
842                    if is_audio_disabled {
843                        sample_requested.Request()?.SetSample(None)?;
844
845                        return Ok(());
846                    }
847
848                    let audio = match audio_receiver.recv() {
849                        Ok(audio) => audio,
850                        Err(e) => panic!("Failed to receive audio from audio sender: {e}"),
851                    };
852
853                    match audio {
854                        Some((source, timespan)) => {
855                            let sample = match source {
856                                AudioEncoderSource::Buffer(buffer_data) => {
857                                    let buffer = buffer_data.0;
858                                    let buffer =
859                                        unsafe { slice::from_raw_parts(buffer.0, buffer_data.1) };
860                                    let buffer = CryptographicBuffer::CreateFromByteArray(buffer)?;
861                                    MediaStreamSample::CreateFromBuffer(&buffer, timespan)?
862                                }
863                            };
864
865                            sample_requested.Request()?.SetSample(&sample)?;
866                        }
867                        None => {
868                            sample_requested.Request()?.SetSample(None)?;
869                        }
870                    }
871
872                    let (lock, cvar) = &*audio_notify;
873                    *lock.lock() = true;
874                    cvar.notify_one();
875                } else {
876                    if is_video_disabled {
877                        sample_requested.Request()?.SetSample(None)?;
878
879                        return Ok(());
880                    }
881
882                    let frame = match frame_receiver.recv() {
883                        Ok(frame) => frame,
884                        Err(e) => panic!("Failed to receive frame from frame sender: {e}"),
885                    };
886
887                    match frame {
888                        Some((source, timespan)) => {
889                            let sample = match source {
890                                VideoEncoderSource::DirectX(surface) => {
891                                    MediaStreamSample::CreateFromDirect3D11Surface(
892                                        &surface.0, timespan,
893                                    )?
894                                }
895                                VideoEncoderSource::Buffer(buffer_data) => {
896                                    let buffer = buffer_data.0;
897                                    let buffer =
898                                        unsafe { slice::from_raw_parts(buffer.0, buffer_data.1) };
899                                    let buffer = CryptographicBuffer::CreateFromByteArray(buffer)?;
900                                    MediaStreamSample::CreateFromBuffer(&buffer, timespan)?
901                                }
902                            };
903
904                            sample_requested.Request()?.SetSample(&sample)?;
905                        }
906                        None => {
907                            sample_requested.Request()?.SetSample(None)?;
908                        }
909                    }
910
911                    let (lock, cvar) = &*frame_notify;
912                    *lock.lock() = true;
913                    cvar.notify_one();
914                }
915
916                Ok(())
917            }
918        }))?;
919
920        let media_transcoder = MediaTranscoder::new()?;
921        media_transcoder.SetHardwareAccelerationEnabled(true)?;
922
923        let transcode = media_transcoder
924            .PrepareMediaStreamSourceTranscodeAsync(
925                &media_stream_source,
926                &stream,
927                &media_encoding_profile,
928            )?
929            .get()?;
930
931        let error_notify = Arc::new(AtomicBool::new(false));
932        let transcode_thread = thread::spawn({
933            let error_notify = error_notify.clone();
934
935            move || -> Result<(), VideoEncoderError> {
936                let result = transcode.TranscodeAsync();
937
938                if result.is_err() {
939                    error_notify.store(true, atomic::Ordering::Relaxed);
940                }
941
942                result?.get()?;
943
944                drop(media_transcoder);
945
946                Ok(())
947            }
948        });
949
950        Ok(Self {
951            first_timespan: None,
952            frame_sender,
953            audio_sender,
954            sample_requested,
955            media_stream_source,
956            starting,
957            transcode_thread: Some(transcode_thread),
958            frame_notify,
959            audio_notify,
960            error_notify,
961            is_video_disabled,
962            is_audio_disabled,
963        })
964    }
965
966    /// Sends a video frame to the video encoder for encoding.
967    ///
968    /// # Arguments
969    ///
970    /// * `frame` - A mutable reference to the `Frame` to be encoded.
971    ///
972    /// # Returns
973    ///
974    /// Returns `Ok(())` if the frame is successfully sent for encoding, or a `VideoEncoderError`
975    /// if an error occurs.
976    #[inline]
977    pub fn send_frame(&mut self, frame: &mut Frame) -> Result<(), VideoEncoderError> {
978        if self.is_video_disabled {
979            return Err(VideoEncoderError::VideoDisabled);
980        }
981
982        let timespan = match self.first_timespan {
983            Some(timespan) => TimeSpan {
984                Duration: frame.timespan().Duration - timespan.Duration,
985            },
986            None => {
987                let timespan = frame.timespan();
988                self.first_timespan = Some(timespan);
989                TimeSpan { Duration: 0 }
990            }
991        };
992
993        self.frame_sender.send(Some((
994            VideoEncoderSource::DirectX(SendDirectX::new(unsafe {
995                frame.as_raw_surface().clone()
996            })),
997            timespan,
998        )))?;
999
1000        let (lock, cvar) = &*self.frame_notify;
1001        let mut processed = lock.lock();
1002        if !*processed {
1003            cvar.wait(&mut processed);
1004        }
1005        *processed = false;
1006        drop(processed);
1007
1008        if self.error_notify.load(atomic::Ordering::Relaxed) {
1009            if let Some(transcode_thread) = self.transcode_thread.take() {
1010                transcode_thread
1011                    .join()
1012                    .expect("Failed to join transcode thread")?;
1013            }
1014        }
1015
1016        Ok(())
1017    }
1018
1019    /// Sends a video frame with audio to the video encoder for encoding.
1020    ///
1021    /// # Arguments
1022    ///
1023    /// * `frame` - A mutable reference to the `Frame` to be encoded.
1024    /// * `audio_buffer` - A reference to the audio byte slice to be encoded.
1025    ///
1026    /// # Returns
1027    ///
1028    /// Returns `Ok(())` if the frame is successfully sent for encoding, or a `VideoEncoderError`
1029    /// if an error occurs.
1030    #[inline]
1031    pub fn send_frame_with_audio(
1032        &mut self,
1033        frame: &mut Frame,
1034        audio_buffer: &[u8],
1035    ) -> Result<(), VideoEncoderError> {
1036        if self.is_video_disabled {
1037            return Err(VideoEncoderError::VideoDisabled);
1038        }
1039
1040        if self.is_audio_disabled {
1041            return Err(VideoEncoderError::AudioDisabled);
1042        }
1043
1044        let timespan = match self.first_timespan {
1045            Some(timespan) => TimeSpan {
1046                Duration: frame.timespan().Duration - timespan.Duration,
1047            },
1048            None => {
1049                let timespan = frame.timespan();
1050                self.first_timespan = Some(timespan);
1051                TimeSpan { Duration: 0 }
1052            }
1053        };
1054
1055        self.frame_sender.send(Some((
1056            VideoEncoderSource::DirectX(SendDirectX::new(unsafe {
1057                frame.as_raw_surface().clone()
1058            })),
1059            timespan,
1060        )))?;
1061
1062        let (lock, cvar) = &*self.frame_notify;
1063        let mut processed = lock.lock();
1064        if !*processed {
1065            cvar.wait(&mut processed);
1066        }
1067        *processed = false;
1068        drop(processed);
1069
1070        if self.error_notify.load(atomic::Ordering::Relaxed) {
1071            if let Some(transcode_thread) = self.transcode_thread.take() {
1072                transcode_thread
1073                    .join()
1074                    .expect("Failed to join transcode thread")?;
1075            }
1076        }
1077
1078        self.audio_sender.send(Some((
1079            AudioEncoderSource::Buffer((
1080                SendDirectX::new(audio_buffer.as_ptr()),
1081                audio_buffer.len(),
1082            )),
1083            timespan,
1084        )))?;
1085
1086        let (lock, cvar) = &*self.audio_notify;
1087        let mut processed = lock.lock();
1088        if !*processed {
1089            cvar.wait(&mut processed);
1090        }
1091        *processed = false;
1092        drop(processed);
1093
1094        if self.error_notify.load(atomic::Ordering::Relaxed) {
1095            if let Some(transcode_thread) = self.transcode_thread.take() {
1096                transcode_thread
1097                    .join()
1098                    .expect("Failed to join transcode thread")?;
1099            }
1100        }
1101
1102        Ok(())
1103    }
1104
1105    /// Sends a video frame to the video encoder for encoding.
1106    ///
1107    /// # Arguments
1108    ///
1109    /// * `buffer` - A reference to the frame byte slice to be encoded Windows API expect this to be Bgra and bottom-top.
1110    /// * `timespan` - The timespan that correlates to the frame buffer.
1111    ///
1112    /// # Returns
1113    ///
1114    /// Returns `Ok(())` if the frame is successfully sent for encoding, or a `VideoEncoderError`
1115    /// if an error occurs.
1116    #[inline]
1117    pub fn send_frame_buffer(
1118        &mut self,
1119        buffer: &[u8],
1120        timespan: i64,
1121    ) -> Result<(), VideoEncoderError> {
1122        if self.is_video_disabled {
1123            return Err(VideoEncoderError::VideoDisabled);
1124        }
1125
1126        let frame_timespan = timespan;
1127        let timespan = match self.first_timespan {
1128            Some(timespan) => TimeSpan {
1129                Duration: frame_timespan - timespan.Duration,
1130            },
1131            None => {
1132                let timespan = frame_timespan;
1133                self.first_timespan = Some(TimeSpan { Duration: timespan });
1134                TimeSpan { Duration: 0 }
1135            }
1136        };
1137
1138        self.frame_sender.send(Some((
1139            VideoEncoderSource::Buffer((SendDirectX::new(buffer.as_ptr()), buffer.len())),
1140            timespan,
1141        )))?;
1142
1143        let (lock, cvar) = &*self.frame_notify;
1144        let mut processed = lock.lock();
1145        if !*processed {
1146            cvar.wait(&mut processed);
1147        }
1148        *processed = false;
1149        drop(processed);
1150
1151        if self.error_notify.load(atomic::Ordering::Relaxed) {
1152            if let Some(transcode_thread) = self.transcode_thread.take() {
1153                transcode_thread
1154                    .join()
1155                    .expect("Failed to join transcode thread")?;
1156            }
1157        }
1158
1159        Ok(())
1160    }
1161
1162    /// Sends a video audio to the video encoder for encoding.
1163    ///
1164    /// # Arguments
1165    ///
1166    /// * `buffer` - A reference to the audio byte slice to be encoded.
1167    /// * `timespan` - The timespan that correlates to the frame buffer.
1168    ///
1169    /// # Returns
1170    ///
1171    /// Returns `Ok(())` if the frame is successfully sent for encoding, or a `VideoEncoderError`
1172    /// if an error occurs.
1173    #[inline]
1174    pub fn send_audio_buffer(
1175        &mut self,
1176        buffer: &[u8],
1177        timespan: i64,
1178    ) -> Result<(), VideoEncoderError> {
1179        if self.is_audio_disabled {
1180            return Err(VideoEncoderError::AudioDisabled);
1181        }
1182
1183        let audio_timespan = timespan;
1184        let timespan = match self.first_timespan {
1185            Some(timespan) => TimeSpan {
1186                Duration: audio_timespan - timespan.Duration,
1187            },
1188            None => {
1189                let timespan = audio_timespan;
1190                self.first_timespan = Some(TimeSpan { Duration: timespan });
1191                TimeSpan { Duration: 0 }
1192            }
1193        };
1194
1195        self.audio_sender.send(Some((
1196            AudioEncoderSource::Buffer((SendDirectX::new(buffer.as_ptr()), buffer.len())),
1197            timespan,
1198        )))?;
1199
1200        let (lock, cvar) = &*self.audio_notify;
1201        let mut processed = lock.lock();
1202        if !*processed {
1203            cvar.wait(&mut processed);
1204        }
1205        *processed = false;
1206        drop(processed);
1207
1208        if self.error_notify.load(atomic::Ordering::Relaxed) {
1209            if let Some(transcode_thread) = self.transcode_thread.take() {
1210                transcode_thread
1211                    .join()
1212                    .expect("Failed to join transcode thread")?;
1213            }
1214        }
1215
1216        Ok(())
1217    }
1218
1219    /// Finishes encoding the video and performs any necessary cleanup.
1220    ///
1221    /// # Returns
1222    ///
1223    /// Returns `Ok(())` if the encoding is successfully finished, or a `VideoEncoderError` if an
1224    /// error occurs.
1225    #[inline]
1226    pub fn finish(mut self) -> Result<(), VideoEncoderError> {
1227        self.frame_sender.send(None)?;
1228        self.audio_sender.send(None)?;
1229
1230        if let Some(transcode_thread) = self.transcode_thread.take() {
1231            transcode_thread
1232                .join()
1233                .expect("Failed to join transcode thread")?;
1234        }
1235
1236        self.media_stream_source.RemoveStarting(self.starting)?;
1237        self.media_stream_source
1238            .RemoveSampleRequested(self.sample_requested)?;
1239
1240        Ok(())
1241    }
1242}
1243
1244impl Drop for VideoEncoder {
1245    #[inline]
1246    fn drop(&mut self) {
1247        let _ = self.frame_sender.send(None);
1248
1249        if let Some(transcode_thread) = self.transcode_thread.take() {
1250            let _ = transcode_thread.join();
1251        }
1252    }
1253}
1254
1255#[allow(clippy::non_send_fields_in_send_ty)]
1256unsafe impl Send for VideoEncoder {}
windows_capture/encoder.rs

windows_capture/
encoder.rs
