windows_capture/
encoder.rs

1use std::{
2    fs::{self, File},
3    path::Path,
4    slice,
5    sync::{
6        Arc,
7        atomic::{self, AtomicBool},
8        mpsc,
9    },
10    thread::{self, JoinHandle},
11};
12
13use parking_lot::{Condvar, Mutex};
14use windows::{
15    Foundation::{TimeSpan, TypedEventHandler},
16    Graphics::{
17        DirectX::Direct3D11::IDirect3DSurface,
18        Imaging::{BitmapAlphaMode, BitmapEncoder, BitmapPixelFormat},
19    },
20    Media::{
21        Core::{
22            AudioStreamDescriptor, MediaStreamSample, MediaStreamSource,
23            MediaStreamSourceSampleRequestedEventArgs, MediaStreamSourceStartingEventArgs,
24            VideoStreamDescriptor,
25        },
26        MediaProperties::{
27            AudioEncodingProperties, ContainerEncodingProperties, MediaEncodingProfile,
28            MediaEncodingSubtypes, VideoEncodingProperties,
29        },
30        Transcoding::MediaTranscoder,
31    },
32    Security::Cryptography::CryptographicBuffer,
33    Storage::{
34        FileAccessMode, StorageFile,
35        Streams::{
36            Buffer, DataReader, IRandomAccessStream, InMemoryRandomAccessStream, InputStreamOptions,
37        },
38    },
39    core::{HSTRING, Interface},
40};
41
42use crate::{
43    d3d11::SendDirectX,
44    frame::{Frame, ImageFormat},
45    settings::ColorFormat,
46};
47
48#[derive(thiserror::Error, Eq, PartialEq, Clone, Debug)]
49pub enum ImageEncoderError {
50    #[error("This color format is not supported for saving as image")]
51    UnsupportedFormat,
52    #[error("Windows API Error: {0}")]
53    WindowsError(#[from] windows::core::Error),
54}
55
56/// The `ImageEncoder` struct represents an image encoder that can be used to encode image buffers to image bytes with a specified format and color format.
57pub struct ImageEncoder {
58    format: ImageFormat,
59    color_format: ColorFormat,
60}
61
62impl ImageEncoder {
63    /// Create a new ImageEncoder with the specified format and color format.
64    ///
65    /// # Arguments
66    ///
67    /// * `format` - The desired image format.
68    /// * `color_format` - The desired color format.
69    ///
70    /// # Returns
71    ///
72    /// A new `ImageEncoder` instance.
73    #[must_use]
74    #[inline]
75    pub const fn new(format: ImageFormat, color_format: ColorFormat) -> Self {
76        Self {
77            format,
78            color_format,
79        }
80    }
81
82    /// Encode the image buffer to image bytes with the specified format.
83    ///
84    /// # Arguments
85    ///
86    /// * `image_buffer` - The image buffer to encode.
87    /// * `width` - The width of the image.
88    /// * `height` - The height of the image.
89    ///
90    /// # Returns
91    ///
92    /// The encoded image bytes as a `Vec<u8>`.
93    ///
94    /// # Errors
95    ///
96    /// Returns an `Error` if the encoding fails or if the color format is unsupported.
97    #[inline]
98    pub fn encode(
99        &self,
100        image_buffer: &[u8],
101        width: u32,
102        height: u32,
103    ) -> Result<Vec<u8>, ImageEncoderError> {
104        let encoder = match self.format {
105            ImageFormat::Jpeg => BitmapEncoder::JpegEncoderId()?,
106            ImageFormat::Png => BitmapEncoder::PngEncoderId()?,
107            ImageFormat::Gif => BitmapEncoder::GifEncoderId()?,
108            ImageFormat::Tiff => BitmapEncoder::TiffEncoderId()?,
109            ImageFormat::Bmp => BitmapEncoder::BmpEncoderId()?,
110            ImageFormat::JpegXr => BitmapEncoder::JpegXREncoderId()?,
111        };
112
113        let stream = InMemoryRandomAccessStream::new()?;
114        let encoder = BitmapEncoder::CreateAsync(encoder, &stream)?.get()?;
115
116        let pixelformat = match self.color_format {
117            ColorFormat::Bgra8 => BitmapPixelFormat::Bgra8,
118            ColorFormat::Rgba8 => BitmapPixelFormat::Rgba8,
119            ColorFormat::Rgba16F => return Err(ImageEncoderError::UnsupportedFormat),
120        };
121
122        encoder.SetPixelData(
123            pixelformat,
124            BitmapAlphaMode::Premultiplied,
125            width,
126            height,
127            1.0,
128            1.0,
129            image_buffer,
130        )?;
131
132        encoder.FlushAsync()?.get()?;
133
134        let buffer = Buffer::Create(u32::try_from(stream.Size()?).unwrap())?;
135        stream
136            .ReadAsync(&buffer, buffer.Capacity()?, InputStreamOptions::None)?
137            .get()?;
138
139        let data_reader = DataReader::FromBuffer(&buffer)?;
140        let length = data_reader.UnconsumedBufferLength()?;
141        let mut bytes = vec![0u8; length as usize];
142        data_reader.ReadBytes(&mut bytes)?;
143
144        Ok(bytes)
145    }
146}
147
148#[derive(thiserror::Error, Debug)]
149pub enum VideoEncoderError {
150    #[error("Windows API Error: {0}")]
151    WindowsError(#[from] windows::core::Error),
152    #[error("Frame send error: {0}")]
153    FrameSendError(#[from] mpsc::SendError<Option<(VideoEncoderSource, TimeSpan)>>),
154    #[error("Audio send error: {0}")]
155    AudioSendError(#[from] mpsc::SendError<Option<(AudioEncoderSource, TimeSpan)>>),
156    #[error("Video is disabled")]
157    VideoDisabled,
158    #[error("Audio is disabled")]
159    AudioDisabled,
160    #[error("IO Error: {0}")]
161    IoError(#[from] std::io::Error),
162}
163
164unsafe impl Send for VideoEncoderError {}
165unsafe impl Sync for VideoEncoderError {}
166
167/// The `VideoEncoderSource` struct represents all the types that can be send to the encoder.
168pub enum VideoEncoderSource {
169    DirectX(SendDirectX<IDirect3DSurface>),
170    Buffer((SendDirectX<*const u8>, usize)),
171}
172
173/// The `AudioEncoderSource` struct represents all the types that can be send to the encoder.
174pub enum AudioEncoderSource {
175    Buffer((SendDirectX<*const u8>, usize)),
176}
177
178/// The `VideoSettings` struct represents the settings for the video encoder.
179pub struct VideoSettingsBuilder {
180    sub_type: VideoSettingsSubType,
181    bitrate: u32,
182    width: u32,
183    height: u32,
184    frame_rate: u32,
185    pixel_aspect_ratio: (u32, u32),
186    disabled: bool,
187}
188
189impl VideoSettingsBuilder {
190    pub const fn new(width: u32, height: u32) -> Self {
191        Self {
192            bitrate: 15000000,
193            frame_rate: 60,
194            pixel_aspect_ratio: (1, 1),
195            sub_type: VideoSettingsSubType::HEVC,
196            width,
197            height,
198            disabled: false,
199        }
200    }
201
202    pub const fn sub_type(mut self, sub_type: VideoSettingsSubType) -> Self {
203        self.sub_type = sub_type;
204        self
205    }
206
207    pub const fn bitrate(mut self, bitrate: u32) -> Self {
208        self.bitrate = bitrate;
209        self
210    }
211
212    pub const fn width(mut self, width: u32) -> Self {
213        self.width = width;
214        self
215    }
216
217    pub const fn height(mut self, height: u32) -> Self {
218        self.height = height;
219        self
220    }
221
222    pub const fn frame_rate(mut self, frame_rate: u32) -> Self {
223        self.frame_rate = frame_rate;
224        self
225    }
226
227    pub const fn pixel_aspect_ratio(mut self, pixel_aspect_ratio: (u32, u32)) -> Self {
228        self.pixel_aspect_ratio = pixel_aspect_ratio;
229        self
230    }
231
232    pub const fn disabled(mut self, disabled: bool) -> Self {
233        self.disabled = disabled;
234        self
235    }
236
237    fn build(self) -> Result<(VideoEncodingProperties, bool), VideoEncoderError> {
238        let properties = VideoEncodingProperties::new()?;
239
240        properties.SetSubtype(&self.sub_type.to_hstring())?;
241        properties.SetBitrate(self.bitrate)?;
242        properties.SetWidth(self.width)?;
243        properties.SetHeight(self.height)?;
244        properties.FrameRate()?.SetNumerator(self.frame_rate)?;
245        properties.FrameRate()?.SetDenominator(1)?;
246        properties
247            .PixelAspectRatio()?
248            .SetNumerator(self.pixel_aspect_ratio.0)?;
249        properties
250            .PixelAspectRatio()?
251            .SetDenominator(self.pixel_aspect_ratio.1)?;
252
253        Ok((properties, self.disabled))
254    }
255}
256
257/// The `AudioSettingsSubType` enum represents the settings for the audio encoder.
258pub struct AudioSettingsBuilder {
259    bitrate: u32,
260    channel_count: u32,
261    sample_rate: u32,
262    bit_per_sample: u32,
263    sub_type: AudioSettingsSubType,
264    disabled: bool,
265}
266
267impl AudioSettingsBuilder {
268    pub const fn new() -> Self {
269        Self {
270            bitrate: 192000,
271            channel_count: 2,
272            sample_rate: 48000,
273            bit_per_sample: 16,
274            sub_type: AudioSettingsSubType::AAC,
275            disabled: false,
276        }
277    }
278    pub const fn bitrate(mut self, bitrate: u32) -> Self {
279        self.bitrate = bitrate;
280        self
281    }
282
283    pub const fn channel_count(mut self, channel_count: u32) -> Self {
284        self.channel_count = channel_count;
285        self
286    }
287
288    pub const fn sample_rate(mut self, sample_rate: u32) -> Self {
289        self.sample_rate = sample_rate;
290        self
291    }
292
293    pub const fn bit_per_sample(mut self, bit_per_sample: u32) -> Self {
294        self.bit_per_sample = bit_per_sample;
295        self
296    }
297
298    pub const fn sub_type(mut self, sub_type: AudioSettingsSubType) -> Self {
299        self.sub_type = sub_type;
300        self
301    }
302
303    pub const fn disabled(mut self, disabled: bool) -> Self {
304        self.disabled = disabled;
305        self
306    }
307
308    fn build(self) -> Result<(AudioEncodingProperties, bool), VideoEncoderError> {
309        let properties = AudioEncodingProperties::new()?;
310        properties.SetBitrate(self.bitrate)?;
311        properties.SetChannelCount(self.channel_count)?;
312        properties.SetSampleRate(self.sample_rate)?;
313        properties.SetBitsPerSample(self.bit_per_sample)?;
314        properties.SetSubtype(&self.sub_type.to_hstring())?;
315
316        Ok((properties, self.disabled))
317    }
318}
319
320impl Default for AudioSettingsBuilder {
321    fn default() -> Self {
322        Self::new()
323    }
324}
325
326/// The `ContainerSettingsSubType` enum represents the settings for the container encoder.
327pub struct ContainerSettingsBuilder {
328    sub_type: ContainerSettingsSubType,
329}
330
331impl ContainerSettingsBuilder {
332    pub const fn new() -> Self {
333        Self {
334            sub_type: ContainerSettingsSubType::MPEG4,
335        }
336    }
337
338    pub const fn sub_type(mut self, sub_type: ContainerSettingsSubType) -> Self {
339        self.sub_type = sub_type;
340        self
341    }
342
343    fn build(self) -> Result<ContainerEncodingProperties, VideoEncoderError> {
344        let properties = ContainerEncodingProperties::new()?;
345        properties.SetSubtype(&self.sub_type.to_hstring())?;
346        Ok(properties)
347    }
348}
349
350impl Default for ContainerSettingsBuilder {
351    fn default() -> Self {
352        Self::new()
353    }
354}
355
356/// The `VideoSettingsSubType` enum represents the subtypes for the video encoder.
357#[derive(Eq, PartialEq, Clone, Copy, Debug)]
358pub enum VideoSettingsSubType {
359    ARGB32,
360    BGRA8,
361    D16,
362    H263,
363    H264,
364    H264ES,
365    HEVC,
366    HEVCES,
367    IYUV,
368    L8,
369    L16,
370    MJPG,
371    NV12,
372    MPEG1,
373    MPEG2,
374    RGB24,
375    RGB32,
376    WMV3,
377    WVC1,
378    VP9,
379    YUY2,
380    YV12,
381}
382
383impl VideoSettingsSubType {
384    pub fn to_hstring(&self) -> HSTRING {
385        let s = match self {
386            Self::ARGB32 => "ARGB32",
387            Self::BGRA8 => "BGRA8",
388            Self::D16 => "D16",
389            Self::H263 => "H263",
390            Self::H264 => "H264",
391            Self::H264ES => "H264ES",
392            Self::HEVC => "HEVC",
393            Self::HEVCES => "HEVCES",
394            Self::IYUV => "IYUV",
395            Self::L8 => "L8",
396            Self::L16 => "L16",
397            Self::MJPG => "MJPG",
398            Self::NV12 => "NV12",
399            Self::MPEG1 => "MPEG1",
400            Self::MPEG2 => "MPEG2",
401            Self::RGB24 => "RGB24",
402            Self::RGB32 => "RGB32",
403            Self::WMV3 => "WMV3",
404            Self::WVC1 => "WVC1",
405            Self::VP9 => "VP9",
406            Self::YUY2 => "YUY2",
407            Self::YV12 => "YV12",
408        };
409
410        HSTRING::from(s)
411    }
412}
413
414/// The `AudioSettingsSubType` enum represents the subtypes for the audio encoder.
415#[derive(Eq, PartialEq, Clone, Copy, Debug)]
416pub enum AudioSettingsSubType {
417    AAC,
418    AC3,
419    AACADTS,
420    AACHDCP,
421    AC3SPDIF,
422    AC3HDCP,
423    ADTS,
424    ALAC,
425    AMRNB,
426    AWRWB,
427    DTS,
428    EAC3,
429    FLAC,
430    Float,
431    MP3,
432    MPEG,
433    OPUS,
434    PCM,
435    WMA8,
436    WMA9,
437    Vorbis,
438}
439
440impl AudioSettingsSubType {
441    pub fn to_hstring(&self) -> HSTRING {
442        let s = match self {
443            Self::AAC => "AAC",
444            Self::AC3 => "AC3",
445            Self::AACADTS => "AACADTS",
446            Self::AACHDCP => "AACHDCP",
447            Self::AC3SPDIF => "AC3SPDIF",
448            Self::AC3HDCP => "AC3HDCP",
449            Self::ADTS => "ADTS",
450            Self::ALAC => "ALAC",
451            Self::AMRNB => "AMRNB",
452            Self::AWRWB => "AWRWB",
453            Self::DTS => "DTS",
454            Self::EAC3 => "EAC3",
455            Self::FLAC => "FLAC",
456            Self::Float => "Float",
457            Self::MP3 => "MP3",
458            Self::MPEG => "MPEG",
459            Self::OPUS => "OPUS",
460            Self::PCM => "PCM",
461            Self::WMA8 => "WMA8",
462            Self::WMA9 => "WMA9",
463            Self::Vorbis => "Vorbis",
464        };
465
466        HSTRING::from(s)
467    }
468}
469
470/// The `Subtype` enum represents the subtypes for the video encoder.
471#[derive(Eq, PartialEq, Clone, Copy, Debug)]
472pub enum ContainerSettingsSubType {
473    ASF,
474    MP3,
475    MPEG4,
476    AVI,
477    MPEG2,
478    WAVE,
479    AACADTS,
480    ADTS,
481    GP3,
482    AMR,
483    FLAC,
484}
485
486impl ContainerSettingsSubType {
487    pub fn to_hstring(&self) -> HSTRING {
488        match self {
489            Self::ASF => HSTRING::from("ASF"),
490            Self::MP3 => HSTRING::from("MP3"),
491            Self::MPEG4 => HSTRING::from("MPEG4"),
492            Self::AVI => HSTRING::from("AVI"),
493            Self::MPEG2 => HSTRING::from("MPEG2"),
494            Self::WAVE => HSTRING::from("WAVE"),
495            Self::AACADTS => HSTRING::from("AACADTS"),
496            Self::ADTS => HSTRING::from("ADTS"),
497            Self::GP3 => HSTRING::from("3GP"),
498            Self::AMR => HSTRING::from("AMR"),
499            Self::FLAC => HSTRING::from("FLAC"),
500        }
501    }
502}
503
504/// The `VideoEncoder` struct represents a video encoder that can be used to encode video frames and save them to a specified file path.
505pub struct VideoEncoder {
506    first_timespan: Option<TimeSpan>,
507    frame_sender: mpsc::Sender<Option<(VideoEncoderSource, TimeSpan)>>,
508    audio_sender: mpsc::Sender<Option<(AudioEncoderSource, TimeSpan)>>,
509    sample_requested: i64,
510    media_stream_source: MediaStreamSource,
511    starting: i64,
512    transcode_thread: Option<JoinHandle<Result<(), VideoEncoderError>>>,
513    frame_notify: Arc<(Mutex<bool>, Condvar)>,
514    audio_notify: Arc<(Mutex<bool>, Condvar)>,
515    error_notify: Arc<AtomicBool>,
516    is_video_disabled: bool,
517    is_audio_disabled: bool,
518}
519
520impl VideoEncoder {
521    /// Creates a new `VideoEncoder` instance with the specified parameters.
522    ///
523    /// # Arguments
524    ///
525    /// * `encoder_type` - The type of video encoder to use.
526    /// * `encoder_quality` - The quality of the video encoder.
527    /// * `width` - The width of the video frames.
528    /// * `height` - The height of the video frames.
529    /// * `path` - The file path where the encoded video will be saved.
530    ///
531    /// # Returns
532    ///
533    /// Returns a `Result` containing the `VideoEncoder` instance if successful, or a
534    /// `VideoEncoderError` if an error occurs.
535    #[inline]
536    pub fn new<P: AsRef<Path>>(
537        video_settings: VideoSettingsBuilder,
538        audio_settings: AudioSettingsBuilder,
539        container_settings: ContainerSettingsBuilder,
540        path: P,
541    ) -> Result<Self, VideoEncoderError> {
542        let path = path.as_ref();
543        let media_encoding_profile = MediaEncodingProfile::new()?;
544
545        let (video_encoding_properties, is_video_disabled) = video_settings.build()?;
546        media_encoding_profile.SetVideo(&video_encoding_properties)?;
547        let (audio_encoding_properties, is_audio_disabled) = audio_settings.build()?;
548        media_encoding_profile.SetAudio(&audio_encoding_properties)?;
549        let container_encoding_properties = container_settings.build()?;
550        media_encoding_profile.SetContainer(&container_encoding_properties)?;
551
552        let video_encoding_properties = VideoEncodingProperties::CreateUncompressed(
553            &MediaEncodingSubtypes::Bgra8()?,
554            video_encoding_properties.Width()?,
555            video_encoding_properties.Height()?,
556        )?;
557        let video_stream_descriptor = VideoStreamDescriptor::Create(&video_encoding_properties)?;
558
559        let audio_encoding_properties = AudioEncodingProperties::CreateAac(
560            audio_encoding_properties.SampleRate()?,
561            audio_encoding_properties.ChannelCount()?,
562            audio_encoding_properties.Bitrate()?,
563        )?;
564        let audio_stream_descriptor = AudioStreamDescriptor::Create(&audio_encoding_properties)?;
565
566        let media_stream_source = MediaStreamSource::CreateFromDescriptors(
567            &video_stream_descriptor,
568            &audio_stream_descriptor,
569        )?;
570        media_stream_source.SetBufferTime(TimeSpan::default())?;
571
572        let starting = media_stream_source.Starting(&TypedEventHandler::<
573            MediaStreamSource,
574            MediaStreamSourceStartingEventArgs,
575        >::new(move |_, stream_start| {
576            let stream_start = stream_start
577                .as_ref()
578                .expect("MediaStreamSource Starting parameter was None This Should Not Happen.");
579
580            stream_start
581                .Request()?
582                .SetActualStartPosition(TimeSpan { Duration: 0 })?;
583            Ok(())
584        }))?;
585
586        let (frame_sender, frame_receiver) =
587            mpsc::channel::<Option<(VideoEncoderSource, TimeSpan)>>();
588
589        let (audio_sender, audio_receiver) =
590            mpsc::channel::<Option<(AudioEncoderSource, TimeSpan)>>();
591
592        let frame_notify = Arc::new((Mutex::new(false), Condvar::new()));
593        let audio_notify = Arc::new((Mutex::new(false), Condvar::new()));
594
595        let sample_requested = media_stream_source.SampleRequested(&TypedEventHandler::<
596            MediaStreamSource,
597            MediaStreamSourceSampleRequestedEventArgs,
598        >::new({
599            let frame_receiver = frame_receiver;
600            let frame_notify = frame_notify.clone();
601
602            let audio_receiver = audio_receiver;
603            let audio_notify = audio_notify.clone();
604
605            move |_, sample_requested| {
606                let sample_requested = sample_requested.as_ref().expect(
607                    "MediaStreamSource SampleRequested parameter was None This Should Not Happen.",
608                );
609
610                if sample_requested
611                    .Request()?
612                    .StreamDescriptor()?
613                    .cast::<AudioStreamDescriptor>()
614                    .is_ok()
615                {
616                    if is_audio_disabled {
617                        sample_requested.Request()?.SetSample(None)?;
618
619                        return Ok(());
620                    }
621
622                    let audio = match audio_receiver.recv() {
623                        Ok(audio) => audio,
624                        Err(e) => panic!("Failed to receive audio from audio sender: {e}"),
625                    };
626
627                    match audio {
628                        Some((source, timespan)) => {
629                            let sample = match source {
630                                AudioEncoderSource::Buffer(buffer_data) => {
631                                    let buffer = buffer_data.0;
632                                    let buffer =
633                                        unsafe { slice::from_raw_parts(buffer.0, buffer_data.1) };
634                                    let buffer = CryptographicBuffer::CreateFromByteArray(buffer)?;
635                                    MediaStreamSample::CreateFromBuffer(&buffer, timespan)?
636                                }
637                            };
638
639                            sample_requested.Request()?.SetSample(&sample)?;
640                        }
641                        None => {
642                            sample_requested.Request()?.SetSample(None)?;
643                        }
644                    }
645
646                    let (lock, cvar) = &*audio_notify;
647                    *lock.lock() = true;
648                    cvar.notify_one();
649                } else {
650                    if is_video_disabled {
651                        sample_requested.Request()?.SetSample(None)?;
652
653                        return Ok(());
654                    }
655
656                    let frame = match frame_receiver.recv() {
657                        Ok(frame) => frame,
658                        Err(e) => panic!("Failed to receive frame from frame sender: {e}"),
659                    };
660
661                    match frame {
662                        Some((source, timespan)) => {
663                            let sample = match source {
664                                VideoEncoderSource::DirectX(surface) => {
665                                    MediaStreamSample::CreateFromDirect3D11Surface(
666                                        &surface.0, timespan,
667                                    )?
668                                }
669                                VideoEncoderSource::Buffer(buffer_data) => {
670                                    let buffer = buffer_data.0;
671                                    let buffer =
672                                        unsafe { slice::from_raw_parts(buffer.0, buffer_data.1) };
673                                    let buffer = CryptographicBuffer::CreateFromByteArray(buffer)?;
674                                    MediaStreamSample::CreateFromBuffer(&buffer, timespan)?
675                                }
676                            };
677
678                            sample_requested.Request()?.SetSample(&sample)?;
679                        }
680                        None => {
681                            sample_requested.Request()?.SetSample(None)?;
682                        }
683                    }
684
685                    let (lock, cvar) = &*frame_notify;
686                    *lock.lock() = true;
687                    cvar.notify_one();
688                }
689
690                Ok(())
691            }
692        }))?;
693
694        let media_transcoder = MediaTranscoder::new()?;
695        media_transcoder.SetHardwareAccelerationEnabled(true)?;
696
697        File::create(path)?;
698        let path = fs::canonicalize(path).unwrap().to_string_lossy()[4..].to_string();
699        let path = Path::new(&path);
700
701        let path = &HSTRING::from(path.as_os_str().to_os_string());
702
703        let file = StorageFile::GetFileFromPathAsync(path)?.get()?;
704        let media_stream_output = file.OpenAsync(FileAccessMode::ReadWrite)?.get()?;
705
706        let transcode = media_transcoder
707            .PrepareMediaStreamSourceTranscodeAsync(
708                &media_stream_source,
709                &media_stream_output,
710                &media_encoding_profile,
711            )?
712            .get()?;
713
714        let error_notify = Arc::new(AtomicBool::new(false));
715        let transcode_thread = thread::spawn({
716            let error_notify = error_notify.clone();
717
718            move || -> Result<(), VideoEncoderError> {
719                let result = transcode.TranscodeAsync();
720
721                if result.is_err() {
722                    error_notify.store(true, atomic::Ordering::Relaxed);
723                }
724
725                result?.get()?;
726
727                drop(media_transcoder);
728
729                Ok(())
730            }
731        });
732
733        Ok(Self {
734            first_timespan: None,
735            frame_sender,
736            audio_sender,
737            sample_requested,
738            media_stream_source,
739            starting,
740            transcode_thread: Some(transcode_thread),
741            frame_notify,
742            audio_notify,
743            error_notify,
744            is_video_disabled,
745            is_audio_disabled,
746        })
747    }
748
749    /// Creates a new `VideoEncoder` instance with the specified parameters.
750    ///
751    /// # Arguments
752    ///
753    /// * `encoder_type` - The type of video encoder to use.
754    /// * `encoder_quality` - The quality of the video encoder.
755    /// * `width` - The width of the video frames.
756    /// * `height` - The height of the video frames.
757    /// * `stream` - The stream where the encoded video will be saved.
758    ///
759    /// # Returns
760    ///
761    /// Returns a `Result` containing the `VideoEncoder` instance if successful, or a
762    /// `VideoEncoderError` if an error occurs.
763    #[inline]
764    pub fn new_from_stream(
765        video_settings: VideoSettingsBuilder,
766        audio_settings: AudioSettingsBuilder,
767        container_settings: ContainerSettingsBuilder,
768        stream: IRandomAccessStream,
769    ) -> Result<Self, VideoEncoderError> {
770        let media_encoding_profile = MediaEncodingProfile::new()?;
771
772        let (video_encoding_properties, is_video_disabled) = video_settings.build()?;
773        media_encoding_profile.SetVideo(&video_encoding_properties)?;
774        let (audio_encoding_properties, is_audio_disabled) = audio_settings.build()?;
775        media_encoding_profile.SetAudio(&audio_encoding_properties)?;
776        let container_encoding_properties = container_settings.build()?;
777        media_encoding_profile.SetContainer(&container_encoding_properties)?;
778
779        let video_encoding_properties = VideoEncodingProperties::CreateUncompressed(
780            &MediaEncodingSubtypes::Bgra8()?,
781            video_encoding_properties.Width()?,
782            video_encoding_properties.Height()?,
783        )?;
784        let video_stream_descriptor = VideoStreamDescriptor::Create(&video_encoding_properties)?;
785
786        let audio_encoding_properties = AudioEncodingProperties::CreateAac(
787            audio_encoding_properties.SampleRate()?,
788            audio_encoding_properties.ChannelCount()?,
789            audio_encoding_properties.Bitrate()?,
790        )?;
791        let audio_stream_descriptor = AudioStreamDescriptor::Create(&audio_encoding_properties)?;
792
793        let media_stream_source = MediaStreamSource::CreateFromDescriptors(
794            &video_stream_descriptor,
795            &audio_stream_descriptor,
796        )?;
797        media_stream_source.SetBufferTime(TimeSpan::default())?;
798
799        let starting = media_stream_source.Starting(&TypedEventHandler::<
800            MediaStreamSource,
801            MediaStreamSourceStartingEventArgs,
802        >::new(move |_, stream_start| {
803            let stream_start = stream_start
804                .as_ref()
805                .expect("MediaStreamSource Starting parameter was None This Should Not Happen.");
806
807            stream_start
808                .Request()?
809                .SetActualStartPosition(TimeSpan { Duration: 0 })?;
810            Ok(())
811        }))?;
812
813        let (frame_sender, frame_receiver) =
814            mpsc::channel::<Option<(VideoEncoderSource, TimeSpan)>>();
815
816        let (audio_sender, audio_receiver) =
817            mpsc::channel::<Option<(AudioEncoderSource, TimeSpan)>>();
818
819        let frame_notify = Arc::new((Mutex::new(false), Condvar::new()));
820        let audio_notify = Arc::new((Mutex::new(false), Condvar::new()));
821
822        let sample_requested = media_stream_source.SampleRequested(&TypedEventHandler::<
823            MediaStreamSource,
824            MediaStreamSourceSampleRequestedEventArgs,
825        >::new({
826            let frame_receiver = frame_receiver;
827            let frame_notify = frame_notify.clone();
828
829            let audio_receiver = audio_receiver;
830            let audio_notify = audio_notify.clone();
831
832            move |_, sample_requested| {
833                let sample_requested = sample_requested.as_ref().expect(
834                    "MediaStreamSource SampleRequested parameter was None This Should Not Happen.",
835                );
836
837                if sample_requested
838                    .Request()?
839                    .StreamDescriptor()?
840                    .cast::<AudioStreamDescriptor>()
841                    .is_ok()
842                {
843                    if is_audio_disabled {
844                        sample_requested.Request()?.SetSample(None)?;
845
846                        return Ok(());
847                    }
848
849                    let audio = match audio_receiver.recv() {
850                        Ok(audio) => audio,
851                        Err(e) => panic!("Failed to receive audio from audio sender: {e}"),
852                    };
853
854                    match audio {
855                        Some((source, timespan)) => {
856                            let sample = match source {
857                                AudioEncoderSource::Buffer(buffer_data) => {
858                                    let buffer = buffer_data.0;
859                                    let buffer =
860                                        unsafe { slice::from_raw_parts(buffer.0, buffer_data.1) };
861                                    let buffer = CryptographicBuffer::CreateFromByteArray(buffer)?;
862                                    MediaStreamSample::CreateFromBuffer(&buffer, timespan)?
863                                }
864                            };
865
866                            sample_requested.Request()?.SetSample(&sample)?;
867                        }
868                        None => {
869                            sample_requested.Request()?.SetSample(None)?;
870                        }
871                    }
872
873                    let (lock, cvar) = &*audio_notify;
874                    *lock.lock() = true;
875                    cvar.notify_one();
876                } else {
877                    if is_video_disabled {
878                        sample_requested.Request()?.SetSample(None)?;
879
880                        return Ok(());
881                    }
882
883                    let frame = match frame_receiver.recv() {
884                        Ok(frame) => frame,
885                        Err(e) => panic!("Failed to receive frame from frame sender: {e}"),
886                    };
887
888                    match frame {
889                        Some((source, timespan)) => {
890                            let sample = match source {
891                                VideoEncoderSource::DirectX(surface) => {
892                                    MediaStreamSample::CreateFromDirect3D11Surface(
893                                        &surface.0, timespan,
894                                    )?
895                                }
896                                VideoEncoderSource::Buffer(buffer_data) => {
897                                    let buffer = buffer_data.0;
898                                    let buffer =
899                                        unsafe { slice::from_raw_parts(buffer.0, buffer_data.1) };
900                                    let buffer = CryptographicBuffer::CreateFromByteArray(buffer)?;
901                                    MediaStreamSample::CreateFromBuffer(&buffer, timespan)?
902                                }
903                            };
904
905                            sample_requested.Request()?.SetSample(&sample)?;
906                        }
907                        None => {
908                            sample_requested.Request()?.SetSample(None)?;
909                        }
910                    }
911
912                    let (lock, cvar) = &*frame_notify;
913                    *lock.lock() = true;
914                    cvar.notify_one();
915                }
916
917                Ok(())
918            }
919        }))?;
920
921        let media_transcoder = MediaTranscoder::new()?;
922        media_transcoder.SetHardwareAccelerationEnabled(true)?;
923
924        let transcode = media_transcoder
925            .PrepareMediaStreamSourceTranscodeAsync(
926                &media_stream_source,
927                &stream,
928                &media_encoding_profile,
929            )?
930            .get()?;
931
932        let error_notify = Arc::new(AtomicBool::new(false));
933        let transcode_thread = thread::spawn({
934            let error_notify = error_notify.clone();
935
936            move || -> Result<(), VideoEncoderError> {
937                let result = transcode.TranscodeAsync();
938
939                if result.is_err() {
940                    error_notify.store(true, atomic::Ordering::Relaxed);
941                }
942
943                result?.get()?;
944
945                drop(media_transcoder);
946
947                Ok(())
948            }
949        });
950
951        Ok(Self {
952            first_timespan: None,
953            frame_sender,
954            audio_sender,
955            sample_requested,
956            media_stream_source,
957            starting,
958            transcode_thread: Some(transcode_thread),
959            frame_notify,
960            audio_notify,
961            error_notify,
962            is_video_disabled,
963            is_audio_disabled,
964        })
965    }
966
967    /// Sends a video frame to the video encoder for encoding.
968    ///
969    /// # Arguments
970    ///
971    /// * `frame` - A mutable reference to the `Frame` to be encoded.
972    ///
973    /// # Returns
974    ///
975    /// Returns `Ok(())` if the frame is successfully sent for encoding, or a `VideoEncoderError`
976    /// if an error occurs.
977    #[inline]
978    pub fn send_frame(&mut self, frame: &mut Frame) -> Result<(), VideoEncoderError> {
979        if self.is_video_disabled {
980            return Err(VideoEncoderError::VideoDisabled);
981        }
982
983        let timespan = match self.first_timespan {
984            Some(timespan) => TimeSpan {
985                Duration: frame.timespan().Duration - timespan.Duration,
986            },
987            None => {
988                let timespan = frame.timespan();
989                self.first_timespan = Some(timespan);
990                TimeSpan { Duration: 0 }
991            }
992        };
993
994        self.frame_sender.send(Some((
995            VideoEncoderSource::DirectX(SendDirectX::new(unsafe {
996                frame.as_raw_surface().clone()
997            })),
998            timespan,
999        )))?;
1000
1001        let (lock, cvar) = &*self.frame_notify;
1002        let mut processed = lock.lock();
1003        if !*processed {
1004            cvar.wait(&mut processed);
1005        }
1006        *processed = false;
1007        drop(processed);
1008
1009        if self.error_notify.load(atomic::Ordering::Relaxed) {
1010            if let Some(transcode_thread) = self.transcode_thread.take() {
1011                transcode_thread
1012                    .join()
1013                    .expect("Failed to join transcode thread")?;
1014            }
1015        }
1016
1017        Ok(())
1018    }
1019
1020    /// Sends a video frame with audio to the video encoder for encoding.
1021    ///
1022    /// # Arguments
1023    ///
1024    /// * `frame` - A mutable reference to the `Frame` to be encoded.
1025    /// * `audio_buffer` - A reference to the audio byte slice to be encoded.
1026    ///
1027    /// # Returns
1028    ///
1029    /// Returns `Ok(())` if the frame is successfully sent for encoding, or a `VideoEncoderError`
1030    /// if an error occurs.
1031    #[inline]
1032    pub fn send_frame_with_audio(
1033        &mut self,
1034        frame: &mut Frame,
1035        audio_buffer: &[u8],
1036    ) -> Result<(), VideoEncoderError> {
1037        if self.is_video_disabled {
1038            return Err(VideoEncoderError::VideoDisabled);
1039        }
1040
1041        if self.is_audio_disabled {
1042            return Err(VideoEncoderError::AudioDisabled);
1043        }
1044
1045        let timespan = match self.first_timespan {
1046            Some(timespan) => TimeSpan {
1047                Duration: frame.timespan().Duration - timespan.Duration,
1048            },
1049            None => {
1050                let timespan = frame.timespan();
1051                self.first_timespan = Some(timespan);
1052                TimeSpan { Duration: 0 }
1053            }
1054        };
1055
1056        self.frame_sender.send(Some((
1057            VideoEncoderSource::DirectX(SendDirectX::new(unsafe {
1058                frame.as_raw_surface().clone()
1059            })),
1060            timespan,
1061        )))?;
1062
1063        let (lock, cvar) = &*self.frame_notify;
1064        let mut processed = lock.lock();
1065        if !*processed {
1066            cvar.wait(&mut processed);
1067        }
1068        *processed = false;
1069        drop(processed);
1070
1071        if self.error_notify.load(atomic::Ordering::Relaxed) {
1072            if let Some(transcode_thread) = self.transcode_thread.take() {
1073                transcode_thread
1074                    .join()
1075                    .expect("Failed to join transcode thread")?;
1076            }
1077        }
1078
1079        self.audio_sender.send(Some((
1080            AudioEncoderSource::Buffer((
1081                SendDirectX::new(audio_buffer.as_ptr()),
1082                audio_buffer.len(),
1083            )),
1084            timespan,
1085        )))?;
1086
1087        let (lock, cvar) = &*self.audio_notify;
1088        let mut processed = lock.lock();
1089        if !*processed {
1090            cvar.wait(&mut processed);
1091        }
1092        *processed = false;
1093        drop(processed);
1094
1095        if self.error_notify.load(atomic::Ordering::Relaxed) {
1096            if let Some(transcode_thread) = self.transcode_thread.take() {
1097                transcode_thread
1098                    .join()
1099                    .expect("Failed to join transcode thread")?;
1100            }
1101        }
1102
1103        Ok(())
1104    }
1105
1106    /// Sends a video frame to the video encoder for encoding.
1107    ///
1108    /// # Arguments
1109    ///
1110    /// * `buffer` - A reference to the frame byte slice to be encoded Windows API expect this to be Bgra and bottom-top.
1111    /// * `timespan` - The timespan that correlates to the frame buffer.
1112    ///
1113    /// # Returns
1114    ///
1115    /// Returns `Ok(())` if the frame is successfully sent for encoding, or a `VideoEncoderError`
1116    /// if an error occurs.
1117    #[inline]
1118    pub fn send_frame_buffer(
1119        &mut self,
1120        buffer: &[u8],
1121        timespan: i64,
1122    ) -> Result<(), VideoEncoderError> {
1123        if self.is_video_disabled {
1124            return Err(VideoEncoderError::VideoDisabled);
1125        }
1126
1127        let frame_timespan = timespan;
1128        let timespan = match self.first_timespan {
1129            Some(timespan) => TimeSpan {
1130                Duration: frame_timespan - timespan.Duration,
1131            },
1132            None => {
1133                let timespan = frame_timespan;
1134                self.first_timespan = Some(TimeSpan { Duration: timespan });
1135                TimeSpan { Duration: 0 }
1136            }
1137        };
1138
1139        self.frame_sender.send(Some((
1140            VideoEncoderSource::Buffer((SendDirectX::new(buffer.as_ptr()), buffer.len())),
1141            timespan,
1142        )))?;
1143
1144        let (lock, cvar) = &*self.frame_notify;
1145        let mut processed = lock.lock();
1146        if !*processed {
1147            cvar.wait(&mut processed);
1148        }
1149        *processed = false;
1150        drop(processed);
1151
1152        if self.error_notify.load(atomic::Ordering::Relaxed) {
1153            if let Some(transcode_thread) = self.transcode_thread.take() {
1154                transcode_thread
1155                    .join()
1156                    .expect("Failed to join transcode thread")?;
1157            }
1158        }
1159
1160        Ok(())
1161    }
1162
1163    /// Sends a video audio to the video encoder for encoding.
1164    ///
1165    /// # Arguments
1166    ///
1167    /// * `buffer` - A reference to the audio byte slice to be encoded.
1168    /// * `timespan` - The timespan that correlates to the frame buffer.
1169    ///
1170    /// # Returns
1171    ///
1172    /// Returns `Ok(())` if the frame is successfully sent for encoding, or a `VideoEncoderError`
1173    /// if an error occurs.
1174    #[inline]
1175    pub fn send_audio_buffer(
1176        &mut self,
1177        buffer: &[u8],
1178        timespan: i64,
1179    ) -> Result<(), VideoEncoderError> {
1180        if self.is_audio_disabled {
1181            return Err(VideoEncoderError::AudioDisabled);
1182        }
1183
1184        let audio_timespan = timespan;
1185        let timespan = match self.first_timespan {
1186            Some(timespan) => TimeSpan {
1187                Duration: audio_timespan - timespan.Duration,
1188            },
1189            None => {
1190                let timespan = audio_timespan;
1191                self.first_timespan = Some(TimeSpan { Duration: timespan });
1192                TimeSpan { Duration: 0 }
1193            }
1194        };
1195
1196        self.audio_sender.send(Some((
1197            AudioEncoderSource::Buffer((SendDirectX::new(buffer.as_ptr()), buffer.len())),
1198            timespan,
1199        )))?;
1200
1201        let (lock, cvar) = &*self.audio_notify;
1202        let mut processed = lock.lock();
1203        if !*processed {
1204            cvar.wait(&mut processed);
1205        }
1206        *processed = false;
1207        drop(processed);
1208
1209        if self.error_notify.load(atomic::Ordering::Relaxed) {
1210            if let Some(transcode_thread) = self.transcode_thread.take() {
1211                transcode_thread
1212                    .join()
1213                    .expect("Failed to join transcode thread")?;
1214            }
1215        }
1216
1217        Ok(())
1218    }
1219
1220    /// Finishes encoding the video and performs any necessary cleanup.
1221    ///
1222    /// # Returns
1223    ///
1224    /// Returns `Ok(())` if the encoding is successfully finished, or a `VideoEncoderError` if an
1225    /// error occurs.
1226    #[inline]
1227    pub fn finish(mut self) -> Result<(), VideoEncoderError> {
1228        self.frame_sender.send(None)?;
1229        self.audio_sender.send(None)?;
1230
1231        if let Some(transcode_thread) = self.transcode_thread.take() {
1232            transcode_thread
1233                .join()
1234                .expect("Failed to join transcode thread")?;
1235        }
1236
1237        self.media_stream_source.RemoveStarting(self.starting)?;
1238        self.media_stream_source
1239            .RemoveSampleRequested(self.sample_requested)?;
1240
1241        Ok(())
1242    }
1243}
1244
1245impl Drop for VideoEncoder {
1246    #[inline]
1247    fn drop(&mut self) {
1248        let _ = self.frame_sender.send(None);
1249
1250        if let Some(transcode_thread) = self.transcode_thread.take() {
1251            let _ = transcode_thread.join();
1252        }
1253    }
1254}
1255
1256#[allow(clippy::non_send_fields_in_send_ty)]
1257unsafe impl Send for VideoEncoder {}
windows_capture/encoder.rs

windows_capture/
encoder.rs
