media-pp 0.2.0

A small, GStreamer-flavored media pipeline library built on FFmpeg. Capture, composite and encode without leaving the GPU, on D3D11 and CUDA.
Documentation
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
//! The crate-wide error type.
//!
//! [`enum@Error`] is the aggregate an element's own error converts into with `?`,
//! so that a pipeline built from unrelated elements still has one return type.
//! Each variant wraps a component error — `thiserror` enums that stay actionable
//! on their own, documented next to the element that produces them.
//!
//! Backend variants are behind the same Cargo features as the elements that
//! raise them, so this enum is exactly as wide as the build it belongs to.

use std::io;

use thiserror::Error;

#[cfg(all(target_os = "windows", feature = "dxgi-capture"))]
use crate::elements::DxgiCaptureSourceError;
#[cfg(all(target_os = "windows", feature = "mf-capture"))]
use crate::elements::MfCaptureSourceError;
#[cfg(feature = "ort")]
use crate::elements::OrtDetectorError;
#[cfg(all(target_os = "linux", feature = "pipewire-audio-capture"))]
use crate::elements::PipeWireAudioCaptureSourceError;
#[cfg(all(target_os = "linux", feature = "pipewire-audio-renderer"))]
use crate::elements::PipeWireAudioRendererError;
#[cfg(all(target_os = "linux", feature = "pipewire-screen-capture"))]
use crate::elements::PipeWireScreenCaptureSourceError;
use crate::elements::RtspSinkError;
#[cfg(all(target_os = "linux", feature = "v4l2-capture"))]
use crate::elements::V4l2CaptureSourceError;
#[cfg(all(target_os = "windows", feature = "wasapi-capture"))]
use crate::elements::WasapiCaptureSourceError;
#[cfg(all(target_os = "windows", feature = "wasapi-renderer"))]
use crate::elements::WasapiRendererError;
#[cfg(feature = "webrtc")]
use crate::elements::WebRtcError;
#[cfg(all(target_os = "windows", feature = "wgc-capture"))]
use crate::elements::WgcCaptureSourceError;
#[cfg(feature = "cuda")]
use crate::elements::{
    CudaConverterError, CudaDecoderError, CudaDownloadError, CudaEncoderError, CudaRendererError,
    CudaScalerError, CudaUploadError, CudaVideoCompositorError,
};
#[cfg(all(target_os = "windows", feature = "d3d11"))]
use crate::elements::{
    D3d11ChromaKeyError, D3d11DecoderError, D3d11DownloadError, D3d11RendererError,
    D3d11ScalerError, D3d11TextLayerError, D3d11UploadError, D3d11VideoCompositorError,
    D3d11VideoEncoderError,
};
#[cfg(all(target_os = "windows", feature = "d3d12"))]
use crate::elements::{
    D3d12DecoderError, D3d12DownloadError, D3d12RendererError, D3d12ScalerError, D3d12UploadError,
};
use crate::{
    control::{PrerollError, SeekError},
    elements::{
        AppSourceError, AudioMixerError, AudioResamplerError, AudioVolumeError, FileDemuxError,
        FileMuxerError, HlsMuxerError, PacerError, RtspSourceError, SwAudioEncoderError,
        SwChromaKeyError, SwDecoderError, SwEncoderError, SwScalerError, SwVideoCompositorError,
        TestAudioSourceError, TestVideoSourceError, VideoSynchronizerError,
    },
    graph::GraphError,
    log::LogInitError,
    queue::QueueError,
};

/// Failure to create one of the background threads owned by this crate.
///
/// The operation that requested the thread returns this error before claiming
/// that it started successfully. The `thread` field identifies the worker so
/// callers can distinguish pipeline, queue, and standalone-driver failures.
#[derive(Debug, Error)]
#[error("failed to spawn {thread} thread: {source}")]
pub struct ThreadSpawnError {
    thread: String,
    #[source]
    source: io::Error,
}

impl ThreadSpawnError {
    pub(crate) fn new(thread: impl Into<String>, source: io::Error) -> Self {
        Self {
            thread: thread.into(),
            source,
        }
    }

    /// Name of the worker that could not be created.
    pub fn thread(&self) -> &str {
        &self.thread
    }
}

/// FFmpeg could not allocate the reference-counted buffer that owns a D3D11
/// texture attached to an `AVFrame`.
#[cfg(all(target_os = "windows", feature = "d3d11"))]
#[derive(Debug, Error)]
#[error("FFmpeg could not allocate a D3D11 texture buffer wrapper")]
pub struct D3d11FrameWrapError;

/// A D3D11 device cannot be shared by the elements of one pipeline.
///
/// Every D3D11 element here funnels its GPU commands through the one immediate
/// context its device owns, and a `Queue` deliberately puts elements on
/// different threads. That context is not free-threaded, so each element
/// enables the runtime's `ID3D11Multithread` protection on the device it is
/// handed and refuses a device that cannot be protected — rather than leaving
/// the resulting data race to a caller who has no way to see it.
#[cfg(all(target_os = "windows", feature = "d3d11"))]
#[derive(Debug, Clone, Error)]
pub enum D3d11SharedDeviceError {
    /// The device was created with `D3D11_CREATE_DEVICE_SINGLETHREADED`, which
    /// promises the runtime that it is used from one thread only. Nothing can
    /// make that device safe here; create it without the flag.
    #[error(
        "the D3D11 device was created with D3D11_CREATE_DEVICE_SINGLETHREADED and cannot be shared across a pipeline's threads"
    )]
    SingleThreaded,

    /// The runtime accepted the request but the protection did not take
    /// effect, so cross-thread use would still be undefined.
    #[error("the D3D11 runtime did not enable multithread protection on the shared context")]
    ProtectionRefused,

    /// The immediate context or its `ID3D11Multithread` interface could not be
    /// obtained from the device.
    #[error("windows error: {0}")]
    Windows(#[from] windows::core::Error),
}

/// Crate-wide error. Each element defines its own `{Element}Error` (see
/// [`FileDemuxError`], [`SwDecoderError`], [`QueueError`]) for its own
/// domain-specific failures; this enum just aggregates them so trait
/// methods (`Sink::consume`, `SourceElement::run`, ...) — which have to
/// return one common error type to stay object-safe across arbitrary
/// `Box<dyn Sink>` — can report any of them. `?` chains through
/// automatically: an element's own function returns its own error type,
/// and the moment that gets used with `?` inside a function returning
/// this top-level `Result`, it's converted here via `#[from]`.
#[derive(Debug, Error)]
pub enum Error {
    /// Waiting for a pipeline-wide preroll failed.
    #[error(transparent)]
    PrerollError(#[from] PrerollError),

    /// One or more elements rejected a pipeline-wide seek check.
    #[error(transparent)]
    SeekError(#[from] SeekError),

    /// A pipeline, queue, or driver worker thread could not be created.
    #[error(transparent)]
    ThreadSpawnError(#[from] ThreadSpawnError),

    /// FFmpeg could not allocate a D3D11 frame buffer wrapper.
    #[cfg(all(target_os = "windows", feature = "d3d11"))]
    #[error(transparent)]
    D3d11FrameWrapError(#[from] D3d11FrameWrapError),

    /// A D3D11 device cannot be shared across a pipeline's threads.
    #[cfg(all(target_os = "windows", feature = "d3d11"))]
    #[error(transparent)]
    D3d11SharedDeviceError(#[from] D3d11SharedDeviceError),

    /// A file demuxer operation failed.
    #[error(transparent)]
    FileDemuxError(#[from] FileDemuxError),

    /// An application source channel is closed.
    #[error(transparent)]
    AppSourceError(#[from] AppSourceError),

    /// An RTSP source operation failed.
    #[error(transparent)]
    RtspSourceError(#[from] RtspSourceError),

    /// A synthetic video source rejected an operation.
    #[error(transparent)]
    TestVideoSourceError(#[from] TestVideoSourceError),

    /// A synthetic audio source rejected an operation.
    #[error(transparent)]
    TestAudioSourceError(#[from] TestAudioSourceError),

    /// A software decoder operation failed.
    #[error(transparent)]
    SwDecoderError(#[from] SwDecoderError),

    /// A CUDA decoder operation failed.
    #[cfg(feature = "cuda")]
    #[error(transparent)]
    CudaDecoderError(#[from] CudaDecoderError),

    /// A CUDA renderer operation failed.
    #[cfg(feature = "cuda")]
    #[error(transparent)]
    CudaRendererError(#[from] CudaRendererError),

    /// Uploading a frame to CUDA failed.
    #[cfg(feature = "cuda")]
    #[error(transparent)]
    CudaUploadError(#[from] CudaUploadError),

    /// Downloading a frame from CUDA failed.
    #[cfg(feature = "cuda")]
    #[error(transparent)]
    CudaDownloadError(#[from] CudaDownloadError),

    /// A CUDA scaling operation failed.
    #[cfg(feature = "cuda")]
    #[error(transparent)]
    CudaScalerError(#[from] CudaScalerError),

    /// A CUDA pixel-format conversion failed.
    #[cfg(feature = "cuda")]
    #[error(transparent)]
    CudaConverterError(#[from] CudaConverterError),

    /// A CUDA compositor operation failed.
    #[cfg(feature = "cuda")]
    #[error(transparent)]
    CudaVideoCompositorError(#[from] CudaVideoCompositorError),

    /// A CUDA encoder operation failed.
    #[cfg(feature = "cuda")]
    #[error(transparent)]
    CudaEncoderError(#[from] CudaEncoderError),

    /// A software video encoder operation failed.
    #[error(transparent)]
    SwEncoderError(#[from] SwEncoderError),

    /// A pacer could not schedule an input timestamp.
    #[error(transparent)]
    PacerError(#[from] PacerError),

    /// A video synchronizer could not schedule an input frame.
    #[error(transparent)]
    VideoSynchronizerError(#[from] VideoSynchronizerError),

    /// A software audio encoder operation failed.
    #[error(transparent)]
    SwAudioEncoderError(#[from] SwAudioEncoderError),

    /// An audio resampling operation failed.
    #[error(transparent)]
    AudioResamplerError(#[from] AudioResamplerError),

    /// An audio gain operation failed.
    #[error(transparent)]
    AudioVolumeError(#[from] AudioVolumeError),

    /// A software scaling operation failed.
    #[error(transparent)]
    SwScalerError(#[from] SwScalerError),

    /// A software chroma-key operation failed.
    #[error(transparent)]
    SwChromaKeyError(#[from] SwChromaKeyError),

    /// A queue worker or capacity policy failed.
    #[error(transparent)]
    QueueError(#[from] QueueError),

    /// See [`crate::elements::PipelineBridgeError`].
    #[error(transparent)]
    PipelineBridgeError(#[from] crate::elements::PipelineBridgeError),

    /// A pipeline graph mutation violated a topology invariant.
    #[error(transparent)]
    GraphError(#[from] GraphError),

    /// Private file logging could not be initialized.
    #[error(transparent)]
    LogInitError(#[from] LogInitError),

    /// An audio mixer operation failed.
    #[error(transparent)]
    AudioMixerError(#[from] AudioMixerError),

    /// A software video compositor operation failed.
    #[error(transparent)]
    SwVideoCompositorError(#[from] SwVideoCompositorError),

    /// Writing a container file failed.
    #[error(transparent)]
    FileMuxerError(#[from] FileMuxerError),

    /// HLS muxing or option validation failed.
    #[error(transparent)]
    HlsMuxerError(#[from] HlsMuxerError),

    /// Sending a stream to an RTSP endpoint failed.
    #[error(transparent)]
    RtspSinkError(#[from] RtspSinkError),

    /// A D3D12 renderer operation failed.
    #[cfg(all(target_os = "windows", feature = "d3d12"))]
    #[error(transparent)]
    D3d12RendererError(#[from] D3d12RendererError),

    /// A D3D12 decoder operation failed.
    #[cfg(all(target_os = "windows", feature = "d3d12"))]
    #[error(transparent)]
    D3d12DecoderError(#[from] D3d12DecoderError),

    /// Uploading a frame to D3D12 failed.
    #[cfg(all(target_os = "windows", feature = "d3d12"))]
    #[error(transparent)]
    D3d12UploadError(#[from] D3d12UploadError),

    /// Downloading a frame from D3D12 failed.
    #[cfg(all(target_os = "windows", feature = "d3d12"))]
    #[error(transparent)]
    D3d12DownloadError(#[from] D3d12DownloadError),

    /// A D3D12 scaling operation failed.
    #[cfg(all(target_os = "windows", feature = "d3d12"))]
    #[error(transparent)]
    D3d12ScalerError(#[from] D3d12ScalerError),

    /// A D3D11 decoder operation failed.
    #[cfg(all(target_os = "windows", feature = "d3d11"))]
    #[error(transparent)]
    D3d11DecoderError(#[from] D3d11DecoderError),

    /// Uploading a frame to D3D11 failed.
    #[cfg(all(target_os = "windows", feature = "d3d11"))]
    #[error(transparent)]
    D3d11UploadError(#[from] D3d11UploadError),

    /// Downloading a frame from D3D11 failed.
    #[cfg(all(target_os = "windows", feature = "d3d11"))]
    #[error(transparent)]
    D3d11DownloadError(#[from] D3d11DownloadError),

    /// A D3D11 scaling operation failed.
    #[cfg(all(target_os = "windows", feature = "d3d11"))]
    #[error(transparent)]
    D3d11ScalerError(#[from] D3d11ScalerError),

    /// A D3D11 chroma-key operation failed.
    #[cfg(all(target_os = "windows", feature = "d3d11"))]
    #[error(transparent)]
    D3d11ChromaKeyError(#[from] D3d11ChromaKeyError),

    /// A D3D11-backed NVENC operation failed.
    #[cfg(all(target_os = "windows", feature = "d3d11"))]
    #[error(transparent)]
    D3d11VideoEncoderError(#[from] D3d11VideoEncoderError),

    /// A D3D11 renderer operation failed.
    #[cfg(all(target_os = "windows", feature = "d3d11"))]
    #[error(transparent)]
    D3d11RendererError(#[from] D3d11RendererError),

    /// A D3D11 compositor operation failed.
    #[cfg(all(target_os = "windows", feature = "d3d11"))]
    #[error(transparent)]
    D3d11VideoCompositorError(#[from] D3d11VideoCompositorError),

    /// A D3D11 text-layer operation failed.
    #[cfg(all(target_os = "windows", feature = "d3d11"))]
    #[error(transparent)]
    D3d11TextLayerError(#[from] D3d11TextLayerError),

    /// Desktop duplication capture failed.
    #[cfg(all(target_os = "windows", feature = "dxgi-capture"))]
    #[error(transparent)]
    DxgiCaptureSourceError(#[from] DxgiCaptureSourceError),

    /// Windows Graphics Capture failed.
    #[cfg(all(target_os = "windows", feature = "wgc-capture"))]
    #[error(transparent)]
    WgcCaptureSourceError(#[from] WgcCaptureSourceError),

    /// PipeWire audio capture failed.
    #[cfg(all(target_os = "linux", feature = "pipewire-audio-capture"))]
    #[error(transparent)]
    PipeWireAudioCaptureSourceError(#[from] PipeWireAudioCaptureSourceError),

    /// PipeWire audio rendering failed.
    #[cfg(all(target_os = "linux", feature = "pipewire-audio-renderer"))]
    #[error(transparent)]
    PipeWireAudioRendererError(#[from] PipeWireAudioRendererError),

    /// PipeWire screen capture failed.
    #[cfg(all(target_os = "linux", feature = "pipewire-screen-capture"))]
    #[error(transparent)]
    PipeWireScreenCaptureSourceError(#[from] PipeWireScreenCaptureSourceError),

    /// Media Foundation camera capture failed.
    #[cfg(all(target_os = "windows", feature = "mf-capture"))]
    #[error(transparent)]
    MfCaptureSourceError(#[from] MfCaptureSourceError),

    /// V4L2 camera capture failed.
    #[cfg(all(target_os = "linux", feature = "v4l2-capture"))]
    #[error(transparent)]
    V4l2CaptureSourceError(#[from] V4l2CaptureSourceError),

    /// WASAPI audio capture failed.
    #[cfg(all(target_os = "windows", feature = "wasapi-capture"))]
    #[error(transparent)]
    WasapiCaptureSourceError(#[from] WasapiCaptureSourceError),

    /// WASAPI audio rendering failed.
    #[cfg(all(target_os = "windows", feature = "wasapi-renderer"))]
    #[error(transparent)]
    WasapiRendererError(#[from] WasapiRendererError),

    /// ONNX Runtime inference or detector processing failed.
    #[cfg(feature = "ort")]
    #[error(transparent)]
    OrtDetectorError(#[from] OrtDetectorError),

    /// A WebRTC peer operation failed.
    #[cfg(feature = "webrtc")]
    #[error(transparent)]
    WebRtcError(#[from] WebRtcError),

    /// An FFmpeg error not assigned to a more specific element error.
    #[error("ffmpeg error: {0}")]
    Ffmpeg(#[from] ffmpeg_next::Error),

    /// An application-defined error message without a more specific category.
    #[error("{0}")]
    Other(String),
}

/// The crate's `Result`, with [`enum@Error`] as the error type.
pub type Result<T> = std::result::Result<T, Error>;