threers_codec/lib.rs
1//! Pure-Rust media codecs: video and audio encoders, decoders, muxers and
2//! demuxers, with hardware acceleration where the platform has it and no C
3//! dependencies anywhere else.
4//!
5//! Everything builds for `wasm32` as well as native and depends on nothing
6//! beyond `std` (`rayon` with `parallel`). Outside the opt-in hardware
7//! modules (`videotoolbox`, `nvidia`, `vaapi`) there are no OS, process or
8//! filesystem calls: encoders return `Vec<u8>`, and the caller owns I/O. It
9//! is the media half of [threers](https://crates.io/crates/threers), which
10//! re-exports it as `threers::codec`, split out so programs that only encode
11//! or decode media do not link a renderer.
12//!
13//! # Start here
14//!
15//! [`prelude`] has the types most programs need. Three entry points cover
16//! the common jobs:
17//!
18//! ```no_run
19//! # #[cfg(feature = "decode")] {
20//! use threers_codec::prelude::*;
21//!
22//! // Encode RGBA frames to MP4 — on the hardware encoder where there is one.
23//! let mut enc = Mp4Encoder::new(640, 360, EncodeOptions::default()).unwrap();
24//! enc.push_rgba(&vec![128; 640 * 360 * 4]).unwrap();
25//! std::fs::write("out.mp4", enc.finish().unwrap()).unwrap();
26//!
27//! // Decode any audio file this crate reads to planar f32.
28//! let pcm = audio_file::read(&std::fs::read("song.flac").unwrap()).unwrap();
29//! println!("{} channels at {} Hz", pcm.channels.len(), pcm.sample_rate);
30//!
31//! // Decode a video track (feature `decode`), again hardware first.
32//! let file = std::fs::read("clip.mp4").unwrap();
33//! let track = demux_video(&file).unwrap();
34//! let mut dec = Decoder::new(&track).unwrap();
35//! for s in &track.samples {
36//! dec.push(&file[s.offset as usize..][..s.size as usize], s.pts, s.dts, s.duration as i64)
37//! .unwrap();
38//! if let Some(_bgra) = dec.take(s.pts) {}
39//! }
40//! # }
41//! ```
42//!
43//! # What is here
44//!
45//! | Kind | Modules |
46//! |------|---------|
47//! | Video encoders | [`h264`] (CAVLC/CABAC, intra + inter), [`hevc`] (intra, alpha), [`vp9`] (intra + inter, alpha) |
48//! | Video decoders | [`decode`]: H.264, HEVC, VP9, AV1 — bit-exact against the reference decoders (feature `decode`) |
49//! | Audio encoders | [`aac::encoder`] (AAC-LC to raw units or ADTS, 8 to 96 kHz, mono and stereo) and [`aac::m4a`] (the same in an `.m4a`, with the priming edit list); [`mp3::encoder`] (MPEG-1 Layer III, 32 to 320 kbit/s, with a gapless `Info` frame) |
50//! | Audio decoders | [`aac`] (LC, HE-AAC SBR + PS), [`mp3`], [`opus`] (SILK, CELT, hybrid), [`vorbis`], [`flac`], [`pcm`] (WAV/RF64, AIFF); [`audio_file`] sniffs and dispatches |
51//! | Animated images | [`gif`] (encode and decode, feature `gif`), [`apng`]; [`animation`] picks one for a browser |
52//! | Containers | [`mp4`] / [`mp4_stream`] (write), [`mp4_demux`] (read), [`webm`] (write), [`mkv_demux`] (read, plus IVF), [`ogg`] (read) |
53//! | Hardware | [`hw`]: one encoder and one decoder over VideoToolbox, NVENC/NVDEC, VAAPI and the native codecs; `videotoolbox`, `videotoolbox_decode` (macOS), `nvidia` (Linux, Windows) and `vaapi` (Linux) directly |
54//! | Rate control | [`rate`]: a quantizer or a bitrate target |
55//! | Conversion | [`swr`] (resampling and sample formats), [`ffcompat`] (`swscale`, `.mov`/`.mkv` readers, the shapes ffmpeg's tools produce) |
56//! | Captions | [`captions`]: tracks and cues, SRT and WebVTT, muxed by `mp4` and `webm` as soft subtitles |
57//! | Primitives | [`bitstream`] (bit writer, Exp-Golomb, emulation prevention), [`contrast`] (WCAG luminance) |
58//!
59//! # Deep access
60//!
61//! Every stage of every codec is a public module, so the building blocks are
62//! there for research, tooling and tests as well as the high-level calls: an
63//! entropy coder ([`h264::cabac`], [`hevc::cabac`], [`vp9::bool_coder`],
64//! [`decode::av1::msac`]), a transform ([`hevc::transform`],
65//! [`decode::av1::itx`]), a loop filter ([`h264::deblock`],
66//! [`decode::vp9::lpf`]), a parameter-set parser ([`decode::h264::params`],
67//! [`decode::hevc::params`]), an MDCT ([`aac::dsp`], [`opus::celt`]). Each
68//! module's docs say which section of which specification it implements.
69//! These inner modules follow the structure of their specifications rather
70//! than a stability promise; the high-level types re-exported from each
71//! codec's root and from [`prelude`] are the stable surface.
72//!
73//! # Hardware acceleration
74//!
75//! [`hw::Mp4Encoder`] and [`decode::Decoder`] use VideoToolbox on macOS (the
76//! `videotoolbox` feature), an NVIDIA GPU's NVENC and NVDEC engines on
77//! Linux and Windows (the `nvidia` feature), and VA drivers — AMD, Intel —
78//! on Linux (the `vaapi` feature), and fall back to the
79//! pure-Rust codecs when the feature is off, the machine has no such
80//! hardware, or the hardware refuses a stream. The NVIDIA and VA libraries
81//! are loaded at run time, so a binary built with those features still runs,
82//! natively, where they are missing.
83//! The software paths use every core with `parallel`: frames of an all-intra
84//! stream are independent, so batches of them encode at once with
85//! byte-identical output (see [`set_frame_memory_budget`]).
86//!
87//! # Features
88//!
89//! | Feature | Default | What it adds |
90//! |---------|---------|--------------|
91//! | `decode` | **on** | The video decoders in [`decode`] (H.264, HEVC, VP9, AV1) |
92//! | `gif` | **on** | GIF89a encode and decode ([`gif`]), and GIF in [`animation`] |
93//! | `parallel` | | Frame-parallel encoding, and AV1 decoding across frames, tiles and loop filters, on `rayon` |
94//! | `videotoolbox` | | VideoToolbox encode and decode on macOS (links Apple's frameworks; inert elsewhere) |
95//! | `nvidia` | | NVENC encode and NVDEC decode on Linux and Windows (driver loaded at run time; implies `decode`; inert elsewhere) |
96//! | `vaapi` | | VAAPI HEVC encode and H.264/HEVC/VP9/AV1 decode on Linux (libva loaded at run time; implies `decode`; inert elsewhere) |
97//!
98//! Encoders, audio decoders, muxers, demuxers and captions are always
99//! compiled; none of them has a dependency to make optional.
100
101// The codecs follow their specifications' notation: loops over indices that
102// address several parallel arrays, argument lists that mirror the spec's
103// processes, and the reference decoders' own nested types.
104#![allow(
105 clippy::too_many_arguments,
106 clippy::needless_range_loop,
107 clippy::type_complexity
108)]
109#![cfg_attr(docsrs, feature(doc_cfg))]
110
111pub mod aac;
112pub mod animation;
113pub mod apng;
114pub mod audio_file;
115#[cfg(feature = "decode")]
116pub mod av1;
117pub mod bitstream;
118pub mod bytes;
119/// The caption data model, from [`threers_captions`].
120pub use threers_captions as captions;
121/// WCAG luminance and contrast, from [`threers_captions`].
122pub use threers_captions::contrast;
123pub mod decode;
124pub mod ffcompat;
125pub mod flac;
126#[cfg(feature = "gif")]
127pub mod gif;
128pub mod h264;
129pub mod hevc;
130pub mod hw;
131pub mod mkv_demux;
132pub mod mp3;
133pub mod mp4;
134pub mod mp4_demux;
135pub mod mp4_stream;
136#[cfg(all(
137 feature = "nvidia",
138 any(target_os = "linux", target_os = "windows"),
139 target_pointer_width = "64"
140))]
141pub mod nvidia;
142pub mod ogg;
143pub mod opus;
144pub mod pcm;
145pub mod prelude;
146pub mod rate;
147pub mod swr;
148#[cfg(all(feature = "vaapi", target_os = "linux"))]
149pub mod vaapi;
150#[cfg(all(feature = "v4l2", target_os = "linux"))]
151pub mod v4l2;
152#[cfg(all(feature = "videotoolbox", target_os = "macos"))]
153pub mod videotoolbox;
154// The decoder hands pictures over in `decode`'s types, so it needs `decode` too.
155#[cfg(all(feature = "videotoolbox", feature = "decode", target_os = "macos"))]
156pub mod videotoolbox_decode;
157pub mod vorbis;
158pub mod vp9;
159pub mod webm;
160
161pub use animation::{
162 encode_animation_rgba, encode_animation_rgba_with_progress, format_animation_progress,
163 AnimationEncodeError, AnimationEncodeOptions, AnimationExportPhase, AnimationExportProgress,
164 BrowserCodec,
165};
166pub use apng::ApngEncoder;
167pub use bitstream::{emulation_prevention, rbsp_trailing_bits, BitWriter};
168#[cfg(feature = "gif")]
169#[allow(deprecated)]
170pub use gif::DecodedFrame;
171#[cfg(feature = "gif")]
172pub use gif::{
173 decode_gif, encode_gif, DisposalMethod, DisposalMode, GifDecoder, GifEncoder, GifError,
174 GifFrame, GifFrameMeta, GifInfo, GifOptions, GifVersion, GifWriter, LzwClearMode, PaletteMode,
175 QuantizerKind,
176};
177pub use vp9::{encode_intra_frame, encode_intra_gray, Reconstruction};
178pub use webm::{encode_gray_webm, encode_webm, mux_webm, WebmCodec, WebmFrame, WebmParams};
179
180// --------------------------------------------------------------------------
181// Frame-level parallelism
182// --------------------------------------------------------------------------
183
184/// How many frames to hold in flight at once.
185///
186/// Every frame of an all-intra stream is independent, so the only thing bounding
187/// the batch is memory: an 8K frame is 50 MB of input and the encoder allocates
188/// a reconstruction of its own alongside it. Encoding every frame at once would
189/// undo the streaming work that got export memory down in the first place, so
190/// the batch is whichever is smaller — the thread count, or what fits in a
191/// fixed working-set budget.
192pub(crate) fn frames_in_flight(width: u32, height: u32) -> usize {
193 frames_by_memory(width, height).min(worker_threads())
194}
195
196/// How many frames the memory budget allows in flight, irrespective of how many
197/// cores there are.
198///
199/// Rate control sizes its groups from this rather than from
200/// [`frames_in_flight`], because the group is how often the controller gets to
201/// correct itself — and a clip must not encode to a different size on a machine
202/// with more cores.
203pub(crate) fn frames_by_memory(width: u32, height: u32) -> usize {
204 (frame_memory_budget() / frame_working_set(width, height).max(1)).max(1)
205}
206
207/// Roughly what one frame costs while it is being encoded: the input planes,
208/// the reconstruction, and the padded copies the encoder makes of both.
209pub(crate) fn frame_working_set(width: u32, height: u32) -> usize {
210 (width as usize * height as usize * 3 / 2) * 5 / 2
211}
212
213/// Default working-set budget for frame-parallel encoding.
214///
215/// Deliberately conservative: it binds only at 4K and above, and only there
216/// because a frame is large enough that holding a core's worth of them is real
217/// memory. See [`set_frame_memory_budget`].
218pub const DEFAULT_FRAME_MEMORY_BUDGET: usize = 256 << 20;
219
220static FRAME_MEMORY_BUDGET: std::sync::atomic::AtomicUsize =
221 std::sync::atomic::AtomicUsize::new(DEFAULT_FRAME_MEMORY_BUDGET);
222
223/// How much memory frame-parallel encoding may hold in flight.
224pub fn frame_memory_budget() -> usize {
225 FRAME_MEMORY_BUDGET.load(std::sync::atomic::Ordering::Relaxed)
226}
227
228/// Set how much memory frame-parallel encoding may hold in flight.
229///
230/// Frames of an all-intra stream are independent, so the only thing limiting how
231/// many encode at once is memory — about 4.5 bytes per pixel per frame in
232/// flight, which is 149 MB at 8K. The default of 256 MB keeps peak usage close
233/// to where the streaming work left it, at the cost of capping 8K to two frames
234/// at a time.
235///
236/// Raising it is the cheapest speedup available at high resolution and it costs
237/// nothing in bitrate: measured on a 4-frame 8K clip, going from 256 MB to 1 GB
238/// took the encode from 49.9 s to 33.0 s with byte-identical output. Tiling the
239/// picture would reach the same parallelism at a bitrate cost; this does not.
240///
241/// ```
242/// // Let an 8K export use eight frames at a time.
243/// threers_codec::set_frame_memory_budget(1024 << 20);
244/// ```
245pub fn set_frame_memory_budget(bytes: usize) {
246 FRAME_MEMORY_BUDGET.store(bytes.max(1), std::sync::atomic::Ordering::Relaxed);
247}
248
249#[cfg(feature = "parallel")]
250fn worker_threads() -> usize {
251 rayon::current_num_threads().max(1)
252}
253
254#[cfg(not(feature = "parallel"))]
255fn worker_threads() -> usize {
256 1
257}
258
259/// Encode a batch of frames, in order, using every available thread.
260///
261/// The output is byte-identical to mapping `encode` over the frames one at a
262/// time — the `parallel` feature changes how long it takes and nothing else.
263#[cfg(feature = "parallel")]
264pub(crate) fn map_frames<T, R>(frames: &[T], encode: impl Fn(&T) -> R + Sync + Send) -> Vec<R>
265where
266 T: Sync,
267 R: Send,
268{
269 use rayon::prelude::*;
270 frames.par_iter().map(encode).collect()
271}
272
273#[cfg(not(feature = "parallel"))]
274pub(crate) fn map_frames<T, R>(frames: &[T], encode: impl Fn(&T) -> R + Sync + Send) -> Vec<R> {
275 frames.iter().map(encode).collect()
276}
277
278/// `map_frames`, public so benchmarks and callers with their own frame source
279/// can use the same batching the export paths do.
280pub fn map_frames_pub<T, R>(frames: &[T], encode: impl Fn(&T) -> R + Sync + Send) -> Vec<R>
281where
282 T: Sync,
283 R: Send,
284{
285 map_frames(frames, encode)
286}
287
288/// [`map_frames`] where the work needs to know its own index — tiles do, to
289/// tell the last one from the rest.
290#[cfg(feature = "parallel")]
291pub(crate) fn map_indexed<T, R>(items: &[T], f: impl Fn(usize, &T) -> R + Sync + Send) -> Vec<R>
292where
293 T: Sync,
294 R: Send,
295{
296 use rayon::prelude::*;
297 items.par_iter().enumerate().map(|(i, t)| f(i, t)).collect()
298}
299
300#[cfg(not(feature = "parallel"))]
301pub(crate) fn map_indexed<T, R>(items: &[T], f: impl Fn(usize, &T) -> R + Sync + Send) -> Vec<R> {
302 items.iter().enumerate().map(|(i, t)| f(i, t)).collect()
303}
304
305/// Run `f` on each of `items`, on every available thread with `parallel`.
306/// Callers give each item its own disjoint piece of the output (a band of
307/// rows, say), so the result is the same whatever the schedule.
308///
309/// On wasm this needs a build with `+atomics` and a thread pool the page has
310/// started (`wasm-bindgen-rayon`); without atomics it runs in order, as a
311/// plain wasm32 build has no threads to spawn.
312#[cfg(all(
313 feature = "parallel",
314 any(not(target_arch = "wasm32"), target_feature = "atomics")
315))]
316pub(crate) fn for_each_owned<T: Send>(items: Vec<T>, f: impl Fn(T) + Sync + Send) {
317 use rayon::prelude::*;
318 items.into_par_iter().for_each(f)
319}
320
321/// Run `f` on the thread pool, returning at once.
322#[cfg(feature = "decode")]
323#[cfg(all(
324 feature = "parallel",
325 any(not(target_arch = "wasm32"), target_feature = "atomics")
326))]
327pub(crate) fn spawn(f: impl FnOnce() + Send + 'static) {
328 rayon::spawn(f)
329}
330
331#[cfg(feature = "decode")]
332#[cfg(not(all(
333 feature = "parallel",
334 any(not(target_arch = "wasm32"), target_feature = "atomics")
335)))]
336pub(crate) fn spawn(f: impl FnOnce() + Send + 'static) {
337 f()
338}
339
340/// Whether the caller can block while the pool does work for it: there must
341/// be a pool, the caller must not be one of its threads, and with one thread
342/// there would be nothing to gain.
343#[cfg(feature = "decode")]
344#[cfg(all(
345 feature = "parallel",
346 any(not(target_arch = "wasm32"), target_feature = "atomics")
347))]
348pub(crate) fn pool_can_serve_caller() -> bool {
349 rayon::current_thread_index().is_none() && rayon::current_num_threads() > 1
350}
351
352#[cfg(feature = "decode")]
353#[cfg(not(all(
354 feature = "parallel",
355 any(not(target_arch = "wasm32"), target_feature = "atomics")
356)))]
357pub(crate) fn pool_can_serve_caller() -> bool {
358 false
359}
360
361/// How many frames to decode at once by default.
362#[cfg(feature = "decode")]
363#[cfg(all(
364 feature = "parallel",
365 any(not(target_arch = "wasm32"), target_feature = "atomics")
366))]
367pub(crate) fn frame_threads() -> usize {
368 rayon::current_num_threads().min(8)
369}
370
371#[cfg(feature = "decode")]
372#[cfg(not(all(
373 feature = "parallel",
374 any(not(target_arch = "wasm32"), target_feature = "atomics")
375)))]
376pub(crate) fn frame_threads() -> usize {
377 1
378}
379
380/// Whether [`for_each_owned`] runs its items on more than one thread, for
381/// callers whose parallel path has a setup cost worth skipping otherwise.
382pub(crate) fn threaded() -> bool {
383 cfg!(all(
384 feature = "parallel",
385 any(not(target_arch = "wasm32"), target_feature = "atomics")
386 ))
387}
388
389#[cfg(not(all(
390 feature = "parallel",
391 any(not(target_arch = "wasm32"), target_feature = "atomics")
392)))]
393pub(crate) fn for_each_owned<T: Send>(items: Vec<T>, f: impl Fn(T) + Sync + Send) {
394 items.into_iter().for_each(f)
395}