Skip to main content

threers_codec/
lib.rs

1//! Pure-Rust media codecs: video and audio encoders, decoders, muxers and
2//! demuxers, with hardware acceleration where the platform has it and no C
3//! dependencies anywhere else.
4//!
5//! Everything builds for `wasm32` as well as native and depends on nothing
6//! beyond `std` (`rayon` with `parallel`). Outside the opt-in hardware
7//! modules (`videotoolbox`, `nvidia`, `vaapi`) there are no OS, process or
8//! filesystem calls: encoders return `Vec<u8>`, and the caller owns I/O. It
9//! is the media half of [threers](https://crates.io/crates/threers), which
10//! re-exports it as `threers::codec`, split out so programs that only encode
11//! or decode media do not link a renderer.
12//!
13//! # Start here
14//!
15//! [`prelude`] has the types most programs need. Three entry points cover
16//! the common jobs:
17//!
18//! ```no_run
19//! # #[cfg(feature = "decode")] {
20//! use threers_codec::prelude::*;
21//!
22//! // Encode RGBA frames to MP4 — on the hardware encoder where there is one.
23//! let mut enc = Mp4Encoder::new(640, 360, EncodeOptions::default()).unwrap();
24//! enc.push_rgba(&vec![128; 640 * 360 * 4]).unwrap();
25//! std::fs::write("out.mp4", enc.finish().unwrap()).unwrap();
26//!
27//! // Decode any audio file this crate reads to planar f32.
28//! let pcm = audio_file::read(&std::fs::read("song.flac").unwrap()).unwrap();
29//! println!("{} channels at {} Hz", pcm.channels.len(), pcm.sample_rate);
30//!
31//! // Decode a video track (feature `decode`), again hardware first.
32//! let file = std::fs::read("clip.mp4").unwrap();
33//! let track = demux_video(&file).unwrap();
34//! let mut dec = Decoder::new(&track).unwrap();
35//! for s in &track.samples {
36//!     dec.push(&file[s.offset as usize..][..s.size as usize], s.pts, s.dts, s.duration as i64)
37//!         .unwrap();
38//!     if let Some(_bgra) = dec.take(s.pts) {}
39//! }
40//! # }
41//! ```
42//!
43//! # What is here
44//!
45//! | Kind | Modules |
46//! |------|---------|
47//! | Video encoders | [`h264`] (CAVLC/CABAC, intra + inter), [`hevc`] (intra, alpha), [`vp9`] (intra + inter, alpha) |
48//! | Video decoders | [`decode`]: H.264, HEVC, VP9, AV1 — bit-exact against the reference decoders (feature `decode`) |
49//! | Audio encoders | [`aac::encoder`] (AAC-LC to raw units or ADTS, 8 to 96 kHz, mono and stereo) and [`aac::m4a`] (the same in an `.m4a`, with the priming edit list); [`mp3::encoder`] (MPEG-1 Layer III, 32 to 320 kbit/s, with a gapless `Info` frame) |
50//! | Audio decoders | [`aac`] (LC, HE-AAC SBR + PS), [`mp3`], [`opus`] (SILK, CELT, hybrid), [`vorbis`], [`flac`], [`pcm`] (WAV/RF64, AIFF); [`audio_file`] sniffs and dispatches |
51//! | Animated images | [`gif`] (encode and decode, feature `gif`), [`apng`]; [`animation`] picks one for a browser |
52//! | Containers | [`mp4`] / [`mp4_stream`] (write), [`mp4_demux`] (read), [`webm`] (write), [`mkv_demux`] (read, plus IVF), [`ogg`] (read) |
53//! | Hardware | [`hw`]: one encoder and one decoder over VideoToolbox, NVENC/NVDEC, VAAPI and the native codecs; `videotoolbox`, `videotoolbox_decode` (macOS), `nvidia` (Linux, Windows) and `vaapi` (Linux) directly |
54//! | Rate control | [`rate`]: a quantizer or a bitrate target |
55//! | Conversion | [`swr`] (resampling and sample formats), [`ffcompat`] (`swscale`, `.mov`/`.mkv` readers, the shapes ffmpeg's tools produce) |
56//! | Captions | [`captions`]: tracks and cues, SRT and WebVTT, muxed by `mp4` and `webm` as soft subtitles |
57//! | Primitives | [`bitstream`] (bit writer, Exp-Golomb, emulation prevention), [`contrast`] (WCAG luminance) |
58//!
59//! # Deep access
60//!
61//! Every stage of every codec is a public module, so the building blocks are
62//! there for research, tooling and tests as well as the high-level calls: an
63//! entropy coder ([`h264::cabac`], [`hevc::cabac`], [`vp9::bool_coder`],
64//! [`decode::av1::msac`]), a transform ([`hevc::transform`],
65//! [`decode::av1::itx`]), a loop filter ([`h264::deblock`],
66//! [`decode::vp9::lpf`]), a parameter-set parser ([`decode::h264::params`],
67//! [`decode::hevc::params`]), an MDCT ([`aac::dsp`], [`opus::celt`]). Each
68//! module's docs say which section of which specification it implements.
69//! These inner modules follow the structure of their specifications rather
70//! than a stability promise; the high-level types re-exported from each
71//! codec's root and from [`prelude`] are the stable surface.
72//!
73//! # Hardware acceleration
74//!
75//! [`hw::Mp4Encoder`] and [`decode::Decoder`] use VideoToolbox on macOS (the
76//! `videotoolbox` feature), an NVIDIA GPU's NVENC and NVDEC engines on
77//! Linux and Windows (the `nvidia` feature), and VA drivers — AMD, Intel —
78//! on Linux (the `vaapi` feature), and fall back to the
79//! pure-Rust codecs when the feature is off, the machine has no such
80//! hardware, or the hardware refuses a stream. The NVIDIA and VA libraries
81//! are loaded at run time, so a binary built with those features still runs,
82//! natively, where they are missing.
83//! The software paths use every core with `parallel`: frames of an all-intra
84//! stream are independent, so batches of them encode at once with
85//! byte-identical output (see [`set_frame_memory_budget`]).
86//!
87//! # Features
88//!
89//! | Feature | Default | What it adds |
90//! |---------|---------|--------------|
91//! | `decode` | **on** | The video decoders in [`decode`] (H.264, HEVC, VP9, AV1) |
92//! | `gif` | **on** | GIF89a encode and decode ([`gif`]), and GIF in [`animation`] |
93//! | `parallel` | | Frame-parallel encoding, and AV1 decoding across frames, tiles and loop filters, on `rayon` |
94//! | `videotoolbox` | | VideoToolbox encode and decode on macOS (links Apple's frameworks; inert elsewhere) |
95//! | `nvidia` | | NVENC encode and NVDEC decode on Linux and Windows (driver loaded at run time; implies `decode`; inert elsewhere) |
96//! | `vaapi` | | VAAPI HEVC encode and H.264/HEVC/VP9/AV1 decode on Linux (libva loaded at run time; implies `decode`; inert elsewhere) |
97//!
98//! Encoders, audio decoders, muxers, demuxers and captions are always
99//! compiled; none of them has a dependency to make optional.
100
101// The codecs follow their specifications' notation: loops over indices that
102// address several parallel arrays, argument lists that mirror the spec's
103// processes, and the reference decoders' own nested types.
104#![allow(
105    clippy::too_many_arguments,
106    clippy::needless_range_loop,
107    clippy::type_complexity
108)]
109#![cfg_attr(docsrs, feature(doc_cfg))]
110
111pub mod aac;
112pub mod animation;
113pub mod apng;
114pub mod audio_file;
115#[cfg(feature = "decode")]
116pub mod av1;
117pub mod bitstream;
118pub mod bytes;
119/// The caption data model, from [`threers_captions`].
120pub use threers_captions as captions;
121/// WCAG luminance and contrast, from [`threers_captions`].
122pub use threers_captions::contrast;
123pub mod decode;
124pub mod ffcompat;
125pub mod flac;
126#[cfg(feature = "gif")]
127pub mod gif;
128pub mod h264;
129pub mod hevc;
130pub mod hw;
131pub mod mkv_demux;
132pub mod mp3;
133pub mod mp4;
134pub mod mp4_demux;
135pub mod mp4_stream;
136#[cfg(all(
137    feature = "nvidia",
138    any(target_os = "linux", target_os = "windows"),
139    target_pointer_width = "64"
140))]
141pub mod nvidia;
142pub mod ogg;
143pub mod opus;
144pub mod pcm;
145pub mod prelude;
146pub mod rate;
147pub mod swr;
148#[cfg(all(feature = "vaapi", target_os = "linux"))]
149pub mod vaapi;
150#[cfg(all(feature = "videotoolbox", target_os = "macos"))]
151pub mod videotoolbox;
152// The decoder hands pictures over in `decode`'s types, so it needs `decode` too.
153#[cfg(all(feature = "videotoolbox", feature = "decode", target_os = "macos"))]
154pub mod videotoolbox_decode;
155pub mod vorbis;
156pub mod vp9;
157pub mod webm;
158
159pub use animation::{
160    encode_animation_rgba, encode_animation_rgba_with_progress, format_animation_progress,
161    AnimationEncodeError, AnimationEncodeOptions, AnimationExportPhase, AnimationExportProgress,
162    BrowserCodec,
163};
164pub use apng::ApngEncoder;
165pub use bitstream::{emulation_prevention, rbsp_trailing_bits, BitWriter};
166#[cfg(feature = "gif")]
167#[allow(deprecated)]
168pub use gif::DecodedFrame;
169#[cfg(feature = "gif")]
170pub use gif::{
171    decode_gif, encode_gif, DisposalMethod, DisposalMode, GifDecoder, GifEncoder, GifError,
172    GifFrame, GifFrameMeta, GifInfo, GifOptions, GifVersion, GifWriter, LzwClearMode, PaletteMode,
173    QuantizerKind,
174};
175pub use vp9::{encode_intra_frame, encode_intra_gray, Reconstruction};
176pub use webm::{encode_gray_webm, encode_webm, mux_webm, WebmCodec, WebmFrame, WebmParams};
177
178// --------------------------------------------------------------------------
179// Frame-level parallelism
180// --------------------------------------------------------------------------
181
182/// How many frames to hold in flight at once.
183///
184/// Every frame of an all-intra stream is independent, so the only thing bounding
185/// the batch is memory: an 8K frame is 50 MB of input and the encoder allocates
186/// a reconstruction of its own alongside it. Encoding every frame at once would
187/// undo the streaming work that got export memory down in the first place, so
188/// the batch is whichever is smaller — the thread count, or what fits in a
189/// fixed working-set budget.
190pub(crate) fn frames_in_flight(width: u32, height: u32) -> usize {
191    frames_by_memory(width, height).min(worker_threads())
192}
193
194/// How many frames the memory budget allows in flight, irrespective of how many
195/// cores there are.
196///
197/// Rate control sizes its groups from this rather than from
198/// [`frames_in_flight`], because the group is how often the controller gets to
199/// correct itself — and a clip must not encode to a different size on a machine
200/// with more cores.
201pub(crate) fn frames_by_memory(width: u32, height: u32) -> usize {
202    (frame_memory_budget() / frame_working_set(width, height).max(1)).max(1)
203}
204
205/// Roughly what one frame costs while it is being encoded: the input planes,
206/// the reconstruction, and the padded copies the encoder makes of both.
207pub(crate) fn frame_working_set(width: u32, height: u32) -> usize {
208    (width as usize * height as usize * 3 / 2) * 5 / 2
209}
210
211/// Default working-set budget for frame-parallel encoding.
212///
213/// Deliberately conservative: it binds only at 4K and above, and only there
214/// because a frame is large enough that holding a core's worth of them is real
215/// memory. See [`set_frame_memory_budget`].
216pub const DEFAULT_FRAME_MEMORY_BUDGET: usize = 256 << 20;
217
218static FRAME_MEMORY_BUDGET: std::sync::atomic::AtomicUsize =
219    std::sync::atomic::AtomicUsize::new(DEFAULT_FRAME_MEMORY_BUDGET);
220
221/// How much memory frame-parallel encoding may hold in flight.
222pub fn frame_memory_budget() -> usize {
223    FRAME_MEMORY_BUDGET.load(std::sync::atomic::Ordering::Relaxed)
224}
225
226/// Set how much memory frame-parallel encoding may hold in flight.
227///
228/// Frames of an all-intra stream are independent, so the only thing limiting how
229/// many encode at once is memory — about 4.5 bytes per pixel per frame in
230/// flight, which is 149 MB at 8K. The default of 256 MB keeps peak usage close
231/// to where the streaming work left it, at the cost of capping 8K to two frames
232/// at a time.
233///
234/// Raising it is the cheapest speedup available at high resolution and it costs
235/// nothing in bitrate: measured on a 4-frame 8K clip, going from 256 MB to 1 GB
236/// took the encode from 49.9 s to 33.0 s with byte-identical output. Tiling the
237/// picture would reach the same parallelism at a bitrate cost; this does not.
238///
239/// ```
240/// // Let an 8K export use eight frames at a time.
241/// threers_codec::set_frame_memory_budget(1024 << 20);
242/// ```
243pub fn set_frame_memory_budget(bytes: usize) {
244    FRAME_MEMORY_BUDGET.store(bytes.max(1), std::sync::atomic::Ordering::Relaxed);
245}
246
247#[cfg(feature = "parallel")]
248fn worker_threads() -> usize {
249    rayon::current_num_threads().max(1)
250}
251
252#[cfg(not(feature = "parallel"))]
253fn worker_threads() -> usize {
254    1
255}
256
257/// Encode a batch of frames, in order, using every available thread.
258///
259/// The output is byte-identical to mapping `encode` over the frames one at a
260/// time — the `parallel` feature changes how long it takes and nothing else.
261#[cfg(feature = "parallel")]
262pub(crate) fn map_frames<T, R>(frames: &[T], encode: impl Fn(&T) -> R + Sync + Send) -> Vec<R>
263where
264    T: Sync,
265    R: Send,
266{
267    use rayon::prelude::*;
268    frames.par_iter().map(encode).collect()
269}
270
271#[cfg(not(feature = "parallel"))]
272pub(crate) fn map_frames<T, R>(frames: &[T], encode: impl Fn(&T) -> R + Sync + Send) -> Vec<R> {
273    frames.iter().map(encode).collect()
274}
275
276/// `map_frames`, public so benchmarks and callers with their own frame source
277/// can use the same batching the export paths do.
278pub fn map_frames_pub<T, R>(frames: &[T], encode: impl Fn(&T) -> R + Sync + Send) -> Vec<R>
279where
280    T: Sync,
281    R: Send,
282{
283    map_frames(frames, encode)
284}
285
286/// [`map_frames`] where the work needs to know its own index — tiles do, to
287/// tell the last one from the rest.
288#[cfg(feature = "parallel")]
289pub(crate) fn map_indexed<T, R>(items: &[T], f: impl Fn(usize, &T) -> R + Sync + Send) -> Vec<R>
290where
291    T: Sync,
292    R: Send,
293{
294    use rayon::prelude::*;
295    items.par_iter().enumerate().map(|(i, t)| f(i, t)).collect()
296}
297
298#[cfg(not(feature = "parallel"))]
299pub(crate) fn map_indexed<T, R>(items: &[T], f: impl Fn(usize, &T) -> R + Sync + Send) -> Vec<R> {
300    items.iter().enumerate().map(|(i, t)| f(i, t)).collect()
301}
302
303/// Run `f` on each of `items`, on every available thread with `parallel`.
304/// Callers give each item its own disjoint piece of the output (a band of
305/// rows, say), so the result is the same whatever the schedule.
306///
307/// On wasm this needs a build with `+atomics` and a thread pool the page has
308/// started (`wasm-bindgen-rayon`); without atomics it runs in order, as a
309/// plain wasm32 build has no threads to spawn.
310#[cfg(all(
311    feature = "parallel",
312    any(not(target_arch = "wasm32"), target_feature = "atomics")
313))]
314pub(crate) fn for_each_owned<T: Send>(items: Vec<T>, f: impl Fn(T) + Sync + Send) {
315    use rayon::prelude::*;
316    items.into_par_iter().for_each(f)
317}
318
319/// Run `f` on the thread pool, returning at once.
320#[cfg(feature = "decode")]
321#[cfg(all(
322    feature = "parallel",
323    any(not(target_arch = "wasm32"), target_feature = "atomics")
324))]
325pub(crate) fn spawn(f: impl FnOnce() + Send + 'static) {
326    rayon::spawn(f)
327}
328
329#[cfg(feature = "decode")]
330#[cfg(not(all(
331    feature = "parallel",
332    any(not(target_arch = "wasm32"), target_feature = "atomics")
333)))]
334pub(crate) fn spawn(f: impl FnOnce() + Send + 'static) {
335    f()
336}
337
338/// Whether the caller can block while the pool does work for it: there must
339/// be a pool, the caller must not be one of its threads, and with one thread
340/// there would be nothing to gain.
341#[cfg(feature = "decode")]
342#[cfg(all(
343    feature = "parallel",
344    any(not(target_arch = "wasm32"), target_feature = "atomics")
345))]
346pub(crate) fn pool_can_serve_caller() -> bool {
347    rayon::current_thread_index().is_none() && rayon::current_num_threads() > 1
348}
349
350#[cfg(feature = "decode")]
351#[cfg(not(all(
352    feature = "parallel",
353    any(not(target_arch = "wasm32"), target_feature = "atomics")
354)))]
355pub(crate) fn pool_can_serve_caller() -> bool {
356    false
357}
358
359/// How many frames to decode at once by default.
360#[cfg(feature = "decode")]
361#[cfg(all(
362    feature = "parallel",
363    any(not(target_arch = "wasm32"), target_feature = "atomics")
364))]
365pub(crate) fn frame_threads() -> usize {
366    rayon::current_num_threads().min(8)
367}
368
369#[cfg(feature = "decode")]
370#[cfg(not(all(
371    feature = "parallel",
372    any(not(target_arch = "wasm32"), target_feature = "atomics")
373)))]
374pub(crate) fn frame_threads() -> usize {
375    1
376}
377
378/// Whether [`for_each_owned`] runs its items on more than one thread, for
379/// callers whose parallel path has a setup cost worth skipping otherwise.
380pub(crate) fn threaded() -> bool {
381    cfg!(all(
382        feature = "parallel",
383        any(not(target_arch = "wasm32"), target_feature = "atomics")
384    ))
385}
386
387#[cfg(not(all(
388    feature = "parallel",
389    any(not(target_arch = "wasm32"), target_feature = "atomics")
390)))]
391pub(crate) fn for_each_owned<T: Send>(items: Vec<T>, f: impl Fn(T) + Sync + Send) {
392    items.into_iter().for_each(f)
393}