Skip to main content

threers_codec/
lib.rs

1//! Pure-Rust media codecs: video and audio encoders, decoders, muxers and
2//! demuxers, with hardware acceleration where the platform has it and no C
3//! dependencies anywhere else.
4//!
5//! Everything builds for `wasm32` as well as native and depends on nothing
6//! beyond `std` (`rayon` with `parallel`). Outside the opt-in hardware
7//! modules (`videotoolbox`, `nvidia`, `vaapi`) there are no OS, process or
8//! filesystem calls: encoders return `Vec<u8>`, and the caller owns I/O. It
9//! is the media half of [threers](https://crates.io/crates/threers), which
10//! re-exports it as `threers::codec`, split out so programs that only encode
11//! or decode media do not link a renderer.
12//!
13//! # Start here
14//!
15//! [`prelude`] has the types most programs need. Three entry points cover
16//! the common jobs:
17//!
18//! ```no_run
19//! # #[cfg(feature = "decode")] {
20//! use threers_codec::prelude::*;
21//!
22//! // Encode RGBA frames to MP4 — on the hardware encoder where there is one.
23//! let mut enc = Mp4Encoder::new(640, 360, EncodeOptions::default()).unwrap();
24//! enc.push_rgba(&vec![128; 640 * 360 * 4]).unwrap();
25//! std::fs::write("out.mp4", enc.finish().unwrap()).unwrap();
26//!
27//! // Decode any audio file this crate reads to planar f32.
28//! let pcm = audio_file::read(&std::fs::read("song.flac").unwrap()).unwrap();
29//! println!("{} channels at {} Hz", pcm.channels.len(), pcm.sample_rate);
30//!
31//! // Decode a video track (feature `decode`), again hardware first.
32//! let file = std::fs::read("clip.mp4").unwrap();
33//! let track = demux_video(&file).unwrap();
34//! let mut dec = Decoder::new(&track).unwrap();
35//! for s in &track.samples {
36//!     dec.push(&file[s.offset as usize..][..s.size as usize], s.pts, s.dts, s.duration as i64)
37//!         .unwrap();
38//!     if let Some(_bgra) = dec.take(s.pts) {}
39//! }
40//! # }
41//! ```
42//!
43//! # What is here
44//!
45//! | Kind | Modules |
46//! |------|---------|
47//! | Video encoders | [`h264`] (CAVLC/CABAC, intra + inter), [`hevc`] (intra, alpha), [`vp9`] (intra + inter, alpha) |
48//! | Video decoders | [`decode`]: H.264, HEVC, VP9, AV1 — bit-exact against the reference decoders (feature `decode`) |
49//! | Audio decoders | [`aac`] (LC, HE-AAC SBR + PS), [`mp3`], [`opus`] (SILK, CELT, hybrid), [`vorbis`], [`flac`], [`pcm`] (WAV/RF64, AIFF); [`audio_file`] sniffs and dispatches |
50//! | Animated images | [`gif`] (encode and decode, feature `gif`), [`apng`]; [`animation`] picks one for a browser |
51//! | Containers | [`mp4`] / [`mp4_stream`] (write), [`mp4_demux`] (read), [`webm`] (write), [`mkv_demux`] (read, plus IVF), [`ogg`] (read) |
52//! | Hardware | [`hw`]: one encoder and one decoder over VideoToolbox, NVENC/NVDEC, VAAPI and the native codecs; `videotoolbox`, `videotoolbox_decode` (macOS), `nvidia` (Linux, Windows) and `vaapi` (Linux) directly |
53//! | Rate control | [`rate`]: a quantizer or a bitrate target |
54//! | Conversion | [`swr`] (resampling and sample formats), [`ffcompat`] (`swscale`, `.mov`/`.mkv` readers, the shapes ffmpeg's tools produce) |
55//! | Captions | [`captions`]: tracks and cues, SRT and WebVTT, muxed by `mp4` and `webm` as soft subtitles |
56//! | Primitives | [`bitstream`] (bit writer, Exp-Golomb, emulation prevention), [`contrast`] (WCAG luminance) |
57//!
58//! # Deep access
59//!
60//! Every stage of every codec is a public module, so the building blocks are
61//! there for research, tooling and tests as well as the high-level calls: an
62//! entropy coder ([`h264::cabac`], [`hevc::cabac`], [`vp9::bool_coder`],
63//! [`decode::av1::msac`]), a transform ([`hevc::transform`],
64//! [`decode::av1::itx`]), a loop filter ([`h264::deblock`],
65//! [`decode::vp9::lpf`]), a parameter-set parser ([`decode::h264::params`],
66//! [`decode::hevc::params`]), an MDCT ([`aac::dsp`], [`opus::celt`]). Each
67//! module's docs say which section of which specification it implements.
68//! These inner modules follow the structure of their specifications rather
69//! than a stability promise; the high-level types re-exported from each
70//! codec's root and from [`prelude`] are the stable surface.
71//!
72//! # Hardware acceleration
73//!
74//! [`hw::Mp4Encoder`] and [`decode::Decoder`] use VideoToolbox on macOS (the
75//! `videotoolbox` feature), an NVIDIA GPU's NVENC and NVDEC engines on
76//! Linux and Windows (the `nvidia` feature), and VA drivers — AMD, Intel —
77//! on Linux (the `vaapi` feature), and fall back to the
78//! pure-Rust codecs when the feature is off, the machine has no such
79//! hardware, or the hardware refuses a stream. The NVIDIA and VA libraries
80//! are loaded at run time, so a binary built with those features still runs,
81//! natively, where they are missing.
82//! The software paths use every core with `parallel`: frames of an all-intra
83//! stream are independent, so batches of them encode at once with
84//! byte-identical output (see [`set_frame_memory_budget`]).
85//!
86//! # Features
87//!
88//! | Feature | Default | What it adds |
89//! |---------|---------|--------------|
90//! | `decode` | **on** | The video decoders in [`decode`] (H.264, HEVC, VP9, AV1) |
91//! | `gif` | **on** | GIF89a encode and decode ([`gif`]), and GIF in [`animation`] |
92//! | `parallel` | | Frame-parallel encoding on `rayon` |
93//! | `videotoolbox` | | VideoToolbox encode and decode on macOS (links Apple's frameworks; inert elsewhere) |
94//! | `nvidia` | | NVENC encode and NVDEC decode on Linux and Windows (driver loaded at run time; implies `decode`; inert elsewhere) |
95//! | `vaapi` | | VAAPI HEVC encode and H.264/HEVC/VP9/AV1 decode on Linux (libva loaded at run time; implies `decode`; inert elsewhere) |
96//!
97//! Encoders, audio decoders, muxers, demuxers and captions are always
98//! compiled; none of them has a dependency to make optional.
99
100// The codecs follow their specifications' notation: loops over indices that
101// address several parallel arrays, argument lists that mirror the spec's
102// processes, and the reference decoders' own nested types.
103#![allow(clippy::too_many_arguments, clippy::needless_range_loop, clippy::type_complexity)]
104#![cfg_attr(docsrs, feature(doc_cfg))]
105
106pub mod aac;
107pub mod animation;
108pub mod apng;
109pub mod audio_file;
110pub mod bitstream;
111pub mod captions;
112pub mod contrast;
113pub mod decode;
114pub mod ffcompat;
115pub mod flac;
116#[cfg(feature = "gif")]
117pub mod gif;
118pub mod h264;
119pub mod hevc;
120pub mod hw;
121pub mod mkv_demux;
122pub mod mp3;
123pub mod mp4;
124pub mod mp4_demux;
125pub mod mp4_stream;
126#[cfg(all(feature = "nvidia", any(target_os = "linux", target_os = "windows"), target_pointer_width = "64"))]
127pub mod nvidia;
128#[cfg(all(feature = "vaapi", target_os = "linux"))]
129pub mod vaapi;
130pub mod ogg;
131pub mod opus;
132pub mod pcm;
133pub mod prelude;
134pub mod rate;
135pub mod swr;
136#[cfg(all(feature = "videotoolbox", target_os = "macos"))]
137pub mod videotoolbox;
138#[cfg(all(feature = "videotoolbox", target_os = "macos"))]
139pub mod videotoolbox_decode;
140pub mod vorbis;
141pub mod vp9;
142pub mod webm;
143
144pub use animation::{
145    encode_animation_rgba, encode_animation_rgba_with_progress, format_animation_progress,
146    AnimationEncodeError, AnimationEncodeOptions, AnimationExportPhase, AnimationExportProgress,
147    BrowserCodec,
148};
149pub use apng::ApngEncoder;
150pub use bitstream::{emulation_prevention, rbsp_trailing_bits, BitWriter};
151#[cfg(feature = "gif")]
152pub use gif::{
153    decode_gif, encode_gif, DisposalMethod, DisposalMode, GifDecoder, GifEncoder, GifError,
154    GifFrame, GifFrameMeta, GifInfo, GifOptions, GifVersion, GifWriter, LzwClearMode, PaletteMode,
155    QuantizerKind,
156};
157#[cfg(feature = "gif")]
158#[allow(deprecated)]
159pub use gif::DecodedFrame;
160pub use vp9::{encode_intra_frame, encode_intra_gray, Reconstruction};
161pub use webm::{encode_gray_webm, encode_webm, mux_webm, WebmCodec, WebmFrame, WebmParams};
162
163// --------------------------------------------------------------------------
164// Frame-level parallelism
165// --------------------------------------------------------------------------
166
167/// How many frames to hold in flight at once.
168///
169/// Every frame of an all-intra stream is independent, so the only thing bounding
170/// the batch is memory: an 8K frame is 50 MB of input and the encoder allocates
171/// a reconstruction of its own alongside it. Encoding every frame at once would
172/// undo the streaming work that got export memory down in the first place, so
173/// the batch is whichever is smaller — the thread count, or what fits in a
174/// fixed working-set budget.
175pub(crate) fn frames_in_flight(width: u32, height: u32) -> usize {
176    frames_by_memory(width, height).min(worker_threads())
177}
178
179/// How many frames the memory budget allows in flight, irrespective of how many
180/// cores there are.
181///
182/// Rate control sizes its groups from this rather than from
183/// [`frames_in_flight`], because the group is how often the controller gets to
184/// correct itself — and a clip must not encode to a different size on a machine
185/// with more cores.
186pub(crate) fn frames_by_memory(width: u32, height: u32) -> usize {
187    (frame_memory_budget() / frame_working_set(width, height).max(1)).max(1)
188}
189
190/// Roughly what one frame costs while it is being encoded: the input planes,
191/// the reconstruction, and the padded copies the encoder makes of both.
192pub(crate) fn frame_working_set(width: u32, height: u32) -> usize {
193    (width as usize * height as usize * 3 / 2) * 5 / 2
194}
195
196/// Default working-set budget for frame-parallel encoding.
197///
198/// Deliberately conservative: it binds only at 4K and above, and only there
199/// because a frame is large enough that holding a core's worth of them is real
200/// memory. See [`set_frame_memory_budget`].
201pub const DEFAULT_FRAME_MEMORY_BUDGET: usize = 256 << 20;
202
203static FRAME_MEMORY_BUDGET: std::sync::atomic::AtomicUsize =
204    std::sync::atomic::AtomicUsize::new(DEFAULT_FRAME_MEMORY_BUDGET);
205
206/// How much memory frame-parallel encoding may hold in flight.
207pub fn frame_memory_budget() -> usize {
208    FRAME_MEMORY_BUDGET.load(std::sync::atomic::Ordering::Relaxed)
209}
210
211/// Set how much memory frame-parallel encoding may hold in flight.
212///
213/// Frames of an all-intra stream are independent, so the only thing limiting how
214/// many encode at once is memory — about 4.5 bytes per pixel per frame in
215/// flight, which is 149 MB at 8K. The default of 256 MB keeps peak usage close
216/// to where the streaming work left it, at the cost of capping 8K to two frames
217/// at a time.
218///
219/// Raising it is the cheapest speedup available at high resolution and it costs
220/// nothing in bitrate: measured on a 4-frame 8K clip, going from 256 MB to 1 GB
221/// took the encode from 49.9 s to 33.0 s with byte-identical output. Tiling the
222/// picture would reach the same parallelism at a bitrate cost; this does not.
223///
224/// ```
225/// // Let an 8K export use eight frames at a time.
226/// threers_codec::set_frame_memory_budget(1024 << 20);
227/// ```
228pub fn set_frame_memory_budget(bytes: usize) {
229    FRAME_MEMORY_BUDGET.store(bytes.max(1), std::sync::atomic::Ordering::Relaxed);
230}
231
232#[cfg(feature = "parallel")]
233fn worker_threads() -> usize {
234    rayon::current_num_threads().max(1)
235}
236
237#[cfg(not(feature = "parallel"))]
238fn worker_threads() -> usize {
239    1
240}
241
242/// Encode a batch of frames, in order, using every available thread.
243///
244/// The output is byte-identical to mapping `encode` over the frames one at a
245/// time — the `parallel` feature changes how long it takes and nothing else.
246#[cfg(feature = "parallel")]
247pub(crate) fn map_frames<T, R>(frames: &[T], encode: impl Fn(&T) -> R + Sync + Send) -> Vec<R>
248where
249    T: Sync,
250    R: Send,
251{
252    use rayon::prelude::*;
253    frames.par_iter().map(encode).collect()
254}
255
256#[cfg(not(feature = "parallel"))]
257pub(crate) fn map_frames<T, R>(frames: &[T], encode: impl Fn(&T) -> R + Sync + Send) -> Vec<R> {
258    frames.iter().map(encode).collect()
259}
260
261/// `map_frames`, public so benchmarks and callers with their own frame source
262/// can use the same batching the export paths do.
263pub fn map_frames_pub<T, R>(frames: &[T], encode: impl Fn(&T) -> R + Sync + Send) -> Vec<R>
264where
265    T: Sync,
266    R: Send,
267{
268    map_frames(frames, encode)
269}
270
271/// [`map_frames`] where the work needs to know its own index — tiles do, to
272/// tell the last one from the rest.
273#[cfg(feature = "parallel")]
274pub(crate) fn map_indexed<T, R>(items: &[T], f: impl Fn(usize, &T) -> R + Sync + Send) -> Vec<R>
275where
276    T: Sync,
277    R: Send,
278{
279    use rayon::prelude::*;
280    items.par_iter().enumerate().map(|(i, t)| f(i, t)).collect()
281}
282
283#[cfg(not(feature = "parallel"))]
284pub(crate) fn map_indexed<T, R>(items: &[T], f: impl Fn(usize, &T) -> R + Sync + Send) -> Vec<R> {
285    items.iter().enumerate().map(|(i, t)| f(i, t)).collect()
286}
287