Skip to main content

threers_codec/nvidia/
nvenc.rs

1//! NVENC: HEVC from RGBA frames on an NVIDIA GPU's encode engine, muxed to
2//! MP4 in process.
3//!
4//! The session runs synchronously with B-frames off, so every frame pushed
5//! comes back as one access unit before `push_rgba` returns — the same
6//! one-in, one-out shape as the native encoder, and the MP4 it writes needs
7//! no composition offsets.
8
9use std::ffi::{c_char, c_void, CStr};
10
11use super::nvenc_layout::*;
12use super::{Context, Lib};
13use crate::hevc::hvcc::{build_hvcc, HvccArray, HvccProfile};
14
15type Status = i32;
16type Handle = *mut c_void;
17
18#[repr(C)]
19#[derive(Clone, Copy)]
20struct Guid(u32, u16, u16, [u8; 8]);
21
22const CODEC_HEVC: Guid = Guid(
23    0x790cdc88,
24    0x4522,
25    0x4d7b,
26    [0x94, 0x25, 0xbd, 0xa9, 0x97, 0x5f, 0x76, 0x03],
27);
28const PRESET_P4: Guid = Guid(
29    0x90a7b826,
30    0xdf06,
31    0x4862,
32    [0xb9, 0xd2, 0xcd, 0x6d, 0x73, 0xa0, 0x86, 0x81],
33);
34const PRESET_P2: Guid = Guid(
35    0xf581cfb8,
36    0x88d6,
37    0x4381,
38    [0x93, 0xf0, 0xdf, 0x13, 0xf9, 0xc2, 0x7d, 0xab],
39);
40
41/// A zeroed C struct of a known size, with fields written at known offsets.
42struct Blob(Vec<u64>);
43
44impl Blob {
45    fn new(size: usize, version: u32) -> Blob {
46        let mut b = Blob(vec![0; size.div_ceil(8)]);
47        b.u32(0, version);
48        b
49    }
50    fn ptr(&mut self) -> *mut c_void {
51        self.0.as_mut_ptr().cast()
52    }
53    fn bytes(&mut self) -> &mut [u8] {
54        let n = self.0.len() * 8;
55        unsafe { std::slice::from_raw_parts_mut(self.0.as_mut_ptr().cast(), n) }
56    }
57    fn u32(&mut self, at: usize, v: u32) {
58        self.bytes()[at..at + 4].copy_from_slice(&v.to_ne_bytes());
59    }
60    fn u64(&mut self, at: usize, v: u64) {
61        self.bytes()[at..at + 8].copy_from_slice(&v.to_ne_bytes());
62    }
63    fn set_ptr(&mut self, at: usize, p: *mut c_void) {
64        self.u64(at, p as usize as u64);
65    }
66    fn guid(&mut self, at: usize, g: Guid) {
67        let mut raw = [0u8; 16];
68        raw[0..4].copy_from_slice(&g.0.to_ne_bytes());
69        raw[4..6].copy_from_slice(&g.1.to_ne_bytes());
70        raw[6..8].copy_from_slice(&g.2.to_ne_bytes());
71        raw[8..].copy_from_slice(&g.3);
72        self.bytes()[at..at + 16].copy_from_slice(&raw);
73    }
74    fn get_u32(&mut self, at: usize) -> u32 {
75        u32::from_ne_bytes(self.bytes()[at..at + 4].try_into().unwrap())
76    }
77    fn get_ptr(&mut self, at: usize) -> *mut c_void {
78        u64::from_ne_bytes(self.bytes()[at..at + 8].try_into().unwrap()) as usize as *mut c_void
79    }
80}
81
82/// The function table `NvEncodeAPICreateInstance` fills.
83struct Api {
84    _lib: Lib,
85    table: Blob,
86}
87
88impl Api {
89    fn load() -> Result<Api, String> {
90        let lib = Lib::open(&[
91            "libnvidia-encode.so.1",
92            "libnvidia-encode.so",
93            "nvEncodeAPI64.dll",
94        ])
95        .ok_or("no NVENC library (libnvidia-encode) in this driver")?;
96        unsafe {
97            let max: unsafe extern "C" fn(*mut u32) -> Status = lib
98                .sym("NvEncodeAPIGetMaxSupportedVersion")
99                .ok_or("NVENC library without a version query")?;
100            let mut v = 0u32;
101            max(&mut v);
102            let have = ((v >> 4) & 0xfff, v & 0xf);
103            if have < (API_VERSION & 0xff, API_VERSION >> 24) {
104                return Err(format!(
105                    "the driver's NVENC is API {}.{}; this needs 12.0 (driver 520 or newer)",
106                    have.0, have.1
107                ));
108            }
109            let create: unsafe extern "C" fn(*mut c_void) -> Status = lib
110                .sym("NvEncodeAPICreateInstance")
111                .ok_or("NVENC library without NvEncodeAPICreateInstance")?;
112            let mut table = Blob::new(FUNCTION_LIST_SIZE, FUNCTION_LIST_VER);
113            let s = create(table.ptr());
114            if s != 0 {
115                return Err(format!(
116                    "NvEncodeAPICreateInstance failed (NVENC status {s})"
117                ));
118            }
119            Ok(Api { _lib: lib, table })
120        }
121    }
122
123    /// The entry at `offset` in the table, as a function of type `F`.
124    unsafe fn f<F: Copy>(&mut self, offset: usize) -> F {
125        let p = self.table.get_ptr(offset);
126        assert!(!p.is_null(), "NVENC table entry {offset} is empty");
127        unsafe { std::mem::transmute_copy(&p) }
128    }
129}
130
131/// A hardware HEVC encoder on an NVIDIA GPU's NVENC engine: RGBA frames in,
132/// an `hvc1` MP4 out.
133pub struct NvencEncoder {
134    api: Api,
135    enc: Handle,
136    input: Handle,
137    output: Handle,
138    // After the session, which uses it, in drop order.
139    ctx: Context,
140    width: u32,
141    height: u32,
142    fps: u32,
143    frame: u32,
144    vps: Vec<u8>,
145    sps: Vec<u8>,
146    pps: Vec<u8>,
147    samples: Vec<Vec<u8>>,
148}
149
150/// How an [`NvencEncoder`] spends bits.
151#[derive(Debug, Clone, Copy, PartialEq)]
152pub enum NvencRate {
153    /// A fixed quantizer for every frame (0–51, lower is better).
154    ConstQp(u32),
155    /// Variable bitrate around this many bits per second.
156    Bitrate(u32),
157}
158
159impl NvencEncoder {
160    /// Open an encoder for `width × height` frames at `fps`, rate-controlled
161    /// by `rate`, a keyframe every `keyframe_secs` seconds. `realtime` picks
162    /// the low-latency tuning and a faster preset; otherwise high quality.
163    pub fn new(
164        width: u32,
165        height: u32,
166        fps: u32,
167        rate: NvencRate,
168        keyframe_secs: u32,
169        realtime: bool,
170    ) -> Result<Self, String> {
171        let ctx = Context::new()?;
172        let mut api = Api::load()?;
173        let fps = fps.max(1);
174        let mut enc: Handle = std::ptr::null_mut();
175        unsafe {
176            let open: unsafe extern "C" fn(*mut c_void, *mut Handle) -> Status =
177                api.f(FN_OPEN_ENCODE_SESSION_EX);
178            let mut p = Blob::new(OPEN_SIZE, OPEN_VER);
179            p.u32(OPEN_DEVICE_TYPE, DEVICE_TYPE_CUDA);
180            p.set_ptr(OPEN_DEVICE, ctx.raw());
181            p.u32(OPEN_API_VERSION, API_VERSION);
182            let s = open(p.ptr(), &mut enc);
183            if s != 0 {
184                return Err(format!(
185                    "nvEncOpenEncodeSessionEx failed (NVENC status {s})"
186                ));
187            }
188        }
189        let mut me = NvencEncoder {
190            api,
191            enc,
192            input: std::ptr::null_mut(),
193            output: std::ptr::null_mut(),
194            ctx,
195            width,
196            height,
197            fps,
198            frame: 0,
199            vps: Vec::new(),
200            sps: Vec::new(),
201            pps: Vec::new(),
202            samples: Vec::new(),
203        };
204        me.initialize(rate, keyframe_secs, realtime)?;
205        Ok(me)
206    }
207
208    fn status(&mut self, what: &str, s: Status) -> Result<(), String> {
209        if s == 0 {
210            return Ok(());
211        }
212        let detail = unsafe {
213            let last: unsafe extern "C" fn(Handle) -> *const c_char =
214                self.api.f(FN_GET_LAST_ERROR_STRING);
215            let p = last(self.enc);
216            if p.is_null() {
217                String::new()
218            } else {
219                CStr::from_ptr(p).to_string_lossy().into_owned()
220            }
221        };
222        Err(format!("{what} failed (NVENC status {s}): {detail}"))
223    }
224
225    fn initialize(
226        &mut self,
227        rate: NvencRate,
228        keyframe_secs: u32,
229        realtime: bool,
230    ) -> Result<(), String> {
231        let (preset, tuning) = if realtime {
232            (PRESET_P2, TUNING_LOW_LATENCY)
233        } else {
234            (PRESET_P4, TUNING_HIGH_QUALITY)
235        };
236        let gop = (self.fps * keyframe_secs.max(1)).max(1);
237        unsafe {
238            // The preset's configuration, then the few fields this changes.
239            let get: unsafe extern "C" fn(Handle, Guid, Guid, u32, *mut c_void) -> Status =
240                self.api.f(FN_GET_ENCODE_PRESET_CONFIG_EX);
241            let mut preset_cfg = Blob::new(PRESET_SIZE, PRESET_VER);
242            preset_cfg.u32(PRESET_CFG, CONFIG_VER);
243            let s = get(self.enc, CODEC_HEVC, preset, tuning, preset_cfg.ptr());
244            self.status("nvEncGetEncodePresetConfigEx", s)?;
245            let mut cfg = Blob::new(CONFIG_SIZE, CONFIG_VER);
246            cfg.bytes()[..CONFIG_SIZE]
247                .copy_from_slice(&preset_cfg.bytes()[PRESET_CFG..PRESET_CFG + CONFIG_SIZE]);
248            cfg.u32(0, CONFIG_VER);
249            cfg.u32(CONFIG_GOP_LENGTH, gop);
250            cfg.u32(CONFIG_FRAME_INTERVAL_P, 1);
251            cfg.u32(CONFIG_HEVC_IDR_PERIOD, gop);
252            match rate {
253                NvencRate::ConstQp(qp) => {
254                    let qp = qp.min(51);
255                    cfg.u32(CONFIG_RC_MODE, RC_CONSTQP);
256                    for i in 0..3 {
257                        cfg.u32(CONFIG_RC_CONST_QP + 4 * i, qp);
258                    }
259                }
260                NvencRate::Bitrate(bps) => {
261                    cfg.u32(CONFIG_RC_MODE, RC_VBR);
262                    cfg.u32(CONFIG_RC_AVG_BITRATE, bps);
263                    cfg.u32(CONFIG_RC_MAX_BITRATE, bps.saturating_mul(3) / 2);
264                }
265            }
266
267            let init: unsafe extern "C" fn(Handle, *mut c_void) -> Status =
268                self.api.f(FN_INITIALIZE_ENCODER);
269            let mut p = Blob::new(INIT_SIZE, INIT_VER);
270            p.guid(INIT_ENCODE_GUID, CODEC_HEVC);
271            p.guid(INIT_PRESET_GUID, preset);
272            p.u32(INIT_WIDTH, self.width);
273            p.u32(INIT_HEIGHT, self.height);
274            p.u32(INIT_DAR_WIDTH, self.width);
275            p.u32(INIT_DAR_HEIGHT, self.height);
276            p.u32(INIT_FRAME_RATE_NUM, self.fps);
277            p.u32(INIT_FRAME_RATE_DEN, 1);
278            p.u32(INIT_ENABLE_PTD, 1);
279            p.set_ptr(INIT_ENCODE_CONFIG, cfg.ptr());
280            p.u32(INIT_MAX_WIDTH, self.width);
281            p.u32(INIT_MAX_HEIGHT, self.height);
282            p.u32(INIT_TUNING_INFO, tuning);
283            let s = init(self.enc, p.ptr());
284            self.status("nvEncInitializeEncoder", s)?;
285
286            let create_in: unsafe extern "C" fn(Handle, *mut c_void) -> Status =
287                self.api.f(FN_CREATE_INPUT_BUFFER);
288            let mut b = Blob::new(INPUT_SIZE, INPUT_VER);
289            b.u32(INPUT_WIDTH, self.width);
290            b.u32(INPUT_HEIGHT, self.height);
291            b.u32(INPUT_FORMAT, BUFFER_FORMAT_ABGR);
292            let s = create_in(self.enc, b.ptr());
293            self.status("nvEncCreateInputBuffer", s)?;
294            self.input = b.get_ptr(INPUT_BUFFER);
295
296            let create_out: unsafe extern "C" fn(Handle, *mut c_void) -> Status =
297                self.api.f(FN_CREATE_BITSTREAM_BUFFER);
298            let mut b = Blob::new(BITSTREAM_SIZE, BITSTREAM_VER);
299            let s = create_out(self.enc, b.ptr());
300            self.status("nvEncCreateBitstreamBuffer", s)?;
301            self.output = b.get_ptr(BITSTREAM_BUFFER);
302        }
303        Ok(())
304    }
305
306    /// Encode one frame of tightly packed RGBA8, `width * height * 4` bytes.
307    pub fn push_rgba(&mut self, rgba: &[u8]) -> Result<(), String> {
308        let (w, h) = (self.width as usize, self.height as usize);
309        if rgba.len() < w * h * 4 {
310            return Err(format!("frame is {} bytes, want {}", rgba.len(), w * h * 4));
311        }
312        unsafe {
313            let lock: unsafe extern "C" fn(Handle, *mut c_void) -> Status =
314                self.api.f(FN_LOCK_INPUT_BUFFER);
315            let mut l = Blob::new(LOCK_INPUT_SIZE, LOCK_INPUT_VER);
316            l.set_ptr(LOCK_INPUT_BUFFER, self.input);
317            let s = lock(self.enc, l.ptr());
318            self.status("nvEncLockInputBuffer", s)?;
319            let base = l.get_ptr(LOCK_INPUT_DATA) as *mut u8;
320            let pitch = l.get_u32(LOCK_INPUT_PITCH) as usize;
321            // `ABGR` is a 32-bit word with R in the low byte: RGBA in memory.
322            for y in 0..h {
323                std::ptr::copy_nonoverlapping(
324                    rgba[y * w * 4..].as_ptr(),
325                    base.add(y * pitch),
326                    w * 4,
327                );
328            }
329            let unlock: unsafe extern "C" fn(Handle, Handle) -> Status =
330                self.api.f(FN_UNLOCK_INPUT_BUFFER);
331            let s = unlock(self.enc, self.input);
332            self.status("nvEncUnlockInputBuffer", s)?;
333
334            let encode: unsafe extern "C" fn(Handle, *mut c_void) -> Status =
335                self.api.f(FN_ENCODE_PICTURE);
336            let mut p = Blob::new(PIC_SIZE, PIC_VER);
337            p.u32(PIC_INPUT_WIDTH, self.width);
338            p.u32(PIC_INPUT_HEIGHT, self.height);
339            p.u32(PIC_INPUT_PITCH, (pitch / 4) as u32);
340            p.u32(PIC_FRAME_IDX, self.frame);
341            p.u64(PIC_TIMESTAMP, self.frame as u64);
342            p.set_ptr(PIC_INPUT_BUFFER, self.input);
343            p.set_ptr(PIC_OUTPUT, self.output);
344            p.u32(PIC_BUFFER_FORMAT, BUFFER_FORMAT_ABGR);
345            p.u32(PIC_STRUCT, PIC_STRUCT_FRAME);
346            let s = encode(self.enc, p.ptr());
347            self.status("nvEncEncodePicture", s)?;
348        }
349        self.frame += 1;
350        self.collect()
351    }
352
353    /// Read the access unit the last frame produced into the sample list.
354    fn collect(&mut self) -> Result<(), String> {
355        let au = unsafe {
356            let lock: unsafe extern "C" fn(Handle, *mut c_void) -> Status =
357                self.api.f(FN_LOCK_BITSTREAM);
358            let mut l = Blob::new(LOCK_BS_SIZE, LOCK_BS_VER);
359            l.set_ptr(LOCK_BS_OUTPUT, self.output);
360            let s = lock(self.enc, l.ptr());
361            self.status("nvEncLockBitstream", s)?;
362            let n = l.get_u32(LOCK_BS_SIZE_BYTES) as usize;
363            let p = l.get_ptr(LOCK_BS_PTR) as *const u8;
364            let au = std::slice::from_raw_parts(p, n).to_vec();
365            let unlock: unsafe extern "C" fn(Handle, Handle) -> Status =
366                self.api.f(FN_UNLOCK_BITSTREAM);
367            unlock(self.enc, self.output);
368            au
369        };
370        let mut sample = Vec::with_capacity(au.len());
371        for nal in crate::h264::nal::split_annexb(&au) {
372            match nal.first().map(|b| (b >> 1) & 0x3f) {
373                Some(32) if self.vps.is_empty() => self.vps = nal,
374                Some(33) if self.sps.is_empty() => self.sps = nal,
375                Some(34) if self.pps.is_empty() => self.pps = nal,
376                // Repeated parameter sets live in `hvcC`, not the samples.
377                Some(32..=34) => {}
378                _ => {
379                    sample.extend_from_slice(&(nal.len() as u32).to_be_bytes());
380                    sample.extend_from_slice(&nal);
381                }
382            }
383        }
384        if !sample.is_empty() {
385            self.samples.push(sample);
386        }
387        Ok(())
388    }
389
390    /// Flush the encoder and return the MP4 file.
391    pub fn finish(mut self) -> Result<Vec<u8>, String> {
392        unsafe {
393            let encode: unsafe extern "C" fn(Handle, *mut c_void) -> Status =
394                self.api.f(FN_ENCODE_PICTURE);
395            let mut p = Blob::new(PIC_SIZE, PIC_VER);
396            p.u32(PIC_FLAGS, PIC_FLAG_EOS);
397            let s = encode(self.enc, p.ptr());
398            self.status("end of stream", s)?;
399        }
400        if self.samples.is_empty() {
401            return Err("no frames were pushed".into());
402        }
403        if self.vps.is_empty() || self.sps.is_empty() || self.pps.is_empty() {
404            return Err("NVENC did not emit HEVC parameter sets".into());
405        }
406        let profile = HvccProfile::from_sps(&self.sps).ok_or("NVENC's SPS did not parse")?;
407        let (vps, sps, pps) = ([self.vps.clone()], [self.sps.clone()], [self.pps.clone()]);
408        let hvcc = build_hvcc(
409            &profile,
410            &[
411                HvccArray {
412                    nal_type: 32,
413                    complete: true,
414                    nals: &vps,
415                },
416                HvccArray {
417                    nal_type: 33,
418                    complete: true,
419                    nals: &sps,
420                },
421                HvccArray {
422                    nal_type: 34,
423                    complete: true,
424                    nals: &pps,
425                },
426            ],
427        );
428        Ok(crate::mp4::mux_hevc(&crate::mp4::Mp4Params {
429            width: self.width,
430            height: self.height,
431            timescale: self.fps * 1000,
432            frame_duration: 1000,
433            hvcc_payload: &hvcc,
434            almo_payload: None,
435            samples: &self.samples,
436        }))
437    }
438}
439
440impl Drop for NvencEncoder {
441    fn drop(&mut self) {
442        unsafe {
443            if !self.input.is_null() {
444                let f: unsafe extern "C" fn(Handle, Handle) -> Status =
445                    self.api.f(FN_DESTROY_INPUT_BUFFER);
446                f(self.enc, self.input);
447            }
448            if !self.output.is_null() {
449                let f: unsafe extern "C" fn(Handle, Handle) -> Status =
450                    self.api.f(FN_DESTROY_BITSTREAM_BUFFER);
451                f(self.enc, self.output);
452            }
453            if !self.enc.is_null() {
454                let f: unsafe extern "C" fn(Handle) -> Status = self.api.f(FN_DESTROY_ENCODER);
455                f(self.enc);
456            }
457        }
458        let _ = &self.ctx;
459    }
460}