use crate::bitstream::{emulation_prevention, BitWriter};
use crate::captions::CaptionTrack;
use crate::hevc::cabac::{CabacEncoder, CtxModel};
use crate::hevc::deblock::{deblock, EdgeMap};
use crate::hevc::encoder::pad_plane;
use crate::hevc::hvcc::{build_hvcc, HvccArray, HvccProfile};
use crate::hevc::nal::{nal_unit_base, push_annexb, NalUnitType};
use crate::hevc::params::{
write_pps_cu_qp_delta, write_pps_cu_qp_delta_no_deblock, write_pps_tiled, write_sps, write_vps,
HevcConfig,
};
use crate::hevc::residual::ResidualCtx;
use crate::hevc::transform::{forward_into, inverse_into, TransformKind, MAX_N};
use crate::hevc::{intra, quant, residual, Yuv420Frame};
use crate::mp4::{mux_hevc, mux_hevc_with_captions, Mp4Params};
use crate::rate::{Quality, RateControl, Reach, PROBE_QP};
pub struct CompressedEncoder {
cfg: HevcConfig,
headers_emitted: bool,
residual: bool,
aq_strength: f32,
deblock: bool,
}
impl CompressedEncoder {
pub fn new(width: u32, height: u32, qp: i32) -> Self {
let mut cfg = HevcConfig::new(width, height)
.with_pcm(false)
.with_level(180)
.with_coding_tree(6, 3, 5)
.with_transform_depth(3)
.with_auto_tiles();
cfg.qp = qp;
Self {
cfg,
headers_emitted: false,
residual: true,
aq_strength: 0.0,
deblock: true,
}
}
pub fn deblock(mut self, on: bool) -> Self {
self.deblock = on;
self
}
pub fn aq(mut self, strength: f32) -> Self {
self.aq_strength = strength;
self
}
pub fn residual(mut self, on: bool) -> Self {
self.residual = on;
self
}
pub fn config(&self) -> &HevcConfig {
&self.cfg
}
pub fn encode_frame(&mut self, frame: &Yuv420Frame) -> (Vec<u8>, Reconstruction) {
assert_eq!(
(frame.width, frame.height),
(self.cfg.width, self.cfg.height),
"frame size"
);
let (cw, ch) = (self.cfg.coded_width, self.cfg.coded_height);
let y = pad_plane(&frame.y, frame.width, frame.height, cw, ch);
let u = pad_plane(&frame.u, frame.width / 2, frame.height / 2, cw / 2, ch / 2);
let v = pad_plane(&frame.v, frame.width / 2, frame.height / 2, cw / 2, ch / 2);
let (slice, recon) = compressed_slice_rbsp(
&self.cfg,
&y,
&u,
&v,
self.residual,
self.aq_strength,
self.deblock,
);
let mut au = Vec::new();
if !self.headers_emitted {
push_annexb(&mut au, &nal_unit_base(NalUnitType::Vps, &write_vps(false)));
push_annexb(
&mut au,
&nal_unit_base(NalUnitType::Sps, &write_sps(&self.cfg)),
);
push_annexb(
&mut au,
&nal_unit_base(NalUnitType::Pps, &self.pps_payload()),
);
self.headers_emitted = true;
}
push_annexb(&mut au, &nal_unit_base(NalUnitType::IdrNLp, &slice));
(au, recon)
}
pub fn encode_slice_au(&self, frame: &Yuv420Frame) -> Vec<u8> {
self.encode_slice_au_at(frame, self.cfg.qp)
}
pub fn encode_slice_au_at(&self, frame: &Yuv420Frame, qp: i32) -> Vec<u8> {
let mut cfg = self.cfg;
cfg.qp = qp.clamp(0, 51);
self.encode_with(&cfg, frame)
}
fn encode_with(&self, cfg: &HevcConfig, frame: &Yuv420Frame) -> Vec<u8> {
let (cw, ch) = (cfg.coded_width, cfg.coded_height);
let y = pad_plane(&frame.y, frame.width, frame.height, cw, ch);
let u = pad_plane(&frame.u, frame.width / 2, frame.height / 2, cw / 2, ch / 2);
let v = pad_plane(&frame.v, frame.width / 2, frame.height / 2, cw / 2, ch / 2);
let (slice, _recon) = compressed_slice_rbsp(
cfg,
&y,
&u,
&v,
self.residual,
self.aq_strength,
self.deblock,
);
let mut au = Vec::new();
push_annexb(&mut au, &nal_unit_base(NalUnitType::IdrNLp, &slice));
au
}
fn pps_payload(&self) -> Vec<u8> {
if self.cfg.tiles() > 1 {
write_pps_tiled(self.cfg.tile_grid, self.deblock)
} else if self.deblock {
write_pps_cu_qp_delta()
} else {
write_pps_cu_qp_delta_no_deblock()
}
}
pub fn tiles(mut self, cols: u32, rows: u32) -> Self {
self.cfg = self.cfg.with_tiles(cols, rows);
self
}
pub fn parameter_sets(&self) -> (Vec<u8>, Vec<u8>, Vec<u8>) {
(
nal_unit_base(NalUnitType::Vps, &write_vps(false)),
nal_unit_base(NalUnitType::Sps, &write_sps(&self.cfg)),
nal_unit_base(NalUnitType::Pps, &self.pps_payload()),
)
}
}
pub fn encode_mp4(width: u32, height: u32, fps: u32, qp: i32, frames: &[Yuv420Frame]) -> Vec<u8> {
encode_mp4_with_captions(width, height, fps, qp, frames, None)
}
pub fn encode_mp4_with_captions(
width: u32,
height: u32,
fps: u32,
qp: i32,
frames: &[Yuv420Frame],
captions: Option<&CaptionTrack>,
) -> Vec<u8> {
encode_mp4_from_iter(width, height, fps, qp, captions, frames.iter().cloned())
}
pub fn encode_mp4_from_iter<I>(
width: u32,
height: u32,
fps: u32,
qp: i32,
captions: Option<&CaptionTrack>,
frames: I,
) -> Vec<u8>
where
I: IntoIterator<Item = Yuv420Frame>,
{
encode_mp4_inner(width, height, fps, qp, captions, &mut frames.into_iter())
}
pub fn encode_mp4_streaming<F>(
width: u32,
height: u32,
fps: u32,
qp: i32,
frame_count: usize,
captions: Option<&CaptionTrack>,
next: F,
) -> Vec<u8>
where
F: FnMut(usize) -> Yuv420Frame,
{
encode_mp4_inner(
width,
height,
fps,
qp,
captions,
&mut (0..frame_count).map(next),
)
}
fn encode_mp4_inner(
width: u32,
height: u32,
fps: u32,
qp: i32,
captions: Option<&CaptionTrack>,
frames: &mut dyn Iterator<Item = Yuv420Frame>,
) -> Vec<u8> {
encode_mp4_quality(width, height, fps, Quality::Qp(qp), captions, frames, None)
}
fn report_reach(reach: Reach) {
match reach {
Reach::TooComplex { times } => eprintln!(
"threers: this content will not fit that bitrate — even at the coarsest \
quantizer it is about {times:.1}x larger. The file is as small as the \
encoder can make it."
),
Reach::TooSimple { times } => eprintln!(
"threers: this content cannot fill that bitrate — even losslessly quantized \
it is about {times:.1}x smaller, so the file is smaller than asked for. \
That is the content, not a failure."
),
}
}
#[allow(clippy::too_many_arguments)]
pub fn encode_mp4_quality(
width: u32,
height: u32,
fps: u32,
quality: Quality,
captions: Option<&CaptionTrack>,
frames: &mut dyn Iterator<Item = Yuv420Frame>,
total: Option<u32>,
) -> Vec<u8> {
let enc = CompressedEncoder::new(width, height, PROBE_QP);
let level_idc = enc.config().level_idc;
let (vps_nal, sps_nal, pps_nal) = enc.parameter_sets();
let (vps, sps, pps) = (vec![vps_nal], vec![sps_nal], vec![pps_nal]);
let mut batch = crate::frames_in_flight(width, height);
let mut samples: Vec<Vec<u8>> = Vec::new();
let mut rc = match quality {
Quality::Qp(_) => None,
Quality::Bitrate(bps) => Some(RateControl::new(bps, fps.max(1), total)),
};
if let Some(r) = rc.as_ref() {
batch = r.group_size(crate::frames_by_memory(width, height));
}
let base_qp = match quality {
Quality::Qp(q) => q,
Quality::Bitrate(_) => PROBE_QP,
};
let mut pending: Vec<Yuv420Frame> = Vec::with_capacity(batch);
if let Some(r) = rc.as_mut()
&& let Some(first) = frames.next()
{
let probe = enc.encode_slice_au_at(&first, PROBE_QP);
r.seed(PROBE_QP, probe.len());
if let Some(q2) = r.probe_again() {
let again = enc.encode_slice_au_at(&first, q2);
r.seed_pair(q2, again.len());
}
pending.push(first);
}
loop {
while pending.len() < batch {
match frames.next() {
Some(f) => pending.push(f),
None => break,
}
}
if pending.is_empty() {
break;
}
let qp = rc.as_ref().map_or(base_qp, |r| r.next_qp());
let aus = crate::map_frames(&pending, |f| enc.encode_slice_au_at(f, qp));
pending.clear();
for au in aus {
if let Some(r) = rc.as_mut() {
r.observe(qp, au.len());
}
let mut sample = Vec::new();
for nal in split_annexb(&au) {
sample.extend_from_slice(&(nal.len() as u32).to_be_bytes());
sample.extend_from_slice(&nal);
}
samples.push(sample);
}
}
if let Some(reach) = rc.as_ref().and_then(|r| r.outcome()) {
report_reach(reach);
}
let hvcc = build_hvcc(
&HvccProfile::main_420(level_idc),
&[
HvccArray {
nal_type: 32,
complete: true,
nals: &vps,
},
HvccArray {
nal_type: 33,
complete: true,
nals: &sps,
},
HvccArray {
nal_type: 34,
complete: true,
nals: &pps,
},
],
);
let timescale = 600u32;
let params = Mp4Params {
width,
height,
timescale,
frame_duration: timescale / fps.max(1),
hvcc_payload: &hvcc,
almo_payload: None,
samples: &samples,
};
match captions {
Some(track) => mux_hevc_with_captions(¶ms, track),
None => mux_hevc(¶ms),
}
}
fn split_annexb(stream: &[u8]) -> Vec<Vec<u8>> {
let mut starts = Vec::new();
let mut i = 0;
while i + 3 <= stream.len() {
if stream[i] == 0 && stream[i + 1] == 0 && stream[i + 2] == 1 {
starts.push((i, 3));
i += 3;
} else if i + 4 <= stream.len()
&& stream[i] == 0
&& stream[i + 1] == 0
&& stream[i + 2] == 0
&& stream[i + 3] == 1
{
starts.push((i, 4));
i += 4;
} else {
i += 1;
}
}
let mut nals = Vec::new();
for (n, &(off, len)) in starts.iter().enumerate() {
let end = starts.get(n + 1).map(|&(o, _)| o).unwrap_or(stream.len());
nals.push(stream[off + len..end].to_vec());
}
nals
}
#[derive(Clone, Copy)]
struct Ctx {
part_mode: CtxModel,
prev_intra: CtxModel,
chroma_mode: CtxModel,
cbf_luma: [CtxModel; 2],
cbf_chroma: [CtxModel; 4],
split_tu: [CtxModel; 3],
split_cu: [CtxModel; 3],
cu_qp_delta: [CtxModel; 2],
res: ResidualCtx,
}
impl Ctx {
fn new(qp: i32) -> Self {
Self {
part_mode: CtxModel::init(184, qp),
prev_intra: CtxModel::init(184, qp),
chroma_mode: CtxModel::init(63, qp),
cbf_luma: [CtxModel::init(111, qp), CtxModel::init(141, qp)],
cbf_chroma: [
CtxModel::init(94, qp),
CtxModel::init(138, qp),
CtxModel::init(182, qp),
CtxModel::init(154, qp),
],
split_tu: [
CtxModel::init(153, qp),
CtxModel::init(138, qp),
CtxModel::init(138, qp),
],
split_cu: [
CtxModel::init(139, qp),
CtxModel::init(141, qp),
CtxModel::init(157, qp),
],
cu_qp_delta: [CtxModel::init(154, qp), CtxModel::init(154, qp)],
res: ResidualCtx::new(qp),
}
}
}
fn chroma_qp(qp_luma: i32) -> i32 {
let qpi = qp_luma.clamp(0, 51);
if qpi < 30 {
qpi
} else {
const T: [i32; 14] = [29, 30, 31, 32, 33, 33, 34, 34, 35, 35, 36, 36, 37, 37];
if qpi <= 43 {
T[(qpi - 30) as usize]
} else {
qpi - 6
}
}
}
pub struct Reconstruction {
pub y: Vec<u8>,
pub u: Vec<u8>,
pub v: Vec<u8>,
pub coded_width: u32,
pub coded_height: u32,
}
pub fn compressed_slice_rbsp(
cfg: &HevcConfig,
y: &[u8],
u: &[u8],
v: &[u8],
residual: bool,
aq_strength: f32,
deblocking: bool,
) -> (Vec<u8>, Reconstruction) {
if cfg.tiles() > 1 {
return tiled_slice_rbsp(cfg, y, u, v, residual, aq_strength, deblocking);
}
let mut rbsp = slice_header(cfg, &[]);
let mut coded = encode_ctbs(cfg, y, u, v, residual, aq_strength, true);
if deblocking {
coded.deblock(cfg);
}
rbsp.extend_from_slice(&coded.data);
(rbsp, coded.recon)
}
fn tile_spans(n: u32, parts: u32) -> Vec<(u32, u32)> {
(0..parts)
.map(|i| {
let a = (i * n) / parts;
let b = ((i + 1) * n) / parts;
(a, b - a)
})
.collect()
}
fn tiled_slice_rbsp(
cfg: &HevcConfig,
y: &[u8],
u: &[u8],
v: &[u8],
residual: bool,
aq_strength: f32,
deblocking: bool,
) -> (Vec<u8>, Reconstruction) {
let (cw, ch) = (cfg.coded_width, cfg.coded_height);
let (cwc, chc) = (cw / 2, ch / 2);
let (nx, ny) = cfg.ctbs();
let ctb = 1u32 << cfg.ctb_log2;
let cols = tile_spans(nx, cfg.tile_grid.0);
let rows = tile_spans(ny, cfg.tile_grid.1);
let mut jobs = Vec::with_capacity(cols.len() * rows.len());
for (ry0, rh) in &rows {
for (cx0, cwd) in &cols {
let x0 = cx0 * ctb;
let y0 = ry0 * ctb;
let tw = (cwd * ctb).min(cw - x0);
let th = (rh * ctb).min(ch - y0);
let mut tcfg = *cfg;
tcfg.width = tw;
tcfg.height = th;
tcfg.coded_width = tw;
tcfg.coded_height = th;
tcfg.tile_grid = (1, 1);
jobs.push((
tcfg,
(x0, y0, tw, th),
crop(y, cw, x0, y0, tw, th),
crop(u, cwc, x0 / 2, y0 / 2, tw / 2, th / 2),
crop(v, cwc, x0 / 2, y0 / 2, tw / 2, th / 2),
));
}
}
let last = jobs.len() - 1;
let coded = crate::map_indexed(&jobs, |i, (tcfg, _, ty, tu, tv)| {
encode_ctbs(tcfg, ty, tu, tv, residual, aq_strength, i == last)
});
let mut recon = Reconstruction {
y: vec![0u8; (cw * ch) as usize],
u: vec![128u8; (cwc * chc) as usize],
v: vec![128u8; (cwc * chc) as usize],
coded_width: cw,
coded_height: ch,
};
let mut edges = EdgeMap::new(cw, ch);
let mut qp = vec![cfg.qp as i8; ((cw >> cfg.min_cb_log2) * (ch >> cfg.min_cb_log2)) as usize];
let gl = cfg.min_cb_log2;
let gw = (cw >> gl) as usize;
for (job, c) in jobs.iter().zip(&coded) {
let (x0, y0, tw, th) = job.1;
paste(&mut recon.y, cw, &c.recon.y, tw, x0, y0, tw, th);
paste(
&mut recon.u,
cwc,
&c.recon.u,
tw / 2,
x0 / 2,
y0 / 2,
tw / 2,
th / 2,
);
paste(
&mut recon.v,
cwc,
&c.recon.v,
tw / 2,
x0 / 2,
y0 / 2,
tw / 2,
th / 2,
);
for ty in (0..th).step_by(1 << gl) {
for tx in (0..tw).step_by(1 << gl) {
let src = (ty >> gl) as usize * (tw >> gl) as usize + (tx >> gl) as usize;
let dst = ((y0 + ty) >> gl) as usize * gw + ((x0 + tx) >> gl) as usize;
qp[dst] = c.qp[src];
}
}
edges.paste(&c.edges, tw, th, x0, y0);
}
if deblocking {
deblock(
&mut recon.y,
&mut recon.u,
&mut recon.v,
cw,
ch,
&edges,
|x, y| i32::from(qp[(y >> gl) as usize * gw + (x >> gl) as usize]),
chroma_qp,
);
}
let mut entry = Vec::new();
for c in &coded[..coded.len() - 1] {
entry.push(emulation_prevention(&c.data).len());
}
let mut rbsp = slice_header(cfg, &entry);
for c in &coded {
rbsp.extend_from_slice(&c.data);
}
(rbsp, recon)
}
fn crop(plane: &[u8], stride: u32, x: u32, y: u32, w: u32, h: u32) -> Vec<u8> {
let mut out = Vec::with_capacity((w * h) as usize);
for r in 0..h {
let o = ((y + r) * stride + x) as usize;
out.extend_from_slice(&plane[o..o + w as usize]);
}
out
}
#[allow(clippy::too_many_arguments)]
fn paste(dst: &mut [u8], stride: u32, src: &[u8], src_stride: u32, x: u32, y: u32, w: u32, h: u32) {
for r in 0..h {
let d = ((y + r) * stride + x) as usize;
let s = (r * src_stride) as usize;
dst[d..d + w as usize].copy_from_slice(&src[s..s + w as usize]);
}
}
struct Coded {
data: Vec<u8>,
recon: Reconstruction,
edges: EdgeMap,
qp: Vec<i8>,
}
impl Coded {
fn deblock(&mut self, cfg: &HevcConfig) {
let (cw, ch) = (cfg.coded_width, cfg.coded_height);
let gl = cfg.min_cb_log2;
let gw = (cw >> gl) as usize;
let qp = &self.qp;
deblock(
&mut self.recon.y,
&mut self.recon.u,
&mut self.recon.v,
cw,
ch,
&self.edges,
|x, y| i32::from(qp[(y >> gl) as usize * gw + (x >> gl) as usize]),
chroma_qp,
);
}
}
fn slice_header(cfg: &HevcConfig, entry_points: &[usize]) -> Vec<u8> {
let mut h = BitWriter::new();
h.flag(true); h.flag(false); h.write_ue(0); h.write_ue(2); h.write_se(cfg.qp - 26); if cfg.tiles() > 1 {
h.write_ue(entry_points.len() as u32);
if !entry_points.is_empty() {
let bits = entry_points
.iter()
.map(|&o| 32 - (o as u32).leading_zeros())
.max()
.unwrap_or(1)
.max(1);
h.write_ue(bits - 1); for &o in entry_points {
h.write_bits((o - 1) as u32, bits);
}
}
}
h.write_bit(1); h.finish()
}
fn encode_ctbs(
cfg: &HevcConfig,
y: &[u8],
u: &[u8],
v: &[u8],
residual: bool,
aq_strength: f32,
last_subset: bool,
) -> Coded {
let cw = cfg.coded_width;
let ch = cfg.coded_height;
let (cwc, chc) = (cw / 2, ch / 2);
let mut ry = vec![0u8; (cw * ch) as usize];
let mut ru = vec![128u8; (cwc * chc) as usize];
let mut rv = vec![128u8; (cwc * chc) as usize];
let lim = TreeLimits {
max_depth: cfg.max_transform_hierarchy_depth_intra,
max_tb_log2: cfg.max_tb_log2,
};
let lambda = 0.57 * 2f64.powf((cfg.qp - 12) as f64 / 3.0);
let mut cabac = CabacEncoder::new();
let mut ctx = Ctx::new(cfg.qp);
let (nx, ny) = cfg.ctbs();
let ctb = 1u32 << cfg.ctb_log2;
let g = Geo { cw, ch, ctb, nx };
let mut grid = Grid::new(cw, ch, cfg.min_cb_log2);
let mut scratch = Scratch::default();
let total = nx * ny;
let mut coded = 0u32;
let ctb_qp = adaptive_qp(y, cw, ch, ctb, nx, ny, cfg.qp, aq_strength);
let mut qp_prev = cfg.qp;
let mut edges = EdgeMap::new(cw, ch);
for cby in 0..ny {
for cbx in 0..nx {
let (px, py) = (cbx * ctb, cby * ctb);
let qp_ctb = ctb_qp[(cby * nx + cbx) as usize];
let qp_l = qp_ctb;
let qp_c = chroma_qp(qp_ctb);
let delta = qp_ctb - qp_prev;
let saved = grid.save(px, py, cfg.ctb_log2);
let cu = {
let mut planes = Planes {
y,
u,
v,
ry: &mut ry,
ru: &mut ru,
rv: &mut rv,
cw,
ch,
cwc,
chc,
nx,
ctb,
scratch: &mut scratch,
qp_l,
qp_c,
residual,
};
let mut scratch = cabac.probe();
let mut sctx = ctx;
let mut qd = QpDelta::new(qp_prev, delta);
search_cu(
&mut planes,
&mut scratch,
&mut sctx,
&mut grid,
g,
px,
py,
cfg.ctb_log2,
0,
lim,
lambda,
&mut qd,
)
.0
};
grid.restore(px, py, cfg.ctb_log2, &saved);
let mut qd = QpDelta::new(qp_prev, delta);
emit_cu(
&mut cabac,
&mut ctx,
&cu,
&mut grid,
g,
px,
py,
cfg.ctb_log2,
0,
lim,
&mut qd,
);
if qd.sent {
qp_prev = qp_ctb;
}
mark_cu_edges(&cu, &mut edges, g, px, py, cfg.ctb_log2);
coded += 1;
if coded == total && !last_subset {
cabac.encode_terminate(0); cabac.encode_terminate(1); } else {
cabac.encode_terminate((coded == total) as u32);
}
}
}
Coded {
data: cabac.finish(),
recon: Reconstruction {
y: ry,
u: ru,
v: rv,
coded_width: cw,
coded_height: ch,
},
edges,
qp: grid.into_qp(),
}
}
#[derive(Clone, Copy)]
struct TreeLimits {
max_depth: u32,
max_tb_log2: u32,
}
struct Tu {
log2: u32,
split: bool,
children: Vec<Tu>,
cbf_luma: bool,
levels_y: Vec<i32>,
cbf_cb: bool,
cbf_cr: bool,
levels_cb: Vec<i32>,
levels_cr: Vec<i32>,
}
struct Planes<'a> {
y: &'a [u8],
u: &'a [u8],
v: &'a [u8],
ry: &'a mut [u8],
ru: &'a mut [u8],
rv: &'a mut [u8],
cw: u32,
ch: u32,
cwc: u32,
chc: u32,
nx: u32,
ctb: u32,
scratch: &'a mut Scratch,
qp_l: i32,
qp_c: i32,
residual: bool,
}
#[allow(clippy::too_many_arguments)]
fn build_tu(
p: &mut Planes,
cabac: &CabacEncoder,
ctx: &Ctx,
x: u32,
y: u32,
log2: u32,
depth: u32,
lim: TreeLimits,
mode: u8,
cmode: u8,
lambda: f64,
qd: &QpDelta,
) -> Tu {
if log2 > lim.max_tb_log2 {
return build_split(
p, cabac, ctx, x, y, log2, depth, lim, mode, cmode, lambda, qd,
);
}
if !(depth < lim.max_depth && log2 > 2) {
return build_leaf(p, x, y, log2, mode, cmode);
}
let before = save_rect(p, x, y, log2);
let leaf = build_leaf(p, x, y, log2, mode, cmode);
let leaf_cost = node_cost(
p, x, y, log2, &leaf, cabac, ctx, depth, lim, mode, cmode, lambda, qd,
);
let leaf_recon = save_rect(p, x, y, log2);
restore_rect(p, x, y, log2, &before);
let split = build_split(
p, cabac, ctx, x, y, log2, depth, lim, mode, cmode, lambda, qd,
);
let split_cost = node_cost(
p, x, y, log2, &split, cabac, ctx, depth, lim, mode, cmode, lambda, qd,
);
if leaf_cost <= split_cost {
restore_rect(p, x, y, log2, &leaf_recon);
leaf
} else {
split
}
}
fn build_leaf(p: &mut Planes, x: u32, y: u32, log2: u32, mode: u8, cmode: u8) -> Tu {
let n = 1usize << log2;
let (cbf_luma, levels_y) = code_leaf(
p.y, p.ry, p.cw, p.ch, x, y, n, p.ctb, p.nx, mode, p.qp_l, true, p.residual, p.scratch,
);
let mut tu = Tu {
log2,
split: false,
children: Vec::new(),
cbf_luma,
levels_y,
cbf_cb: false,
cbf_cr: false,
levels_cb: Vec::new(),
levels_cr: Vec::new(),
};
if log2 > 2 {
code_node_chroma(p, &mut tu, x, y, log2, cmode);
}
tu
}
#[allow(clippy::too_many_arguments)]
fn build_split(
p: &mut Planes,
cabac: &CabacEncoder,
ctx: &Ctx,
x: u32,
y: u32,
log2: u32,
depth: u32,
lim: TreeLimits,
mode: u8,
cmode: u8,
lambda: f64,
qd: &QpDelta,
) -> Tu {
let half = 1u32 << (log2 - 1);
let mut children = Vec::with_capacity(4);
for (dx, dy) in [(0, 0), (half, 0), (0, half), (half, half)] {
children.push(build_tu(
p,
cabac,
ctx,
x + dx,
y + dy,
log2 - 1,
depth + 1,
lim,
mode,
cmode,
lambda,
qd,
));
}
let mut tu = Tu {
log2,
split: true,
children,
cbf_luma: false,
levels_y: Vec::new(),
cbf_cb: false,
cbf_cr: false,
levels_cb: Vec::new(),
levels_cr: Vec::new(),
};
if log2 == 3 {
code_node_chroma(p, &mut tu, x, y, log2, cmode);
} else {
tu.cbf_cb = tu.children.iter().any(|c| c.cbf_cb);
tu.cbf_cr = tu.children.iter().any(|c| c.cbf_cr);
}
tu
}
fn code_node_chroma(p: &mut Planes, tu: &mut Tu, x: u32, y: u32, log2: u32, cmode: u8) {
let cn = 1usize << (log2 - 1);
let (cx, cy) = (x / 2, y / 2);
let (cbf_cb, levels_cb) = code_leaf(
p.u,
p.ru,
p.cwc,
p.chc,
cx,
cy,
cn,
p.ctb / 2,
p.nx,
cmode,
p.qp_c,
false,
p.residual,
p.scratch,
);
let (cbf_cr, levels_cr) = code_leaf(
p.v,
p.rv,
p.cwc,
p.chc,
cx,
cy,
cn,
p.ctb / 2,
p.nx,
cmode,
p.qp_c,
false,
p.residual,
p.scratch,
);
tu.cbf_cb = cbf_cb;
tu.cbf_cr = cbf_cr;
tu.levels_cb = levels_cb;
tu.levels_cr = levels_cr;
}
struct Scratch {
refs: Vec<i32>,
smooth: Vec<i32>,
above: Vec<i32>,
left: Vec<i32>,
pred: Vec<i32>,
orig: Vec<i32>,
resid: Vec<i32>,
coeff: Vec<i32>,
deq: Vec<i32>,
res: Vec<i32>,
}
impl Default for Scratch {
fn default() -> Self {
const N: usize = MAX_N * MAX_N;
const R: usize = 4 * MAX_N + 1;
Self {
refs: vec![0; R],
smooth: vec![0; R],
above: vec![0; 2 * MAX_N + 1],
left: vec![0; 2 * MAX_N + 1],
pred: vec![0; N],
orig: vec![0; N],
resid: vec![0; N],
coeff: vec![0; N],
deq: vec![0; N],
res: vec![0; N],
}
}
}
#[allow(clippy::too_many_arguments)]
fn code_leaf(
orig: &[u8],
recon: &mut [u8],
w: u32,
h: u32,
x: u32,
y: u32,
n: usize,
ctb: u32,
nx: u32,
mode: u8,
qp: i32,
luma: bool,
residual: bool,
sc: &mut Scratch,
) -> (bool, Vec<i32>) {
let (nn, nr) = (n * n, 4 * n + 1);
refs_generic_into(recon, w, h, x, y, n as u32, ctb, nx, &mut sc.refs[..nr]);
let src: &[i32] = if luma && filter_flag(mode, n) {
smooth_refs_into(&sc.refs[..nr], &mut sc.smooth[..nr]);
&sc.smooth[..nr]
} else {
&sc.refs[..nr]
};
split_refs_into(
src,
n,
&mut sc.above[..2 * n + 1],
&mut sc.left[..2 * n + 1],
);
intra::predict_into(
mode,
n,
&sc.above[..2 * n + 1],
&sc.left[..2 * n + 1],
luma,
&mut sc.pred[..nn],
);
gather_into(orig, w, x, y, n as u32, &mut sc.orig[..nn]);
for i in 0..nn {
sc.resid[i] = sc.orig[i] - sc.pred[i];
}
let kind = if luma && n == 4 {
TransformKind::Dst
} else {
TransformKind::Dct
};
forward_into(&sc.resid[..nn], &mut sc.coeff[..nn], n, kind);
let mut levels = vec![0i32; nn];
quant::quant_into(&sc.coeff[..nn], n, qp, true, &mut levels);
let cbf = residual && levels.iter().any(|&c| c != 0);
if cbf {
quant::dequant_into(&levels, n, qp, &mut sc.deq[..nn]);
inverse_into(&sc.deq[..nn], &mut sc.res[..nn], n, kind);
for i in 0..nn {
sc.pred[i] += sc.res[i];
}
}
store(recon, w, x, y, n as u32, &sc.pred[..nn]);
(cbf, levels)
}
#[allow(clippy::too_many_arguments)]
fn emit_tu(
cabac: &mut CabacEncoder,
ctx: &mut Ctx,
tu: &Tu,
depth: u32,
lim: TreeLimits,
mode: u8,
cmode: u8,
parent_cb: bool,
parent_cr: bool,
blk3_chroma: Option<&Tu>,
qd: &mut QpDelta,
) {
if tu.log2 <= lim.max_tb_log2 && depth < lim.max_depth && tu.log2 > 2 {
cabac.encode_bin(&mut ctx.split_tu[(5 - tu.log2) as usize], tu.split as u32);
}
if tu.log2 > 2 {
if depth == 0 || parent_cb {
cabac.encode_bin(&mut ctx.cbf_chroma[depth as usize], tu.cbf_cb as u32);
}
if depth == 0 || parent_cr {
cabac.encode_bin(&mut ctx.cbf_chroma[depth as usize], tu.cbf_cr as u32);
}
}
if tu.split {
for (i, child) in tu.children.iter().enumerate() {
let carry = (tu.log2 == 3 && i == 3).then_some(tu);
emit_tu(
cabac,
ctx,
child,
depth + 1,
lim,
mode,
cmode,
tu.cbf_cb,
tu.cbf_cr,
carry,
qd,
);
}
return;
}
cabac.encode_bin(
&mut ctx.cbf_luma[usize::from(depth == 0)],
tu.cbf_luma as u32,
);
let cbf_chroma = if tu.log2 > 2 {
tu.cbf_cb || tu.cbf_cr
} else {
parent_cb || parent_cr
};
if tu.cbf_luma || cbf_chroma {
qd.emit(cabac, ctx);
}
let n = 1usize << tu.log2;
if tu.cbf_luma {
residual::encode(
cabac,
&mut ctx.res,
&tu.levels_y,
n,
false,
scan_idx(mode, tu.log2, false),
);
}
if tu.log2 > 2 {
let cscan = scan_idx(cmode, tu.log2 - 1, true);
if tu.cbf_cb {
residual::encode(cabac, &mut ctx.res, &tu.levels_cb, n / 2, true, cscan);
}
if tu.cbf_cr {
residual::encode(cabac, &mut ctx.res, &tu.levels_cr, n / 2, true, cscan);
}
} else if let Some(parent) = blk3_chroma {
let cscan = scan_idx(cmode, 2, true);
if parent.cbf_cb {
residual::encode(cabac, &mut ctx.res, &parent.levels_cb, 4, true, cscan);
}
if parent.cbf_cr {
residual::encode(cabac, &mut ctx.res, &parent.levels_cr, 4, true, cscan);
}
}
}
fn save_rect(p: &Planes, x: u32, y: u32, log2: u32) -> Vec<u8> {
let n = 1u32 << log2;
let cn = n / 2;
let mut out = Vec::with_capacity((n * n * 3 / 2) as usize);
for r in 0..n {
let o = ((y + r) * p.cw + x) as usize;
out.extend_from_slice(&p.ry[o..o + n as usize]);
}
let (cx, cy) = (x / 2, y / 2);
for plane in [&*p.ru, &*p.rv] {
for r in 0..cn {
let o = ((cy + r) * p.cwc + cx) as usize;
out.extend_from_slice(&plane[o..o + cn as usize]);
}
}
out
}
fn restore_rect(p: &mut Planes, x: u32, y: u32, log2: u32, saved: &[u8]) {
let n = 1u32 << log2;
let cn = n / 2;
let mut k = 0;
for r in 0..n {
let o = ((y + r) * p.cw + x) as usize;
p.ry[o..o + n as usize].copy_from_slice(&saved[k..k + n as usize]);
k += n as usize;
}
let (cx, cy) = (x / 2, y / 2);
for ci in 0..2 {
let plane: &mut [u8] = if ci == 0 { p.ru } else { p.rv };
for r in 0..cn {
let o = ((cy + r) * p.cwc + cx) as usize;
plane[o..o + cn as usize].copy_from_slice(&saved[k..k + cn as usize]);
k += cn as usize;
}
}
}
#[allow(clippy::too_many_arguments)]
fn node_cost(
p: &Planes,
x: u32,
y: u32,
log2: u32,
tu: &Tu,
cabac: &CabacEncoder,
ctx: &Ctx,
depth: u32,
lim: TreeLimits,
mode: u8,
cmode: u8,
lambda: f64,
qd: &QpDelta,
) -> f64 {
let n = 1u32 << log2;
let cn = n / 2;
let mut ssd = 0f64;
for r in 0..n {
let o = ((y + r) * p.cw + x) as usize;
for c in 0..n as usize {
let d = p.ry[o + c] as f64 - p.y[o + c] as f64;
ssd += d * d;
}
}
let (cx, cy) = (x / 2, y / 2);
for (rec, src) in [(&*p.ru, p.u), (&*p.rv, p.v)] {
for r in 0..cn {
let o = ((cy + r) * p.cwc + cx) as usize;
for c in 0..cn as usize {
let d = rec[o + c] as f64 - src[o + c] as f64;
ssd += d * d;
}
}
}
let mut trial = cabac.probe();
let mut tctx = *ctx;
let mut tqd = *qd;
let before = trial.bit_len();
emit_tu(
&mut trial, &mut tctx, tu, depth, lim, mode, cmode, true, true, None, &mut tqd,
);
let bits = (trial.bit_len() - before) as f64;
ssd + lambda * bits
}
#[derive(Clone, Copy)]
struct QpDelta {
pred: i32,
value: i32,
sent: bool,
}
impl QpDelta {
fn new(pred: i32, value: i32) -> Self {
Self {
pred,
value,
sent: false,
}
}
fn qp_now(&self) -> i32 {
self.pred + if self.sent { self.value } else { 0 }
}
fn emit(&mut self, cabac: &mut CabacEncoder, ctx: &mut Ctx) {
if self.sent {
return;
}
self.sent = true;
let abs = self.value.unsigned_abs();
let prefix = abs.min(5);
for k in 0..prefix {
cabac.encode_bin(&mut ctx.cu_qp_delta[usize::from(k > 0)], 1);
}
if prefix < 5 {
cabac.encode_bin(&mut ctx.cu_qp_delta[usize::from(prefix > 0)], 0);
} else {
let mut rest = abs - 5;
let mut k = 0u32;
while rest >= (1 << k) {
cabac.encode_bypass(1);
rest -= 1 << k;
k += 1;
}
cabac.encode_bypass(0);
for b in (0..k).rev() {
cabac.encode_bypass((rest >> b) & 1);
}
}
if abs > 0 {
cabac.encode_bypass(u32::from(self.value < 0));
}
}
}
#[allow(clippy::too_many_arguments)]
fn adaptive_qp(
y: &[u8],
cw: u32,
ch: u32,
ctb: u32,
nx: u32,
ny: u32,
base: i32,
strength: f32,
) -> Vec<i32> {
let n = (nx * ny) as usize;
if strength == 0.0 {
return vec![base; n];
}
let mut energy = Vec::with_capacity(n);
for cby in 0..ny {
for cbx in 0..nx {
let (ox, oy) = (cbx * ctb, cby * ctb);
let (w, h) = (ctb.min(cw - ox), ctb.min(ch - oy));
let (mut sum, mut sq) = (0i64, 0i64);
for j in 0..h {
let row = ((oy + j) * cw + ox) as usize;
for &p in &y[row..row + w as usize] {
sum += i64::from(p);
sq += i64::from(p) * i64::from(p);
}
}
let count = i64::from(w * h).max(1);
let var = (sq - sum * sum / count).max(0) as f32 / count as f32;
energy.push((var + 1.0).log2());
}
}
let mean = energy.iter().sum::<f32>() / energy.len() as f32;
energy
.iter()
.map(|e| {
let adj = (strength * (e - mean)).round().clamp(-5.0, 5.0) as i32;
(base + adj).clamp(0, 51)
})
.collect()
}
#[derive(Clone, Copy)]
struct Geo {
cw: u32,
ch: u32,
ctb: u32,
nx: u32,
}
struct Grid {
modes: Vec<u8>,
depth: Vec<u8>,
qp: Vec<i8>,
log2: u32,
w: usize,
h: usize,
}
impl Grid {
fn new(cw: u32, ch: u32, log2: u32) -> Self {
let (w, h) = ((cw >> log2) as usize, (ch >> log2) as usize);
Self {
modes: vec![255; w * h],
depth: vec![0; w * h],
qp: vec![0; w * h],
log2,
w,
h,
}
}
fn idx(&self, x: u32, y: u32) -> usize {
(y >> self.log2) as usize * self.w + (x >> self.log2) as usize
}
fn mode_at(&self, x: u32, y: u32) -> u8 {
match self.modes[self.idx(x, y)] {
255 => intra::DC,
m => m,
}
}
fn depth_at(&self, x: u32, y: u32) -> u32 {
u32::from(self.depth[self.idx(x, y)])
}
fn into_qp(self) -> Vec<i8> {
self.qp
}
#[allow(dead_code)]
fn qp_at(&self, x: u32, y: u32) -> i32 {
i32::from(self.qp[self.idx(x, y)])
}
fn set_cu(&mut self, x: u32, y: u32, log2: u32, mode: u8, depth: u32, qp: i32) {
let n = 1u32 << log2;
for dy in (0..n).step_by(1 << self.log2) {
for dx in (0..n).step_by(1 << self.log2) {
let (gx, gy) = ((x + dx) >> self.log2, (y + dy) >> self.log2);
if (gx as usize) < self.w && (gy as usize) < self.h {
let i = gy as usize * self.w + gx as usize;
self.modes[i] = mode;
self.depth[i] = depth as u8;
self.qp[i] = qp as i8;
}
}
}
}
fn save(&self, x: u32, y: u32, log2: u32) -> Vec<(u8, u8, i8)> {
let n = 1u32 << log2;
let mut out = Vec::new();
for dy in (0..n).step_by(1 << self.log2) {
for dx in (0..n).step_by(1 << self.log2) {
let (gx, gy) = ((x + dx) >> self.log2, (y + dy) >> self.log2);
if (gx as usize) < self.w && (gy as usize) < self.h {
let i = gy as usize * self.w + gx as usize;
out.push((self.modes[i], self.depth[i], self.qp[i]));
}
}
}
out
}
fn restore(&mut self, x: u32, y: u32, log2: u32, saved: &[(u8, u8, i8)]) {
let n = 1u32 << log2;
let mut k = 0;
for dy in (0..n).step_by(1 << self.log2) {
for dx in (0..n).step_by(1 << self.log2) {
let (gx, gy) = ((x + dx) >> self.log2, (y + dy) >> self.log2);
if (gx as usize) < self.w && (gy as usize) < self.h {
let i = gy as usize * self.w + gx as usize;
(self.modes[i], self.depth[i], self.qp[i]) = saved[k];
k += 1;
}
}
}
}
}
enum Cu {
Split(Vec<Cu>),
Leaf(Box<CuLeaf>),
}
struct CuLeaf {
mode: u8,
cmode: u8,
tu: Tu,
}
fn mark_cu_edges(cu: &Cu, edges: &mut EdgeMap, g: Geo, x: u32, y: u32, log2: u32) {
match cu {
Cu::Split(kids) => {
let half = 1u32 << (log2 - 1);
let mut k = 0;
for (dx, dy) in [(0, 0), (half, 0), (0, half), (half, half)] {
let (kx, ky) = (x + dx, y + dy);
if kx >= g.cw || ky >= g.ch {
continue;
}
mark_cu_edges(&kids[k], edges, g, kx, ky, log2 - 1);
k += 1;
}
}
Cu::Leaf(l) => mark_tu_edges(&l.tu, edges, x, y),
}
}
fn mark_tu_edges(tu: &Tu, edges: &mut EdgeMap, x: u32, y: u32) {
if tu.split {
let half = 1u32 << (tu.log2 - 1);
for (i, (dx, dy)) in [(0, 0), (half, 0), (0, half), (half, half)]
.into_iter()
.enumerate()
{
if let Some(child) = tu.children.get(i) {
mark_tu_edges(child, edges, x + dx, y + dy);
}
}
return;
}
edges.mark(x, y, 1 << tu.log2);
}
fn split_ctx(grid: &Grid, g: Geo, x: u32, y: u32, depth: u32) -> usize {
let (xi, yi) = (x as i32, y as i32);
let left =
available(xi - 1, yi, x, y, g.cw, g.ch, g.ctb, g.nx) && grid.depth_at(x - 1, y) > depth;
let above =
available(xi, yi - 1, x, y, g.cw, g.ch, g.ctb, g.nx) && grid.depth_at(x, y - 1) > depth;
usize::from(left) + usize::from(above)
}
fn ssd_rect(p: &Planes, x: u32, y: u32, log2: u32) -> f64 {
let n = (1u32 << log2).min(p.cw - x).min(p.ch - y);
let mut ssd = 0f64;
for r in 0..n {
let o = ((y + r) * p.cw + x) as usize;
for c in 0..n as usize {
let d = p.ry[o + c] as f64 - p.y[o + c] as f64;
ssd += d * d;
}
}
let (cx, cy) = (x / 2, y / 2);
let cn = n / 2;
for (rec, src) in [(&*p.ru, p.u), (&*p.rv, p.v)] {
for r in 0..cn {
let o = ((cy + r) * p.cwc + cx) as usize;
for c in 0..cn as usize {
let d = rec[o + c] as f64 - src[o + c] as f64;
ssd += d * d;
}
}
}
ssd
}
#[allow(clippy::too_many_arguments)]
fn search_cu(
p: &mut Planes,
cabac: &mut CabacEncoder,
ctx: &mut Ctx,
grid: &mut Grid,
g: Geo,
x: u32,
y: u32,
log2: u32,
depth: u32,
lim: TreeLimits,
lambda: f64,
qd: &mut QpDelta,
) -> (Cu, f64) {
let size = 1u32 << log2;
let inside = x + size <= g.cw && y + size <= g.ch;
if !inside {
return split_cu(
p, cabac, ctx, grid, g, x, y, log2, depth, lim, lambda, false, qd,
);
}
if log2 == grid.log2 {
return leaf_cu(
p, cabac, ctx, grid, g, x, y, log2, depth, lim, lambda, false, qd,
);
}
let before = save_rect(p, x, y, log2);
let gbefore = grid.save(x, y, log2);
let mut cabac_one = cabac.clone();
let mut ctx_one = *ctx;
let mut qd_one = *qd;
let (cu_one, cost_one) = leaf_cu(
p,
&mut cabac_one,
&mut ctx_one,
grid,
g,
x,
y,
log2,
depth,
lim,
lambda,
true,
&mut qd_one,
);
let recon_one = save_rect(p, x, y, log2);
let grid_one = grid.save(x, y, log2);
restore_rect(p, x, y, log2, &before);
grid.restore(x, y, log2, &gbefore);
let mut cabac_four = cabac.clone();
let mut ctx_four = *ctx;
let mut qd_four = *qd;
let (cu_four, cost_four) = split_cu(
p,
&mut cabac_four,
&mut ctx_four,
grid,
g,
x,
y,
log2,
depth,
lim,
lambda,
true,
&mut qd_four,
);
if cost_one <= cost_four {
restore_rect(p, x, y, log2, &recon_one);
grid.restore(x, y, log2, &grid_one);
*cabac = cabac_one;
*ctx = ctx_one;
*qd = qd_one;
(cu_one, cost_one)
} else {
*cabac = cabac_four;
*ctx = ctx_four;
*qd = qd_four;
(cu_four, cost_four)
}
}
#[allow(clippy::too_many_arguments)]
fn leaf_cu(
p: &mut Planes,
cabac: &mut CabacEncoder,
ctx: &mut Ctx,
grid: &mut Grid,
g: Geo,
x: u32,
y: u32,
log2: u32,
depth: u32,
lim: TreeLimits,
lambda: f64,
flag: bool,
qd: &mut QpDelta,
) -> (Cu, f64) {
let start = cabac.bit_len();
if flag {
let inc = split_ctx(grid, g, x, y, depth);
cabac.encode_bin(&mut ctx.split_cu[inc], 0);
}
let mn = 1u32 << log2.min(lim.max_tb_log2);
let lvals = refs_luma(p.ry, p.cw, p.ch, x, y, mn, p.nx, p.ctb);
let orig = gather(p.y, p.cw, x, y, mn);
let mode = best_luma_mode(&orig, mn as usize, &lvals);
let cmode = chroma_mode_from_luma(mode);
let tu = build_tu(p, cabac, ctx, x, y, log2, 0, lim, mode, cmode, lambda, qd);
if log2 == grid.log2 {
cabac.encode_bin(&mut ctx.part_mode, 1); }
code_luma_mode(cabac, ctx, mode, grid, g, x, y);
cabac.encode_bin(&mut ctx.chroma_mode, 0); emit_tu(cabac, ctx, &tu, 0, lim, mode, cmode, true, true, None, qd);
grid.set_cu(x, y, log2, mode, depth, qd.qp_now());
let bits = (cabac.bit_len() - start) as f64;
let cost = ssd_rect(p, x, y, log2) + lambda * bits;
(Cu::Leaf(Box::new(CuLeaf { mode, cmode, tu })), cost)
}
#[allow(clippy::too_many_arguments)]
fn split_cu(
p: &mut Planes,
cabac: &mut CabacEncoder,
ctx: &mut Ctx,
grid: &mut Grid,
g: Geo,
x: u32,
y: u32,
log2: u32,
depth: u32,
lim: TreeLimits,
lambda: f64,
flag: bool,
qd: &mut QpDelta,
) -> (Cu, f64) {
let start = cabac.bit_len();
if flag {
let inc = split_ctx(grid, g, x, y, depth);
cabac.encode_bin(&mut ctx.split_cu[inc], 1);
}
let mut cost = lambda * (cabac.bit_len() - start) as f64;
let half = 1u32 << (log2 - 1);
let mut kids = Vec::with_capacity(4);
for (dx, dy) in [(0, 0), (half, 0), (0, half), (half, half)] {
let (kx, ky) = (x + dx, y + dy);
if kx >= g.cw || ky >= g.ch {
continue; }
let (kid, kc) = search_cu(
p,
cabac,
ctx,
grid,
g,
kx,
ky,
log2 - 1,
depth + 1,
lim,
lambda,
qd,
);
kids.push(kid);
cost += kc;
}
(Cu::Split(kids), cost)
}
#[allow(clippy::too_many_arguments)]
fn emit_cu(
cabac: &mut CabacEncoder,
ctx: &mut Ctx,
cu: &Cu,
grid: &mut Grid,
g: Geo,
x: u32,
y: u32,
log2: u32,
depth: u32,
lim: TreeLimits,
qd: &mut QpDelta,
) {
let size = 1u32 << log2;
let codes_flag = x + size <= g.cw && y + size <= g.ch && log2 > grid.log2;
match cu {
Cu::Split(kids) => {
if codes_flag {
let inc = split_ctx(grid, g, x, y, depth);
cabac.encode_bin(&mut ctx.split_cu[inc], 1);
}
let half = 1u32 << (log2 - 1);
let mut k = 0;
for (dx, dy) in [(0, 0), (half, 0), (0, half), (half, half)] {
let (kx, ky) = (x + dx, y + dy);
if kx >= g.cw || ky >= g.ch {
continue;
}
emit_cu(
cabac,
ctx,
&kids[k],
grid,
g,
kx,
ky,
log2 - 1,
depth + 1,
lim,
qd,
);
k += 1;
}
}
Cu::Leaf(l) => {
if codes_flag {
let inc = split_ctx(grid, g, x, y, depth);
cabac.encode_bin(&mut ctx.split_cu[inc], 0);
}
if log2 == grid.log2 {
cabac.encode_bin(&mut ctx.part_mode, 1);
}
code_luma_mode(cabac, ctx, l.mode, grid, g, x, y);
cabac.encode_bin(&mut ctx.chroma_mode, 0);
emit_tu(
cabac, ctx, &l.tu, 0, lim, l.mode, l.cmode, true, true, None, qd,
);
grid.set_cu(x, y, log2, l.mode, depth, qd.qp_now());
}
}
}
fn chroma_mode_from_luma(luma: u8) -> u8 {
luma
}
fn best_luma_mode(orig: &[i32], n: usize, vals: &[i32]) -> u8 {
let plain = split_refs(vals, n);
let smooth = split_refs(&smooth_refs(vals), n);
let mut best = (u32::MAX, intra::DC);
let mut pred = vec![0i32; n * n];
for mode in 0..=34u8 {
let (above, left) = if filter_flag(mode, n) {
&smooth
} else {
&plain
};
intra::predict_into(mode, n, above, left, true, &mut pred[..]);
let sad: u32 = orig
.iter()
.zip(&pred)
.map(|(&o, &p)| (o - p).unsigned_abs())
.sum();
if sad < best.0 {
best = (sad, mode);
}
}
best.1
}
#[allow(clippy::too_many_arguments)]
fn code_luma_mode(
cabac: &mut CabacEncoder,
ctx: &mut Ctx,
mode: u8,
grid: &Grid,
g: Geo,
x: u32,
y: u32,
) {
let (xi, yi) = (x as i32, y as i32);
let cand_a = if available(xi - 1, yi, x, y, g.cw, g.ch, g.ctb, g.nx) {
grid.mode_at(x - 1, y)
} else {
intra::DC
};
let ctb_top = ((y >> g.ctb.trailing_zeros()) << g.ctb.trailing_zeros()) as i32;
let same_ctb_row = yi > ctb_top;
let cand_b = if same_ctb_row && available(xi, yi - 1, x, y, g.cw, g.ch, g.ctb, g.nx) {
grid.mode_at(x, y - 1)
} else {
intra::DC
};
let list = mpm_list(cand_a, cand_b);
if let Some(idx) = list.iter().position(|&m| m == mode) {
cabac.encode_bin(&mut ctx.prev_intra, 1);
match idx {
0 => cabac.encode_bypass(0),
1 => {
cabac.encode_bypass(1);
cabac.encode_bypass(0);
}
_ => {
cabac.encode_bypass(1);
cabac.encode_bypass(1);
}
}
} else {
cabac.encode_bin(&mut ctx.prev_intra, 0);
let mut sorted = list;
sorted.sort_unstable();
let mut rem = mode as i32;
for &c in sorted.iter().rev() {
if rem > c as i32 {
rem -= 1;
}
}
cabac.encode_bypass_bits(rem as u32, 5); }
}
fn mpm_list(a: u8, b: u8) -> [u8; 3] {
if a == b {
if a < 2 {
[intra::PLANAR, intra::DC, 26]
} else {
[
a,
2 + ((a as i32 + 29) % 32) as u8,
2 + ((a as i32 - 2 + 1) % 32) as u8,
]
}
} else {
let mut l = [a, b, 0];
l[2] = if a != intra::PLANAR && b != intra::PLANAR {
intra::PLANAR
} else if a != intra::DC && b != intra::DC {
intra::DC
} else {
26
};
l
}
}
fn available(px: i32, py: i32, bx: u32, by: u32, w: u32, h: u32, ctb: u32, nx: u32) -> bool {
available_at(px, py, decode_order(bx, by, ctb, nx), w, h, ctb, nx)
}
#[inline]
fn available_at(px: i32, py: i32, here: (u32, u32), w: u32, h: u32, ctb: u32, nx: u32) -> bool {
if px < 0 || py < 0 || px >= w as i32 || py >= h as i32 {
return false;
}
decode_order(px as u32, py as u32, ctb, nx) < here
}
#[inline]
fn decode_order(px: u32, py: u32, ctb: u32, nx: u32) -> (u32, u32) {
let sh = ctb.trailing_zeros();
let mask = ctb - 1;
((py >> sh) * nx + (px >> sh), morton(px & mask, py & mask))
}
#[inline]
fn morton(x: u32, y: u32) -> u32 {
spread(x) | (spread(y) << 1)
}
#[inline]
fn spread(v: u32) -> u32 {
let mut v = v & 0xFFFF;
v = (v | (v << 8)) & 0x00FF_00FF;
v = (v | (v << 4)) & 0x0F0F_0F0F;
v = (v | (v << 2)) & 0x3333_3333;
v = (v | (v << 1)) & 0x5555_5555;
v
}
fn scan_idx(mode: u8, log2: u32, chroma: bool) -> u8 {
if !(log2 == 2 || (log2 == 3 && !chroma)) {
return 0;
}
match mode {
6..=14 => 2, 22..=30 => 1, _ => 0,
}
}
#[allow(clippy::too_many_arguments)]
fn refs_generic(
recon: &[u8],
w: u32,
h: u32,
x: u32,
y: u32,
n: u32,
ctb: u32,
nx: u32,
) -> Vec<i32> {
let mut out = vec![0i32; (4 * n + 1) as usize];
refs_generic_into(recon, w, h, x, y, n, ctb, nx, &mut out);
out
}
fn refs_generic_into(
recon: &[u8],
w: u32,
h: u32,
x: u32,
y: u32,
n: u32,
ctb: u32,
nx: u32,
out: &mut [i32],
) {
let ni = n as i32;
let (xi, yi) = (x as i32, y as i32);
let sample = |px: i32, py: i32| recon[(py as u32 * w + px as u32) as usize] as i32;
let here = decode_order(x, y, ctb, nx);
let vals = out;
vals.fill(0);
let mut avail = vec![false; (4 * n + 1) as usize];
for k in 0..(2 * ni) {
let py = yi + (2 * ni - 1 - k); if available_at(xi - 1, py, here, w, h, ctb, nx) {
vals[k as usize] = sample(xi - 1, py);
avail[k as usize] = true;
}
}
if available_at(xi - 1, yi - 1, here, w, h, ctb, nx) {
vals[(2 * ni) as usize] = sample(xi - 1, yi - 1); avail[(2 * ni) as usize] = true;
}
for k in 0..(2 * ni) {
let px = xi + k; if available_at(px, yi - 1, here, w, h, ctb, nx) {
vals[(2 * ni + 1 + k) as usize] = sample(px, yi - 1);
avail[(2 * ni + 1 + k) as usize] = true;
}
}
let len = vals.len();
if avail.iter().all(|&a| !a) {
vals.iter_mut().for_each(|v| *v = 128);
} else {
if !avail[0] {
let first = (0..len).find(|&i| avail[i]).unwrap();
vals[0] = vals[first];
}
for i in 1..len {
if !avail[i] {
vals[i] = vals[i - 1];
}
}
}
}
#[allow(clippy::too_many_arguments)]
fn refs_luma(recon: &[u8], w: u32, h: u32, x: u32, y: u32, n: u32, nx: u32, ctb: u32) -> Vec<i32> {
refs_generic(recon, w, h, x, y, n, ctb, nx)
}
fn filter_flag(mode: u8, n: usize) -> bool {
if mode == intra::DC || n == 4 {
return false;
}
let min_dist = (mode as i32 - 26).abs().min((mode as i32 - 10).abs());
let thres = match n {
8 => 7,
16 => 1,
32 => 0,
_ => 8,
};
min_dist > thres
}
#[allow(dead_code)]
fn extract(vals: &[i32], n: usize, mode: u8, luma: bool) -> (Vec<i32>, Vec<i32>) {
let (mut a, mut l) = (vec![0i32; 2 * n + 1], vec![0i32; 2 * n + 1]);
extract_into(vals, n, mode, luma, &mut a, &mut l);
(a, l)
}
fn extract_into(vals: &[i32], n: usize, mode: u8, luma: bool, above: &mut [i32], left: &mut [i32]) {
if luma && filter_flag(mode, n) {
split_refs_into(&smooth_refs(vals), n, above, left);
} else {
split_refs_into(vals, n, above, left);
}
}
fn smooth_refs(vals: &[i32]) -> Vec<i32> {
let mut f = vec![0i32; vals.len()];
smooth_refs_into(vals, &mut f);
f
}
fn smooth_refs_into(vals: &[i32], out: &mut [i32]) {
let last = vals.len() - 1;
out[0] = vals[0];
out[last] = vals[last];
for i in 1..last {
out[i] = (vals[i - 1] + 2 * vals[i] + vals[i + 1] + 2) >> 2;
}
}
fn split_refs(src: &[i32], n: usize) -> (Vec<i32>, Vec<i32>) {
let (mut a, mut l) = (vec![0i32; 2 * n + 1], vec![0i32; 2 * n + 1]);
split_refs_into(src, n, &mut a, &mut l);
(a, l)
}
fn split_refs_into(src: &[i32], n: usize, above: &mut [i32], left: &mut [i32]) {
let corner = 2 * n;
above[0] = src[corner];
left[0] = src[corner];
for k in 1..=2 * n {
above[k] = src[corner + k];
left[k] = src[corner - k];
}
}
fn gather_into(plane: &[u8], w: u32, x: u32, y: u32, n: u32, out: &mut [i32]) {
for yy in 0..n {
let row = ((y + yy) * w + x) as usize;
let dst = (yy * n) as usize;
for (o, &v) in out[dst..dst + n as usize]
.iter_mut()
.zip(&plane[row..row + n as usize])
{
*o = i32::from(v);
}
}
}
fn gather(plane: &[u8], w: u32, x: u32, y: u32, n: u32) -> Vec<i32> {
let mut b = vec![0i32; (n * n) as usize];
for yy in 0..n {
for xx in 0..n {
b[(yy * n + xx) as usize] = plane[((y + yy) * w + (x + xx)) as usize] as i32;
}
}
b
}
fn store(plane: &mut [u8], w: u32, x: u32, y: u32, n: u32, block: &[i32]) {
for yy in 0..n {
for xx in 0..n {
plane[((y + yy) * w + (x + xx)) as usize] =
block[(yy * n + xx) as usize].clamp(0, 255) as u8;
}
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn mpm_list_cases() {
assert_eq!(mpm_list(1, 1), [intra::PLANAR, intra::DC, 26]);
assert_eq!(mpm_list(10, 26), [10, 26, intra::PLANAR]);
assert_eq!(mpm_list(intra::PLANAR, 26), [0, 26, intra::DC]);
let l = mpm_list(20, 20);
assert_eq!(l[0], 20);
assert!(l[1] >= 2 && l[2] >= 2 && l[1] != l[2]);
}
#[test]
fn filter_flag_rules() {
assert!(!filter_flag(intra::DC, 16)); assert!(!filter_flag(10, 4)); assert!(!filter_flag(26, 16)); assert!(filter_flag(intra::PLANAR, 16)); assert!(filter_flag(18, 16)); }
#[test]
fn rem_mode_roundtrips_through_mpm() {
let list = mpm_list(1, 1);
let mut sorted = list;
sorted.sort_unstable();
for mode in 0..=34u8 {
if list.contains(&mode) {
continue;
}
let mut rem = mode as i32;
for &c in sorted.iter().rev() {
if rem > c as i32 {
rem -= 1;
}
}
let mut dec = rem;
for &c in sorted.iter() {
if dec >= c as i32 {
dec += 1;
}
}
assert_eq!(dec as u8, mode, "rem roundtrip for mode {mode}");
}
}
}