1use super::backend::SubtitleRenderer;
4use super::blend::{self, ColorMatrix, ColorRange, OverlayImage, PlaneView, SampleFormat};
5use super::layout::{self, ColorModel, FormatSpec};
6use super::options::SubtitleFilterBuilder;
7use crate::core::filter::frame_filter::FrameFilter;
8use crate::core::filter::frame_filter_context::FrameFilterContext;
9use crate::util::ffmpeg_utils::av_err2str;
10use ffmpeg_next::Frame;
11use ffmpeg_sys_next::{
12 av_frame_make_writable, av_get_pix_fmt_name, av_rescale_q, AVColorRange, AVColorSpace, AVFrame,
13 AVMediaType, AVPixelFormat, AVRational, AV_NOPTS_VALUE,
14};
15use std::ffi::CStr;
16
17pub struct SubtitleFilter {
44 renderer: Box<dyn SubtitleRenderer>,
47 original_size: Option<(u32, u32)>,
49 configured_dims: Option<(i32, i32)>,
51 blend_scratch: BlendScratch,
53 warned_missing_time: bool,
54 warned_colorspace: bool,
55 locked_color: Option<(ColorMatrix, Option<ColorRange>)>,
59}
60
61impl std::fmt::Debug for SubtitleFilter {
62 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
63 f.debug_struct("SubtitleFilter")
64 .field("original_size", &self.original_size)
65 .finish_non_exhaustive()
66 }
67}
68
69impl SubtitleFilter {
70 pub fn builder() -> SubtitleFilterBuilder {
72 SubtitleFilterBuilder::default()
73 }
74
75 #[cfg(test)]
77 pub(crate) fn renderer_mut(&mut self) -> &mut dyn SubtitleRenderer {
78 self.renderer.as_mut()
79 }
80
81 pub(crate) fn new(
82 renderer: Box<dyn SubtitleRenderer>,
83 original_size: Option<(u32, u32)>,
84 ) -> Self {
85 Self {
86 renderer,
87 original_size,
88 configured_dims: None,
89 blend_scratch: BlendScratch::default(),
90 warned_missing_time: false,
91 warned_colorspace: false,
92 locked_color: None,
93 }
94 }
95
96 fn configure_renderer(&mut self, width: i32, height: i32) {
100 if self.configured_dims == Some((width, height)) {
101 return;
102 }
103 self.configured_dims = Some((width, height));
104 self.renderer.set_frame_size(width, height);
105 match self.original_size {
106 Some((original_w, original_h)) if original_w > 0 && original_h > 0 => {
107 let par = (f64::from(width) / f64::from(height))
108 / (f64::from(original_w) / f64::from(original_h));
109 self.renderer.set_pixel_aspect(par);
110 self.renderer
111 .set_storage_size(original_w as i32, original_h as i32);
112 }
113 _ => self.renderer.set_storage_size(width, height),
114 }
115 }
116
117 fn frame_color(
125 &mut self,
126 colorspace: AVColorSpace,
127 color_range: AVColorRange,
128 ) -> (ColorMatrix, Option<ColorRange>) {
129 if let Some(locked) = self.locked_color {
130 return locked;
131 }
132 let matrix = match colorspace {
133 AVColorSpace::AVCOL_SPC_BT709 => ColorMatrix::Bt709,
134 AVColorSpace::AVCOL_SPC_SMPTE170M | AVColorSpace::AVCOL_SPC_BT470BG => {
135 ColorMatrix::Bt601
136 }
137 AVColorSpace::AVCOL_SPC_BT2020_NCL | AVColorSpace::AVCOL_SPC_BT2020_CL => {
138 ColorMatrix::Bt2020
139 }
140 AVColorSpace::AVCOL_SPC_FCC => ColorMatrix::Fcc,
141 AVColorSpace::AVCOL_SPC_SMPTE240M => ColorMatrix::Smpte240m,
142 AVColorSpace::AVCOL_SPC_YCGCO => ColorMatrix::YCoCg,
143 AVColorSpace::AVCOL_SPC_UNSPECIFIED => ColorMatrix::Bt601,
144 other => {
145 if !self.warned_colorspace {
146 self.warned_colorspace = true;
147 log::warn!(
148 "subtitle filter: colorspace {other:?} not supported for subtitle \
149 color conversion, falling back to BT.601 (logged once; FFmpeg's \
150 subtitles filter leaves its draw context uninitialized here)"
151 );
152 }
153 ColorMatrix::Bt601
154 }
155 };
156 let range = match color_range {
157 AVColorRange::AVCOL_RANGE_JPEG => Some(ColorRange::Full),
158 AVColorRange::AVCOL_RANGE_MPEG => Some(ColorRange::Limited),
159 _ => None,
160 };
161 self.locked_color = Some((matrix, range));
162 (matrix, range)
163 }
164
165 unsafe fn blend_images(
179 frame: *mut AVFrame,
180 images: &[OverlayImage<'_>],
181 spec: &FormatSpec,
182 matrix: ColorMatrix,
183 range: Option<ColorRange>,
184 scratch: &mut BlendScratch,
185 parallel: bool,
186 ) {
187 let default_full = spec.force_full_range || matches!(spec.model, ColorModel::Rgb);
193 let range = range.unwrap_or(if default_full {
194 ColorRange::Full
195 } else {
196 ColorRange::Limited
197 });
198 let width = (*frame).width as usize;
199 let height = (*frame).height as usize;
200
201 scratch.preps.clear();
207 scratch.preps.reserve(images.len());
208 let mut colors = [(u32::MAX, [0u32; 3]); 8]; let mut color_count = 0usize;
210 for overlay in images {
211 let alpha = spec.sample.alpha_fixed(overlay.opacity());
212 if alpha == 0 {
213 scratch.preps.push(OverlayPrep { alpha, src: [0; 3] });
214 continue;
215 }
216 let key = overlay.color >> 8; let src = match colors[..color_count].iter().find(|(k, _)| *k == key) {
218 Some(&(_, hit)) => hit,
219 None => {
220 let converted = match spec.model {
221 ColorModel::Yuv => {
222 blend::yuv_components(overlay.rgb(), matrix, range, spec.scale_bits)
223 }
224 ColorModel::Rgb => {
225 blend::rgb_components(overlay.rgb(), range, spec.scale_bits)
226 }
227 };
228 if color_count < colors.len() {
229 colors[color_count] = (key, converted);
230 color_count += 1;
231 }
232 converted
233 }
234 };
235 scratch.preps.push(OverlayPrep { alpha, src });
236 }
237
238 let mut tasks: Vec<CompTask> = Vec::with_capacity(spec.comps.len());
242 for (source, placement) in spec.comps {
243 let plane_w = (width + (1usize << placement.hsub) - 1) >> placement.hsub;
244 let plane_h = (height + (1usize << placement.vsub) - 1) >> placement.vsub;
245 let linesize = (*frame).linesize[placement.plane];
246 let data = (*frame).data[placement.plane];
247 if linesize <= 0 || data.is_null() || plane_w == 0 || plane_h == 0 {
248 continue;
249 }
250 let linesize = linesize as usize;
251 tasks.push(CompTask {
252 plane: placement.plane,
253 data: data.add(placement.offset),
254 len: linesize * (plane_h - 1)
262 + (plane_w - 1) * placement.pixel_step
263 + spec.sample.bytes(),
264 linesize,
265 pixel_step: placement.pixel_step,
266 source: *source,
267 hsub: placement.hsub,
268 vsub: placement.vsub,
269 });
270 }
271
272 let dims = (width, height);
273 let sample = spec.sample;
274 let preps = &scratch.preps;
275 if parallel {
276 if let Some((group_a, group_b)) = split_tasks(&tasks) {
277 let (pool_a, pool_b) = (&mut scratch.pool_a, &mut scratch.pool_b);
278 std::thread::scope(|s| {
279 s.spawn(move || blend_task_group(group_a, images, preps, sample, dims, pool_a));
287 blend_task_group(group_b, images, preps, sample, dims, pool_b);
288 });
289 return;
290 }
291 }
292 blend_task_group(&tasks, images, preps, sample, dims, &mut scratch.pool_a);
293 }
294}
295
296#[derive(Default)]
299struct BlendScratch {
300 preps: Vec<OverlayPrep>,
302 pool_a: Vec<u16>,
304 pool_b: Vec<u16>,
307}
308
309struct OverlayPrep {
312 alpha: u32,
313 src: [u32; 3],
314}
315
316#[derive(Clone, Copy)]
319struct CompTask {
320 plane: usize,
322 data: *mut u8,
324 len: usize,
328 linesize: usize,
329 pixel_step: usize,
330 source: usize,
332 hsub: u32,
333 vsub: u32,
334}
335
336unsafe impl Send for CompTask {}
342unsafe impl Sync for CompTask {}
345
346impl CompTask {
347 unsafe fn view(&self) -> PlaneView<'_> {
357 PlaneView {
358 data: std::slice::from_raw_parts_mut(self.data, self.len),
359 linesize: self.linesize,
360 pixel_step: self.pixel_step,
361 }
362 }
363}
364
365fn split_tasks(tasks: &[CompTask]) -> Option<(&[CompTask], &[CompTask])> {
371 let first_plane = tasks.first()?.plane;
372 let split = tasks.iter().position(|task| task.plane != first_plane)?;
373 let (a, b) = tasks.split_at(split);
374 if b.iter().any(|task| task.plane == first_plane) {
375 return None;
376 }
377 let disjoint = a.iter().all(|ta| {
378 b.iter().all(|tb| {
379 let (a0, a1) = (ta.data as usize, ta.data as usize + ta.len);
380 let (b0, b1) = (tb.data as usize, tb.data as usize + tb.len);
381 a1 <= b0 || b1 <= a0
382 })
383 });
384 disjoint.then_some((a, b))
385}
386
387fn blend_task_group(
393 tasks: &[CompTask],
394 images: &[OverlayImage<'_>],
395 preps: &[OverlayPrep],
396 sample: SampleFormat,
397 (width, height): (usize, usize),
398 pool: &mut Vec<u16>,
399) {
400 let mut pooled = [usize::MAX; 2];
405 let mut pooled_count = 0usize;
406 for (index, task) in tasks.iter().enumerate() {
407 if task.hsub != 0 || task.vsub != 0 {
408 if pooled_count < 2 {
409 pooled[pooled_count] = index;
410 }
411 pooled_count += 1;
412 }
413 }
414 let share_pooling = pooled_count == 2 && blend::two_phase_pooled_preferred(sample) && {
415 let (a, b) = (&tasks[pooled[0]], &tasks[pooled[1]]);
416 a.hsub == 1 && b.hsub == 1 && a.vsub == b.vsub && a.vsub <= 1
417 };
418
419 for (overlay, prep) in images.iter().zip(preps) {
420 if prep.alpha == 0 {
421 continue;
422 }
423 if share_pooling {
424 if let Some(rect) =
425 blend::pool_sums_h2(width, height, overlay, tasks[pooled[0]].vsub, pool)
426 {
427 for &index in &pooled {
428 let task = &tasks[index];
429 let mut plane = unsafe { task.view() };
432 blend::blend_pooled_from_sums(
433 &mut plane,
434 pool,
435 rect,
436 prep.src[task.source],
437 prep.alpha,
438 sample,
439 );
440 }
441 }
442 }
443 for (index, task) in tasks.iter().enumerate() {
444 if share_pooling && (index == pooled[0] || index == pooled[1]) {
445 continue;
446 }
447 let mut plane = unsafe { task.view() };
450 blend::blend_component(
451 &mut plane,
452 width,
453 height,
454 overlay,
455 prep.src[task.source],
456 prep.alpha,
457 task.hsub,
458 task.vsub,
459 sample,
460 );
461 }
462 }
463}
464
465const PARALLEL_MASK_PX_THRESHOLD: usize = 256 * 1024;
473
474fn should_parallelize(clipped_mask_px: usize) -> bool {
476 clipped_mask_px > PARALLEL_MASK_PX_THRESHOLD
477}
478
479fn clipped_mask_px(images: &[OverlayImage<'_>], width: usize, height: usize) -> usize {
481 images
482 .iter()
483 .map(|image| blend::clipped_area(width, height, image))
484 .sum()
485}
486
487impl FrameFilter for SubtitleFilter {
488 fn media_type(&self) -> AVMediaType {
489 AVMediaType::AVMEDIA_TYPE_VIDEO
490 }
491
492 fn filter_frame(
493 &mut self,
494 mut frame: Frame,
495 _ctx: &FrameFilterContext,
496 ) -> Result<Option<Frame>, String> {
497 unsafe {
502 if frame.as_ptr().is_null() || frame.is_empty() {
503 return Ok(Some(frame));
504 }
505 }
506 let (width, height, format, pts, time_base, is_hw, colorspace, color_range) = unsafe {
508 let p = frame.as_ptr();
509 (
510 (*p).width,
511 (*p).height,
512 (*p).format,
513 (*p).pts,
514 (*p).time_base,
515 !(*p).hw_frames_ctx.is_null(),
516 (*p).colorspace,
517 (*p).color_range,
518 )
519 };
520 if width <= 0 || height <= 0 {
521 return Ok(Some(frame));
522 }
523 if is_hw {
524 return Err(
525 "SubtitleFilter requires software frames; do not set hwaccel_output_format to a \
526 hardware format (frames must be downloaded before this filter)"
527 .to_string(),
528 );
529 }
530 let Some(spec) = layout::format_spec(format) else {
535 return Err(format!(
536 "SubtitleFilter: unsupported pixel format {}; convert first, e.g. \
537 .filter_desc(\"format=yuv420p\") or Output::set_pix_fmt(\"yuv420p\"). \
538 Supported: {}",
539 pix_fmt_name(format),
540 layout::SUPPORTED_LIST
541 ));
542 };
543
544 if pts == AV_NOPTS_VALUE || time_base.num <= 0 || time_base.den <= 0 {
545 if !self.warned_missing_time {
546 self.warned_missing_time = true;
547 log::warn!(
548 "subtitle filter: frame without pts/time_base cannot be timed, \
549 passing through (logged once)"
550 );
551 }
552 return Ok(Some(frame));
553 }
554 let now_ms = unsafe { av_rescale_q(pts, time_base, AVRational { num: 1, den: 1000 }) };
556
557 self.configure_renderer(width, height);
558 let (matrix, range) = self.frame_color(colorspace, color_range);
561
562 let images = self.renderer.render_frame(now_ms);
563 if images.is_empty() {
564 return Ok(Some(frame));
567 }
568 if !any_visible_image(&images, width, height, spec.sample) {
572 return Ok(Some(frame));
573 }
574
575 let ret = unsafe { av_frame_make_writable(frame.as_mut_ptr()) };
578 if ret < 0 {
579 return Err(format!(
580 "av_frame_make_writable failed: {}",
581 av_err2str(ret)
582 ));
583 }
584
585 let negative_linesize = unsafe {
591 let p = frame.as_ptr();
592 spec.comps
593 .iter()
594 .any(|(_, placement)| (*p).linesize[placement.plane] < 0)
595 };
596 if negative_linesize {
597 return Err(
598 "SubtitleFilter: negative linesize (bottom-up/vertically flipped frames) is \
599 not supported; apply vflip after this filter or convert the frame first"
600 .to_string(),
601 );
602 }
603
604 let parallel =
605 should_parallelize(clipped_mask_px(&images, width as usize, height as usize));
606 unsafe {
610 Self::blend_images(
611 frame.as_mut_ptr(),
612 &images,
613 spec,
614 matrix,
615 range,
616 &mut self.blend_scratch,
617 parallel,
618 )
619 };
620
621 Ok(Some(frame))
622 }
623
624 fn uninit(&mut self, _ctx: &FrameFilterContext) {
625 self.renderer.teardown();
627 }
628}
629
630fn any_visible_image(
633 images: &[OverlayImage<'_>],
634 width: i32,
635 height: i32,
636 sample: SampleFormat,
637) -> bool {
638 images.iter().any(|image| {
639 if sample.alpha_fixed(image.opacity()) == 0 {
640 return false;
641 }
642 let (x0, y0) = (i64::from(image.dst_x), i64::from(image.dst_y));
643 x0 < i64::from(width)
644 && y0 < i64::from(height)
645 && x0 + image.w as i64 > 0
646 && y0 + image.h as i64 > 0
647 })
648}
649
650fn pix_fmt_name(format: i32) -> String {
651 if (0..AVPixelFormat::AV_PIX_FMT_NB as i32).contains(&format) {
655 let name =
657 unsafe { av_get_pix_fmt_name(std::mem::transmute::<i32, AVPixelFormat>(format)) };
658 if !name.is_null() {
659 return unsafe { CStr::from_ptr(name) }
660 .to_string_lossy()
661 .into_owned();
662 }
663 }
664 format!("#{format}")
665}
666
667#[cfg(test)]
668mod tests {
669 use super::*;
670 use crate::subtitle::test_util::{self, diff_stats, temp_path, transcode_test_mp4};
671 use crate::subtitle::FontProvider;
672 use ffmpeg_sys_next::{av_frame_alloc, av_frame_free, av_frame_get_buffer};
673
674 const W: usize = 320;
675 const H: usize = 240;
676 const FRAME_BYTES: usize = W * H + 2 * ((W / 2) * (H / 2));
677
678 fn quarter_box_script(start: &str, end: &str) -> String {
682 test_util::minimal_ass(&format!(
683 "Dialogue: 0,{start},{end},Default,,0,0,0,,{{\\an7\\pos(0,0)\\p1}}m 0 0 l 320 0 320 180 0 180{{\\p0}}\n"
684 ))
685 }
686
687 fn build_filter(script: String, font: &str) -> SubtitleFilter {
688 SubtitleFilter::builder()
689 .ass_content(script)
690 .default_font_file(font)
691 .font_provider(FontProvider::None)
692 .build()
693 .expect("build subtitle filter")
694 }
695
696 #[test]
699 #[allow(clippy::manual_is_multiple_of)] fn burns_subtitles_into_the_pipeline_output() {
701 let Some(font) = test_util::test_font() else {
702 eprintln!("skipping: no known test font present on this machine");
703 return;
704 };
705
706 let baseline_path = temp_path("baseline.yuv");
707 let burned_path = temp_path("burned.yuv");
708 let unburned_path = temp_path("unburned.yuv");
709
710 transcode_test_mp4(&baseline_path, None, "yuv420p", None);
711 transcode_test_mp4(
712 &burned_path,
713 Some(build_filter(
714 quarter_box_script("0:00:00.00", "0:00:10.00"),
715 font,
716 )),
717 "yuv420p",
718 None,
719 );
720 transcode_test_mp4(
723 &unburned_path,
724 Some(build_filter(
725 quarter_box_script("0:00:20.00", "0:00:25.00"),
726 font,
727 )),
728 "yuv420p",
729 None,
730 );
731
732 let baseline = std::fs::read(&baseline_path).expect("read baseline");
733 let burned = std::fs::read(&burned_path).expect("read burned");
734 let unburned = std::fs::read(&unburned_path).expect("read unburned");
735 let _ = std::fs::remove_file(&baseline_path);
736 let _ = std::fs::remove_file(&burned_path);
737 let _ = std::fs::remove_file(&unburned_path);
738
739 assert!(!baseline.is_empty() && baseline.len() % FRAME_BYTES == 0);
740 assert_eq!(baseline.len(), burned.len(), "same frame count");
741 assert_eq!(
742 baseline, unburned,
743 "out-of-window subtitles must not alter any byte"
744 );
745 assert_ne!(baseline, burned, "in-window subtitles must alter the video");
746
747 let inside = 60 * W + 80; let outside = 230 * W + 300; assert_ne!(
752 baseline[inside], burned[inside],
753 "pixel inside the subtitle box should change"
754 );
755 assert_eq!(
756 baseline[outside], burned[outside],
757 "pixel far outside the subtitle box must not change"
758 );
759 assert!(
761 burned[inside] >= 230,
762 "expected near-white luma inside the box, got {}",
763 burned[inside]
764 );
765 }
766
767 #[test]
770 fn burns_into_nv12_yuv444p_and_rgb24() {
771 let Some(font) = test_util::test_font() else {
772 eprintln!("skipping: no known test font present on this machine");
773 return;
774 };
775 let script = || quarter_box_script("0:00:00.00", "0:00:10.00");
776 let inside_luma = 60 * W + 80;
777 let outside_luma = 230 * W + 300;
778
779 for (pix_fmt, white_check) in [("nv12", None), ("yuv444p", None), ("rgb24", Some(250u8))] {
780 let baseline_path = temp_path(&format!("base_{pix_fmt}"));
781 let burned_path = temp_path(&format!("burn_{pix_fmt}"));
782 transcode_test_mp4(&baseline_path, None, pix_fmt, None);
783 transcode_test_mp4(
784 &burned_path,
785 Some(build_filter(script(), font)),
786 pix_fmt,
787 None,
788 );
789 let baseline = std::fs::read(&baseline_path).expect("read baseline");
790 let burned = std::fs::read(&burned_path).expect("read burned");
791 let _ = std::fs::remove_file(&baseline_path);
792 let _ = std::fs::remove_file(&burned_path);
793
794 assert_eq!(baseline.len(), burned.len(), "{pix_fmt}: frame count");
795 assert_ne!(baseline, burned, "{pix_fmt}: subtitles must alter video");
796 match white_check {
797 None => {
798 assert!(
800 burned[inside_luma] >= 230,
801 "{pix_fmt}: near-white luma expected, got {}",
802 burned[inside_luma]
803 );
804 assert_eq!(
805 baseline[outside_luma], burned[outside_luma],
806 "{pix_fmt}: outside pixel must not change"
807 );
808 }
809 Some(threshold) => {
810 let inside = inside_luma * 3;
812 let outside = outside_luma * 3;
813 for c in 0..3 {
814 assert!(
815 burned[inside + c] >= threshold,
816 "{pix_fmt}: white component {c} expected, got {}",
817 burned[inside + c]
818 );
819 assert_eq!(
820 baseline[outside + c],
821 burned[outside + c],
822 "{pix_fmt}: outside component {c} must not change"
823 );
824 }
825 }
826 }
827 }
828 }
829
830 fn u16le_at(buf: &[u8], sample_index: usize) -> u16 {
831 u16::from_le_bytes([buf[sample_index * 2], buf[sample_index * 2 + 1]])
832 }
833
834 #[test]
837 fn burns_into_10bit_planar_and_p010() {
838 let Some(font) = test_util::test_font() else {
839 eprintln!("skipping: no known test font present on this machine");
840 return;
841 };
842 let script = || quarter_box_script("0:00:00.00", "0:00:10.00");
843 let inside_luma = 60 * W + 80; let outside_luma = 230 * W + 300;
845
846 for (pix_fmt, white_floor) in [("yuv420p10le", 900u16), ("p010le", 57000u16)] {
850 let baseline_path = temp_path(&format!("base_{pix_fmt}"));
851 let burned_path = temp_path(&format!("burn_{pix_fmt}"));
852 transcode_test_mp4(&baseline_path, None, pix_fmt, None);
853 transcode_test_mp4(
854 &burned_path,
855 Some(build_filter(script(), font)),
856 pix_fmt,
857 None,
858 );
859 let baseline = std::fs::read(&baseline_path).expect("read baseline");
860 let burned = std::fs::read(&burned_path).expect("read burned");
861 let _ = std::fs::remove_file(&baseline_path);
862 let _ = std::fs::remove_file(&burned_path);
863
864 assert_eq!(baseline.len(), burned.len(), "{pix_fmt}: frame count");
865 assert_ne!(baseline, burned, "{pix_fmt}: subtitles must alter video");
866 let inside = u16le_at(&burned, inside_luma);
867 assert!(
868 inside >= white_floor,
869 "{pix_fmt}: near-white luma expected inside the box, got {inside}"
870 );
871 assert_eq!(
872 u16le_at(&baseline, outside_luma),
873 u16le_at(&burned, outside_luma),
874 "{pix_fmt}: outside sample must not change"
875 );
876 }
877 }
878
879 fn lcg_bytes(len: usize, seed: &mut u64) -> Vec<u8> {
881 (0..len)
882 .map(|_| {
883 *seed = seed
884 .wrapping_mul(6364136223846793005)
885 .wrapping_add(1442695040888963407);
886 (*seed >> 33) as u8
887 })
888 .collect()
889 }
890
891 fn plane_rows(spec: &FormatSpec, plane: usize, height: usize) -> usize {
894 spec.comps
895 .iter()
896 .filter(|(_, p)| p.plane == plane)
897 .map(|(_, p)| (height + (1usize << p.vsub) - 1) >> p.vsub)
898 .max()
899 .unwrap_or(0)
900 }
901
902 unsafe fn filled_frame(
908 w: i32,
909 h: i32,
910 format: AVPixelFormat,
911 spec: &FormatSpec,
912 mut seed: u64,
913 ) -> *mut AVFrame {
914 let frame = av_frame_alloc();
915 assert!(!frame.is_null());
916 (*frame).width = w;
917 (*frame).height = h;
918 (*frame).format = format as i32;
919 assert!(av_frame_get_buffer(frame, 0) >= 0, "av_frame_get_buffer");
920 for plane in 0..4usize {
921 let rows = plane_rows(spec, plane, h as usize);
922 let data = (*frame).data[plane];
923 if rows == 0 || data.is_null() {
924 continue;
925 }
926 let len = (*frame).linesize[plane] as usize * rows;
927 let bytes = lcg_bytes(len, &mut seed);
928 std::slice::from_raw_parts_mut(data, len).copy_from_slice(&bytes);
929 }
930 frame
931 }
932
933 unsafe fn plane_bytes(frame: *mut AVFrame, spec: &FormatSpec, height: usize) -> Vec<Vec<u8>> {
938 (0..4usize)
939 .map(|plane| {
940 let rows = plane_rows(spec, plane, height);
941 let data = (*frame).data[plane];
942 if rows == 0 || data.is_null() {
943 return Vec::new();
944 }
945 let len = (*frame).linesize[plane] as usize * rows;
946 std::slice::from_raw_parts(data, len).to_vec()
947 })
948 .collect()
949 }
950
951 #[test]
956 fn parallel_blend_matches_serial_bitexact() {
957 use ffmpeg_sys_next::AVPixelFormat::*;
958 let (w, h) = (318i32, 178i32);
959 let mut seed = 0x00C0_FFEE_0DDB_A11Du64;
960
961 let mask_a = {
968 let mut mask = lcg_bytes(220 * 130, &mut seed);
969 for (i, byte) in mask.iter_mut().enumerate() {
970 if (i / 40) % 3 == 0 {
971 *byte = 0;
972 }
973 }
974 mask
975 };
976 let mask_b = lcg_bytes(97 * 53, &mut seed);
977 let mask_c = vec![255u8; 64 * 33];
978 let images = [
979 OverlayImage {
980 w: 220,
981 h: 130,
982 stride: 220,
983 bitmap: &mask_a,
984 color: 0xFFFFFF00,
985 dst_x: -8,
986 dst_y: -6,
987 },
988 OverlayImage {
989 w: 97,
990 h: 53,
991 stride: 97,
992 bitmap: &mask_b,
993 color: 0xFF000040,
994 dst_x: 40,
995 dst_y: 30,
996 },
997 OverlayImage {
998 w: 64,
999 h: 33,
1000 stride: 64,
1001 bitmap: &mask_c,
1002 color: 0x00FF0080,
1003 dst_x: 240,
1004 dst_y: 160,
1005 },
1006 ];
1007
1008 for format in [
1009 AV_PIX_FMT_YUV420P,
1010 AV_PIX_FMT_NV12,
1011 AV_PIX_FMT_YUV444P,
1012 AV_PIX_FMT_YUV420P10LE,
1013 AV_PIX_FMT_P010LE,
1014 AV_PIX_FMT_RGB24,
1015 AV_PIX_FMT_GRAY8,
1016 ] {
1017 let spec = layout::format_spec(format as i32).expect("supported format");
1018 let fill_seed = 0x5EED_0000 ^ format as u64;
1019 unsafe {
1022 let serial = filled_frame(w, h, format, spec, fill_seed);
1023 let parallel = filled_frame(w, h, format, spec, fill_seed);
1024 let original = plane_bytes(serial, spec, h as usize);
1025
1026 let mut scratch_serial = BlendScratch::default();
1027 let mut scratch_parallel = BlendScratch::default();
1028 SubtitleFilter::blend_images(
1029 serial,
1030 &images,
1031 spec,
1032 ColorMatrix::Bt601,
1033 Some(ColorRange::Limited),
1034 &mut scratch_serial,
1035 false,
1036 );
1037 SubtitleFilter::blend_images(
1038 parallel,
1039 &images,
1040 spec,
1041 ColorMatrix::Bt601,
1042 Some(ColorRange::Limited),
1043 &mut scratch_parallel,
1044 true,
1045 );
1046
1047 let serial_planes = plane_bytes(serial, spec, h as usize);
1048 let parallel_planes = plane_bytes(parallel, spec, h as usize);
1049 assert_ne!(
1050 original, serial_planes,
1051 "{format:?}: blend must alter the frame"
1052 );
1053 assert_eq!(
1054 serial_planes, parallel_planes,
1055 "{format:?}: parallel blend diverged from serial"
1056 );
1057
1058 let mut serial = serial;
1059 let mut parallel = parallel;
1060 av_frame_free(&mut serial);
1061 av_frame_free(&mut parallel);
1062 }
1063 }
1064 }
1065
1066 #[test]
1074 fn right_edge_offset_component_stays_in_bounds() {
1075 use ffmpeg_sys_next::AVPixelFormat::*;
1076 let (w, h) = (17i32, 9i32);
1077 for format in [
1078 AV_PIX_FMT_RGB24,
1079 AV_PIX_FMT_BGR24,
1080 AV_PIX_FMT_RGBA,
1081 AV_PIX_FMT_BGRA,
1082 AV_PIX_FMT_ARGB,
1083 AV_PIX_FMT_ABGR,
1084 AV_PIX_FMT_NV12,
1085 AV_PIX_FMT_NV21,
1086 AV_PIX_FMT_P010LE,
1087 AV_PIX_FMT_YUV420P,
1088 ] {
1089 let spec = layout::format_spec(format as i32).expect("supported format");
1090 let mask = vec![255u8; (w * h) as usize];
1092 let images = [OverlayImage {
1093 w: w as usize,
1094 h: h as usize,
1095 stride: w as usize,
1096 bitmap: &mask,
1097 color: 0xFFFFFF00,
1098 dst_x: 0,
1099 dst_y: 0,
1100 }];
1101 unsafe {
1104 let frame = filled_frame(w, h, format, spec, 0xC0DE ^ format as u64);
1105 let before = plane_bytes(frame, spec, h as usize);
1106 let mut scratch = BlendScratch::default();
1107 SubtitleFilter::blend_images(
1109 frame,
1110 &images,
1111 spec,
1112 ColorMatrix::Bt601,
1113 Some(ColorRange::Limited),
1114 &mut scratch,
1115 false,
1116 );
1117 let after = plane_bytes(frame, spec, h as usize);
1118 assert_ne!(
1119 before, after,
1120 "{format:?}: full-cover blend must alter the frame"
1121 );
1122 let mut frame = frame;
1123 av_frame_free(&mut frame);
1124 }
1125 }
1126 }
1127
1128 #[test]
1131 fn rgb_unspecified_range_is_full() {
1132 use ffmpeg_sys_next::AVPixelFormat::AV_PIX_FMT_RGB24;
1133 let (w, h) = (4i32, 2i32);
1134 let spec = layout::format_spec(AV_PIX_FMT_RGB24 as i32).expect("rgb24");
1135 let mask = vec![255u8; (w * h) as usize];
1136 let images = [OverlayImage {
1137 w: w as usize,
1138 h: h as usize,
1139 stride: w as usize,
1140 bitmap: &mask,
1141 color: 0xFFFFFF00,
1142 dst_x: 0,
1143 dst_y: 0,
1144 }];
1145 unsafe {
1148 let frame = filled_frame(w, h, AV_PIX_FMT_RGB24, spec, 0x1);
1149 let mut scratch = BlendScratch::default();
1150 SubtitleFilter::blend_images(
1152 frame,
1153 &images,
1154 spec,
1155 ColorMatrix::Bt601,
1156 None,
1157 &mut scratch,
1158 false,
1159 );
1160 let r = *(*frame).data[0];
1161 assert!(
1162 r > 240,
1163 "unspecified-range RGB white must be full-range (got {r}, limited=235)"
1164 );
1165 let mut frame = frame;
1166 av_frame_free(&mut frame);
1167 }
1168 }
1169
1170 #[test]
1173 fn force_style_blur_is_bounded() {
1174 let Some(font) = test_util::test_font() else {
1175 eprintln!("skipping: no known test font present on this machine");
1176 return;
1177 };
1178 let mut filter = SubtitleFilter::builder()
1179 .ass_content(quarter_box_script("0:00:00.00", "0:00:10.00"))
1180 .force_style("Blur=100000")
1181 .default_font_file(font)
1182 .font_provider(FontProvider::None)
1183 .build()
1184 .expect("build subtitle filter");
1185 filter.configure_renderer(W as i32, H as i32);
1186 let _ = filter.renderer_mut().render_frame(1_000);
1188 }
1189
1190 #[test]
1199 #[ignore = "manual micro-benchmark; run in release with --nocapture"]
1200 fn bench_plane_parallel_blend_images() {
1201 use crate::subtitle::bench_kernels::{capture, dense_events, measure, sparse_events};
1202 use ffmpeg_sys_next::AVPixelFormat::*;
1203
1204 let Some(dense) = capture(dense_events()) else {
1205 eprintln!("skipping: no known test font present on this machine");
1206 return;
1207 };
1208 let sparse = capture(sparse_events()).expect("font present per the check above");
1209 for (name, owned) in [("dense", dense), ("sparse", sparse)] {
1210 let images: Vec<OverlayImage<'_>> = owned.iter().map(|o| o.as_view()).collect();
1211 let px = clipped_mask_px(&images, 1920, 1080);
1212 println!(
1213 "blend_images {name}: mask_px={px} gate_parallel={}",
1214 should_parallelize(px)
1215 );
1216 for (format, label) in [
1217 (AV_PIX_FMT_YUV420P, "yuv420p"),
1218 (AV_PIX_FMT_YUV420P10LE, "yuv420p10le"),
1219 ] {
1220 let spec = layout::format_spec(format as i32).expect("supported format");
1221 unsafe {
1226 let frame = filled_frame(1920, 1080, format, spec, 0xBEEF);
1227 let mut scratch = BlendScratch::default();
1228 let serial = measure(|| {
1229 SubtitleFilter::blend_images(
1230 frame,
1231 &images,
1232 spec,
1233 ColorMatrix::Bt601,
1234 Some(ColorRange::Limited),
1235 &mut scratch,
1236 false,
1237 );
1238 });
1239 let parallel = measure(|| {
1240 SubtitleFilter::blend_images(
1241 frame,
1242 &images,
1243 spec,
1244 ColorMatrix::Bt601,
1245 Some(ColorRange::Limited),
1246 &mut scratch,
1247 true,
1248 );
1249 });
1250 println!(
1251 " {label:<12} serial {serial:>10.0} ns/frame parallel \
1252 {parallel:>10.0} ns/frame {:>5.2}x",
1253 serial / parallel
1254 );
1255 let mut frame = frame;
1256 av_frame_free(&mut frame);
1257 }
1258 }
1259 }
1260 }
1261
1262 #[test]
1264 fn parallel_gate_threshold_is_exclusive() {
1265 assert!(!should_parallelize(0));
1266 assert!(!should_parallelize(PARALLEL_MASK_PX_THRESHOLD));
1267 assert!(should_parallelize(PARALLEL_MASK_PX_THRESHOLD + 1));
1268 }
1269
1270 #[test]
1272 fn clipped_mask_px_counts_only_intersecting_area() {
1273 let bitmap = vec![255u8; 100 * 50];
1274 let image = |dst_x: i32, dst_y: i32| OverlayImage {
1275 w: 100,
1276 h: 50,
1277 stride: 100,
1278 bitmap: &bitmap,
1279 color: 0xFFFFFF00,
1280 dst_x,
1281 dst_y,
1282 };
1283 let on_frame = image(10, 10);
1284 assert_eq!(
1285 clipped_mask_px(std::slice::from_ref(&on_frame), 640, 360),
1286 5000
1287 );
1288 let half_off = image(-50, 0);
1289 assert_eq!(
1290 clipped_mask_px(std::slice::from_ref(&half_off), 640, 360),
1291 2500
1292 );
1293 let fully_off = image(640, 0);
1294 assert_eq!(
1295 clipped_mask_px(std::slice::from_ref(&fully_off), 640, 360),
1296 0
1297 );
1298 let both = [image(10, 10), image(-50, 0)];
1299 assert_eq!(clipped_mask_px(&both, 640, 360), 7500);
1300 }
1301
1302 #[test]
1306 fn split_tasks_by_plane_and_overlap() {
1307 let mut buf_a = vec![0u8; 64];
1308 let mut buf_b = vec![0u8; 64];
1309 let task = |plane: usize, data: *mut u8, len: usize| CompTask {
1310 plane,
1311 data,
1312 len,
1313 linesize: 8,
1314 pixel_step: 1,
1315 source: 0,
1316 hsub: 0,
1317 vsub: 0,
1318 };
1319 let a_ptr = buf_a.as_mut_ptr();
1320 let b_ptr = buf_b.as_mut_ptr();
1321
1322 let chroma2 = unsafe { b_ptr.add(32) };
1325 let tasks = [task(0, a_ptr, 64), task(1, b_ptr, 32), task(2, chroma2, 32)];
1326 let (group_a, group_b) = split_tasks(&tasks).expect("planar split");
1327 assert_eq!((group_a.len(), group_b.len()), (1, 2));
1328
1329 let (rgb1, rgb2) = unsafe { (a_ptr.add(1), a_ptr.add(2)) };
1332 let tasks = [task(0, a_ptr, 62), task(0, rgb1, 62), task(0, rgb2, 62)];
1333 assert!(split_tasks(&tasks).is_none());
1334
1335 assert!(split_tasks(&[task(0, a_ptr, 64)]).is_none());
1337 assert!(split_tasks(&[]).is_none());
1338
1339 let overlap = unsafe { a_ptr.add(32) };
1342 let tasks = [task(0, a_ptr, 64), task(1, overlap, 32)];
1343 assert!(split_tasks(&tasks).is_none());
1344
1345 let tasks = [task(0, a_ptr, 32), task(1, b_ptr, 32), task(0, overlap, 32)];
1347 assert!(split_tasks(&tasks).is_none());
1348 }
1349
1350 fn ffmpeg_cli_has_libass() -> bool {
1351 std::process::Command::new("ffmpeg")
1352 .args(["-hide_banner", "-buildconf"])
1353 .output()
1354 .map(|out| String::from_utf8_lossy(&out.stdout).contains("--enable-libass"))
1355 .unwrap_or(false)
1356 }
1357
1358 #[test]
1362 #[ignore = "requires ffmpeg CLI built with --enable-libass"]
1363 fn parity_with_ffmpeg_cli() {
1364 let Some(font) = test_util::test_font() else {
1365 eprintln!("skipping: no known test font present on this machine");
1366 return;
1367 };
1368 if !ffmpeg_cli_has_libass() {
1369 eprintln!("skipping: no ffmpeg CLI with --enable-libass on PATH");
1370 return;
1371 }
1372
1373 let fonts_dir = temp_path("parity_fonts");
1375 std::fs::create_dir_all(&fonts_dir).expect("create fonts dir");
1376 let font_file = fonts_dir.join("DejaVuSans.ttf");
1377 std::fs::copy(font, &font_file).expect("copy test font");
1378
1379 const FORCE_STYLE: &str = "PrimaryColour=&H0000FF&";
1382 let fonts_str = fonts_dir.to_str().expect("utf8 path");
1383
1384 let run_pair = |script: &str,
1385 name: &str,
1386 setparams: Option<&str>,
1387 pix_fmt: &str|
1388 -> (Vec<u8>, Vec<u8>) {
1389 let label = format!("{name}_{}", setparams.map_or("default", |_| "bt709"));
1390 let script_path = temp_path(&format!("parity_{label}.ass"));
1391 std::fs::write(&script_path, script).expect("write script");
1392 let script_str = script_path.to_str().expect("utf8 path");
1393 assert!(
1395 !script_str.contains([':', '\'']) && !fonts_str.contains([':', '\'']),
1396 "temp paths must not need lavfi escaping"
1397 );
1398 let reference_path = temp_path(&format!("parity_ref_{label}.yuv"));
1399 let ours_path = temp_path(&format!("parity_ours_{label}.yuv"));
1400
1401 let mut chain = String::new();
1402 if let Some(setparams) = setparams {
1403 chain.push_str(setparams);
1404 chain.push(',');
1405 }
1406 chain.push_str(&format!(
1407 "format={pix_fmt},subtitles=filename='{script_str}':fontsdir='{fonts_str}':force_style='{FORCE_STYLE}'"
1408 ));
1409 let status = std::process::Command::new("ffmpeg")
1410 .args(["-y", "-v", "error", "-i", "test.mp4", "-an", "-vf"])
1411 .arg(&chain)
1412 .args(["-f", "rawvideo"])
1413 .arg(&reference_path)
1414 .status()
1415 .expect("run ffmpeg CLI");
1416 assert!(status.success(), "ffmpeg CLI failed for {label}");
1417
1418 let filter = SubtitleFilter::builder()
1419 .ass_content(script)
1420 .fonts_dir(&fonts_dir)
1421 .default_font_file(&font_file)
1422 .font_provider(FontProvider::None)
1423 .force_style(FORCE_STYLE)
1424 .build()
1425 .expect("build subtitle filter");
1426 transcode_test_mp4(&ours_path, Some(filter), pix_fmt, setparams);
1427
1428 let reference = std::fs::read(&reference_path).expect("read reference");
1429 let ours = std::fs::read(&ours_path).expect("read ours");
1430 if std::env::var_os("EZ_PARITY_KEEP").is_none() {
1432 let _ = std::fs::remove_file(&script_path);
1433 let _ = std::fs::remove_file(&reference_path);
1434 let _ = std::fs::remove_file(&ours_path);
1435 } else {
1436 eprintln!("kept: {reference_path:?} vs {ours_path:?}");
1437 }
1438 assert_eq!(reference.len(), ours.len(), "{label}: frame count");
1439 (reference, ours)
1440 };
1441
1442 let drawing = test_util::minimal_ass(test_util::DRAWING_EVENT);
1449 let mut ours_by_matrix = Vec::new();
1450 for setparams in [None, Some("setparams=colorspace=bt709")] {
1451 let (reference, ours) = run_pair(&drawing, "draw", setparams, "yuv420p");
1452 let (max, mean) = diff_stats(&reference, &ours);
1453 let big_diffs = reference
1454 .iter()
1455 .zip(&ours)
1456 .filter(|(a, b)| a.abs_diff(**b) > 8)
1457 .count();
1458 let big_fraction = big_diffs as f64 / reference.len() as f64;
1459 assert!(
1460 max <= 16 && mean < 0.05 && big_fraction < 0.0005,
1461 "drawing parity diverged (setparams {setparams:?}: max {max}, mean {mean:.4}, \
1462 >8-diff fraction {big_fraction:.6}) — interiors must stay byte-exact, edges \
1463 within cross-rasterizer AA noise"
1464 );
1465 ours_by_matrix.push(ours);
1466 }
1467 assert_ne!(
1470 ours_by_matrix[0], ours_by_matrix[1],
1471 "BT.601 and BT.709 renders should differ for a red subtitle"
1472 );
1473
1474 let (reference, ours) = run_pair(&drawing, "draw10", None, "yuv420p10le");
1478 let (max, mean) = test_util::diff_stats_u16le(&reference, &ours);
1479 assert!(
1480 max <= 64 && mean < 0.2,
1481 "10-bit drawing parity diverged (max {max}, mean {mean:.4})"
1482 );
1483
1484 let text = test_util::minimal_ass(&format!(
1493 "{}{}",
1494 test_util::HELLO_EVENT,
1495 test_util::DRAWING_EVENT
1496 ));
1497 let (reference, ours) = run_pair(&text, "text", None, "yuv420p");
1498 let (_, mean) = diff_stats(&reference, &ours);
1499 let outliers = reference
1500 .iter()
1501 .zip(&ours)
1502 .filter(|(a, b)| a.abs_diff(**b) > 8)
1503 .count();
1504 let outlier_fraction = outliers as f64 / reference.len() as f64;
1505 assert!(
1506 mean < 1.0 && outlier_fraction < 0.03,
1507 "text parity diverged (mean {mean:.4}, >8-diff fraction {outlier_fraction:.5})"
1508 );
1509
1510 let _ = std::fs::remove_file(&font_file);
1511 let _ = std::fs::remove_dir(&fonts_dir);
1512 }
1513}