1use crate::video::error::{VideoError, VideoResult};
12use crate::video::scheduler::ScheduledFrame;
13use crossbeam_channel::{Receiver, SendTimeoutError, Sender};
14use ffmpeg_next as ffmpeg;
15use std::ffi::{CString, c_char};
16use std::path::{Path, PathBuf};
17use std::sync::Arc;
18use std::sync::atomic::{AtomicBool, AtomicU8, Ordering};
19use std::thread;
20use std::time::Duration;
21
22pub trait VideoSource: Send {
24 fn metadata(&self) -> &VideoMetadata;
26
27 fn decoder_info(&self) -> DecoderInfo;
29
30 fn next_frame(&self) -> Option<VideoFrame>;
32
33 fn is_video_file(path: &Path) -> bool;
35}
36
37#[derive(Debug, Clone)]
39pub struct VideoMetadata {
40 pub width: u32,
41 pub height: u32,
42 pub duration: Duration,
43 pub fps: f64,
44 pub codec: String,
45 pub format: String,
46 pub total_frames: u64,
47}
48
49#[derive(Debug, Clone)]
51pub struct DecoderInfo {
52 pub codec_name: String,
53 pub hardware_accel: Option<String>,
54 pub pixel_format: String,
55}
56
57#[derive(Debug, Clone)]
59pub struct VideoFrame {
60 pub data: Vec<u8>,
62 pub width: u32,
64 pub height: u32,
66 pub pts: Duration,
68 pub index: u64,
70}
71
72impl VideoFrame {
73 pub fn into_scheduled(self) -> ScheduledFrame {
75 ScheduledFrame::new(self.data, self.width, self.height, self.pts, self.index)
76 }
77}
78
79#[derive(Debug, Clone, Copy, PartialEq, Eq)]
81pub enum HwAccel {
82 Vaapi,
84 Nvdec,
86 VideoToolbox,
88 Software,
90}
91
92impl HwAccel {
93 pub fn name(&self) -> &'static str {
95 match self {
96 HwAccel::Vaapi => "VAAPI",
97 HwAccel::Nvdec => "NVDEC",
98 HwAccel::VideoToolbox => "VideoToolbox",
99 HwAccel::Software => "Software",
100 }
101 }
102
103 const fn code(self) -> u8 {
105 match self {
106 HwAccel::Software => 1,
107 HwAccel::Vaapi => 2,
108 HwAccel::Nvdec => 3,
109 HwAccel::VideoToolbox => 4,
110 }
111 }
112
113 const fn from_code(code: u8) -> HwAccel {
115 match code {
116 2 => HwAccel::Vaapi,
117 3 => HwAccel::Nvdec,
118 4 => HwAccel::VideoToolbox,
119 _ => HwAccel::Software,
120 }
121 }
122 pub fn from_config(value: &str) -> HwAccel {
125 match value.trim().to_ascii_lowercase().as_str() {
126 "vaapi" => HwAccel::Vaapi,
127 "nvdec" | "nvidia" | "cuda" => HwAccel::Nvdec,
128 "software" | "none" | "off" => HwAccel::Software,
129 _ => HwAccel::detect_available(),
130 }
131 }
132
133 pub fn detect_available() -> HwAccel {
135 #[cfg(target_os = "linux")]
136 {
137 if Path::new("/dev/dri/renderD128").exists() {
138 return HwAccel::Vaapi;
139 }
140 if Path::new("/dev/nvidia0").exists() {
141 return HwAccel::Nvdec;
142 }
143 }
144
145 #[cfg(target_os = "macos")]
146 {
147 return HwAccel::VideoToolbox;
148 }
149
150 HwAccel::Software
151 }
152}
153
154#[derive(Debug, Clone, Copy)]
156enum DecoderControl {
157 Pause,
158 Resume,
159 Seek(Duration),
160}
161
162pub struct VideoDecoder {
164 metadata: VideoMetadata,
165 frame_rx: Receiver<VideoFrame>,
166 control_tx: Sender<DecoderControl>,
167 stop_flag: Arc<AtomicBool>,
168 hw_in_use: Arc<AtomicU8>,
170 decode_thread: Option<thread::JoinHandle<()>>,
171}
172
173impl VideoDecoder {
174 pub fn new<P: AsRef<Path>>(path: P, hw_accel: HwAccel) -> VideoResult<Self> {
179 let path = path.as_ref().to_path_buf();
180
181 ffmpeg::init().map_err(|e| {
183 VideoError::SoftwareDecoderInit(anyhow::anyhow!("FFmpeg init failed: {}", e))
184 })?;
185
186 let metadata = Self::extract_metadata(&path)?;
188
189 tracing::info!(
190 "Opened video: {}x{} @ {:.2} fps, duration: {:?}, codec: {}",
191 metadata.width,
192 metadata.height,
193 metadata.fps,
194 metadata.duration,
195 metadata.codec
196 );
197
198 let (frame_tx, frame_rx) = crossbeam_channel::bounded(3);
201 let (control_tx, control_rx) = crossbeam_channel::unbounded();
203
204 let stop_flag = Arc::new(AtomicBool::new(false));
205 let stop_flag_clone = stop_flag.clone();
206 let hw_in_use = Arc::new(AtomicU8::new(0));
207 let hw_in_use_clone = hw_in_use.clone();
208
209 let decode_thread = thread::Builder::new()
211 .name("wallr-video-decoder".to_string())
212 .spawn(move || {
213 let hw_report = hw_in_use_clone.clone();
214 let used = std::panic::catch_unwind(|| {
215 Self::decode_loop(
216 path,
217 hw_accel,
218 frame_tx,
219 control_rx,
220 stop_flag_clone,
221 hw_report,
222 )
223 });
224 let used = match used {
225 Ok(Ok(used)) => used,
226 Ok(Err(e)) => {
227 tracing::error!("Video decode loop error: {}", e);
228 HwAccel::Software
229 }
230 Err(panic) => {
231 let msg = panic
232 .downcast_ref::<&str>()
233 .map(|s| s.to_string())
234 .or_else(|| panic.downcast_ref::<String>().cloned())
235 .unwrap_or_else(|| "unknown panic".to_string());
236 tracing::error!("Video decode thread panicked: {}", msg);
237 HwAccel::Software
238 }
239 };
240 tracing::info!("Video decode thread exited (backend: {})", used.name());
241 hw_in_use_clone.store(used.code(), Ordering::Relaxed);
242 })
243 .map_err(|e| VideoError::SoftwareDecoderInit(e.into()))?;
244
245 Ok(Self {
246 metadata,
247 frame_rx,
248 control_tx,
249 stop_flag,
250 hw_in_use,
251 decode_thread: Some(decode_thread),
252 })
253 }
254
255 fn extract_metadata(path: &Path) -> VideoResult<VideoMetadata> {
257 let ictx = ffmpeg::format::input(&path).map_err(|e| VideoError::FileOpen {
258 path: path.to_path_buf(),
259 source: std::io::Error::other(e.to_string()),
260 })?;
261
262 let stream = ictx
263 .streams()
264 .best(ffmpeg::media::Type::Video)
265 .ok_or_else(|| VideoError::NoVideoStream(path.to_path_buf()))?;
266
267 let decoder = ffmpeg::codec::context::Context::from_parameters(stream.parameters())
268 .and_then(|ctx| ctx.decoder().video())
269 .map_err(|e| VideoError::SoftwareDecoderInit(e.into()))?;
270
271 let width = decoder.width();
272 let height = decoder.height();
273 let codec = decoder
274 .codec()
275 .map(|c| c.name().to_string())
276 .unwrap_or_else(|| "unknown".to_string());
277
278 let fps = {
280 let frame_rate = stream.avg_frame_rate();
281 if frame_rate.numerator() > 0 {
282 frame_rate.numerator() as f64 / frame_rate.denominator() as f64
283 } else {
284 30.0 }
286 };
287
288 let duration = {
290 let duration_ts = stream.duration();
291 let time_base = stream.time_base();
292 if duration_ts > 0 {
293 Duration::from_secs_f64(
294 duration_ts as f64 * time_base.numerator() as f64
295 / time_base.denominator() as f64,
296 )
297 } else {
298 let container_duration = ictx.duration() as f64 / ffmpeg::ffi::AV_TIME_BASE as f64;
300 Duration::from_secs_f64(container_duration)
301 }
302 };
303
304 let total_frames = if fps > 0.0 {
305 (duration.as_secs_f64() * fps) as u64
306 } else {
307 0
308 };
309
310 let format = ictx.format().name().to_string();
311
312 Ok(VideoMetadata {
313 width,
314 height,
315 duration,
316 fps,
317 codec,
318 format,
319 total_frames,
320 })
321 }
322
323 fn init_hw_device(hw_accel: HwAccel) -> Option<(*mut ffmpeg::ffi::AVBufferRef, &'static str)> {
329 let (type_name, device) = match hw_accel {
330 HwAccel::Vaapi => ("vaapi", Some(c"/dev/dri/renderD128")),
331 HwAccel::Nvdec => ("cuda", Some(c"0")),
332 HwAccel::VideoToolbox => ("videotoolbox", None),
333 HwAccel::Software => return None,
334 };
335
336 let type_name_c = match CString::new(type_name) {
337 Ok(c) => c,
338 Err(_) => return None,
339 };
340
341 unsafe {
344 let hw_type = ffmpeg::ffi::av_hwdevice_find_type_by_name(type_name_c.as_ptr());
345 if hw_type == ffmpeg::ffi::AVHWDeviceType::AV_HWDEVICE_TYPE_NONE {
346 tracing::warn!("{} hardware type unavailable", hw_accel.name());
347 return None;
348 }
349
350 let mut device_ctx: *mut ffmpeg::ffi::AVBufferRef = std::ptr::null_mut();
351 let device_ptr: *const c_char = device.map(|d| d.as_ptr()).unwrap_or(std::ptr::null());
352 let ret = ffmpeg::ffi::av_hwdevice_ctx_create(
353 &mut device_ctx,
354 hw_type,
355 device_ptr,
356 std::ptr::null_mut(),
357 0,
358 );
359 if ret < 0 || device_ctx.is_null() {
360 tracing::warn!(
361 "{} device init failed (error {}), falling back to software",
362 hw_accel.name(),
363 ret
364 );
365 return None;
366 }
367 Some((device_ctx, type_name))
368 }
369 }
370
371 fn build_decoder(
376 stream: &ffmpeg::format::stream::Stream,
377 hw_accel: HwAccel,
378 ) -> (ffmpeg::codec::decoder::Video, HwAccel) {
379 if hw_accel != HwAccel::Software
380 && let Some((device_ctx, _type_name)) = Self::init_hw_device(hw_accel)
381 {
382 let mut context =
387 match ffmpeg::codec::context::Context::from_parameters(stream.parameters()) {
388 Ok(ctx) => ctx,
389 Err(e) => {
390 tracing::warn!(
391 "codec context creation failed ({}), falling back to software",
392 e
393 );
394 let decoder =
395 ffmpeg::codec::context::Context::from_parameters(stream.parameters())
396 .and_then(|ctx| ctx.decoder().video())
397 .expect("software decoder must be available");
398 return (decoder, HwAccel::Software);
399 }
400 };
401 unsafe {
402 (*context.as_mut_ptr()).hw_device_ctx = device_ctx;
403 }
404 match context.decoder().video() {
405 Ok(decoder) => {
406 tracing::info!("Hardware decode active: {}", hw_accel.name());
407 return (decoder, hw_accel);
408 }
409 Err(e) => {
410 tracing::warn!(
411 "{} decode init failed ({}), falling back to software",
412 hw_accel.name(),
413 e
414 );
415 }
418 }
419 }
420
421 let decoder = ffmpeg::codec::context::Context::from_parameters(stream.parameters())
422 .and_then(|ctx| ctx.decoder().video())
423 .expect("software decoder must be available");
424 (decoder, HwAccel::Software)
425 }
426
427 #[allow(clippy::too_many_arguments)]
431 fn decode_loop(
432 path: PathBuf,
433 hw_accel: HwAccel,
434 frame_tx: Sender<VideoFrame>,
435 control_rx: Receiver<DecoderControl>,
436 stop_flag: Arc<AtomicBool>,
437 hw_in_use: Arc<AtomicU8>,
438 ) -> VideoResult<HwAccel> {
439 let mut ictx = ffmpeg::format::input(&path).map_err(|e| VideoError::FileOpen {
440 path: path.clone(),
441 source: std::io::Error::other(e.to_string()),
442 })?;
443
444 let stream = ictx
445 .streams()
446 .best(ffmpeg::media::Type::Video)
447 .ok_or_else(|| VideoError::NoVideoStream(path.clone()))?;
448
449 let video_stream_index = stream.index();
450 let time_base = stream.time_base();
451
452 let (mut decoder, used_hw) = Self::build_decoder(&stream, hw_accel);
453 hw_in_use.store(used_hw.code(), Ordering::Relaxed);
454 tracing::info!("Decoder in use: {}", used_hw.name());
455
456 let mut scaler: Option<ffmpeg::software::scaling::Context> = None;
460 let mut scaler_src: Option<ffmpeg::format::Pixel> = None;
461
462 let mut paused = false;
463 let mut pending_seek: Option<Duration> = None;
464 let mut frame_index = 0u64;
465 let mut loop_count = 0u64;
466 let mut decoded_frame = ffmpeg::frame::Video::empty();
467 let mut sw_frame = ffmpeg::frame::Video::empty();
468 let mut rgb_frame = ffmpeg::frame::Video::empty();
469
470 'outer: loop {
471 loop_count += 1;
472 tracing::debug!("Decode loop iteration {}", loop_count);
473 if stop_flag.load(Ordering::Relaxed) {
474 tracing::info!("Decode loop exiting (stop flag)");
475 break;
476 }
477
478 while let Ok(control) = control_rx.try_recv() {
481 match control {
482 DecoderControl::Pause => paused = true,
483 DecoderControl::Resume => paused = false,
484 DecoderControl::Seek(ts) => pending_seek = Some(ts),
485 }
486 }
487
488 if let Some(ts) = pending_seek.take() {
491 Self::apply_seek(&mut ictx, &mut decoder, time_base, ts);
492 }
493
494 if paused {
496 thread::sleep(Duration::from_millis(10));
497 continue;
498 }
499
500 for (stream, packet) in ictx.packets() {
502 if stop_flag.load(Ordering::Relaxed) {
503 break 'outer;
504 }
505
506 while let Ok(control) = control_rx.try_recv() {
508 match control {
509 DecoderControl::Pause => paused = true,
510 DecoderControl::Resume => paused = false,
511 DecoderControl::Seek(ts) => pending_seek = Some(ts),
512 }
513 }
514 if paused || pending_seek.is_some() {
515 break;
519 }
520
521 if stream.index() != video_stream_index {
522 continue;
523 }
524
525 decoder.send_packet(&packet).map_err(|e| {
526 VideoError::DecodeFailed(anyhow::anyhow!("send_packet failed: {}", e))
527 })?;
528
529 while decoder.receive_frame(&mut decoded_frame).is_ok() {
530 if stop_flag.load(Ordering::Relaxed) {
531 break 'outer;
532 }
533
534 let is_hw_frame = unsafe { (*decoded_frame.as_ptr()).data[0].is_null() };
537
538 let src_frame = if is_hw_frame {
539 let ret = unsafe {
543 ffmpeg::ffi::av_hwframe_transfer_data(
544 sw_frame.as_mut_ptr(),
545 decoded_frame.as_ptr(),
546 0,
547 )
548 };
549 if ret < 0 {
550 tracing::warn!("hwframe transfer failed: {ret}");
551 continue;
552 }
553 &sw_frame
554 } else {
555 &decoded_frame
556 };
557
558 let src_format = src_frame.format();
559 if scaler_src != Some(src_format) {
560 scaler = Some(
561 ffmpeg::software::scaling::context::Context::get(
562 src_format,
563 src_frame.width(),
564 src_frame.height(),
565 ffmpeg::format::Pixel::RGBA,
566 src_frame.width(),
567 src_frame.height(),
568 ffmpeg::software::scaling::Flags::BILINEAR,
569 )
570 .map_err(|e| VideoError::FormatConversionFailed(e.into()))?,
571 );
572 scaler_src = Some(src_format);
573 tracing::debug!("Scaler initialized for format {:?}", src_format);
574 }
575
576 scaler
577 .as_mut()
578 .expect("scaler initialized above")
579 .run(src_frame, &mut rgb_frame)
580 .map_err(|e| VideoError::FormatConversionFailed(e.into()))?;
581
582 let pts_duration = if let Some(pts) = decoded_frame.timestamp() {
584 Duration::from_secs_f64(
585 pts as f64 * time_base.numerator() as f64
586 / time_base.denominator() as f64,
587 )
588 } else {
589 Duration::from_secs_f64(frame_index as f64 / 30.0) };
591
592 let data = rgb_frame.data(0).to_vec();
594
595 let video_frame = VideoFrame {
596 data,
597 width: rgb_frame.width(),
598 height: rgb_frame.height(),
599 pts: pts_duration,
600 index: frame_index,
601 };
602 frame_index = frame_index.wrapping_add(1);
603
604 match frame_tx.send_timeout(video_frame, Duration::from_millis(20)) {
607 Ok(()) => {}
608 Err(SendTimeoutError::Timeout(_)) => {
609 break;
614 }
615 Err(SendTimeoutError::Disconnected(_)) => {
616 tracing::warn!("Frame queue disconnected, ending decode loop");
618 return Ok(used_hw);
619 }
620 }
621 }
622 }
623
624 tracing::debug!(
627 "End of stream after {} frames, seeking back to 0",
628 frame_index
629 );
630 while decoder.receive_frame(&mut decoded_frame).is_ok() {
631 if stop_flag.load(Ordering::Relaxed) {
632 break 'outer;
633 }
634 }
635 if stop_flag.load(Ordering::Relaxed) {
636 break;
637 }
638 if !paused {
639 ictx.seek(0, ..)
640 .map_err(|e| VideoError::SeekFailed(Duration::ZERO, e.into()))?;
641 decoder.flush();
642 }
643 }
644
645 Ok(used_hw)
646 }
647
648 fn apply_seek(
650 ictx: &mut ffmpeg::format::context::Input,
651 decoder: &mut ffmpeg::codec::decoder::Video,
652 time_base: ffmpeg::Rational,
653 ts: Duration,
654 ) {
655 let tb_sec = time_base.numerator() as f64 / time_base.denominator() as f64;
656 let ts_tb = if tb_sec > 0.0 {
657 (ts.as_secs_f64() / tb_sec) as i64
658 } else {
659 0
660 };
661 match ictx.seek(ts_tb, ..) {
662 Ok(()) => {
663 decoder.flush();
664 let mut drain = ffmpeg::frame::Video::empty();
667 while decoder.receive_frame(&mut drain).is_ok() {}
668 tracing::info!("Decoder seeked to {:?}", ts);
669 }
670 Err(e) => tracing::warn!("Decoder seek to {:?} failed: {}", ts, e),
671 }
672 }
673
674 pub fn next_frame(&self) -> Option<VideoFrame> {
676 self.frame_rx.try_recv().ok()
677 }
678
679 pub fn metadata(&self) -> &VideoMetadata {
681 &self.metadata
682 }
683
684 pub fn decoder_info(&self) -> DecoderInfo {
687 DecoderInfo {
688 codec_name: self.metadata.codec.clone(),
689 hardware_accel: if self.hw_accel_in_use() != HwAccel::Software {
690 Some(self.hw_accel_in_use().name().to_string())
691 } else {
692 None
693 },
694 pixel_format: "RGBA".to_string(),
695 }
696 }
697
698 pub fn hw_accel_in_use(&self) -> HwAccel {
703 for _ in 0..50 {
705 let code = self.hw_in_use.load(Ordering::Relaxed);
706 if code != 0 {
707 return HwAccel::from_code(code);
708 }
709 std::thread::sleep(Duration::from_millis(5));
710 }
711 HwAccel::Software
712 }
713
714 pub fn pause(&self) {
716 let _ = self.control_tx.send(DecoderControl::Pause);
717 }
718
719 pub fn resume(&self) {
721 let _ = self.control_tx.send(DecoderControl::Resume);
722 }
723
724 pub fn seek(&self, timestamp: Duration) {
727 let _ = self.control_tx.send(DecoderControl::Seek(timestamp));
728 }
729
730 pub fn drain(&mut self) {
732 while self.frame_rx.try_recv().is_ok() {}
733 }
734
735 pub fn is_video_file<P: AsRef<Path>>(path: P) -> bool {
737 let path = path.as_ref();
738
739 if let Some(ext) = path.extension().and_then(|e| e.to_str()) {
740 matches!(
741 ext.to_lowercase().as_str(),
742 "mp4" | "webm" | "mkv" | "mov" | "avi" | "m4v"
743 )
744 } else {
745 false
746 }
747 }
748}
749
750impl Drop for VideoDecoder {
751 fn drop(&mut self) {
752 self.stop_flag.store(true, Ordering::Relaxed);
754
755 if let Some(thread) = self.decode_thread.take() {
757 let _ = thread.join();
758 }
759 }
760}
761
762impl VideoSource for VideoDecoder {
763 fn metadata(&self) -> &VideoMetadata {
764 &self.metadata
765 }
766
767 fn decoder_info(&self) -> DecoderInfo {
768 self.decoder_info()
769 }
770
771 fn next_frame(&self) -> Option<VideoFrame> {
772 self.next_frame()
773 }
774
775 fn is_video_file(path: &Path) -> bool {
776 VideoDecoder::is_video_file(path)
777 }
778}
779
780#[cfg(test)]
781mod tests {
782 use super::*;
783
784 #[test]
785 fn test_is_video_file() {
786 assert!(VideoDecoder::is_video_file("test.mp4"));
787 assert!(VideoDecoder::is_video_file("test.MP4"));
788 assert!(VideoDecoder::is_video_file("test.webm"));
789 assert!(VideoDecoder::is_video_file("test.mkv"));
790 assert!(VideoDecoder::is_video_file("test.mov"));
791 assert!(!VideoDecoder::is_video_file("test.jpg"));
792 assert!(!VideoDecoder::is_video_file("test.gif"));
793 assert!(!VideoDecoder::is_video_file("test.png"));
794 }
795
796 #[test]
797 fn test_hwaccel_names() {
798 assert_eq!(HwAccel::Vaapi.name(), "VAAPI");
799 assert_eq!(HwAccel::Nvdec.name(), "NVDEC");
800 assert_eq!(HwAccel::Software.name(), "Software");
801 }
802
803 #[test]
804 fn test_hwaccel_codes_roundtrip() {
805 for accel in [
806 HwAccel::Software,
807 HwAccel::Vaapi,
808 HwAccel::Nvdec,
809 HwAccel::VideoToolbox,
810 ] {
811 assert_eq!(HwAccel::from_code(accel.code()), accel);
812 }
813 }
814
815 #[test]
816 fn test_hwaccel_from_config() {
817 assert_eq!(HwAccel::from_config("vaapi"), HwAccel::Vaapi);
818 assert_eq!(HwAccel::from_config("nvdec"), HwAccel::Nvdec);
819 assert_eq!(HwAccel::from_config("software"), HwAccel::Software);
820 assert!(matches!(
822 HwAccel::from_config("auto"),
823 HwAccel::Vaapi | HwAccel::Nvdec | HwAccel::VideoToolbox | HwAccel::Software
824 ));
825 }
826}