1use super::*;
2use rsmpeg::{
3 avcodec::AVCodecContext,
4 avformat::AVFormatContextInput,
5 avutil::{AVChannelLayout, AVDictionary, AVFrame},
6 error::RsmpegError,
7 ffi,
8 swresample::SwrContext,
9 swscale::SwsContext,
10};
11use rusttorch_core::Tensor;
12use rusttorch_data::{ResourceLimits, WorkerContext};
13use std::{ffi::CString, path::Path};
14pub fn capabilities() -> CodecCapabilities {
16 CodecCapabilities {
17 version: rsmpeg::avutil::version_info()
18 .to_string_lossy()
19 .into_owned(),
20 configuration: rsmpeg::avcodec::configuration()
21 .to_string_lossy()
22 .into_owned(),
23 license: rsmpeg::avcodec::license().to_string_lossy().into_owned(),
24 cpu_decode: true,
25 }
26}
27pub struct MediaDecoder {
33 input: AVFormatContextInput,
34 decoder: AVCodecContext,
35 info: StreamInfo,
36 limits: ResourceLimits,
37 draining: bool,
38 done: bool,
39 count: usize,
40 seek_target: Option<f64>,
41 context: Option<WorkerContext>,
42 audio_frames: u64,
43}
44impl MediaDecoder {
45 pub fn open(path: impl AsRef<Path>, kind: StreamKind, limits: ResourceLimits) -> Result<Self> {
49 let path = std::fs::canonicalize(path)?;
50 let metadata = path.metadata()?;
51 if !metadata.is_file() {
52 return Err(CodecError::Invalid(
53 "media input must be a regular local file".into(),
54 ));
55 }
56 limits.check(
57 "media encoded bytes",
58 usize::try_from(metadata.len()).unwrap_or(usize::MAX),
59 limits.max_encoded_bytes,
60 )?;
61 let path = CString::new(
62 path.to_str()
63 .ok_or_else(|| CodecError::Invalid("media path must be UTF-8".into()))?,
64 )
65 .map_err(|_| CodecError::Invalid("media path contains NUL".into()))?;
66 let opts = AVDictionary::new(c"protocol_whitelist", c"file", 0)
67 .set(
68 c"format_whitelist",
69 c"matroska,webm,wav,avi,mov,mp3,flac",
70 0,
71 )
72 .set_int(
73 c"probesize",
74 limits.max_encoded_bytes.clamp(32, 5_000_000) as i64,
75 0,
76 )
77 .set_int(c"analyzeduration", 5_000_000, 0)
78 .set_int(c"rw_timeout", 5_000_000, 0)
79 .set_int(
80 c"max_streams",
81 limits.max_streams.min(i32::MAX as usize) as i64,
82 0,
83 );
84 let mut options = Some(opts);
85 let input = AVFormatContextInput::builder()
86 .url(&path)
87 .options(&mut options)
88 .open()?;
89 limits.check("media streams", input.streams().len(), limits.max_streams)?;
90 let media = match kind {
91 StreamKind::Video => ffi::AVMEDIA_TYPE_VIDEO,
92 StreamKind::Audio => ffi::AVMEDIA_TYPE_AUDIO,
93 };
94 let (index, codec) = input
95 .find_best_stream(media)?
96 .ok_or_else(|| CodecError::Invalid("requested media stream not found".into()))?;
97 let stream = &input.streams()[index];
98 let time_base = TimeBase::new(
99 u32::try_from(stream.time_base.num)
100 .map_err(|_| CodecError::Invalid("invalid stream time numerator".into()))?,
101 u32::try_from(stream.time_base.den)
102 .map_err(|_| CodecError::Invalid("invalid stream time denominator".into()))?,
103 )?;
104 let duration = if stream.duration == ffi::AV_NOPTS_VALUE {
105 None
106 } else {
107 Some(stream.duration)
108 };
109 if duration.is_some_and(|d| d < 0 || time_base.seconds(d) > limits.max_media_seconds as f64)
110 {
111 return Err(CodecError::Invalid(
112 "declared media duration exceeds limit".into(),
113 ));
114 }
115 let params = stream.codecpar();
116 if kind == StreamKind::Video {
117 check_video(params.height, params.width, limits)?;
118 } else if params.ch_layout.nb_channels < 1 || params.ch_layout.nb_channels > 2 {
119 return Err(CodecError::Invalid(
120 "native audio baseline supports mono/stereo only".into(),
121 ));
122 }
123 let mut decoder = AVCodecContext::new(&codec);
124 decoder.apply_codecpar(¶ms)?;
125 decoder.open(Some(AVDictionary::new_int(
126 c"max_pixels",
127 limits.max_tensor_elements.min(i64::MAX as usize) as i64,
128 0,
129 )))?;
130 drop(params);
131 let info = StreamInfo {
132 index,
133 kind,
134 time_base,
135 duration,
136 decoder: codec.name().to_string_lossy().into_owned(),
137 };
138 Ok(Self {
139 input,
140 decoder,
141 info,
142 limits,
143 draining: false,
144 done: false,
145 count: 0,
146 seek_target: None,
147 context: None,
148 audio_frames: 0,
149 })
150 }
151 pub fn stream_info(&self) -> &StreamInfo {
153 &self.info
154 }
155 pub fn with_worker_context(mut self, context: WorkerContext) -> Self {
157 self.context = Some(context);
158 self
159 }
160 pub fn seek(&mut self, seconds: f64) -> Result<()> {
164 if !seconds.is_finite() || seconds < 0. || seconds > self.limits.max_media_seconds as f64 {
165 return Err(CodecError::Invalid("invalid seek time".into()));
166 }
167 let ticks = seconds * self.info.time_base.denominator() as f64
168 / self.info.time_base.numerator() as f64;
169 if ticks >= i64::MAX as f64 {
170 return Err(CodecError::Invalid("seek timestamp overflows".into()));
171 }
172 self.input.seek(
173 self.info.index as i32,
174 ticks.floor() as i64,
175 ffi::AVSEEK_FLAG_BACKWARD as i32,
176 )?;
177 self.decoder.flush_buffers();
178 self.draining = false;
179 self.done = false;
180 self.count = 0;
181 self.audio_frames = 0;
182 self.seek_target = Some(seconds);
183 Ok(())
184 }
185 fn next_frame(&mut self) -> Result<Option<MediaFrame>> {
186 loop {
187 if let Some(context) = &self.context
188 && (context.cancellation.is_cancelled() || context.deadline.is_expired())
189 {
190 return Err(CodecError::Invalid(
191 "media decoding cancelled or timed out".into(),
192 ));
193 }
194 match self.decoder.receive_frame() {
195 Ok(frame) => {
196 self.count = self
197 .count
198 .checked_add(1)
199 .ok_or_else(|| CodecError::Invalid("frame count overflow".into()))?;
200 self.limits.check(
201 "decoded media frames",
202 self.count,
203 self.limits.max_records,
204 )?;
205 let pts = frame.best_effort_timestamp;
206 let timestamp = if pts == ffi::AV_NOPTS_VALUE {
207 None
208 } else {
209 Some(Timestamp {
210 ticks: pts,
211 time_base: self.info.time_base,
212 })
213 };
214 if timestamp
215 .is_some_and(|t| t.seconds().abs() > self.limits.max_media_seconds as f64)
216 {
217 return Err(CodecError::Invalid(
218 "decoded timestamp exceeds duration ceiling".into(),
219 ));
220 }
221 if let Some(target) = self.seek_target {
222 match timestamp {
223 Some(t) if t.seconds() < target => continue,
224 Some(_) => self.seek_target = None,
225 None => {
226 return Err(CodecError::Invalid(
227 "cannot perform time seek on frames without timestamps".into(),
228 ));
229 }
230 }
231 }
232 return Ok(Some(match self.info.kind {
233 StreamKind::Video => {
234 MediaFrame::Video(video(frame, timestamp, self.limits)?)
235 }
236 StreamKind::Audio => {
237 let audio = audio(frame, timestamp, self.limits)?;
238 self.audio_frames = self
239 .audio_frames
240 .checked_add(audio.samples.size()[1] as u64)
241 .ok_or_else(|| {
242 CodecError::Invalid("audio length overflow".into())
243 })?;
244 if self.audio_frames as u128
245 > audio.sample_rate as u128 * self.limits.max_media_seconds as u128
246 {
247 return Err(CodecError::Invalid(
248 "decoded audio duration exceeds limit".into(),
249 ));
250 }
251 MediaFrame::Audio(audio)
252 }
253 }));
254 }
255 Err(RsmpegError::DecoderFlushedError) => return Ok(None),
256 Err(RsmpegError::DecoderDrainError) => {
257 if self.draining {
258 return Err(CodecError::Invalid(
259 "decoder requested packets after end-of-stream flush".into(),
260 ));
261 }
262 }
263 Err(e) => return Err(e.into()),
264 }
265 loop {
266 match self.input.read_packet()? {
267 Some(packet) => {
268 if packet.size < 0 || packet.size as usize > self.limits.max_encoded_bytes {
269 return Err(CodecError::Invalid(
270 "invalid or oversized media packet".into(),
271 ));
272 }
273 if packet.stream_index != self.info.index as i32 {
274 continue;
275 }
276 self.decoder.send_packet(Some(&packet))?;
277 break;
278 }
279 None => {
280 self.decoder.send_packet(None)?;
281 self.draining = true;
282 break;
283 }
284 }
285 }
286 }
287 }
288}
289impl Iterator for MediaDecoder {
290 type Item = Result<MediaFrame>;
291 fn next(&mut self) -> Option<Self::Item> {
292 if self.done {
293 return None;
294 }
295 match self.next_frame() {
296 Ok(Some(f)) => Some(Ok(f)),
297 Ok(None) => {
298 self.done = true;
299 None
300 }
301 Err(e) => {
302 self.done = true;
303 Some(Err(e))
304 }
305 }
306 }
307}
308fn check_video(h: i32, w: i32, limits: ResourceLimits) -> Result<usize> {
309 if h <= 0 || w <= 0 {
310 return Err(CodecError::Invalid(
311 "video dimensions must be positive".into(),
312 ));
313 }
314 limits.check("video height", h as usize, limits.max_image_dimension)?;
315 limits.check("video width", w as usize, limits.max_image_dimension)?;
316 let n = limits.tensor_elements(&[3, h as usize, w as usize])?;
317 limits.check("RGB frame bytes", n, limits.max_decoded_bytes)?;
318 Ok(n)
319}
320fn video(
321 frame: AVFrame,
322 timestamp: Option<Timestamp>,
323 limits: ResourceLimits,
324) -> Result<VideoFrame> {
325 let size = check_video(frame.height, frame.width, limits)?;
326 let mut scaler = SwsContext::get_context(
327 frame.width,
328 frame.height,
329 frame.format,
330 frame.width,
331 frame.height,
332 ffi::AV_PIX_FMT_RGB24,
333 ffi::SWS_BILINEAR,
334 None,
335 None,
336 None,
337 )
338 .ok_or_else(|| CodecError::Invalid("unsupported video pixel conversion".into()))?;
339 let mut output = AVFrame::new();
340 output.set_width(frame.width);
341 output.set_height(frame.height);
342 output.set_format(ffi::AV_PIX_FMT_RGB24);
343 output.alloc_buffer()?;
344 scaler.scale_frame(&frame, 0, frame.height, &mut output)?;
345 let expected = output.image_get_buffer_size(1)?;
346 if expected != size {
347 return Err(CodecError::Invalid(
348 "RGB buffer size differs from checked dimensions".into(),
349 ));
350 }
351 let mut bytes = vec![0u8; size];
352 output.image_copy_to_buffer(&mut bytes, 1)?;
353 Ok(VideoFrame {
354 pixels: Tensor::f_from_slice(&bytes)?
355 .f_reshape([frame.height as i64, frame.width as i64, 3])?
356 .f_permute([2, 0, 1])?,
357 timestamp,
358 })
359}
360fn audio(
361 mut frame: AVFrame,
362 timestamp: Option<Timestamp>,
363 limits: ResourceLimits,
364) -> Result<AudioFrame> {
365 let channels = frame.ch_layout.nb_channels;
366 let samples = frame.nb_samples;
367 if !(1..=2).contains(&channels) || samples < 0 || frame.sample_rate <= 0 {
368 return Err(CodecError::Invalid(
369 "invalid mono/stereo audio frame".into(),
370 ));
371 }
372 let n = limits.tensor_elements(&[channels as usize, samples as usize])?;
373 let bytes = n
374 .checked_mul(std::mem::size_of::<f32>())
375 .ok_or_else(|| CodecError::Invalid("audio frame size overflow".into()))?;
376 limits.check("audio frame bytes", bytes, limits.max_decoded_bytes)?;
377 let layout = AVChannelLayout::from_nb_channels(channels);
378 if frame.ch_layout.order == ffi::AV_CHANNEL_ORDER_UNSPEC {
379 frame.set_ch_layout(*layout);
380 }
381 let mut converter = SwrContext::new(
382 &layout,
383 ffi::AV_SAMPLE_FMT_FLT,
384 frame.sample_rate,
385 &frame.ch_layout,
386 frame.format,
387 frame.sample_rate,
388 )?;
389 converter.init()?;
390 let mut output = AVFrame::new();
391 output.set_ch_layout(*layout);
392 output.set_sample_rate(frame.sample_rate);
393 output.set_format(ffi::AV_SAMPLE_FMT_FLT);
394 output.set_nb_samples(samples);
395 output.alloc_buffer()?;
396 converter.convert_frame(Some(&frame), &mut output)?;
397 if output.nb_samples != samples
398 || output.linesize[0] < 0
399 || (output.linesize[0] as usize) < bytes
400 || output.data[0].is_null()
401 || !(output.data[0] as usize).is_multiple_of(std::mem::align_of::<f32>())
402 {
403 return Err(CodecError::Invalid(
404 "native audio output does not match its allocated layout".into(),
405 ));
406 }
407 let data = unsafe { std::slice::from_raw_parts(output.data[0].cast::<f32>(), n) };
414 Ok(AudioFrame {
415 samples: Tensor::f_from_slice(data)?
416 .f_reshape([samples as i64, channels as i64])?
417 .f_transpose(0, 1)?,
418 sample_rate: frame.sample_rate as u32,
419 timestamp,
420 })
421}