Skip to main content

rusttorch_codec/
native.rs

1use super::*;
2use rsmpeg::{
3    avcodec::AVCodecContext,
4    avformat::AVFormatContextInput,
5    avutil::{AVChannelLayout, AVDictionary, AVFrame},
6    error::RsmpegError,
7    ffi,
8    swresample::SwrContext,
9    swscale::SwsContext,
10};
11use rusttorch_core::Tensor;
12use rusttorch_data::{ResourceLimits, WorkerContext};
13use std::{ffi::CString, path::Path};
14/// Reports the actually linked FFmpeg build, including GPL/nonfree configuration.
15pub fn capabilities() -> CodecCapabilities {
16    CodecCapabilities {
17        version: rsmpeg::avutil::version_info()
18            .to_string_lossy()
19            .into_owned(),
20        configuration: rsmpeg::avcodec::configuration()
21            .to_string_lossy()
22            .into_owned(),
23        license: rsmpeg::avcodec::license().to_string_lossy().into_owned(),
24        cpu_decode: true,
25    }
26}
27/// Owned FFmpeg decoder over one bounded, regular local file and selected stream.
28/// Decode is lazy, one frame at a time; errors are yielded once then iteration
29/// terminates. This adapter does not promise exact checkpoint/resume of codecs.
30/// Native calls cannot be force-cancelled; cooperative context is checked between
31/// packet/frame operations. Network protocols and playlist demuxers are disabled.
32pub struct MediaDecoder {
33    input: AVFormatContextInput,
34    decoder: AVCodecContext,
35    info: StreamInfo,
36    limits: ResourceLimits,
37    draining: bool,
38    done: bool,
39    count: usize,
40    seek_target: Option<f64>,
41    context: Option<WorkerContext>,
42    audio_frames: u64,
43}
44impl MediaDecoder {
45    /// Opens a regular local file and chooses FFmpeg's best stream of this kind.
46    /// Supported demuxers: Matroska/WebM, WAV, AVI, MOV/MP4, MP3 and FLAC. CPU
47    /// video and mono/stereo audio are the explicit baseline; no hardware fallback.
48    pub fn open(path: impl AsRef<Path>, kind: StreamKind, limits: ResourceLimits) -> Result<Self> {
49        let path = std::fs::canonicalize(path)?;
50        let metadata = path.metadata()?;
51        if !metadata.is_file() {
52            return Err(CodecError::Invalid(
53                "media input must be a regular local file".into(),
54            ));
55        }
56        limits.check(
57            "media encoded bytes",
58            usize::try_from(metadata.len()).unwrap_or(usize::MAX),
59            limits.max_encoded_bytes,
60        )?;
61        let path = CString::new(
62            path.to_str()
63                .ok_or_else(|| CodecError::Invalid("media path must be UTF-8".into()))?,
64        )
65        .map_err(|_| CodecError::Invalid("media path contains NUL".into()))?;
66        let opts = AVDictionary::new(c"protocol_whitelist", c"file", 0)
67            .set(
68                c"format_whitelist",
69                c"matroska,webm,wav,avi,mov,mp3,flac",
70                0,
71            )
72            .set_int(
73                c"probesize",
74                limits.max_encoded_bytes.clamp(32, 5_000_000) as i64,
75                0,
76            )
77            .set_int(c"analyzeduration", 5_000_000, 0)
78            .set_int(c"rw_timeout", 5_000_000, 0)
79            .set_int(
80                c"max_streams",
81                limits.max_streams.min(i32::MAX as usize) as i64,
82                0,
83            );
84        let mut options = Some(opts);
85        let input = AVFormatContextInput::builder()
86            .url(&path)
87            .options(&mut options)
88            .open()?;
89        limits.check("media streams", input.streams().len(), limits.max_streams)?;
90        let media = match kind {
91            StreamKind::Video => ffi::AVMEDIA_TYPE_VIDEO,
92            StreamKind::Audio => ffi::AVMEDIA_TYPE_AUDIO,
93        };
94        let (index, codec) = input
95            .find_best_stream(media)?
96            .ok_or_else(|| CodecError::Invalid("requested media stream not found".into()))?;
97        let stream = &input.streams()[index];
98        let time_base = TimeBase::new(
99            u32::try_from(stream.time_base.num)
100                .map_err(|_| CodecError::Invalid("invalid stream time numerator".into()))?,
101            u32::try_from(stream.time_base.den)
102                .map_err(|_| CodecError::Invalid("invalid stream time denominator".into()))?,
103        )?;
104        let duration = if stream.duration == ffi::AV_NOPTS_VALUE {
105            None
106        } else {
107            Some(stream.duration)
108        };
109        if duration.is_some_and(|d| d < 0 || time_base.seconds(d) > limits.max_media_seconds as f64)
110        {
111            return Err(CodecError::Invalid(
112                "declared media duration exceeds limit".into(),
113            ));
114        }
115        let params = stream.codecpar();
116        if kind == StreamKind::Video {
117            check_video(params.height, params.width, limits)?;
118        } else if params.ch_layout.nb_channels < 1 || params.ch_layout.nb_channels > 2 {
119            return Err(CodecError::Invalid(
120                "native audio baseline supports mono/stereo only".into(),
121            ));
122        }
123        let mut decoder = AVCodecContext::new(&codec);
124        decoder.apply_codecpar(&params)?;
125        decoder.open(Some(AVDictionary::new_int(
126            c"max_pixels",
127            limits.max_tensor_elements.min(i64::MAX as usize) as i64,
128            0,
129        )))?;
130        drop(params);
131        let info = StreamInfo {
132            index,
133            kind,
134            time_base,
135            duration,
136            decoder: codec.name().to_string_lossy().into_owned(),
137        };
138        Ok(Self {
139            input,
140            decoder,
141            info,
142            limits,
143            draining: false,
144            done: false,
145            count: 0,
146            seek_target: None,
147            context: None,
148            audio_frames: 0,
149        })
150    }
151    /// Metadata for the selected stream.
152    pub fn stream_info(&self) -> &StreamInfo {
153        &self.info
154    }
155    /// Attaches cooperative loader cancellation/deadline checks between native calls.
156    pub fn with_worker_context(mut self, context: WorkerContext) -> Self {
157        self.context = Some(context);
158        self
159    }
160    /// Seeks to a keyframe then discards frames before the requested presentation
161    /// time. Audio returns the first whole block starting at or after that time.
162    /// Unknown timestamps after seeking are rejected rather than guessed.
163    pub fn seek(&mut self, seconds: f64) -> Result<()> {
164        if !seconds.is_finite() || seconds < 0. || seconds > self.limits.max_media_seconds as f64 {
165            return Err(CodecError::Invalid("invalid seek time".into()));
166        }
167        let ticks = seconds * self.info.time_base.denominator() as f64
168            / self.info.time_base.numerator() as f64;
169        if ticks >= i64::MAX as f64 {
170            return Err(CodecError::Invalid("seek timestamp overflows".into()));
171        }
172        self.input.seek(
173            self.info.index as i32,
174            ticks.floor() as i64,
175            ffi::AVSEEK_FLAG_BACKWARD as i32,
176        )?;
177        self.decoder.flush_buffers();
178        self.draining = false;
179        self.done = false;
180        self.count = 0;
181        self.audio_frames = 0;
182        self.seek_target = Some(seconds);
183        Ok(())
184    }
185    fn next_frame(&mut self) -> Result<Option<MediaFrame>> {
186        loop {
187            if let Some(context) = &self.context
188                && (context.cancellation.is_cancelled() || context.deadline.is_expired())
189            {
190                return Err(CodecError::Invalid(
191                    "media decoding cancelled or timed out".into(),
192                ));
193            }
194            match self.decoder.receive_frame() {
195                Ok(frame) => {
196                    self.count = self
197                        .count
198                        .checked_add(1)
199                        .ok_or_else(|| CodecError::Invalid("frame count overflow".into()))?;
200                    self.limits.check(
201                        "decoded media frames",
202                        self.count,
203                        self.limits.max_records,
204                    )?;
205                    let pts = frame.best_effort_timestamp;
206                    let timestamp = if pts == ffi::AV_NOPTS_VALUE {
207                        None
208                    } else {
209                        Some(Timestamp {
210                            ticks: pts,
211                            time_base: self.info.time_base,
212                        })
213                    };
214                    if timestamp
215                        .is_some_and(|t| t.seconds().abs() > self.limits.max_media_seconds as f64)
216                    {
217                        return Err(CodecError::Invalid(
218                            "decoded timestamp exceeds duration ceiling".into(),
219                        ));
220                    }
221                    if let Some(target) = self.seek_target {
222                        match timestamp {
223                            Some(t) if t.seconds() < target => continue,
224                            Some(_) => self.seek_target = None,
225                            None => {
226                                return Err(CodecError::Invalid(
227                                    "cannot perform time seek on frames without timestamps".into(),
228                                ));
229                            }
230                        }
231                    }
232                    return Ok(Some(match self.info.kind {
233                        StreamKind::Video => {
234                            MediaFrame::Video(video(frame, timestamp, self.limits)?)
235                        }
236                        StreamKind::Audio => {
237                            let audio = audio(frame, timestamp, self.limits)?;
238                            self.audio_frames = self
239                                .audio_frames
240                                .checked_add(audio.samples.size()[1] as u64)
241                                .ok_or_else(|| {
242                                    CodecError::Invalid("audio length overflow".into())
243                                })?;
244                            if self.audio_frames as u128
245                                > audio.sample_rate as u128 * self.limits.max_media_seconds as u128
246                            {
247                                return Err(CodecError::Invalid(
248                                    "decoded audio duration exceeds limit".into(),
249                                ));
250                            }
251                            MediaFrame::Audio(audio)
252                        }
253                    }));
254                }
255                Err(RsmpegError::DecoderFlushedError) => return Ok(None),
256                Err(RsmpegError::DecoderDrainError) => {
257                    if self.draining {
258                        return Err(CodecError::Invalid(
259                            "decoder requested packets after end-of-stream flush".into(),
260                        ));
261                    }
262                }
263                Err(e) => return Err(e.into()),
264            }
265            loop {
266                match self.input.read_packet()? {
267                    Some(packet) => {
268                        if packet.size < 0 || packet.size as usize > self.limits.max_encoded_bytes {
269                            return Err(CodecError::Invalid(
270                                "invalid or oversized media packet".into(),
271                            ));
272                        }
273                        if packet.stream_index != self.info.index as i32 {
274                            continue;
275                        }
276                        self.decoder.send_packet(Some(&packet))?;
277                        break;
278                    }
279                    None => {
280                        self.decoder.send_packet(None)?;
281                        self.draining = true;
282                        break;
283                    }
284                }
285            }
286        }
287    }
288}
289impl Iterator for MediaDecoder {
290    type Item = Result<MediaFrame>;
291    fn next(&mut self) -> Option<Self::Item> {
292        if self.done {
293            return None;
294        }
295        match self.next_frame() {
296            Ok(Some(f)) => Some(Ok(f)),
297            Ok(None) => {
298                self.done = true;
299                None
300            }
301            Err(e) => {
302                self.done = true;
303                Some(Err(e))
304            }
305        }
306    }
307}
308fn check_video(h: i32, w: i32, limits: ResourceLimits) -> Result<usize> {
309    if h <= 0 || w <= 0 {
310        return Err(CodecError::Invalid(
311            "video dimensions must be positive".into(),
312        ));
313    }
314    limits.check("video height", h as usize, limits.max_image_dimension)?;
315    limits.check("video width", w as usize, limits.max_image_dimension)?;
316    let n = limits.tensor_elements(&[3, h as usize, w as usize])?;
317    limits.check("RGB frame bytes", n, limits.max_decoded_bytes)?;
318    Ok(n)
319}
320fn video(
321    frame: AVFrame,
322    timestamp: Option<Timestamp>,
323    limits: ResourceLimits,
324) -> Result<VideoFrame> {
325    let size = check_video(frame.height, frame.width, limits)?;
326    let mut scaler = SwsContext::get_context(
327        frame.width,
328        frame.height,
329        frame.format,
330        frame.width,
331        frame.height,
332        ffi::AV_PIX_FMT_RGB24,
333        ffi::SWS_BILINEAR,
334        None,
335        None,
336        None,
337    )
338    .ok_or_else(|| CodecError::Invalid("unsupported video pixel conversion".into()))?;
339    let mut output = AVFrame::new();
340    output.set_width(frame.width);
341    output.set_height(frame.height);
342    output.set_format(ffi::AV_PIX_FMT_RGB24);
343    output.alloc_buffer()?;
344    scaler.scale_frame(&frame, 0, frame.height, &mut output)?;
345    let expected = output.image_get_buffer_size(1)?;
346    if expected != size {
347        return Err(CodecError::Invalid(
348            "RGB buffer size differs from checked dimensions".into(),
349        ));
350    }
351    let mut bytes = vec![0u8; size];
352    output.image_copy_to_buffer(&mut bytes, 1)?;
353    Ok(VideoFrame {
354        pixels: Tensor::f_from_slice(&bytes)?
355            .f_reshape([frame.height as i64, frame.width as i64, 3])?
356            .f_permute([2, 0, 1])?,
357        timestamp,
358    })
359}
360fn audio(
361    mut frame: AVFrame,
362    timestamp: Option<Timestamp>,
363    limits: ResourceLimits,
364) -> Result<AudioFrame> {
365    let channels = frame.ch_layout.nb_channels;
366    let samples = frame.nb_samples;
367    if !(1..=2).contains(&channels) || samples < 0 || frame.sample_rate <= 0 {
368        return Err(CodecError::Invalid(
369            "invalid mono/stereo audio frame".into(),
370        ));
371    }
372    let n = limits.tensor_elements(&[channels as usize, samples as usize])?;
373    let bytes = n
374        .checked_mul(std::mem::size_of::<f32>())
375        .ok_or_else(|| CodecError::Invalid("audio frame size overflow".into()))?;
376    limits.check("audio frame bytes", bytes, limits.max_decoded_bytes)?;
377    let layout = AVChannelLayout::from_nb_channels(channels);
378    if frame.ch_layout.order == ffi::AV_CHANNEL_ORDER_UNSPEC {
379        frame.set_ch_layout(*layout);
380    }
381    let mut converter = SwrContext::new(
382        &layout,
383        ffi::AV_SAMPLE_FMT_FLT,
384        frame.sample_rate,
385        &frame.ch_layout,
386        frame.format,
387        frame.sample_rate,
388    )?;
389    converter.init()?;
390    let mut output = AVFrame::new();
391    output.set_ch_layout(*layout);
392    output.set_sample_rate(frame.sample_rate);
393    output.set_format(ffi::AV_SAMPLE_FMT_FLT);
394    output.set_nb_samples(samples);
395    output.alloc_buffer()?;
396    converter.convert_frame(Some(&frame), &mut output)?;
397    if output.nb_samples != samples
398        || output.linesize[0] < 0
399        || (output.linesize[0] as usize) < bytes
400        || output.data[0].is_null()
401        || !(output.data[0] as usize).is_multiple_of(std::mem::align_of::<f32>())
402    {
403        return Err(CodecError::Invalid(
404            "native audio output does not match its allocated layout".into(),
405        ));
406    }
407    // SAFETY: `output` is an owned FFmpeg AVFrame allocated above in packed FLT
408    // mono/stereo format. Same-rate swresample initialized all `samples` frames;
409    // the result count, byte length, pointer and f32 alignment were checked.
410    // The immutable slice exists only while output owns that live allocation.
411    // Tensor::f_from_slice copies the values before output can be dropped. No
412    // raw pointer or borrowed native storage escapes this private boundary.
413    let data = unsafe { std::slice::from_raw_parts(output.data[0].cast::<f32>(), n) };
414    Ok(AudioFrame {
415        samples: Tensor::f_from_slice(data)?
416            .f_reshape([samples as i64, channels as i64])?
417            .f_transpose(0, 1)?,
418        sample_rate: frame.sample_rate as u32,
419        timestamp,
420    })
421}