symphonia_format_isomp4/
demuxer.rs

1// Symphonia
2// Copyright (c) 2019-2022 The Project Symphonia Developers.
3//
4// This Source Code Form is subject to the terms of the Mozilla Public
5// License, v. 2.0. If a copy of the MPL was not distributed with this
6// file, You can obtain one at https://mozilla.org/MPL/2.0/.
7
8use symphonia_core::{errors::end_of_stream_error, support_format};
9
10use symphonia_core::codecs::CodecParameters;
11use symphonia_core::errors::{decode_error, seek_error, unsupported_error, Result, SeekErrorKind};
12use symphonia_core::formats::prelude::*;
13use symphonia_core::io::{MediaSource, MediaSourceStream, ReadBytes, SeekBuffered};
14use symphonia_core::meta::{Metadata, MetadataLog};
15use symphonia_core::probe::{Descriptor, Instantiate, QueryDescriptor};
16use symphonia_core::units::Time;
17
18use std::io::{Seek, SeekFrom};
19use std::sync::Arc;
20
21use crate::atoms::{AtomIterator, AtomType};
22use crate::atoms::{FtypAtom, MetaAtom, MoofAtom, MoovAtom, MvexAtom, SidxAtom, TrakAtom};
23use crate::stream::*;
24
25use log::{debug, info, trace, warn};
26
27pub struct TrackState {
28    codec_params: CodecParameters,
29    /// The track number.
30    track_num: usize,
31    /// The current segment.
32    cur_seg: usize,
33    /// The current sample index relative to the track.
34    next_sample: u32,
35    /// The current sample byte position relative to the start of the track.
36    next_sample_pos: u64,
37}
38
39impl TrackState {
40    #[allow(clippy::single_match)]
41    pub fn new(track_num: usize, trak: &TrakAtom) -> Self {
42        let mut codec_params = CodecParameters::new();
43
44        codec_params
45            .with_time_base(TimeBase::new(1, trak.mdia.mdhd.timescale))
46            .with_n_frames(trak.mdia.mdhd.duration);
47
48        // Fill the codec parameters using the sample description atom.
49        trak.mdia.minf.stbl.stsd.fill_codec_params(&mut codec_params);
50
51        Self { codec_params, track_num, cur_seg: 0, next_sample: 0, next_sample_pos: 0 }
52    }
53
54    pub fn codec_params(&self) -> CodecParameters {
55        self.codec_params.clone()
56    }
57}
58
59/// Information regarding the next sample.
60#[derive(Debug)]
61struct NextSampleInfo {
62    /// The track number of the next sample.
63    track_num: usize,
64    /// The timestamp of the next sample.
65    ts: u64,
66    /// The timestamp expressed in seconds.
67    time: Time,
68    /// The duration of the next sample.
69    dur: u32,
70    /// The segment containing the next sample.
71    seg_idx: usize,
72}
73
74/// Information regarding a sample.
75#[derive(Debug)]
76struct SampleDataInfo {
77    /// The position of the sample in the track.
78    pos: u64,
79    /// The length of the sample.
80    len: u32,
81}
82
83/// ISO Base Media File Format (MP4, M4A, MOV, etc.) demultiplexer.
84///
85/// `IsoMp4Reader` implements a demuxer for the ISO Base Media File Format.
86pub struct IsoMp4Reader {
87    iter: AtomIterator<MediaSourceStream>,
88    tracks: Vec<Track>,
89    cues: Vec<Cue>,
90    metadata: MetadataLog,
91    /// Segments of the movie. Sorted in ascending order by sequence number.
92    segs: Vec<Box<dyn StreamSegment>>,
93    /// State tracker for each track.
94    track_states: Vec<TrackState>,
95    /// Optional, movie extends atom used for fragmented streams.
96    mvex: Option<Arc<MvexAtom>>,
97}
98
99impl IsoMp4Reader {
100    /// Idempotently gets information regarding the next sample of the media stream. This function
101    /// selects the next sample with the lowest timestamp of all tracks.
102    fn next_sample_info(&self) -> Result<Option<NextSampleInfo>> {
103        let mut earliest = None;
104
105        // TODO: Consider returning samples based on lowest byte position in the track instead of
106        // timestamp. This may be important if video tracks are ever decoded (i.e., DTS vs. PTS).
107
108        for (state, track) in self.track_states.iter().zip(&self.tracks) {
109            // Get the timebase of the track used to calculate the presentation time.
110            let tb = track.codec_params.time_base.unwrap();
111
112            // Get the next timestamp for the next sample of the current track. The next sample may
113            // be in a future segment.
114            for (seg_idx_delta, seg) in self.segs[state.cur_seg..].iter().enumerate() {
115                // Try to get the timestamp for the next sample of the track from the segment.
116                if let Some(timing) = seg.sample_timing(state.track_num, state.next_sample)? {
117                    // Calculate the presentation time using the timestamp.
118                    let sample_time = tb.calc_time(timing.ts);
119
120                    // Compare the presentation time of the sample from this track to other tracks,
121                    // and select the track with the earliest presentation time.
122                    match earliest {
123                        Some(NextSampleInfo { track_num: _, ts: _, time, dur: _, seg_idx: _ })
124                            if time <= sample_time =>
125                        {
126                            // Earliest is less than or equal to the track's next sample
127                            // presentation time. No need to update earliest.
128                        }
129                        _ => {
130                            // Earliest was either None, or greater than the track's next sample
131                            // presentation time. Update earliest.
132                            earliest = Some(NextSampleInfo {
133                                track_num: state.track_num,
134                                ts: timing.ts,
135                                time: sample_time,
136                                dur: timing.dur,
137                                seg_idx: seg_idx_delta + state.cur_seg,
138                            });
139                        }
140                    }
141
142                    // Either the next sample of the track had the earliest presentation time seen
143                    // thus far, or it was greater than those from other tracks, but there is no
144                    // reason to check samples in future segments.
145                    break;
146                }
147            }
148        }
149
150        Ok(earliest)
151    }
152
153    fn consume_next_sample(&mut self, info: &NextSampleInfo) -> Result<Option<SampleDataInfo>> {
154        // Get the track state.
155        let track = &mut self.track_states[info.track_num];
156
157        // Get the segment associated with the sample.
158        let seg = &self.segs[info.seg_idx];
159
160        // Get the sample data descriptor.
161        let sample_data_desc = seg.sample_data(track.track_num, track.next_sample, false)?;
162
163        // The sample base position in the sample data descriptor remains constant if the sample
164        // followed immediately after the previous sample. In this case, the track state's
165        // next_sample_pos is the position of the current sample. If the base position has jumped,
166        // then the base position is the position of current the sample.
167        let pos = if sample_data_desc.base_pos > track.next_sample_pos {
168            sample_data_desc.base_pos
169        }
170        else {
171            track.next_sample_pos
172        };
173
174        // Advance the track's current segment to the next sample's segment.
175        track.cur_seg = info.seg_idx;
176
177        // Advance the track's next sample number and position.
178        track.next_sample += 1;
179        track.next_sample_pos = pos + u64::from(sample_data_desc.size);
180
181        Ok(Some(SampleDataInfo { pos, len: sample_data_desc.size }))
182    }
183
184    fn try_read_more_segments(&mut self) -> Result<()> {
185        // Continue iterating over atoms until a segment (a moof + mdat atom pair) is found. All
186        // other atoms will be ignored.
187        while let Some(header) = self.iter.next_no_consume()? {
188            match header.atype {
189                AtomType::MediaData => {
190                    // Consume the atom from the iterator so that on the next iteration a new atom
191                    // will be read.
192                    self.iter.consume_atom();
193
194                    return Ok(());
195                }
196                AtomType::MovieFragment => {
197                    let moof = self.iter.read_atom::<MoofAtom>()?;
198
199                    // A moof segment can only be created if the mvex atom is present.
200                    if let Some(mvex) = &self.mvex {
201                        // Get the last segment. Note, there will always be one segment because the
202                        // moov atom is converted into a segment when the reader is instantiated.
203                        let last_seg = self.segs.last().unwrap();
204
205                        // Create a new segment for the moof atom.
206                        let seg = MoofSegment::new(moof, mvex.clone(), last_seg.as_ref());
207
208                        // Segments should have a monotonic sequence number.
209                        if seg.sequence_num() <= last_seg.sequence_num() {
210                            warn!("moof fragment has a non-monotonic sequence number.");
211                        }
212
213                        // Push the segment.
214                        self.segs.push(Box::new(seg));
215                    }
216                    else {
217                        // TODO: This is a fatal error.
218                        return decode_error("isomp4: moof atom present without mvex atom");
219                    }
220                }
221                _ => {
222                    trace!("skipping atom: {:?}.", header.atype);
223                    self.iter.consume_atom();
224                }
225            }
226        }
227
228        // If no atoms were returned above, then the end-of-stream has been reached.
229        end_of_stream_error()
230    }
231
232    fn seek_track_by_time(&mut self, track_num: usize, time: Time) -> Result<SeekedTo> {
233        // Convert time to timestamp for the track.
234        if let Some(track) = self.tracks.get(track_num) {
235            let tb = track.codec_params.time_base.unwrap();
236            self.seek_track_by_ts(track_num, tb.calc_timestamp(time))
237        }
238        else {
239            seek_error(SeekErrorKind::Unseekable)
240        }
241    }
242
243    fn seek_track_by_ts(&mut self, track_num: usize, ts: u64) -> Result<SeekedTo> {
244        debug!("seeking track={} to frame_ts={}", track_num, ts);
245
246        struct SeekLocation {
247            seg_idx: usize,
248            sample_num: u32,
249        }
250
251        let mut seek_loc = None;
252        let mut seg_skip = 0;
253
254        loop {
255            // Iterate over all segments and attempt to find the segment and sample number that
256            // contains the desired timestamp. Skip segments already examined.
257            for (seg_idx, seg) in self.segs.iter().enumerate().skip(seg_skip) {
258                if let Some(sample_num) = seg.ts_sample(track_num, ts)? {
259                    seek_loc = Some(SeekLocation { seg_idx, sample_num });
260                    break;
261                }
262
263                // Mark the segment as examined.
264                seg_skip = seg_idx + 1;
265            }
266
267            // If a seek location is found, break.
268            if seek_loc.is_some() {
269                break;
270            }
271
272            // Otherwise, try to read more segments from the stream.
273            self.try_read_more_segments()?;
274        }
275
276        if let Some(seek_loc) = seek_loc {
277            let seg = &self.segs[seek_loc.seg_idx];
278
279            // Get the sample information.
280            let data_desc = seg.sample_data(track_num, seek_loc.sample_num, true)?;
281
282            // Update the track's next sample information to point to the seeked sample.
283            let track = &mut self.track_states[track_num];
284
285            track.cur_seg = seek_loc.seg_idx;
286            track.next_sample = seek_loc.sample_num;
287            track.next_sample_pos = data_desc.base_pos + data_desc.offset.unwrap();
288
289            // Get the actual timestamp for this sample.
290            let timing = seg.sample_timing(track_num, seek_loc.sample_num)?.unwrap();
291
292            debug!(
293                "seeked track={} to packet_ts={} (delta={})",
294                track_num,
295                timing.ts,
296                timing.ts as i64 - ts as i64
297            );
298
299            Ok(SeekedTo { track_id: track_num as u32, required_ts: ts, actual_ts: timing.ts })
300        }
301        else {
302            // Timestamp was not found.
303            seek_error(SeekErrorKind::OutOfRange)
304        }
305    }
306}
307
308impl QueryDescriptor for IsoMp4Reader {
309    fn query() -> &'static [Descriptor] {
310        &[support_format!(
311            "isomp4",
312            "ISO Base Media File Format",
313            &["mp4", "m4a", "m4p", "m4b", "m4r", "m4v", "mov"],
314            &["video/mp4", "audio/m4a"],
315            &[b"ftyp"] // Top-level atoms
316        )]
317    }
318
319    fn score(_context: &[u8]) -> u8 {
320        255
321    }
322}
323
324impl FormatReader for IsoMp4Reader {
325    fn try_new(mut mss: MediaSourceStream, _options: &FormatOptions) -> Result<Self> {
326        // To get to beginning of the atom.
327        mss.seek_buffered_rel(-4);
328
329        let is_seekable = mss.is_seekable();
330
331        let mut ftyp = None;
332        let mut moov = None;
333        let mut sidx = None;
334
335        // Get the total length of the stream, if possible.
336        let total_len = if is_seekable {
337            let pos = mss.pos();
338            let len = mss.seek(SeekFrom::End(0))?;
339            mss.seek(SeekFrom::Start(pos))?;
340            info!("stream is seekable with len={} bytes.", len);
341            Some(len)
342        }
343        else {
344            None
345        };
346
347        let mut metadata = MetadataLog::default();
348
349        // Parse all atoms if the stream is seekable, otherwise parse all atoms up-to the mdat atom.
350        let mut iter = AtomIterator::new_root(mss, total_len);
351
352        while let Some(header) = iter.next()? {
353            // Top-level atoms.
354            match header.atype {
355                AtomType::FileType => {
356                    ftyp = Some(iter.read_atom::<FtypAtom>()?);
357                }
358                AtomType::Movie => {
359                    moov = Some(iter.read_atom::<MoovAtom>()?);
360                }
361                AtomType::SegmentIndex => {
362                    // If the stream is not seekable, then it can only be assumed that the first
363                    // segment index atom is indeed the first segment index because the format
364                    // reader cannot practically skip past this point.
365                    if !is_seekable {
366                        sidx = Some(iter.read_atom::<SidxAtom>()?);
367                        break;
368                    }
369                    else {
370                        // If the stream is seekable, examine all segment indexes and select the
371                        // index with the earliest presentation timestamp to be the first.
372                        let new_sidx = iter.read_atom::<SidxAtom>()?;
373
374                        let is_earlier = match &sidx {
375                            Some(sidx) => new_sidx.earliest_pts < sidx.earliest_pts,
376                            _ => true,
377                        };
378
379                        if is_earlier {
380                            sidx = Some(new_sidx);
381                        }
382                    }
383                }
384                AtomType::MediaData | AtomType::MovieFragment => {
385                    // The mdat atom contains the codec bitstream data. For segmented streams, a
386                    // moof + mdat pair is required for playback. If the source is unseekable then
387                    // the format reader cannot skip past these atoms without dropping samples.
388                    if !is_seekable {
389                        // If the moov atom hasn't been seen before the moof and/or mdat atom, and
390                        // the stream is not seekable, then the mp4 is not streamable.
391                        if moov.is_none() || ftyp.is_none() {
392                            warn!("mp4 is not streamable.");
393                        }
394
395                        // The remainder of the stream will be read incrementally.
396                        break;
397                    }
398                }
399                AtomType::Meta => {
400                    // Read the metadata atom and append it to the log.
401                    let mut meta = iter.read_atom::<MetaAtom>()?;
402
403                    if let Some(rev) = meta.take_metadata() {
404                        metadata.push(rev);
405                    }
406                }
407                AtomType::Free => (),
408                AtomType::Skip => (),
409                _ => {
410                    info!("skipping top-level atom: {:?}.", header.atype);
411                }
412            }
413        }
414
415        if ftyp.is_none() {
416            return unsupported_error("isomp4: missing ftyp atom");
417        }
418
419        if moov.is_none() {
420            return unsupported_error("isomp4: missing moov atom");
421        }
422
423        // If the stream was seekable, then all atoms in the media source stream were scanned. Seek
424        // back to the first mdat atom for playback. If the stream is not seekable, then the atom
425        // iterator is currently positioned at the first mdat atom.
426        if is_seekable {
427            let mut mss = iter.into_inner();
428            mss.seek(SeekFrom::Start(0))?;
429
430            iter = AtomIterator::new_root(mss, total_len);
431
432            while let Some(header) = iter.next_no_consume()? {
433                match header.atype {
434                    AtomType::MediaData | AtomType::MovieFragment => break,
435                    _ => (),
436                }
437                iter.consume_atom();
438            }
439        }
440
441        let mut moov = moov.unwrap();
442
443        if moov.is_fragmented() {
444            // If a Segment Index (sidx) atom was found, add the segments contained within.
445            if sidx.is_some() {
446                info!("stream is segmented with a segment index.");
447            }
448            else {
449                info!("stream is segmented without a segment index.");
450            }
451        }
452
453        if let Some(rev) = moov.take_metadata() {
454            metadata.push(rev);
455        }
456
457        // Instantiate a TrackState for each track in the stream.
458        let track_states = moov
459            .traks
460            .iter()
461            .enumerate()
462            .map(|(t, trak)| TrackState::new(t, trak))
463            .collect::<Vec<TrackState>>();
464
465        // Instantiate a Tracks for all tracks above.
466        let tracks = track_states
467            .iter()
468            .map(|track| Track::new(track.track_num as u32, track.codec_params()))
469            .collect();
470
471        // A Movie Extends (mvex) atom is required to support segmented streams. If the mvex atom is
472        // present, wrap it in an Arc so it can be shared amongst all segments.
473        let mvex = moov.mvex.take().map(Arc::new);
474
475        // The number of tracks specified in the moov atom must match the number in the mvex atom.
476        if let Some(mvex) = &mvex {
477            if mvex.trexs.len() != moov.traks.len() {
478                return decode_error("isomp4: mvex and moov track number mismatch");
479            }
480        }
481
482        let segs: Vec<Box<dyn StreamSegment>> = vec![Box::new(MoovSegment::new(moov))];
483
484        Ok(IsoMp4Reader {
485            iter,
486            tracks,
487            cues: Default::default(),
488            metadata,
489            track_states,
490            segs,
491            mvex,
492        })
493    }
494
495    fn next_packet(&mut self) -> Result<Packet> {
496        // Get the index of the track with the next-nearest (minimum) timestamp.
497        let next_sample_info = loop {
498            // Using the current set of segments, try to get the next sample info.
499            if let Some(info) = self.next_sample_info()? {
500                break info;
501            }
502            else {
503                // No more segments. If the stream is unseekable, it may be the case that there are
504                // more segments coming. Iterate atoms until a new segment is found or the
505                // end-of-stream is reached.
506                self.try_read_more_segments()?;
507            }
508        };
509
510        // Get the position and length information of the next sample.
511        let sample_info = self.consume_next_sample(&next_sample_info)?.unwrap();
512
513        let reader = self.iter.inner_mut();
514
515        // Attempt a fast seek within the buffer cache.
516        if reader.seek_buffered(sample_info.pos) != sample_info.pos {
517            if reader.is_seekable() {
518                // Fallback to a slow seek if the stream is seekable.
519                reader.seek(SeekFrom::Start(sample_info.pos))?;
520            }
521            else if sample_info.pos > reader.pos() {
522                // The stream is not seekable but the desired seek position is ahead of the reader's
523                // current position, thus the seek can be emulated by ignoring the bytes up to the
524                // the desired seek position.
525                reader.ignore_bytes(sample_info.pos - reader.pos())?;
526            }
527            else {
528                // The stream is not seekable and the desired seek position falls outside the lower
529                // bound of the buffer cache. This sample cannot be read.
530                return decode_error("isomp4: packet out-of-bounds for a non-seekable stream");
531            }
532        }
533
534        Ok(Packet::new_from_boxed_slice(
535            next_sample_info.track_num as u32,
536            next_sample_info.ts,
537            u64::from(next_sample_info.dur),
538            reader.read_boxed_slice_exact(sample_info.len as usize)?,
539        ))
540    }
541
542    fn metadata(&mut self) -> Metadata<'_> {
543        self.metadata.metadata()
544    }
545
546    fn cues(&self) -> &[Cue] {
547        &self.cues
548    }
549
550    fn tracks(&self) -> &[Track] {
551        &self.tracks
552    }
553
554    fn seek(&mut self, _mode: SeekMode, to: SeekTo) -> Result<SeekedTo> {
555        if self.tracks.is_empty() {
556            return seek_error(SeekErrorKind::Unseekable);
557        }
558
559        match to {
560            SeekTo::TimeStamp { ts, track_id } => {
561                let selected_track_id = track_id as usize;
562
563                // The seek timestamp is in timebase units specific to the selected track. Get the
564                // selected track and use the timebase to convert the timestamp into time units so
565                // that the other tracks can be seeked.
566                if let Some(selected_track) = self.tracks().get(selected_track_id) {
567                    // Convert to time units.
568                    let time = selected_track.codec_params.time_base.unwrap().calc_time(ts);
569
570                    // Seek all tracks excluding the primary track to the desired time.
571                    for t in 0..self.track_states.len() {
572                        if t != selected_track_id {
573                            self.seek_track_by_time(t, time)?;
574                        }
575                    }
576
577                    // Seek the primary track and return the result.
578                    self.seek_track_by_ts(selected_track_id, ts)
579                }
580                else {
581                    seek_error(SeekErrorKind::Unseekable)
582                }
583            }
584            SeekTo::Time { time, track_id } => {
585                // Select the first track if a selected track was not provided.
586                let selected_track_id = track_id.unwrap_or(0) as usize;
587
588                // Seek all tracks excluding the selected track and discard the result.
589                for t in 0..self.track_states.len() {
590                    if t != selected_track_id {
591                        self.seek_track_by_time(t, time)?;
592                    }
593                }
594
595                // Seek the primary track and return the result.
596                self.seek_track_by_time(selected_track_id, time)
597            }
598        }
599    }
600
601    fn into_inner(self: Box<Self>) -> MediaSourceStream {
602        self.iter.into_inner()
603    }
604}