use std::io::{Seek, SeekFrom};
use crate::common::Tier;
use crate::errors::{Error, Result, unsupported_error};
use crate::formats::{FormatInfo, FormatOptions, FormatReader};
use crate::io::{MediaSource, MediaSourceStream, ReadBytes, ScopedStream, SeekBuffered};
use crate::meta::{MetadataInfo, MetadataOptions, MetadataReader, MetadataSideData};
use log::{debug, error, trace, warn};
mod bloom {
fn fnv1a32(value: &[u8; 2]) -> u32 {
const INIT: u32 = 0x811c_9dc5;
const PRIME: u32 = 0x0100_0193;
let mut state = INIT;
for byte in value.iter() {
state = (state ^ u32::from(*byte)).wrapping_mul(PRIME);
}
state
}
pub struct BloomFilter {
filter: Box<[u64]>,
}
impl Default for BloomFilter {
fn default() -> Self {
BloomFilter { filter: vec![0; BloomFilter::M >> 6].into_boxed_slice() }
}
}
impl BloomFilter {
const M: usize = 2 * 1024 * 8;
pub fn insert(&mut self, key: &[u8; 2]) {
let hash = fnv1a32(key);
let h0 = (hash >> 16) as u16;
let h1 = (hash >> 0) as u16;
let i0 = h0 as usize & (BloomFilter::M - 1);
let i1 = h0.wrapping_add(h1.wrapping_mul(1)) as usize & (BloomFilter::M - 1);
let i2 = h0.wrapping_add(h1.wrapping_mul(2)) as usize & (BloomFilter::M - 1);
self.filter[i0 >> 6] |= 1 << (i0 & 63);
self.filter[i1 >> 6] |= 1 << (i1 & 63);
self.filter[i2 >> 6] |= 1 << (i2 & 63);
}
pub fn may_contain(&self, key: &[u8; 2]) -> bool {
let hash = fnv1a32(key);
let h0 = (hash >> 16) as u16;
let h1 = (hash >> 0) as u16;
let i0 = h0 as usize & (BloomFilter::M - 1);
let i1 = h0.wrapping_add(h1.wrapping_mul(1)) as usize & (BloomFilter::M - 1);
let i2 = h0.wrapping_add(h1.wrapping_mul(2)) as usize & (BloomFilter::M - 1);
if (self.filter[i0 >> 6] & (1 << (i0 & 63))) == 0 {
return false;
}
if (self.filter[i1 >> 6] & (1 << (i1 & 63))) == 0 {
return false;
}
if (self.filter[i2 >> 6] & (1 << (i2 & 63))) == 0 {
return false;
}
true
}
}
}
#[derive(Copy, Clone)]
pub enum Anchors {
None,
Exclusive(&'static [u32]),
Supplemental(&'static [u32]),
}
#[derive(Copy, Clone)]
pub struct ProbeDataMatchSpec {
pub extensions: &'static [&'static str],
pub mime_types: &'static [&'static str],
pub markers: &'static [&'static [u8]],
}
#[derive(Copy, Clone)]
pub struct ProbeFormatData {
pub spec: ProbeDataMatchSpec,
pub info: FormatInfo,
}
#[derive(Copy, Clone)]
pub struct ProbeMetadataData {
pub spec: ProbeDataMatchSpec,
pub info: MetadataInfo,
pub anchors: Anchors,
}
pub type FormatFactoryFn =
for<'s> fn(MediaSourceStream<'s>, FormatOptions) -> Result<Box<dyn FormatReader + 's>>;
pub type MetadataFactoryFn =
for<'s> fn(MediaSourceStream<'s>, MetadataOptions) -> Result<Box<dyn MetadataReader + 's>>;
#[derive(Copy, Clone)]
enum ProbeMatch {
Format {
info: FormatInfo,
factory: FormatFactoryFn,
},
Metadata {
info: MetadataInfo,
factory: MetadataFactoryFn,
},
}
type ScoreFn = fn(ScopedStream<&mut MediaSourceStream<'_>>) -> Result<Score>;
#[derive(Copy, Clone)]
struct GenericProbeMatch {
spec: ProbeDataMatchSpec,
score: ScoreFn,
specific: ProbeMatch,
anchors: Anchors,
}
impl GenericProbeMatch {
fn should_test(&self, is_trailing_probe: bool) -> bool {
match self.anchors {
Anchors::None => !is_trailing_probe,
Anchors::Exclusive(_) => is_trailing_probe,
Anchors::Supplemental(_) => true,
}
}
}
pub enum Score {
Unsupported,
Supported(u8),
}
pub trait Scoreable {
fn score(src: ScopedStream<&mut MediaSourceStream<'_>>) -> Result<Score>;
}
pub trait ProbeableFormat<'s>: FormatReader + Scoreable {
fn try_probe_new(
mss: MediaSourceStream<'s>,
opts: FormatOptions,
) -> Result<Box<dyn FormatReader + 's>>
where
Self: Sized;
fn probe_data() -> &'static [ProbeFormatData];
}
pub trait ProbeableMetadata<'s>: MetadataReader + Scoreable {
fn try_probe_new(
mss: MediaSourceStream<'s>,
opts: MetadataOptions,
) -> Result<Box<dyn MetadataReader + 's>>
where
Self: Sized;
fn probe_data() -> &'static [ProbeMetadataData];
}
#[derive(Clone, Debug, Default)]
pub struct Hint {
extension: Option<String>,
mime_type: Option<String>,
}
impl Hint {
pub fn new() -> Self {
Hint { extension: None, mime_type: None }
}
pub fn with_extension(&mut self, extension: &str) -> &mut Self {
self.extension = Some(extension.to_owned());
self
}
pub fn mime_type(&mut self, mime_type: &str) -> &mut Self {
self.mime_type = Some(mime_type.to_owned());
self
}
}
#[derive(Copy, Clone, Debug, PartialEq, Eq)]
pub struct ProbeOptions {
pub max_probe_depth: u32,
pub max_score_depth: u16,
}
impl Default for ProbeOptions {
fn default() -> Self {
Self {
max_probe_depth: 1 * 1024 * 1024, max_score_depth: 16 * 1024, }
}
}
#[derive(Default)]
pub struct Probe {
filter: bloom::BloomFilter,
preferred: Vec<GenericProbeMatch>,
standard: Vec<GenericProbeMatch>,
fallback: Vec<GenericProbeMatch>,
anchors: Vec<u32>,
opts: ProbeOptions,
}
impl Probe {
pub fn new() -> Self {
Probe::new_with_options(&Default::default())
}
pub fn new_with_options(opts: &ProbeOptions) -> Self {
Probe { opts: *opts, ..Default::default() }
}
pub fn register_format<P>(&mut self)
where
for<'a> P: ProbeableFormat<'a>,
{
self.register_format_at_tier::<P>(Tier::Standard);
}
pub fn register_metadata<P>(&mut self)
where
for<'a> P: ProbeableMetadata<'a>,
{
self.register_metadata_at_tier::<P>(Tier::Standard);
}
pub fn register_format_at_tier<P>(&mut self, tier: Tier)
where
for<'a> P: ProbeableFormat<'a>,
{
for data in P::probe_data() {
let candidate = GenericProbeMatch {
spec: data.spec,
score: P::score,
specific: ProbeMatch::Format {
info: data.info,
factory: |mss, opts| P::try_probe_new(mss, opts),
},
anchors: Anchors::None,
};
self.register(tier, candidate);
}
}
pub fn register_metadata_at_tier<P>(&mut self, tier: Tier)
where
for<'a> P: ProbeableMetadata<'a>,
{
for data in P::probe_data() {
let candidate = GenericProbeMatch {
spec: data.spec,
score: P::score,
specific: ProbeMatch::Metadata {
info: data.info,
factory: |mss, opts| P::try_probe_new(mss, opts),
},
anchors: data.anchors,
};
match data.anchors {
Anchors::Exclusive(anchors) | Anchors::Supplemental(anchors) => {
self.anchors.extend_from_slice(anchors);
self.anchors.sort_by(|a, b| b.cmp(a));
self.anchors.dedup();
}
_ => (),
}
self.register(tier, candidate);
}
}
fn register(&mut self, tier: Tier, candidate: GenericProbeMatch) {
for marker in candidate.spec.markers {
let mut prefix = [0u8; 2];
match marker.len() {
2..=16 => prefix.copy_from_slice(&marker[0..2]),
_ => panic!("invalid marker length (only 2-16 bytes supported)."),
}
self.filter.insert(&prefix);
}
match tier {
Tier::Preferred => self.preferred.push(candidate),
Tier::Standard => self.standard.push(candidate),
Tier::Fallback => self.fallback.push(candidate),
}
}
pub fn probe<'s>(
&self,
hint: &Hint,
mut mss: MediaSourceStream<'s>,
mut fmt_opts: FormatOptions,
meta_opts: MetadataOptions,
) -> Result<Box<dyn FormatReader + 's>> {
if mss.is_seekable() {
if let Some(end) = mss.byte_len() {
let init_pos = mss.pos();
mss = self.probe_trailing(mss, end, &mut fmt_opts, meta_opts)?;
mss.seek(SeekFrom::Start(init_pos))?;
}
}
loop {
match self.next(&mut mss, hint)? {
ProbeMatch::Format { factory, .. } => {
return factory(mss, fmt_opts);
}
ProbeMatch::Metadata { factory, .. } => {
let mut reader = factory(mss, meta_opts)?;
read_and_append_metadata(&mut reader, &mut fmt_opts)?;
mss = reader.into_inner();
}
}
}
}
fn probe_trailing<'s>(
&self,
mut mss: MediaSourceStream<'s>,
end: u64,
fmt_opts: &mut FormatOptions,
meta_opts: MetadataOptions,
) -> Result<MediaSourceStream<'s>> {
debug_assert!(mss.is_seekable());
let mut last_reader_end = 0;
for anchor in self.anchors.iter().copied().map(u64::from) {
if anchor > end || end - anchor < last_reader_end {
debug!("skipping trailing metadata offset -{anchor}");
continue;
}
trace!("probing for trailing metadata at offset -{} ({})", anchor, end - anchor);
let anchor_pos = end - anchor;
if mss.seek_buffered(anchor_pos) != anchor_pos {
mss.seek(SeekFrom::Start(anchor_pos))?;
}
let win = mss.read_double_bytes()?;
if self.filter.may_contain(&win) {
mss.seek_buffered_rel(-2);
if let Some(ProbeMatch::Metadata { factory, .. }) =
self.find_best_reader(&mut mss, true)?
{
let mut reader = factory(mss, meta_opts)?;
let is_ok = read_and_append_metadata(&mut reader, fmt_opts).is_ok();
if !is_ok {
warn!(
"skipping trailing '{}' metdata due to error while reading it",
reader.metadata_info().short_name
);
}
mss = reader.into_inner();
if is_ok {
last_reader_end = mss.pos();
}
}
}
}
Ok(mss)
}
fn next(&self, mss: &mut MediaSourceStream<'_>, _hint: &Hint) -> Result<ProbeMatch> {
let mut win = 0u16;
let init_pos = mss.pos();
let mut count = 0;
while let Ok(byte) = mss.read_byte() {
win = (win << 8) | u16::from(byte);
count += 1;
if count > self.opts.max_probe_depth {
break;
}
if count % 4096 == 0 {
debug!(
"searching for format marker... {}+{} / {} bytes",
init_pos, count, self.opts.max_probe_depth
);
}
if self.filter.may_contain(&win.to_be_bytes()) {
mss.seek_buffered_rel(-2);
if let Some(probed) = self.find_best_reader(mss, false)? {
warn_junk_bytes(mss.pos(), init_pos);
return Ok(probed);
}
mss.seek_buffered_rel(2);
}
}
if count < self.opts.max_probe_depth {
error!("probe reached EOF at {count} bytes");
}
else {
error!("reached probe limit of {} bytes", self.opts.max_probe_depth);
}
unsupported_error("core (probe): no suitable format reader found")
}
fn find_best_reader(
&self,
mss: &mut MediaSourceStream,
is_trailing: bool,
) -> Result<Option<ProbeMatch>> {
let mut win = [0u8; 16];
let win_len = mss.read_buf(&mut win)?;
mss.seek_buffered_rel(-(win_len as isize));
if let Some(inst) =
find_reader(mss, &self.preferred, win, self.opts.max_score_depth, is_trailing)?
{
return Ok(Some(inst));
}
if let Some(inst) =
find_reader(mss, &self.standard, win, self.opts.max_score_depth, is_trailing)?
{
return Ok(Some(inst));
}
if let Some(inst) =
find_reader(mss, &self.fallback, win, self.opts.max_score_depth, is_trailing)?
{
return Ok(Some(inst));
}
Ok(None)
}
}
fn read_and_append_metadata<'s>(
reader: &mut Box<dyn MetadataReader + 's>,
fmt_opts: &mut FormatOptions,
) -> Result<()> {
let metadata = reader.read_all()?;
debug!("appending '{}' metadata", reader.metadata_info().short_name);
fmt_opts.external_data.metadata.get_or_insert_with(Default::default).push(metadata.revision);
for side_data in metadata.side_data {
match side_data {
MetadataSideData::Chapters(chapters) => {
debug!("appending '{}' chapters", reader.metadata_info().short_name);
fmt_opts.external_data.chapters = Some(chapters)
}
}
}
Ok(())
}
fn find_reader(
mss: &mut MediaSourceStream,
descs: &[GenericProbeMatch],
win: [u8; 16],
max_depth: u16,
is_trailing: bool,
) -> Result<Option<ProbeMatch>> {
mss.ensure_seekback_buffer(usize::from(max_depth));
for desc in descs.iter().filter(|d| d.should_test(is_trailing)) {
let should_score = desc.spec.markers.iter().any(|marker| {
let is_match = win[0..marker.len()] == **marker;
if is_match {
trace!("found the marker {:x?} @ {} bytes", &win[0..marker.len()], mss.pos());
}
is_match
});
if should_score {
if let Score::Supported(score) = score(desc, mss, max_depth)? {
match &desc.specific {
ProbeMatch::Format { info, .. } => {
debug!("selected format reader '{}' with score {}", info.short_name, score)
}
ProbeMatch::Metadata { info, .. } => {
debug!(
"selected metadata reader '{}' with score {}",
info.short_name, score
)
}
}
return Ok(Some(desc.specific));
}
match &desc.specific {
ProbeMatch::Format { info, .. } => {
trace!("format reader '{}' failed scoring", info.short_name)
}
ProbeMatch::Metadata { info, .. } => {
trace!("metadata reader '{}' failed scoring", info.short_name)
}
}
}
}
Ok(None)
}
fn score(
candidate: &GenericProbeMatch,
mss: &mut MediaSourceStream,
max_depth: u16,
) -> Result<Score> {
let init_pos = mss.pos();
let result = match (candidate.score)(ScopedStream::new(mss, u64::from(max_depth))) {
Err(Error::IoError(err)) if err.kind() != std::io::ErrorKind::UnexpectedEof => {
Err(Error::IoError(err))
}
Err(_) => {
Ok(Score::Unsupported)
}
result => result,
};
mss.seek_buffered(init_pos);
result
}
fn warn_junk_bytes(pos: u64, init_pos: u64) {
if pos > init_pos {
warn!("skipped {} bytes of junk at {}", pos - init_pos, init_pos);
}
}
#[macro_export]
macro_rules! support_format {
($info:expr, $exts:expr, $mimes:expr, $markers:expr) => {
symphonia_core::formats::probe::ProbeFormatData {
spec: symphonia_core::formats::probe::ProbeDataMatchSpec {
extensions: $exts,
mime_types: $mimes,
markers: $markers,
},
info: $info,
}
};
}
#[macro_export]
macro_rules! support_metadata {
($info:expr, $exts:expr, $mimes:expr, $markers:expr) => {
symphonia_core::formats::probe::ProbeMetadataData {
spec: symphonia_core::formats::probe::ProbeDataMatchSpec {
extensions: $exts,
mime_types: $mimes,
markers: $markers,
},
info: $info,
anchors: symphonia_core::formats::probe::Anchors::None,
}
};
($info:expr, $exts:expr, $mimes:expr, $markers:expr, $anchors:expr) => {
symphonia_core::formats::probe::ProbeMetadataData {
spec: symphonia_core::formats::probe::ProbeDataMatchSpec {
extensions: $exts,
mime_types: $mimes,
markers: $markers,
},
info: $info,
anchors: $anchors,
}
};
}