use std::collections::HashMap;
use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};
use std::sync::{Arc, Mutex};
use std::time::Duration;
use crossbeam_channel::Sender;
use url::Url;
use super::abr::Abr;
use super::http::Fetcher;
use super::playlist::{self, Playlist, Rendition, VariantStream};
use super::segment::SegmentDemuxer;
use super::timeline::{Role, Timeline};
use super::track::{SegmentData, Start, Track, TrackConfig, TrackEvent};
use crate::backend::Registry;
use crate::demux::{Codec, Demuxer, Packet, StreamInfo, StreamKind};
use crate::player::PlayerEvent;
use crate::{Error, Result};
const VIDEO_ID: u32 = 1;
const AUDIO_ID: u32 = 2;
const POLL: Duration = Duration::from_millis(50);
#[derive(Clone, Debug, PartialEq)]
pub struct VariantInfo {
pub bandwidth: u64,
pub resolution: Option<(u32, u32)>,
pub codecs: Vec<String>,
pub frame_rate: Option<f64>,
}
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub enum Variant {
Auto,
Index(usize),
}
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct AudioRendition {
pub name: String,
pub language: Option<String>,
}
pub struct HlsControl {
variants: Vec<VariantInfo>,
renditions: Vec<AudioRendition>,
current: AtomicUsize,
requested: Mutex<Option<Variant>>,
requested_audio: Mutex<Option<usize>>,
events: Mutex<Option<Sender<PlayerEvent>>>,
shutdown: AtomicBool,
}
impl HlsControl {
pub fn variants(&self) -> Vec<VariantInfo> {
self.variants.clone()
}
pub fn current_variant(&self) -> usize {
self.current.load(Ordering::Relaxed)
}
pub fn set_variant(&self, v: Variant) {
*self.requested.lock().unwrap() = Some(v);
}
pub fn audio_renditions(&self) -> Vec<AudioRendition> {
self.renditions.clone()
}
pub fn set_audio_rendition(&self, index: usize) {
*self.requested_audio.lock().unwrap() = Some(index);
}
pub(crate) fn set_events(&self, events: Sender<PlayerEvent>) {
*self.events.lock().unwrap() = Some(events);
}
fn variant_changed(&self, variant: usize) {
if self.current.swap(variant, Ordering::Relaxed) != variant
&& let Some(tx) = self.events.lock().unwrap().as_ref()
{
let _ = tx.send(PlayerEvent::VariantChanged(variant));
}
}
pub(crate) fn shut_down(&self) {
self.shutdown.store(true, Ordering::Relaxed);
}
}
struct Out {
id: u32,
pts: Duration,
keyframe: bool,
data: Vec<u8>,
info: Arc<StreamInfo>,
}
struct Open {
demux: SegmentDemuxer,
offset: i128,
disc: u64,
}
struct Lane {
role: Role,
track: Track,
seg: Option<Open>,
pending: Option<Out>,
formats: HashMap<StreamKind, Arc<StreamInfo>>,
ts_ref: Option<(u64, u64)>,
ts_params: Option<(crate::demux::VideoParams, u64)>,
ended: bool,
epoch: u64,
use_audio: bool,
wait_key: bool,
skip_before: Option<Duration>,
jump_by_peer: bool,
jumped: bool,
}
impl Lane {
fn new(role: Role, track: Track, use_audio: bool) -> Self {
Self { role, track, seg: None, pending: None, formats: HashMap::new(), ts_ref: None, ts_params: None, ended: false, epoch: 0, use_audio, wait_key: false, skip_before: None, jump_by_peer: false, jumped: false }
}
}
struct Selection {
abr: Abr,
blacklist: Vec<bool>,
fetching: usize,
manual: Option<usize>,
returned: Duration,
}
impl Selection {
fn on_segment(&mut self, s: &SegmentData, control: &HlsControl, fetched_end: Duration) -> Option<usize> {
if s.bytes > 0 {
self.abr.sample(s.bytes, s.elapsed);
}
control.variant_changed(s.variant);
let n = self.blacklist.len();
match control.requested.lock().unwrap().take() {
Some(Variant::Index(i)) if i < n && !self.blacklist[i] => self.manual = Some(i),
Some(Variant::Index(i)) => log::warn!("HLS: variant {i} cannot be chosen"),
Some(Variant::Auto) => self.manual = None,
None => {}
}
let buffer = fetched_end.saturating_sub(self.returned);
let next = match self.manual {
Some(i) => i,
None => self.abr.choose(self.fetching, buffer, s.duration, &self.blacklist),
};
if next == self.fetching {
return None;
}
log::info!("HLS: switching to variant {next} (buffer {buffer:?}, estimate {:?} b/s)", self.abr.estimate());
self.fetching = next;
Some(next)
}
}
pub struct HlsDemuxer {
control: Arc<HlsControl>,
main: Lane,
audio: Option<Lane>,
streams: Vec<StreamInfo>,
timeline: Timeline,
update: Option<StreamInfo>,
reported: HashMap<u32, Arc<StreamInfo>>,
live: bool,
duration: Option<Duration>,
selection: Selection,
rendition_uris: Vec<Option<Url>>,
rendition: Option<usize>,
}
impl HlsDemuxer {
pub fn open(url: &Url, registry: &Registry) -> Result<HlsDemuxer> {
let fetch = Fetcher::new(Arc::new(AtomicBool::new(false)));
let first = fetch.get(url, None)?;
let text = String::from_utf8_lossy(&first.data);
let (variants, renditions) = match playlist::parse(&text, &first.url)? {
Playlist::Media(_) => {
let only = VariantStream {
uri: first.url.clone(),
bandwidth: 0,
average_bandwidth: None,
codecs: Vec::new(),
resolution: None,
frame_rate: None,
audio_group: None,
};
(vec![only], Vec::new())
}
Playlist::Master(m) => (m.variants, m.audio),
};
let mut blacklist: Vec<bool> = variants.iter().map(|v| !decodable(registry, &v.codecs)).collect();
if blacklist.iter().all(|&b| b) {
let mut codecs: Vec<String> = variants.iter().flat_map(|v| v.codecs.clone()).collect();
codecs.sort();
codecs.dedup();
return Err(Error::UnsupportedCodec { codec: codecs.join(", "), tried_backends: registry.names() });
}
let bitrates: Vec<u64> = variants.iter().map(|v| v.bandwidth).collect();
let playlists: Vec<Url> = variants.iter().map(|v| v.uri.clone()).collect();
let (main, chosen) = loop {
let chosen = Abr::initial(&bitrates, first.data.len() as u64, first.elapsed, &blacklist);
let cfg = TrackConfig { playlists: playlists.clone(), variant: chosen, start: Start::LiveEdge, cancel: Arc::new(AtomicBool::new(false)) };
match Track::start(cfg) {
Ok(t) => break (t, chosen),
Err(e @ Error::Unsupported(_)) => {
blacklist[chosen] = true;
if blacklist.iter().all(|&b| b) {
return Err(e);
}
log::warn!("HLS: variant {chosen} cannot be played ({e}); trying another");
}
Err(e) => return Err(e),
}
};
let group: Vec<&Rendition> = match variants[chosen].audio_group.as_deref() {
Some(g) => renditions.iter().filter(|r| r.group == g).collect(),
None => Vec::new(),
};
let rendition = group.iter().position(|r| r.default).or((!group.is_empty()).then_some(0));
let audio = match rendition.and_then(|i| group[i].uri.clone()) {
Some(uri) => {
let cfg = TrackConfig { playlists: vec![uri], variant: 0, start: Start::LiveEdge, cancel: Arc::new(AtomicBool::new(false)) };
Some(Lane::new(Role::Audio, Track::start(cfg)?, true))
}
None => None,
};
let live = main.is_live();
let duration = main.duration();
let tolerance = Duration::from_secs(1).max(main.target_duration() * 2);
let control = Arc::new(HlsControl {
variants: variants
.iter()
.map(|v| VariantInfo { bandwidth: v.bandwidth, resolution: v.resolution, codecs: v.codecs.clone(), frame_rate: v.frame_rate })
.collect(),
renditions: group.iter().map(|r| AudioRendition { name: r.name.clone(), language: r.language.clone() }).collect(),
current: AtomicUsize::new(chosen),
requested: Mutex::new(None),
requested_audio: Mutex::new(None),
events: Mutex::new(None),
shutdown: AtomicBool::new(false),
});
let rendition_uris = group.iter().map(|r| r.uri.clone()).collect();
let mut d = HlsDemuxer {
control,
main: Lane::new(Role::Main, main, audio.is_none()),
audio,
streams: Vec::new(),
timeline: Timeline::new(tolerance),
update: None,
reported: HashMap::new(),
live,
duration,
selection: Selection {
abr: Abr::new(bitrates),
blacklist,
fetching: chosen,
manual: None,
returned: Duration::ZERO,
},
rendition_uris,
rendition,
};
d.fill_all()?;
let video = d.main.formats.get(&StreamKind::Video).cloned();
let audio = match &d.audio {
Some(a) => a.formats.get(&StreamKind::Audio).cloned(),
None => d.main.formats.get(&StreamKind::Audio).cloned(),
};
for (id, info) in [(VIDEO_ID, video), (AUDIO_ID, audio)] {
if let Some(info) = info {
d.reported.insert(id, info.clone());
let mut s = (*info).clone();
s.id = id;
s.duration = duration;
d.streams.push(s);
}
}
if d.streams.is_empty() {
return Err(Error::Demux("HLS stream without audio or video".into()));
}
Ok(d)
}
pub fn control(&self) -> Arc<HlsControl> {
self.control.clone()
}
pub fn is_live(&self) -> bool {
self.live
}
fn fill_all(&mut self) -> Result<()> {
Self::fill(&mut self.main, &mut self.timeline, &self.control, Some(&mut self.selection))?;
if std::mem::take(&mut self.main.jumped)
&& let Some(a) = self.audio.as_mut()
{
Self::follow_jump(a);
}
if let Some(a) = self.audio.as_mut() {
Self::fill(a, &mut self.timeline, &self.control, None)?;
if std::mem::take(&mut a.jumped) {
Self::follow_jump(&mut self.main);
Self::fill(&mut self.main, &mut self.timeline, &self.control, Some(&mut self.selection))?;
}
}
Ok(())
}
fn follow_jump(lane: &mut Lane) {
lane.track.jump_to_live_edge();
lane.epoch += 1;
lane.seg = None;
lane.pending = None;
lane.ended = false;
lane.ts_ref = None;
lane.jump_by_peer = true;
}
fn switch_rendition(&mut self) -> Result<()> {
let Some(i) = self.control.requested_audio.lock().unwrap().take() else { return Ok(()) };
if Some(i) == self.rendition || i >= self.rendition_uris.len() {
return Ok(());
}
self.rendition = Some(i);
let Some(uri) = self.rendition_uris[i].clone() else {
self.audio = None;
self.main.use_audio = true;
return Ok(());
};
let now = self.audio.as_ref().and_then(|a| a.pending.as_ref()).map_or(self.selection.returned, |p| p.pts);
let start = if self.live { Start::LiveEdge } else { Start::At(now) };
let cfg = TrackConfig { playlists: vec![uri], variant: 0, start, cancel: Arc::new(AtomicBool::new(false)) };
let track = Track::start(cfg)?;
self.timeline.forget(Role::Audio);
let mut lane = Lane::new(Role::Audio, track, true);
lane.skip_before = Some(now);
self.audio = Some(lane);
self.main.use_audio = false;
Ok(())
}
fn fill(lane: &mut Lane, timeline: &mut Timeline, control: &HlsControl, mut selection: Option<&mut Selection>) -> Result<()> {
while lane.pending.is_none() && !lane.ended {
if control.shutdown.load(Ordering::Relaxed) {
lane.ended = true;
break;
}
if let Some(open) = lane.seg.as_mut() {
let Some(raw) = open.demux.next()? else {
if let Some(r) = open.demux.ts_last_raw() {
lane.ts_ref = Some((r, open.disc));
}
if let Some(p) = open.demux.ts_video_params() {
lane.ts_params = Some((p, open.disc));
}
lane.seg = None;
continue;
};
if let Some(u) = open.demux.take_update() {
lane.formats.insert(u.kind, Arc::new(u));
}
let wanted = match raw.kind {
StreamKind::Video => lane.role == Role::Main,
StreamKind::Audio => lane.use_audio,
StreamKind::Other => false,
};
if !wanted {
continue;
}
if lane.wait_key {
if raw.kind != StreamKind::Video || !raw.keyframe {
continue;
}
lane.wait_key = false;
}
let (id, info) = match raw.kind {
StreamKind::Video => (VIDEO_ID, lane.formats[&StreamKind::Video].clone()),
_ => (AUDIO_ID, lane.formats[&StreamKind::Audio].clone()),
};
let pts = Timeline::map(open.offset, raw.raw);
if lane.skip_before.is_some_and(|t| pts < t) {
continue;
}
lane.skip_before = None;
lane.pending = Some(Out { id, pts, keyframe: raw.keyframe, data: raw.data, info });
continue;
}
match lane.track.recv(POLL) {
None => {}
Some(TrackEvent::Segment(s)) if s.epoch < lane.epoch => {}
Some(TrackEvent::Segment(s)) => {
if let Some(sel) = selection.as_deref_mut()
&& let Some(v) = sel.on_segment(&s, control, lane.track.fetched_end())
{
lane.track.switch(v);
}
Self::open_segment(lane, timeline, s)?
}
Some(TrackEvent::End(epoch)) if epoch < lane.epoch => {}
Some(TrackEvent::End(_)) => lane.ended = true,
Some(TrackEvent::Failed(_, epoch)) if epoch < lane.epoch => {}
Some(TrackEvent::Failed(e, _)) => return Err(e),
}
}
Ok(())
}
fn open_segment(lane: &mut Lane, timeline: &mut Timeline, s: SegmentData) -> Result<()> {
if s.jumped {
lane.ts_ref = None;
}
let reference = lane.ts_ref.filter(|&(_, disc)| disc == s.discontinuity_seq).map(|(r, _)| r).or_else(|| {
let offset = timeline.offset(lane.role, s.discontinuity_seq)?;
let raw = s.start.as_nanos() as i128 - offset;
(raw >= 0).then(|| (raw * 90_000 / 1_000_000_000) as u64)
});
let params = lane.ts_params.clone().filter(|(_, disc)| *disc == s.discontinuity_seq).map(|(p, _)| p);
let mut demux = match SegmentDemuxer::open_with(s.data, s.init.as_deref().map(Vec::as_slice), reference, params) {
Ok(d) => d,
Err(e) if lane.track.is_live() => {
log::warn!("HLS: skipping segment {}: {e}", s.seq);
return Ok(());
}
Err(e) => return Err(e),
};
for info in &demux.streams {
lane.formats.insert(info.kind, Arc::new(info.clone()));
}
let first = demux.first_raw()?.unwrap_or_default();
let offset = if !s.jumped {
timeline.anchor(lane.role, s.discontinuity_seq, s.start, first)
} else if std::mem::take(&mut lane.jump_by_peer) {
timeline.forget(lane.role);
timeline.anchor(lane.role, s.discontinuity_seq, s.start, first)
} else {
lane.jumped = true;
timeline.reanchor(lane.role, s.discontinuity_seq, s.start, first)
};
lane.seg = Some(Open { demux, offset, disc: s.discontinuity_seq });
Ok(())
}
}
impl Demuxer for HlsDemuxer {
fn streams(&self) -> &[StreamInfo] {
&self.streams
}
fn next_packet(&mut self) -> Result<Option<Packet>> {
self.switch_rendition()?;
self.fill_all()?;
let take_audio = match (&self.main.pending, self.audio.as_ref().and_then(|a| a.pending.as_ref())) {
(None, None) => return Ok(None),
(Some(_), None) => false,
(None, Some(_)) => true,
(Some(m), Some(a)) => a.pts <= m.pts,
};
let out = if take_audio { self.audio.as_mut().unwrap().pending.take() } else { self.main.pending.take() }.unwrap();
if !take_audio {
self.selection.returned = out.pts;
}
let changed = self.reported.get(&out.id).is_none_or(|r| !same_format(r, &out.info));
if changed {
self.reported.insert(out.id, out.info.clone());
let mut info = (*out.info).clone();
info.id = out.id;
info.duration = self.duration;
if let Some(s) = self.streams.iter_mut().find(|s| s.id == out.id) {
*s = info.clone();
}
self.update = Some(info);
}
Ok(Some(Packet { stream: out.id, pts: out.pts, keyframe: out.keyframe, data: out.data, generation: 0 }))
}
fn seek(&mut self, target: Duration) -> Result<Duration> {
if self.live {
return Err(Error::Seek("live HLS streams cannot seek".into()));
}
let has_video = self.streams.iter().any(|s| s.kind == StreamKind::Video);
for lane in std::iter::once(&mut self.main).chain(self.audio.as_mut()) {
lane.track.seek(target);
lane.epoch += 1;
lane.seg = None;
lane.pending = None;
lane.ended = false;
lane.ts_ref = None;
}
self.main.wait_key = has_video;
self.selection.returned = target;
self.reported.clear();
self.fill_all()?;
Ok(self.main.pending.as_ref().or(self.audio.as_ref().and_then(|a| a.pending.as_ref())).map_or(target, |p| p.pts))
}
fn take_stream_update(&mut self) -> Option<StreamInfo> {
self.update.take()
}
}
fn same_format(a: &StreamInfo, b: &StreamInfo) -> bool {
a.codec == b.codec
&& a.extradata == b.extradata
&& (a.width, a.height, a.sample_rate, a.channels) == (b.width, b.height, b.sample_rate, b.channels)
}
fn decodable(registry: &Registry, codecs: &[String]) -> bool {
codecs.iter().all(|c| {
let codec = Codec::from_mp4_codec_string(c);
let kind = match codec {
Codec::Av1 | Codec::Vp8 | Codec::Vp9 | Codec::H264 | Codec::Hevc | Codec::ProRes => StreamKind::Video,
Codec::Other(_) => return true,
_ => StreamKind::Audio,
};
registry.can_decode(&StreamInfo::new(0, kind, codec))
})
}
#[cfg(test)]
mod tests {
use super::*;
fn control(current: usize) -> HlsControl {
HlsControl {
variants: Vec::new(),
renditions: Vec::new(),
current: AtomicUsize::new(current),
requested: Mutex::new(None),
requested_audio: Mutex::new(None),
events: Mutex::new(None),
shutdown: AtomicBool::new(false),
}
}
fn segment(seq: u64, variant: usize) -> SegmentData {
SegmentData {
seq,
discontinuity_seq: 0,
start: Duration::from_secs(seq),
duration: Duration::from_secs(1),
data: Vec::new(),
init: None,
variant,
bytes: 125_000, elapsed: Duration::from_millis(100),
epoch: 0,
jumped: false,
}
}
#[test]
fn switches_up_when_downloads_are_well_ahead_of_playback() {
let mut sel = Selection {
abr: Abr::new(vec![433_000, 143_000]),
blacklist: vec![false; 2],
fetching: 1,
manual: None,
returned: Duration::ZERO,
};
let c = control(1);
assert_eq!(sel.on_segment(&segment(0, 1), &c, Duration::from_secs(4)), None, "one sample is not enough");
assert_eq!(sel.on_segment(&segment(1, 1), &c, Duration::from_secs(4)), None);
sel.returned = Duration::from_secs(2);
assert_eq!(sel.on_segment(&segment(2, 1), &c, Duration::from_secs(5)), Some(0));
}
#[test]
fn a_manual_choice_holds_until_auto() {
let mut sel =
Selection { abr: Abr::new(vec![433_000, 143_000]), blacklist: vec![false; 2], fetching: 0, manual: None, returned: Duration::ZERO };
let c = control(0);
c.set_variant(Variant::Index(1));
assert_eq!(sel.on_segment(&segment(0, 0), &c, Duration::from_secs(4)), Some(1));
for seq in 1..5 {
sel.returned = Duration::from_secs(seq);
assert_eq!(sel.on_segment(&segment(seq, 1), &c, Duration::from_secs(seq + 3)), None, "manual: no ABR");
}
c.set_variant(Variant::Auto);
sel.returned = Duration::from_secs(5);
assert_eq!(sel.on_segment(&segment(5, 1), &c, Duration::from_secs(8)), Some(0));
}
}