use crate::buffer::BufferHandle;
#[cfg(feature = "encryption")]
use crate::cenc::{self, ContentKeys, FragmentRole};
use crate::error::Error;
use std::collections::HashMap;
use std::fmt;
use std::io::{self, Read, Seek, SeekFrom};
use std::ops::{Range, RangeInclusive};
use std::sync::atomic::{AtomicBool, Ordering};
use std::sync::{Arc, Condvar, Mutex, MutexGuard, OnceLock, Weak};
use std::time::Duration;
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct MediaTimeline {
pub timescale: u32,
pub media_durations: Vec<u64>,
}
const SYMPHONIA_COMPAT_MEDIA_FACTOR: u128 = 4;
const SYMPHONIA_COMPAT_BYTES_PER_FRAGMENT: u128 = 64 * 1024;
impl MediaTimeline {
fn ticks_to_duration(&self, ticks: u64) -> Option<Duration> {
if self.timescale == 0 {
return None;
}
let timescale = u64::from(self.timescale);
let seconds = ticks / timescale;
let nanos = (u128::from(ticks % timescale) * 1_000_000_000 / u128::from(timescale)) as u32;
Some(Duration::new(seconds, nanos))
}
pub(crate) fn spans(&self) -> Vec<Range<Duration>> {
let mut ticks = 0u64;
self.media_durations
.iter()
.map_while(|&duration| {
let start = self.ticks_to_duration(ticks)?;
ticks = ticks.saturating_add(duration);
Some(start..self.ticks_to_duration(ticks)?)
})
.collect()
}
fn fragment_start(&self, fragment_index: usize) -> Option<Duration> {
let media_index = fragment_index.checked_sub(1)?;
Some(self.spans().get(media_index)?.start)
}
pub(crate) fn total_duration(&self) -> Option<Duration> {
self.ticks_to_duration(self.media_durations.iter().copied().sum())
}
pub fn estimate_len(&self, init_len: u64, bitrate_bps: u32) -> Option<u64> {
if bitrate_bps == 0 || self.timescale == 0 || self.media_durations.is_empty() {
return None;
}
let ticks: u128 = self.media_durations.iter().copied().map(u128::from).sum();
let fragments = u128::from(self.media_durations.len() as u64);
let denom = u128::from(self.timescale) * 8;
let media = (u128::from(bitrate_bps) * ticks).div_ceil(denom);
let padded = media
.saturating_mul(SYMPHONIA_COMPAT_MEDIA_FACTOR)
.saturating_add(fragments.saturating_mul(SYMPHONIA_COMPAT_BYTES_PER_FRAGMENT))
.saturating_add(u128::from(init_len));
Some(u64::try_from(padded).unwrap_or(u64::MAX))
}
}
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub struct Position {
pub byte: Option<u64>,
pub time: Option<TimePosition>,
}
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub struct TimePosition {
pub timestamp: Duration,
pub accuracy: TimeAccuracy,
}
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub enum TimeAccuracy {
Exact,
Coarse,
}
pub(crate) fn parse_byte_range(value: &str) -> Result<RangeInclusive<u64>, Error> {
value
.trim()
.split_once('-')
.and_then(|(start, end)| {
let (start, end) = (start.parse().ok()?, end.parse().ok()?);
(start <= end).then_some(start..=end)
})
.ok_or_else(|| Error::InvalidByteRange(value.to_owned()))
}
#[derive(Clone, Debug, PartialEq, Eq, Hash)]
pub struct Fragment {
pub url: String,
pub range: Option<RangeInclusive<u64>>,
}
impl Fragment {
pub fn new(url: impl Into<String>) -> Self {
Self {
url: url.into(),
range: None,
}
}
pub fn with_range(url: impl Into<String>, range: RangeInclusive<u64>) -> Self {
Self {
url: url.into(),
range: Some(range),
}
}
pub fn known_size(&self) -> Option<u64> {
self.range
.as_ref()
.and_then(|range| range.end().checked_sub(*range.start())?.checked_add(1))
}
}
impl From<String> for Fragment {
fn from(url: String) -> Self {
Self::new(url)
}
}
impl From<&str> for Fragment {
fn from(url: &str) -> Self {
Self::new(url)
}
}
impl fmt::Display for Fragment {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
match &self.range {
Some(range) => write!(f, "{} (bytes {}-{})", self.url, range.start(), range.end()),
None => f.write_str(&self.url),
}
}
}
pub trait Transport: Send + Sync {
fn get(&self, fragment: &Fragment, size_hint: Option<u64>) -> Result<Arc<[u8]>, Error>;
fn head(&self, fragment: &Fragment) -> Result<u64, Error> {
Err(Error::StreamInitializationError(format!(
"transport cannot HEAD {fragment}"
)))
}
}
impl Transport for ureq::Agent {
fn get(&self, fragment: &Fragment, size_hint: Option<u64>) -> Result<Arc<[u8]>, Error> {
let mut request = ureq::Agent::get(self, &fragment.url);
if let Some(range) = &fragment.range {
request = request.header(
ureq::http::header::RANGE,
format!("bytes={}-{}", range.start(), range.end()),
);
}
let response = request.call()?;
let partial = response.status() == ureq::http::StatusCode::PARTIAL_CONTENT;
let mut bytes = size_hint
.and_then(|len| usize::try_from(len).ok())
.map(Vec::with_capacity)
.unwrap_or_default();
response.into_body().into_reader().read_to_end(&mut bytes)?;
match &fragment.range {
Some(range) if !partial => {
#[cfg(feature = "log")]
log::warn!(
"server ignored the Range request for {fragment}; slicing the full body"
);
Ok(slice_range(&bytes, range).unwrap_or(&bytes).into())
}
_ => Ok(bytes.into()),
}
}
fn head(&self, fragment: &Fragment) -> Result<u64, Error> {
let url = &fragment.url;
let response = ureq::Agent::head(self, url).call()?;
response
.headers()
.get(ureq::http::header::CONTENT_LENGTH)
.and_then(|value| value.to_str().ok())
.and_then(|value| value.parse::<u64>().ok())
.ok_or_else(|| {
Error::StreamInitializationError(format!(
"HEAD for {url} did not return Content-Length"
))
})
}
}
#[derive(Debug)]
pub(crate) struct FragmentData {
fragment: Fragment,
body: OnceLock<Arc<[u8]>>,
probed_size: OnceLock<u64>,
get_lock: Mutex<()>,
head_lock: Mutex<()>,
#[cfg(feature = "encryption")]
role: OnceLock<FragmentRole>,
}
impl FragmentData {
pub(crate) fn new(fragment: Fragment) -> Self {
Self {
fragment,
body: OnceLock::new(),
probed_size: OnceLock::new(),
get_lock: Mutex::new(()),
head_lock: Mutex::new(()),
#[cfg(feature = "encryption")]
role: OnceLock::new(),
}
}
pub(crate) fn body(&self) -> Option<&Arc<[u8]>> {
self.body.get()
}
#[cfg(test)]
pub(crate) fn set_body(&self, body: &[u8]) {
self.body.set(body.into()).expect("body set once");
}
pub(crate) fn size(&self) -> Option<u64> {
self.fragment
.known_size()
.or_else(|| self.body().map(|body| body.len() as u64))
.or_else(|| self.probed_size.get().copied())
}
}
pub struct FragmentCache {
transport: Arc<dyn Transport>,
fragments: Mutex<HashMap<Fragment, Arc<FragmentData>>>,
listeners: Mutex<Vec<Weak<dyn CacheListener>>>,
#[cfg(feature = "encryption")]
keys: Option<Arc<ContentKeys>>,
}
impl FragmentCache {
pub fn new(transport: Arc<dyn Transport>) -> Self {
Self {
transport,
fragments: Mutex::default(),
listeners: Mutex::default(),
#[cfg(feature = "encryption")]
keys: None,
}
}
#[cfg(feature = "encryption")]
pub fn with_keys(mut self, keys: ContentKeys) -> Self {
self.keys = Some(Arc::new(keys));
self
}
pub fn decrypts(&self) -> bool {
#[cfg(feature = "encryption")]
return self.keys.is_some();
#[cfg(not(feature = "encryption"))]
false
}
pub fn cached(&self, fragment: &Fragment) -> Option<Arc<[u8]>> {
self.lock().get(fragment)?.body().cloned()
}
pub fn known_size(&self, fragment: &Fragment) -> Option<u64> {
fragment
.known_size()
.or_else(|| self.lock().get(fragment)?.size())
}
fn lock(&self) -> MutexGuard<'_, HashMap<Fragment, Arc<FragmentData>>> {
self.fragments
.lock()
.unwrap_or_else(|error| error.into_inner())
}
pub(crate) fn subscribe(&self, listener: Weak<dyn CacheListener>) {
self.listeners
.lock()
.unwrap_or_else(|error| error.into_inner())
.push(listener);
}
fn notify(&self, data: &FragmentData) {
let listeners: Vec<_> = {
let mut listeners = self
.listeners
.lock()
.unwrap_or_else(|error| error.into_inner());
listeners.retain(|listener| listener.strong_count() > 0);
listeners.iter().filter_map(Weak::upgrade).collect()
};
for listener in listeners {
listener.fragment_changed(data);
}
}
pub(crate) fn data(&self, fragment: Fragment) -> Arc<FragmentData> {
self.lock()
.entry(fragment)
.or_insert_with_key(|fragment| Arc::new(FragmentData::new(fragment.clone())))
.clone()
}
fn get(&self, data: &FragmentData) -> Result<Arc<[u8]>, Error> {
if let Some(body) = data.body() {
return Ok(body.clone());
}
let _in_flight = data
.get_lock
.lock()
.unwrap_or_else(|error| error.into_inner());
if let Some(body) = data.body() {
return Ok(body.clone());
}
let fragment = &data.fragment;
let body = self.transport.get(fragment, data.size())?;
if let Some(expected) = fragment.known_size()
&& body.len() as u64 != expected
{
return Err(Error::FragmentLength {
fragment: fragment.to_string(),
expected,
actual: body.len() as u64,
});
}
#[cfg(feature = "encryption")]
let body = self.decrypt(data, body)?;
let body = data.body.get_or_init(|| body).clone();
self.notify(data);
Ok(body)
}
#[cfg(feature = "encryption")]
fn decrypt(&self, data: &FragmentData, mut body: Arc<[u8]>) -> Result<Arc<[u8]>, Error> {
let Some(keys) = &self.keys else {
return Ok(body);
};
let role = match data.role.get() {
Some(FragmentRole::Media(tracks)) if tracks.is_empty() => return Ok(body),
Some(role) => role,
None => {
debug_assert!(
false,
"keyed cache fetched {} before a reader set its role",
data.fragment
);
return Ok(body);
}
};
if Arc::get_mut(&mut body).is_none() {
body = Arc::from(&*body);
}
let bytes = Arc::get_mut(&mut body).expect("body was just made unique");
match role {
FragmentRole::Init => cenc::clear_sample_entries(bytes),
FragmentRole::Media(tracks) => cenc::decrypt_fragment(bytes, tracks, keys),
}
.map_err(|problem| problem.at(&data.fragment))?;
Ok(body)
}
#[cfg(feature = "encryption")]
fn assign_media_roles(&self, fragments: &[Arc<FragmentData>]) -> Result<(), Error> {
let Some(keys) = &self.keys else {
return Ok(());
};
let init = &fragments[0];
let body = init.body().expect("init fetched before media roles");
let tracks = cenc::track_protection(body).map_err(|problem| problem.at(&init.fragment))?;
if let Some(track) = tracks.iter().find(|track| !keys.contains(&track.kid)) {
return Err(Error::MissingContentKey(cenc::hex(&track.kid)));
}
let tracks: Arc<[_]> = tracks.into();
for data in &fragments[1..] {
let _ = data.role.set(FragmentRole::Media(tracks.clone()));
}
Ok(())
}
fn probe(&self, data: &FragmentData) -> Result<(), Error> {
if data.size().is_some() {
return Ok(());
}
let _in_flight = data
.head_lock
.lock()
.unwrap_or_else(|error| error.into_inner());
if data.size().is_none() {
let size = self.transport.head(&data.fragment)?;
data.probed_size.get_or_init(|| size);
self.notify(data);
}
Ok(())
}
}
pub(crate) trait CacheListener: Send + Sync {
fn fragment_changed(&self, data: &FragmentData);
}
impl Default for FragmentCache {
fn default() -> Self {
Self::new(Arc::new(ureq_agent()))
}
}
impl fmt::Debug for FragmentCache {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
f.debug_struct("FragmentCache")
.field("fragments", &self.lock().len())
.field("decrypts", &self.decrypts())
.finish_non_exhaustive()
}
}
const POOL_IDLE_AGE: Duration = Duration::from_secs(90);
fn ureq_agent() -> ureq::Agent {
ureq::Agent::new_with_config(
ureq::Agent::config_builder()
.max_idle_age(POOL_IDLE_AGE)
.build(),
)
}
fn slice_range<'a>(body: &'a [u8], range: &RangeInclusive<u64>) -> Option<&'a [u8]> {
let start = usize::try_from(*range.start()).ok()?;
let end = usize::try_from(*range.end()).ok()?;
body.get(start..=end)
}
#[derive(Debug)]
pub struct StreamReader {
shared: Arc<ReaderShared>,
byte_position: Option<u64>,
fragment_index: usize,
offset_in_fragment: usize,
timeline: Option<MediaTimeline>,
symphonia_compat: bool,
length_estimate: Option<u64>,
}
#[derive(Debug)]
struct ReaderShared {
state: Mutex<ReaderState>,
condvar: Condvar,
cancelled: AtomicBool,
cache: Arc<FragmentCache>,
}
#[derive(Debug)]
struct ReaderState {
fragments: Vec<Arc<FragmentData>>,
head_available: bool,
eager_head: bool,
head_worker_running: bool,
cursor_fragment: usize,
priority: Option<usize>,
complete: bool,
error: Option<String>,
}
impl ReaderState {
fn all_sizes_known(&self) -> bool {
self.total_len().is_some()
}
fn missing_body(&self, index: usize) -> bool {
self.fragments
.get(index)
.is_some_and(|data| data.body().is_none())
}
#[cfg_attr(not(feature = "log"), allow(unused_variables))]
fn disable_head(&mut self, reason: &str) {
if self.head_available {
self.head_available = false;
#[cfg(feature = "log")]
log::warn!("MPEG stream disabling HEAD size probes: {reason}");
}
}
fn next_get(&mut self) -> Option<(usize, Arc<FragmentData>)> {
let cursor = self.cursor_fragment.min(self.fragments.len());
let index = self
.priority
.take()
.filter(|&priority| self.missing_body(priority))
.or_else(|| {
(cursor..self.fragments.len())
.chain(0..cursor)
.find(|&index| self.missing_body(index))
})?;
Some((index, self.fragments[index].clone()))
}
fn next_head(&self) -> Option<(usize, Arc<FragmentData>)> {
if !self.head_available {
return None;
}
let index = self
.fragments
.iter()
.position(|data| data.size().is_none())?;
Some((index, self.fragments[index].clone()))
}
fn prefix_len(&self, count: usize) -> Option<u64> {
self.fragments[..count].iter().map(|data| data.size()).sum()
}
fn total_len(&self) -> Option<u64> {
self.prefix_len(self.fragments.len())
}
fn check_probe(&mut self, index: usize) {
let data = &self.fragments[index];
if let (Some(&expected), Some(body)) = (data.probed_size.get(), data.body())
&& expected != body.len() as u64
{
let len = body.len();
self.disable_head(&format!(
"GET length {len} != HEAD length {expected} for fragment {index}"
));
}
}
fn resolve_cursor(&self, target: u64) -> Option<(usize, usize)> {
let mut acc = 0u64;
for (index, data) in self.fragments.iter().enumerate() {
let len = data.size()?;
if target < acc + len {
return Some((index, (target - acc) as usize));
}
acc += len;
}
let last = self.fragments.len().checked_sub(1)?;
Some((last, self.fragments[last].size().unwrap_or(0) as usize))
}
}
impl StreamReader {
pub fn new(
fragments: Vec<Fragment>,
timeline: Option<MediaTimeline>,
eager_head: bool,
) -> Result<Self, Error> {
Self::new_with_cache(fragments, timeline, eager_head, Arc::default())
}
pub fn new_with_transport(
fragments: Vec<Fragment>,
timeline: Option<MediaTimeline>,
eager_head: bool,
transport: Arc<dyn Transport>,
) -> Result<Self, Error> {
let cache = FragmentCache::new(transport);
Self::new_with_cache(fragments, timeline, eager_head, Arc::new(cache))
}
pub fn new_with_cache(
fragments: Vec<Fragment>,
timeline: Option<MediaTimeline>,
eager_head: bool,
cache: Arc<FragmentCache>,
) -> Result<Self, Error> {
if fragments.is_empty() {
return Err(Error::DashManifestMissingUrls);
}
let fragments: Vec<_> = fragments
.into_iter()
.map(|fragment| cache.data(fragment))
.collect();
#[cfg(feature = "encryption")]
let _ = fragments[0].role.set(FragmentRole::Init);
cache.get(&fragments[0])?;
#[cfg(feature = "encryption")]
cache.assign_media_roles(&fragments)?;
let complete = fragments.iter().all(|data| data.body().is_some());
let shared = Arc::new(ReaderShared {
state: Mutex::new(ReaderState {
fragments,
head_available: true,
eager_head: false,
head_worker_running: false,
cursor_fragment: 0,
priority: None,
complete,
error: None,
}),
condvar: Condvar::new(),
cancelled: AtomicBool::new(false),
cache,
});
if !complete {
spawn_get_worker(shared.clone());
}
let reader = Self {
shared,
byte_position: Some(0),
fragment_index: 0,
offset_in_fragment: 0,
timeline,
symphonia_compat: false,
length_estimate: None,
};
if eager_head {
reader.start_eager_head();
}
Ok(reader)
}
pub fn buffer(&self) -> Option<BufferHandle> {
let timeline = self.timeline.as_ref()?;
let media = self.shared.lock().fragments[1..].to_vec();
Some(BufferHandle::new(&self.shared.cache, media, timeline))
}
pub fn start_eager_head(&self) {
let mut inner = self.shared.lock();
inner.eager_head = true;
if inner.head_worker_running
|| inner.error.is_some()
|| inner.next_head().is_none()
|| self.shared.cancelled.load(Ordering::SeqCst)
{
return;
}
inner.head_worker_running = true;
drop(inner);
spawn_head_worker(self.shared.clone());
}
pub fn stop_eager_head(&self) {
self.shared.lock().eager_head = false;
}
pub fn set_symphonia_compat(&mut self, enabled: bool, bitrate_bps: u32) -> Result<(), Error> {
if !enabled {
self.symphonia_compat = false;
self.length_estimate = None;
return Ok(());
}
let timeline = self.timeline.as_ref().ok_or_else(|| {
Error::StreamInitializationError(
"symphonia_compat requires a SegmentTimeline".to_owned(),
)
})?;
let init_len = {
let inner = self.shared.lock();
inner.fragments[0].size().unwrap_or(0)
};
let estimate = timeline
.estimate_len(init_len, bitrate_bps)
.ok_or_else(|| {
Error::StreamInitializationError(
"symphonia_compat requires a non-zero bitrate and timescale".to_owned(),
)
})?;
self.symphonia_compat = true;
self.length_estimate = Some(estimate);
Ok(())
}
fn reported_end_len(&self) -> Option<u64> {
let inner = self.shared.lock();
if let Some(exact) = inner.total_len() {
return Some(exact);
}
if !self.symphonia_compat {
return None;
}
let known: u64 = inner.fragments.iter().filter_map(|data| data.size()).sum();
self.length_estimate.map(|estimate| estimate.max(known))
}
pub fn seek_bytes(&mut self, from: SeekFrom) -> io::Result<Position> {
if let SeekFrom::End(0) = from
&& let Some(len) = self.reported_end_len()
{
return self.park_at_reported_end(len);
}
let target = match from {
SeekFrom::Start(offset) => offset,
SeekFrom::Current(offset) => {
let Some(position) = self.byte_position else {
return Err(io::Error::new(
io::ErrorKind::InvalidInput,
"MPEG stream absolute byte position is unknown; use seek_time_coarse or SeekFrom::Start",
));
};
if offset >= 0 {
position.saturating_add(offset as u64)
} else {
position.saturating_sub(offset.unsigned_abs())
}
}
SeekFrom::End(offset) => {
self.ensure_sizes(ReaderState::all_sizes_known)?;
let size = {
let inner = self.shared.lock();
inner.total_len().unwrap_or(0)
};
if offset >= 0 {
size.saturating_add(offset as u64)
} else {
size.saturating_sub(offset.unsigned_abs())
}
}
};
self.ensure_sizes(|inner| inner.resolve_cursor(target).is_some())?;
let (fragment_index, offset_in_fragment) = {
let inner = self.shared.lock();
match inner.resolve_cursor(target) {
Some(cursor) => cursor,
None => {
return Err(io::Error::other(
"MPEG stream ended before the requested byte offset",
));
}
}
};
self.fragment_index = fragment_index;
self.offset_in_fragment = offset_in_fragment;
self.publish_cursor();
self.prioritize_fragment(fragment_index);
self.byte_position = Some(target);
Ok(Position {
byte: Some(target),
time: self.time_at_fragment(
fragment_index,
if offset_in_fragment == 0 {
TimeAccuracy::Exact
} else {
TimeAccuracy::Coarse
},
),
})
}
pub fn seek_time_coarse(&mut self, time: Duration) -> io::Result<Position> {
let timeline = self.timeline.clone().ok_or_else(|| {
io::Error::new(
io::ErrorKind::InvalidInput,
"MPEG stream has no SegmentTimeline",
)
})?;
if timeline.timescale == 0 || timeline.media_durations.is_empty() {
return Err(io::Error::new(
io::ErrorKind::InvalidInput,
"MPEG stream timeline is missing a timescale or media durations",
));
}
let total_ticks: u64 = timeline.media_durations.iter().sum();
let target_ticks =
((time.as_secs_f64() * f64::from(timeline.timescale)).round() as u64).min(total_ticks);
let mut tick = 0u64;
let mut media_index = timeline.media_durations.len() - 1;
for (index, duration) in timeline.media_durations.iter().enumerate() {
let next = tick.saturating_add(*duration);
if target_ticks < next
|| (target_ticks == total_ticks && index + 1 == timeline.media_durations.len())
{
media_index = index;
break;
}
tick = next;
}
let target_fragment = media_index + 1;
{
let mut inner = self.shared.lock();
if target_fragment >= inner.fragments.len() {
return Err(io::Error::other(
"MPEG stream ended before the requested media time",
));
}
inner.cursor_fragment = target_fragment;
if inner.missing_body(target_fragment) {
inner.priority = Some(target_fragment);
self.shared.condvar.notify_all();
}
}
let byte_position = {
let inner =
self.wait_until(|inner| !inner.missing_body(target_fragment) || inner.complete)?;
if inner.missing_body(target_fragment) {
return Err(io::Error::other(
"MPEG stream ended before the requested media time",
));
}
inner.prefix_len(target_fragment)
};
self.fragment_index = target_fragment;
self.offset_in_fragment = 0;
self.byte_position = byte_position;
Ok(Position {
byte: byte_position,
time: self.time_at_fragment(
target_fragment,
if target_ticks == tick {
TimeAccuracy::Exact
} else {
TimeAccuracy::Coarse
},
),
})
}
fn ensure_sizes(&self, known: impl Fn(&ReaderState) -> bool) -> io::Result<()> {
loop {
let next_head = {
let inner = self.shared.lock();
if known(&inner) {
return Ok(());
}
inner.next_head()
};
let Some((index, data)) = next_head else {
break;
};
self.shared.probe(index, &data);
}
{
let mut inner = self.shared.lock();
if known(&inner) {
return Ok(());
}
if let Some(index) = (0..inner.fragments.len()).find(|&i| inner.missing_body(i)) {
inner.priority = Some(index);
self.shared.condvar.notify_all();
}
}
self.wait_until(|inner| known(inner) || inner.complete)
.map(drop)
}
fn wait_until<F>(&self, mut ready: F) -> io::Result<MutexGuard<'_, ReaderState>>
where
F: FnMut(&ReaderState) -> bool,
{
let mut inner = self.shared.lock();
loop {
if self.shared.cancelled.load(Ordering::SeqCst) {
return Err(io::Error::new(
io::ErrorKind::Interrupted,
"MPEG stream cancelled",
));
}
if let Some(error) = &inner.error {
return Err(io::Error::other(error.clone()));
}
if ready(&inner) {
return Ok(inner);
}
inner = self
.shared
.condvar
.wait(inner)
.unwrap_or_else(|error| error.into_inner());
}
}
fn prioritize_fragment(&self, index: usize) {
let mut inner = self.shared.lock();
if inner.missing_body(index) {
inner.priority = Some(index);
self.shared.condvar.notify_all();
}
}
fn publish_cursor(&self) {
let mut inner = self.shared.lock();
inner.cursor_fragment = self.fragment_index;
}
fn time_at_fragment(
&self,
fragment_index: usize,
accuracy: TimeAccuracy,
) -> Option<TimePosition> {
self.timeline
.as_ref()?
.fragment_start(fragment_index)
.map(|timestamp| TimePosition {
timestamp,
accuracy,
})
}
fn time_at_eof(&self) -> Option<TimePosition> {
self.timeline
.as_ref()?
.total_duration()
.map(|timestamp| TimePosition {
timestamp,
accuracy: TimeAccuracy::Exact,
})
}
fn park_at_reported_end(&mut self, len: u64) -> io::Result<Position> {
let fragment_count = {
let inner = self.shared.lock();
inner.fragments.len()
};
self.fragment_index = fragment_count;
self.offset_in_fragment = 0;
self.byte_position = Some(len);
Ok(Position {
byte: Some(len),
time: self.time_at_eof(),
})
}
}
impl ReaderShared {
fn lock(&self) -> MutexGuard<'_, ReaderState> {
self.state.lock().unwrap_or_else(|error| error.into_inner())
}
fn probe(&self, index: usize, data: &FragmentData) {
let result = self.cache.probe(data);
let mut state = self.lock();
if let Err(error) = result {
state.disable_head(&format!("HEAD failed for fragment {index}: {error}"));
}
self.condvar.notify_all();
}
}
fn spawn_get_worker(shared: Arc<ReaderShared>) {
std::thread::spawn(move || {
while !shared.cancelled.load(Ordering::SeqCst) {
let (index, data) = {
let mut state = shared.lock();
if state.error.is_some() {
return;
}
let Some(work) = state.next_get() else {
state.complete = true;
shared.condvar.notify_all();
return;
};
work
};
let result = shared.cache.get(&data);
let mut state = shared.lock();
match result {
Ok(_) => state.check_probe(index),
Err(error) => state.error = Some(error.to_string()),
}
shared.condvar.notify_all();
}
});
}
fn spawn_head_worker(shared: Arc<ReaderShared>) {
std::thread::spawn(move || {
loop {
if shared.cancelled.load(Ordering::SeqCst) {
return;
}
let Some((index, data)) = ({
let mut inner = shared.lock();
let next = if inner.eager_head && inner.error.is_none() {
inner.next_head()
} else {
None
};
if next.is_none() {
inner.head_worker_running = false;
}
next
}) else {
return;
};
shared.probe(index, &data);
}
});
}
impl Read for StreamReader {
fn read(&mut self, buf: &mut [u8]) -> io::Result<usize> {
if buf.is_empty() {
return Ok(0);
}
loop {
let fragment_count = {
let inner = self.shared.lock();
inner.fragments.len()
};
if self.fragment_index >= fragment_count {
let _inner = self.wait_until(|inner| inner.complete)?;
return Ok(0);
}
self.prioritize_fragment(self.fragment_index);
let index = self.fragment_index;
let offset = self.offset_in_fragment;
let (copied, fragment_len) = {
let inner =
self.wait_until(|inner| !inner.missing_body(index) || inner.complete)?;
let Some(fragment) = inner.fragments[index].body() else {
return Ok(0);
};
let fragment_len = fragment.len();
if offset >= fragment_len {
(0, fragment_len)
} else {
let copied = (fragment_len - offset).min(buf.len());
buf[..copied].copy_from_slice(&fragment[offset..offset + copied]);
(copied, fragment_len)
}
};
if offset >= fragment_len {
self.fragment_index += 1;
self.offset_in_fragment = 0;
self.publish_cursor();
continue;
}
self.offset_in_fragment += copied;
if self.offset_in_fragment >= fragment_len {
self.fragment_index += 1;
self.offset_in_fragment = 0;
self.publish_cursor();
}
if let Some(position) = self.byte_position {
self.byte_position = Some(position + copied as u64);
}
return Ok(copied);
}
}
}
impl Seek for StreamReader {
fn seek(&mut self, from: SeekFrom) -> io::Result<u64> {
self.seek_bytes(from)?
.byte
.ok_or_else(|| io::Error::other("MPEG stream byte position is unknown"))
}
}
impl Drop for StreamReader {
fn drop(&mut self) {
self.shared.cancelled.store(true, Ordering::SeqCst);
self.shared.condvar.notify_all();
}
}
#[cfg(test)]
mod tests {
use super::*;
use crate::BufferStatus;
use std::sync::atomic::AtomicUsize;
use std::thread;
use std::time::Duration;
struct FnTransport<G, H> {
get: G,
head: H,
}
impl<G, H> Transport for FnTransport<G, H>
where
G: Fn(&Fragment, Option<u64>) -> Result<Vec<u8>, Error> + Send + Sync,
H: Fn(&Fragment) -> Result<u64, Error> + Send + Sync,
{
fn get(&self, fragment: &Fragment, size_hint: Option<u64>) -> Result<Arc<[u8]>, Error> {
(self.get)(fragment, size_hint).map(Arc::from)
}
fn head(&self, fragment: &Fragment) -> Result<u64, Error> {
(self.head)(fragment)
}
}
fn closure_reader<G, H>(
fragments: Vec<Fragment>,
timeline: Option<MediaTimeline>,
eager_head: bool,
get: G,
head: H,
) -> Result<StreamReader, Error>
where
G: Fn(&Fragment, Option<u64>) -> Result<Vec<u8>, Error> + Send + Sync + 'static,
H: Fn(&Fragment) -> Result<u64, Error> + Send + Sync + 'static,
{
let transport = Arc::new(FnTransport { get, head });
StreamReader::new_with_transport(fragments, timeline, eager_head, transport)
}
fn position(byte: Option<u64>, time: Option<(Duration, TimeAccuracy)>) -> Position {
Position {
byte,
time: time.map(|(timestamp, accuracy)| TimePosition {
timestamp,
accuracy,
}),
}
}
fn instant_get(
payloads: &'static [(&'static str, &'static [u8])],
) -> impl Fn(&Fragment, Option<u64>) -> Result<Vec<u8>, Error> + Send + Sync {
move |fragment: &Fragment, _expected_len: Option<u64>| {
let url = fragment.url.clone();
payloads
.iter()
.find(|(name, _)| *name == url)
.map(|(_, bytes)| bytes.to_vec())
.ok_or_else(|| Error::StreamInitializationError(format!("unknown url {url}")))
}
}
fn instant_head(
payloads: &'static [(&'static str, &'static [u8])],
) -> impl Fn(&Fragment) -> Result<u64, Error> + Send + Sync {
move |fragment: &Fragment| {
let url = fragment.url.clone();
payloads
.iter()
.find(|(name, _)| *name == url)
.map(|(_, bytes)| bytes.len() as u64)
.ok_or_else(|| Error::StreamInitializationError(format!("unknown url {url}")))
}
}
fn failing_head() -> impl Fn(&Fragment) -> Result<u64, Error> + Send + Sync {
move |_fragment: &Fragment| Err(Error::StreamInitializationError("HEAD unavailable".into()))
}
fn start_instant(
urls: Vec<Fragment>,
payloads: &'static [(&'static str, &'static [u8])],
) -> Result<StreamReader, Error> {
closure_reader(
urls,
None,
true,
instant_get(payloads),
instant_head(payloads),
)
}
fn gated_get(
allow_rest: Arc<AtomicBool>,
payload: impl Fn(&str) -> Vec<u8> + Send + Sync + 'static,
) -> impl Fn(&Fragment, Option<u64>) -> Result<Vec<u8>, Error> + Send + Sync {
move |fragment: &Fragment, _expected_len: Option<u64>| {
let url = fragment.url.clone();
if url != "init" {
while !allow_rest.load(Ordering::SeqCst) {
thread::sleep(Duration::from_millis(5));
}
}
Ok(payload(&url))
}
}
#[test]
fn reads_init_then_media_fragments() {
let mut reader = start_instant(
vec!["init".into(), "a".into(), "b".into()],
&[("init", b"INIT"), ("a", b"AAAA"), ("b", b"BB")],
)
.unwrap();
let mut buf = Vec::new();
reader.read_to_end(&mut buf).unwrap();
assert_eq!(buf, b"INITAAAABB");
}
#[test]
fn blocks_until_later_fragments_arrive() {
let allow_rest = Arc::new(AtomicBool::new(false));
let mut reader = closure_reader(
vec!["init".into(), "more".into()],
None,
true,
gated_get(allow_rest.clone(), |url| url.as_bytes().to_vec()),
failing_head(),
)
.unwrap();
let mut init = [0u8; 4];
assert_eq!(reader.read(&mut init).unwrap(), 4);
assert_eq!(&init, b"init");
let handle = thread::spawn(move || {
let mut rest = Vec::new();
reader.read_to_end(&mut rest).unwrap();
rest
});
thread::sleep(Duration::from_millis(30));
assert!(!handle.is_finished());
allow_rest.store(true, Ordering::SeqCst);
assert_eq!(handle.join().unwrap(), b"more");
}
#[test]
fn seek_bytes_within_downloaded_data() {
let mut reader = start_instant(
vec!["init".into(), "body".into()],
&[("init", b"0123"), ("body", b"4567")],
)
.unwrap();
let mut all = Vec::new();
reader.read_to_end(&mut all).unwrap();
assert_eq!(all, b"01234567");
reader.seek_bytes(SeekFrom::Start(2)).unwrap();
let mut buf = [0u8; 4];
reader.read_exact(&mut buf).unwrap();
assert_eq!(&buf, b"2345");
reader.seek_bytes(SeekFrom::Current(-2)).unwrap();
reader.read_exact(&mut buf[..2]).unwrap();
assert_eq!(&buf[..2], b"45");
reader.seek_bytes(SeekFrom::End(-3)).unwrap();
let mut tail = Vec::new();
reader.read_to_end(&mut tail).unwrap();
assert_eq!(tail, b"567");
}
#[test]
fn seek_bytes_from_end_falls_back_without_head() {
let allow_rest = Arc::new(AtomicBool::new(false));
let mut reader = closure_reader(
vec!["init".into(), "tail".into()],
None,
true,
gated_get(allow_rest.clone(), |url| vec![url.as_bytes()[0]; 4]),
failing_head(),
)
.unwrap();
let handle = thread::spawn(move || {
reader.seek_bytes(SeekFrom::End(-2)).unwrap();
let mut buf = [0u8; 2];
reader.read_exact(&mut buf).unwrap();
buf
});
thread::sleep(Duration::from_millis(30));
assert!(!handle.is_finished());
allow_rest.store(true, Ordering::SeqCst);
assert_eq!(&handle.join().unwrap(), b"tt");
}
#[test]
fn seek_bytes_uses_head_without_downloading_prefix() {
let allow_get = Arc::new(AtomicBool::new(false));
let get_order = Arc::new(Mutex::new(Vec::new()));
let head_calls = Arc::new(Mutex::new(Vec::new()));
let mut reader = closure_reader(
vec!["init".into(), "a".into(), "b".into()],
None,
true,
{
let allow_get = allow_get.clone();
let get_order = get_order.clone();
move |fragment: &Fragment, _expected_len: Option<u64>| {
let url = fragment.url.clone();
if url != "init" {
while !allow_get.load(Ordering::SeqCst) {
thread::sleep(Duration::from_millis(5));
}
}
get_order.lock().unwrap().push(url.clone());
Ok(vec![url.as_bytes()[0]; 4])
}
},
{
let head_calls = head_calls.clone();
move |fragment: &Fragment| {
let url = fragment.url.clone();
head_calls.lock().unwrap().push(url.clone());
Ok(4)
}
},
)
.unwrap();
assert_eq!(
reader.seek_bytes(SeekFrom::Start(9)).unwrap(),
position(Some(9), None)
);
assert!(
get_order.lock().unwrap().iter().all(|url| url == "init"),
"seek via HEAD must not GET intervening bodies yet: {:?}",
get_order.lock().unwrap()
);
assert_eq!(head_calls.lock().unwrap().as_slice(), &["a", "b"]);
allow_get.store(true, Ordering::SeqCst);
let mut buf = [0u8; 2];
reader.read_exact(&mut buf).unwrap();
assert_eq!(&buf, b"bb");
}
#[test]
fn seek_bytes_disables_head_on_length_mismatch() {
let allow_get = Arc::new(AtomicBool::new(false));
let mut reader = closure_reader(
vec!["init".into(), "a".into(), "b".into()],
None,
true,
{
let allow_get = allow_get.clone();
move |fragment: &Fragment, _expected_len: Option<u64>| {
let url = fragment.url.clone();
if url != "init" {
while !allow_get.load(Ordering::SeqCst) {
thread::sleep(Duration::from_millis(5));
}
}
match url.as_str() {
"init" => Ok(b"INIT".to_vec()),
"a" => Ok(b"AAAA".to_vec()),
"b" => Ok(b"BBBB".to_vec()),
other => Err(Error::StreamInitializationError(format!(
"unknown url {other}"
))),
}
}
},
move |fragment: &Fragment| {
let url = fragment.url.clone();
match url.as_str() {
"init" => Ok(4),
"a" => Ok(99),
"b" => Ok(4),
other => Err(Error::StreamInitializationError(format!(
"unknown url {other}"
))),
}
},
)
.unwrap();
reader.seek_bytes(SeekFrom::Start(5)).unwrap();
allow_get.store(true, Ordering::SeqCst);
let mut buf = [0u8; 1];
reader.read_exact(&mut buf).unwrap();
assert_eq!(&buf, b"A");
assert!(
!reader.shared.lock().head_available,
"HEAD should be disabled after GET/HEAD length mismatch"
);
}
#[test]
fn seek_bytes_current_requires_known_position() {
let allow_a = Arc::new(AtomicBool::new(false));
let allow_b = Arc::new(AtomicBool::new(false));
let allow_c = Arc::new(AtomicBool::new(false));
let mut reader = closure_reader(
vec!["init".into(), "a".into(), "b".into(), "c".into()],
Some(MediaTimeline {
timescale: 1,
media_durations: vec![1, 1, 1],
}),
true,
{
let allow_a = allow_a.clone();
let allow_b = allow_b.clone();
let allow_c = allow_c.clone();
move |fragment: &Fragment, _expected_len: Option<u64>| match fragment.url.as_str() {
"init" => Ok(b"INIT".to_vec()),
"a" => {
while !allow_a.load(Ordering::SeqCst) {
thread::sleep(Duration::from_millis(5));
}
Ok(b"AAAA".to_vec())
}
"b" => {
while !allow_b.load(Ordering::SeqCst) {
thread::sleep(Duration::from_millis(5));
}
Ok(b"BBBB".to_vec())
}
"c" => {
while !allow_c.load(Ordering::SeqCst) {
thread::sleep(Duration::from_millis(5));
}
Ok(b"CCCC".to_vec())
}
other => Err(Error::StreamInitializationError(format!(
"unknown url {other}"
))),
}
},
failing_head(),
)
.unwrap();
let handle = thread::spawn(move || {
let position = reader.seek_time_coarse(Duration::from_secs(2)).unwrap();
let error = reader.seek_bytes(SeekFrom::Current(1)).unwrap_err();
(position, error.kind())
});
thread::sleep(Duration::from_millis(20));
allow_a.store(true, Ordering::SeqCst);
allow_c.store(true, Ordering::SeqCst);
let (position, kind) = handle.join().unwrap();
assert_eq!(
position,
Position {
byte: None,
time: Some(TimePosition {
timestamp: Duration::from_secs(2),
accuracy: TimeAccuracy::Exact,
}),
}
);
assert_eq!(kind, io::ErrorKind::InvalidInput);
allow_b.store(true, Ordering::SeqCst);
}
#[test]
fn seek_time_coarse_maps_timeline_to_fragment_starts() {
let timeline = MediaTimeline {
timescale: 100,
media_durations: vec![100, 100],
};
let mut reader = closure_reader(
vec!["init".into(), "a".into(), "b".into()],
Some(timeline),
true,
instant_get(&[("init", b"INIT"), ("a", b"AAAA"), ("b", b"BBBB")]),
instant_head(&[("init", b"INIT"), ("a", b"AAAA"), ("b", b"BBBB")]),
)
.unwrap();
reader.read_to_end(&mut Vec::new()).unwrap();
assert_eq!(
reader.seek_time_coarse(Duration::from_secs(0)).unwrap(),
position(Some(4), Some((Duration::ZERO, TimeAccuracy::Exact))),
"time 0 is the first media fragment, after init"
);
let mut buf = [0u8; 4];
reader.read_exact(&mut buf).unwrap();
assert_eq!(&buf, b"AAAA");
assert_eq!(
reader.seek_time_coarse(Duration::from_secs(1)).unwrap(),
position(Some(8), Some((Duration::from_secs(1), TimeAccuracy::Exact)))
);
reader.read_exact(&mut buf).unwrap();
assert_eq!(&buf, b"BBBB");
assert_eq!(
reader.seek_time_coarse(Duration::from_millis(500)).unwrap(),
position(Some(4), Some((Duration::ZERO, TimeAccuracy::Coarse))),
"coarse seeks land at the containing fragment's moof"
);
reader.read_exact(&mut buf[..2]).unwrap();
assert_eq!(&buf[..2], b"AA");
}
#[test]
fn seek_time_coarse_prioritizes_target_fragment() {
let allow_a = Arc::new(AtomicBool::new(false));
let allow_b = Arc::new(AtomicBool::new(false));
let allow_c = Arc::new(AtomicBool::new(false));
let a_started = Arc::new(AtomicBool::new(false));
let order = Arc::new(Mutex::new(Vec::new()));
let mut reader = closure_reader(
vec!["init".into(), "a".into(), "b".into(), "c".into()],
Some(MediaTimeline {
timescale: 1,
media_durations: vec![1, 1, 1],
}),
true,
{
let allow_a = allow_a.clone();
let allow_b = allow_b.clone();
let allow_c = allow_c.clone();
let a_started = a_started.clone();
let order = order.clone();
move |fragment: &Fragment, _expected_len: Option<u64>| match fragment.url.as_str() {
"init" => Ok(b"INIT".to_vec()),
"a" => {
a_started.store(true, Ordering::SeqCst);
while !allow_a.load(Ordering::SeqCst) {
thread::sleep(Duration::from_millis(5));
}
order.lock().unwrap().push("a");
Ok(b"AAAA".to_vec())
}
"b" => {
while !allow_b.load(Ordering::SeqCst) {
thread::sleep(Duration::from_millis(5));
}
order.lock().unwrap().push("b");
Ok(b"BBBB".to_vec())
}
"c" => {
while !allow_c.load(Ordering::SeqCst) {
thread::sleep(Duration::from_millis(5));
}
order.lock().unwrap().push("c");
Ok(b"CCCC".to_vec())
}
other => Err(Error::StreamInitializationError(format!(
"unknown url {other}"
))),
}
},
failing_head(),
)
.unwrap();
while !a_started.load(Ordering::SeqCst) {
thread::sleep(Duration::from_millis(1));
}
let handle = thread::spawn(move || {
let position = reader.seek_time_coarse(Duration::from_secs(2)).unwrap();
let mut buf = [0u8; 4];
reader.read_exact(&mut buf).unwrap();
(position, buf)
});
thread::sleep(Duration::from_millis(30));
assert!(!handle.is_finished());
allow_a.store(true, Ordering::SeqCst);
allow_c.store(true, Ordering::SeqCst);
let (position, buf) = handle.join().unwrap();
assert_eq!(
position,
Position {
byte: None,
time: Some(TimePosition {
timestamp: Duration::from_secs(2),
accuracy: TimeAccuracy::Exact,
}),
}
);
assert_eq!(&buf, b"CCCC");
assert_eq!(order.lock().unwrap().as_slice(), &["a", "c"]);
allow_b.store(true, Ordering::SeqCst);
}
#[test]
fn get_worker_fills_ahead_of_cursor_before_backfill() {
let release_media = Arc::new(AtomicBool::new(false));
let order = Arc::new(Mutex::new(Vec::new()));
let mut reader = closure_reader(
vec![
"init".into(),
"a".into(),
"b".into(),
"c".into(),
"d".into(),
],
Some(MediaTimeline {
timescale: 1,
media_durations: vec![1, 1, 1, 1],
}),
true,
{
let release_media = release_media.clone();
let order = order.clone();
move |fragment: &Fragment, _expected_len: Option<u64>| {
let url = fragment.url.clone();
if url != "init" {
while !release_media.load(Ordering::SeqCst) {
thread::sleep(Duration::from_millis(5));
}
}
order.lock().unwrap().push(url.clone());
Ok(vec![url.as_bytes()[0]; 4])
}
},
failing_head(),
)
.unwrap();
let cache = reader.shared.clone();
let release_media_when_seeked = release_media.clone();
thread::spawn(move || {
loop {
if cache.lock().cursor_fragment >= 3 {
release_media_when_seeked.store(true, Ordering::SeqCst);
return;
}
thread::sleep(Duration::from_millis(5));
}
});
reader.seek_time_coarse(Duration::from_secs(2)).unwrap();
let _inner = reader.wait_until(|inner| inner.complete).unwrap();
let got = order.lock().unwrap().clone();
let b = got.iter().position(|url| url == "b").unwrap();
let c = got.iter().position(|url| url == "c").unwrap();
let d = got.iter().position(|url| url == "d").unwrap();
assert!(
c < b && d < b,
"suffix ahead of the playhead must download before backfill: {got:?}"
);
}
#[test]
fn seek_time_coarse_returns_known_when_prefix_filled() {
let timeline = MediaTimeline {
timescale: 1,
media_durations: vec![1, 1],
};
let mut reader = closure_reader(
vec!["init".into(), "a".into(), "b".into()],
Some(timeline),
true,
instant_get(&[("init", b"INIT"), ("a", b"AAAA"), ("b", b"BBBB")]),
instant_head(&[("init", b"INIT"), ("a", b"AAAA"), ("b", b"BBBB")]),
)
.unwrap();
reader.read_to_end(&mut Vec::new()).unwrap();
assert_eq!(
reader.seek_time_coarse(Duration::from_secs(1)).unwrap(),
position(Some(8), Some((Duration::from_secs(1), TimeAccuracy::Exact)))
);
}
#[test]
fn seek_bytes_backfills_leading_fragments_without_head() {
let allow_rest = Arc::new(AtomicBool::new(false));
let mut reader = closure_reader(
vec!["init".into(), "aaaa".into(), "bbbb".into()],
None,
true,
gated_get(allow_rest.clone(), |url| vec![url.as_bytes()[0]; 4]),
failing_head(),
)
.unwrap();
let handle = thread::spawn(move || {
reader.seek_bytes(SeekFrom::Start(6)).unwrap();
let mut buf = [0u8; 2];
reader.read_exact(&mut buf).unwrap();
buf
});
thread::sleep(Duration::from_millis(30));
assert!(!handle.is_finished());
allow_rest.store(true, Ordering::SeqCst);
assert_eq!(&handle.join().unwrap(), b"aa");
}
#[test]
fn seek_time_coarse_requires_timeline() {
let mut reader = start_instant(vec!["init".into()], &[("init", b"xyz")]).unwrap();
let error = reader.seek_time_coarse(Duration::ZERO).unwrap_err();
assert_eq!(error.kind(), io::ErrorKind::InvalidInput);
}
#[test]
fn eof_after_complete_download() {
let mut reader = start_instant(vec!["init".into()], &[("init", b"xyz")]).unwrap();
let mut buf = [0u8; 8];
assert_eq!(reader.read(&mut buf).unwrap(), 3);
assert_eq!(reader.read(&mut buf).unwrap(), 0);
}
#[test]
fn surfaces_download_failure() {
let calls = Arc::new(AtomicUsize::new(0));
let get = {
let calls = calls.clone();
move |fragment: &Fragment, _expected_len: Option<u64>| {
let url = fragment.url.clone();
let n = calls.fetch_add(1, Ordering::SeqCst);
if n == 0 {
Ok(b"INIT".to_vec())
} else {
Err(Error::StreamInitializationError(format!(
"failed to fetch {url}"
)))
}
}
};
let mut reader = closure_reader(
vec!["init".into(), "bad".into()],
None,
true,
get,
failing_head(),
)
.unwrap();
let mut buf = Vec::new();
let error = loop {
match reader.read_to_end(&mut buf) {
Ok(_) => {
buf.clear();
thread::sleep(Duration::from_millis(5));
}
Err(error) => break error,
}
};
assert_eq!(error.kind(), io::ErrorKind::Other);
assert!(error.to_string().contains("failed to fetch bad"));
}
#[test]
fn empty_url_list_is_rejected() {
let result = closure_reader(Vec::new(), None, true, instant_get(&[]), failing_head());
assert!(matches!(result, Err(Error::DashManifestMissingUrls)));
}
#[test]
fn head_worker_fills_sizes_without_bodies() {
let allow_late = Arc::new(AtomicBool::new(false));
let head_calls = Arc::new(Mutex::new(Vec::new()));
let get_expected = Arc::new(Mutex::new(Vec::new()));
let reader = closure_reader(
vec![
"init".into(),
"a".into(),
"b".into(),
"c".into(),
"d".into(),
"e".into(),
],
None,
true,
{
let allow_late = allow_late.clone();
let get_expected = get_expected.clone();
move |fragment: &Fragment, expected_len: Option<u64>| {
let url = fragment.url.clone();
get_expected
.lock()
.unwrap()
.push((url.clone(), expected_len));
if url == "d" || url == "e" {
while !allow_late.load(Ordering::SeqCst) {
thread::sleep(Duration::from_millis(5));
}
}
Ok(vec![url.as_bytes()[0]; 4])
}
},
{
let head_calls = head_calls.clone();
move |fragment: &Fragment| {
let url = fragment.url.clone();
head_calls.lock().unwrap().push(url.clone());
Ok(4)
}
},
)
.unwrap();
let inner = reader
.wait_until(|inner| {
inner.fragments[4].size().is_some() && inner.fragments[5].size().is_some()
})
.unwrap();
assert!(
inner.missing_body(4) && inner.missing_body(5),
"HEAD worker should learn sizes before those bodies are fetched"
);
drop(inner);
let heads = head_calls.lock().unwrap().clone();
assert!(heads.contains(&"d".to_string()), "{heads:?}");
assert!(heads.contains(&"e".to_string()), "{heads:?}");
allow_late.store(true, Ordering::SeqCst);
let _inner = reader.wait_until(|inner| inner.complete).unwrap();
let expected = get_expected.lock().unwrap().clone();
assert!(
expected
.iter()
.any(|(url, len)| url == "e" && *len == Some(4)),
"GET started after HEAD should receive the known length: {expected:?}"
);
}
#[test]
fn head_worker_heads_immediately_without_body_buffer() {
let allow_get = Arc::new(AtomicBool::new(false));
let head_calls = Arc::new(Mutex::new(Vec::new()));
let reader = closure_reader(
vec!["init".into(), "a".into(), "b".into(), "c".into()],
None,
true,
{
let allow_get = allow_get.clone();
move |fragment: &Fragment, _expected_len: Option<u64>| {
let url = fragment.url.clone();
if url != "init" {
while !allow_get.load(Ordering::SeqCst) {
thread::sleep(Duration::from_millis(5));
}
}
Ok(vec![url.as_bytes()[0]; 4])
}
},
{
let head_calls = head_calls.clone();
move |fragment: &Fragment| {
let url = fragment.url.clone();
head_calls.lock().unwrap().push(url.clone());
Ok(4)
}
},
)
.unwrap();
let inner = reader.wait_until(|inner| inner.all_sizes_known()).unwrap();
assert!(
(1..=3).all(|index| inner.missing_body(index)),
"HEAD worker should fill sizes without waiting for a body buffer"
);
drop(inner);
let heads = head_calls.lock().unwrap().clone();
assert!(heads.contains(&"a".to_string()), "{heads:?}");
assert!(heads.contains(&"b".to_string()), "{heads:?}");
assert!(heads.contains(&"c".to_string()), "{heads:?}");
allow_get.store(true, Ordering::SeqCst);
}
#[test]
fn get_worker_proceeds_while_head_is_blocked() {
let allow_heads = Arc::new(AtomicBool::new(false));
let reader = closure_reader(
vec!["init".into(), "a".into(), "b".into(), "c".into()],
None,
true,
instant_get(&[
("init", b"INIT"),
("a", b"AAAA"),
("b", b"BBBB"),
("c", b"CCCC"),
]),
{
let allow_heads = allow_heads.clone();
move |_fragment: &Fragment| {
while !allow_heads.load(Ordering::SeqCst) {
thread::sleep(Duration::from_millis(5));
}
Ok(4)
}
},
)
.unwrap();
let _inner = reader.wait_until(|inner| inner.complete).unwrap();
allow_heads.store(true, Ordering::SeqCst);
}
#[test]
fn head_worker_failure_disables_head() {
let allow_get = Arc::new(AtomicBool::new(false));
let head_calls = Arc::new(AtomicUsize::new(0));
let reader = closure_reader(
vec![
"init".into(),
"a".into(),
"b".into(),
"c".into(),
"d".into(),
],
None,
true,
{
let allow_get = allow_get.clone();
move |fragment: &Fragment, _expected_len: Option<u64>| {
let url = fragment.url.clone();
if url != "init" {
while !allow_get.load(Ordering::SeqCst) {
thread::sleep(Duration::from_millis(5));
}
}
Ok(vec![url.as_bytes()[0]; 4])
}
},
{
let head_calls = head_calls.clone();
move |_fragment: &Fragment| {
head_calls.fetch_add(1, Ordering::SeqCst);
Err(Error::StreamInitializationError("HEAD unavailable".into()))
}
},
)
.unwrap();
let inner = reader.wait_until(|inner| !inner.head_available).unwrap();
assert!(
head_calls.load(Ordering::SeqCst) >= 1,
"HEAD worker should attempt probes after the initialization GET"
);
drop(inner);
allow_get.store(true, Ordering::SeqCst);
let _inner = reader.wait_until(|inner| inner.complete).unwrap();
}
fn recording_head(
calls: Arc<Mutex<Vec<String>>>,
) -> impl Fn(&Fragment) -> Result<u64, Error> + Send + Sync {
move |fragment: &Fragment| {
let url = fragment.url.clone();
calls.lock().unwrap().push(url);
Ok(4)
}
}
#[test]
fn stopped_eager_head_spawns_no_worker() {
let head_calls = Arc::new(Mutex::new(Vec::new()));
let reader = closure_reader(
vec!["init".into(), "a".into(), "b".into()],
None,
false,
instant_get(&[("init", b"INIT"), ("a", b"AAAA"), ("b", b"BBBB")]),
recording_head(head_calls.clone()),
)
.unwrap();
let inner = reader.wait_until(|inner| inner.complete).unwrap();
assert!(!inner.eager_head && !inner.head_worker_running);
drop(inner);
assert!(head_calls.lock().unwrap().is_empty());
}
#[test]
fn start_eager_head_fills_sizes_after_stopped_construction() {
let allow_get = Arc::new(AtomicBool::new(false));
let head_calls = Arc::new(Mutex::new(Vec::new()));
let reader = closure_reader(
vec!["init".into(), "a".into(), "b".into(), "c".into()],
None,
false,
{
let allow_get = allow_get.clone();
move |fragment: &Fragment, _expected_len: Option<u64>| {
let url = fragment.url.clone();
if url != "init" {
while !allow_get.load(Ordering::SeqCst) {
thread::sleep(Duration::from_millis(5));
}
}
Ok(vec![url.as_bytes()[0]; 4])
}
},
recording_head(head_calls.clone()),
)
.unwrap();
thread::sleep(Duration::from_millis(20));
assert!(head_calls.lock().unwrap().is_empty());
reader.start_eager_head();
let inner = reader.wait_until(|inner| inner.all_sizes_known()).unwrap();
assert!(
inner.fragments[1..]
.iter()
.all(|data| data.body().is_none())
);
drop(inner);
let _inner = reader
.wait_until(|inner| !inner.head_worker_running)
.unwrap();
assert_eq!(head_calls.lock().unwrap().as_slice(), &["a", "b", "c"]);
allow_get.store(true, Ordering::SeqCst);
}
#[test]
fn stop_eager_head_halts_after_in_flight_request() {
let allow_get = Arc::new(AtomicBool::new(false));
let head_entered = Arc::new(AtomicUsize::new(0));
let release_head = Arc::new(AtomicBool::new(false));
let reader = closure_reader(
vec![
"init".into(),
"a".into(),
"b".into(),
"c".into(),
"d".into(),
],
None,
true,
{
let allow_get = allow_get.clone();
move |fragment: &Fragment, _expected_len: Option<u64>| {
let url = fragment.url.clone();
if url != "init" {
while !allow_get.load(Ordering::SeqCst) {
thread::sleep(Duration::from_millis(5));
}
}
Ok(vec![url.as_bytes()[0]; 4])
}
},
{
let head_entered = head_entered.clone();
let release_head = release_head.clone();
move |_fragment: &Fragment| {
head_entered.fetch_add(1, Ordering::SeqCst);
while !release_head.load(Ordering::SeqCst) {
thread::sleep(Duration::from_millis(5));
}
Ok(4)
}
},
)
.unwrap();
while head_entered.load(Ordering::SeqCst) == 0 {
thread::sleep(Duration::from_millis(5));
}
reader.stop_eager_head();
release_head.store(true, Ordering::SeqCst);
let inner = reader
.wait_until(|inner| !inner.head_worker_running)
.unwrap();
assert_eq!(head_entered.load(Ordering::SeqCst), 1);
assert_eq!(
inner.fragments[1].size(),
Some(4),
"in-flight HEAD should be kept"
);
assert!(
inner.fragments[2..]
.iter()
.all(|data| data.size().is_none())
);
drop(inner);
allow_get.store(true, Ordering::SeqCst);
}
#[test]
fn restart_during_in_flight_head_reuses_worker() {
let allow_get = Arc::new(AtomicBool::new(false));
let release_head = Arc::new(AtomicBool::new(false));
let active = Arc::new(AtomicUsize::new(0));
let max_active = Arc::new(AtomicUsize::new(0));
let reader = closure_reader(
vec!["init".into(), "a".into(), "b".into(), "c".into()],
None,
true,
{
let allow_get = allow_get.clone();
move |fragment: &Fragment, _expected_len: Option<u64>| {
let url = fragment.url.clone();
if url != "init" {
while !allow_get.load(Ordering::SeqCst) {
thread::sleep(Duration::from_millis(5));
}
}
Ok(vec![url.as_bytes()[0]; 4])
}
},
{
let release_head = release_head.clone();
let active = active.clone();
let max_active = max_active.clone();
move |_fragment: &Fragment| {
let now = active.fetch_add(1, Ordering::SeqCst) + 1;
max_active.fetch_max(now, Ordering::SeqCst);
while !release_head.load(Ordering::SeqCst) {
thread::sleep(Duration::from_millis(5));
}
active.fetch_sub(1, Ordering::SeqCst);
Ok(4)
}
},
)
.unwrap();
while active.load(Ordering::SeqCst) == 0 {
thread::sleep(Duration::from_millis(5));
}
for _ in 0..5 {
reader.stop_eager_head();
reader.start_eager_head();
}
release_head.store(true, Ordering::SeqCst);
let _inner = reader.wait_until(|inner| inner.all_sizes_known()).unwrap();
assert_eq!(max_active.load(Ordering::SeqCst), 1);
allow_get.store(true, Ordering::SeqCst);
}
#[test]
fn start_eager_head_is_noop_when_head_disabled() {
let head_calls = Arc::new(Mutex::new(Vec::new()));
let reader = closure_reader(
vec!["init".into(), "a".into()],
None,
false,
gated_get(Arc::new(AtomicBool::new(false)), |url| {
url.as_bytes().to_vec()
}),
recording_head(head_calls.clone()),
)
.unwrap();
reader.shared.lock().disable_head("test");
reader.start_eager_head();
assert!(!reader.shared.lock().head_worker_running);
assert!(head_calls.lock().unwrap().is_empty());
}
#[test]
fn stopped_eager_head_still_heads_on_demand_for_seeks() {
let head_calls = Arc::new(Mutex::new(Vec::new()));
let mut reader = closure_reader(
vec!["init".into(), "a".into(), "b".into()],
None,
false,
gated_get(Arc::new(AtomicBool::new(false)), |url| {
url.as_bytes().to_vec()
}),
recording_head(head_calls.clone()),
)
.unwrap();
let position = reader.seek_bytes(SeekFrom::End(0)).unwrap();
assert_eq!(position.byte, Some(4 + 4 + 4));
assert_eq!(head_calls.lock().unwrap().as_slice(), &["a", "b"]);
assert!(!reader.shared.lock().head_worker_running);
}
fn two_second_timeline() -> MediaTimeline {
MediaTimeline {
timescale: 44_100,
media_durations: vec![44_100, 44_100],
}
}
#[test]
fn estimate_len_errs_larger_than_advertised_media() {
let timeline = two_second_timeline();
let init_len = 4;
let bitrate = 128_000;
let raw_media = u64::from(bitrate) * 2 / 8;
let estimate = timeline.estimate_len(init_len, bitrate).unwrap();
assert!(
estimate >= init_len + raw_media * 4 + 2 * 64 * 1024,
"estimate {estimate} should pad raw media {raw_media}"
);
assert!(timeline.estimate_len(init_len, 0).is_none());
}
#[test]
fn symphonia_compat_end_zero_does_not_wait_for_bodies() {
let allow_rest = Arc::new(AtomicBool::new(false));
let mut reader = closure_reader(
vec!["init".into(), "a".into(), "b".into()],
Some(two_second_timeline()),
true,
gated_get(allow_rest.clone(), |url| vec![url.as_bytes()[0]; 4]),
failing_head(),
)
.unwrap();
reader.set_symphonia_compat(true, 128_000).unwrap();
let end = reader.seek_bytes(SeekFrom::End(0)).unwrap();
let estimate = two_second_timeline().estimate_len(4, 128_000).unwrap();
assert_eq!(end.byte, Some(estimate));
assert_eq!(
end.time,
Some(TimePosition {
timestamp: Duration::from_secs(2),
accuracy: TimeAccuracy::Exact,
})
);
reader.seek_bytes(SeekFrom::Start(0)).unwrap();
let mut init = [0u8; 4];
assert_eq!(reader.read(&mut init).unwrap(), 4);
assert_eq!(&init, b"iiii");
assert!(
!allow_rest.load(Ordering::SeqCst),
"End(0) must not wait on media bodies"
);
}
#[test]
fn symphonia_compat_end_zero_uses_exact_len_once_known() {
let mut reader = closure_reader(
vec!["init".into(), "a".into()],
Some(two_second_timeline()),
true,
instant_get(&[("init", b"INIT"), ("a", b"AAAA")]),
instant_head(&[("init", b"INIT"), ("a", b"AAAA")]),
)
.unwrap();
reader.set_symphonia_compat(true, 1_411_200).unwrap();
drop(reader.wait_until(|inner| inner.all_sizes_known()).unwrap());
let end = reader.seek_bytes(SeekFrom::End(0)).unwrap();
assert_eq!(end.byte, Some(8));
assert!(
end.byte.unwrap() < reader.length_estimate.unwrap(),
"exact total should replace the padded estimate"
);
}
#[test]
fn symphonia_compat_requires_timeline() {
let mut reader = start_instant(
vec!["init".into(), "a".into()],
&[("init", b"INIT"), ("a", b"AAAA")],
)
.unwrap();
let error = reader.set_symphonia_compat(true, 128_000).unwrap_err();
assert!(
matches!(error, Error::StreamInitializationError(message) if message.contains("SegmentTimeline"))
);
}
#[test]
fn byte_range_parses_dash_form() {
let range = parse_byte_range(" 100-199 ").unwrap();
assert_eq!(range, 100..=199);
assert_eq!(
Fragment::with_range("a", range.clone()).known_size(),
Some(100)
);
assert_eq!(Fragment::with_range("a", 5..=5).known_size(), Some(1));
for invalid in ["", "5", "5-", "-5", "9-3", "a-b"] {
assert!(
matches!(parse_byte_range(invalid), Err(Error::InvalidByteRange(value)) if value == invalid),
"{invalid:?} should be rejected"
);
}
assert_eq!(
Fragment::with_range("media.mp4", range).to_string(),
"media.mp4 (bytes 100-199)"
);
}
fn ranged_get(
file: &'static [u8],
) -> impl Fn(&Fragment, Option<u64>) -> Result<Vec<u8>, Error> + Send + Sync {
move |fragment: &Fragment, _expected_len: Option<u64>| {
let range = fragment.range.as_ref().expect("test fragments are ranged");
Ok(file[*range.start() as usize..=*range.end() as usize].to_vec())
}
}
fn ranged(url: &str, start: u64, end: u64) -> Fragment {
Fragment::with_range(url, start..=end)
}
#[test]
fn ranged_fragments_know_sizes_without_head() {
const FILE: &[u8] = b"INITaaaaBBBBBBcc";
let allow_media = Arc::new(AtomicBool::new(false));
let head_calls = Arc::new(Mutex::new(Vec::new()));
let mut reader = closure_reader(
vec![
ranged("file", 0, 3),
ranged("file", 4, 7),
ranged("file", 8, 13),
ranged("file", 14, 15),
],
None,
true,
{
let allow_media = allow_media.clone();
let get = ranged_get(FILE);
move |fragment: &Fragment, expected_len: Option<u64>| {
assert_eq!(expected_len, fragment.known_size());
if *fragment.range.as_ref().unwrap().start() > 0 {
while !allow_media.load(Ordering::SeqCst) {
thread::sleep(Duration::from_millis(5));
}
}
get(fragment, expected_len)
}
},
recording_head(head_calls.clone()),
)
.unwrap();
assert_eq!(reader.seek_bytes(SeekFrom::End(0)).unwrap().byte, Some(16));
assert_eq!(reader.seek_bytes(SeekFrom::Start(9)).unwrap().byte, Some(9));
assert!(reader.shared.lock().missing_body(2));
allow_media.store(true, Ordering::SeqCst);
let mut rest = Vec::new();
reader.read_to_end(&mut rest).unwrap();
assert_eq!(rest, b"BBBBBcc");
reader.seek_bytes(SeekFrom::Start(0)).unwrap();
let mut all = Vec::new();
reader.read_to_end(&mut all).unwrap();
assert_eq!(all, FILE);
assert!(head_calls.lock().unwrap().is_empty());
}
#[test]
fn ranged_fragment_with_wrong_length_fails() {
let result = closure_reader(
vec![ranged("file", 0, 7)],
None,
false,
|_fragment: &Fragment, _expected_len: Option<u64>| Ok(b"INIT".to_vec()),
failing_head(),
);
assert!(matches!(
result,
Err(Error::FragmentLength { fragment, expected: 8, actual: 4 })
if fragment == "file (bytes 0-7)"
));
let mut reader = closure_reader(
vec![ranged("file", 0, 3), ranged("file", 4, 7)],
None,
false,
|fragment: &Fragment, _expected_len: Option<u64>| match *fragment
.range
.as_ref()
.unwrap()
.start()
{
0 => Ok(b"INIT".to_vec()),
_ => Ok(b"short".to_vec()),
},
failing_head(),
)
.unwrap();
let mut bytes = Vec::new();
let error = reader.read_to_end(&mut bytes).unwrap_err();
assert!(error.to_string().contains("returned 5 bytes, expected 4"));
}
fn spawn_range_server(connections: usize) -> String {
use std::io::{BufRead, BufReader, Write};
use std::net::TcpListener;
const FILE: &[u8] = b"INITaaaaBBBBBBcc";
let listener = TcpListener::bind("127.0.0.1:0").unwrap();
let address = listener.local_addr().unwrap();
thread::spawn(move || {
for _ in 0..connections {
let (stream, _) = listener.accept().unwrap();
let mut reader = BufReader::new(stream);
let mut request_line = String::new();
reader.read_line(&mut request_line).unwrap();
let mut range = None;
let mut header = String::new();
while reader.read_line(&mut header).unwrap() > 2 {
if let Some(value) = header.to_ascii_lowercase().strip_prefix("range: bytes=") {
range = Some(parse_byte_range(value).unwrap());
}
header.clear();
}
let (status, body) = match range {
Some(range) if request_line.starts_with("GET /ranged ") => (
"206 Partial Content",
&FILE[*range.start() as usize..=*range.end() as usize],
),
_ => ("200 OK", FILE),
};
let head = format!(
"HTTP/1.1 {status}\r\nContent-Length: {}\r\nConnection: close\r\n\r\n",
body.len()
);
let stream = reader.get_mut();
stream.write_all(head.as_bytes()).unwrap();
stream.write_all(body).unwrap();
}
});
format!("http://{address}")
}
#[test]
fn default_transport_sends_range_and_slices_ignored_ranges() {
let base = spawn_range_server(3);
let mut reader = StreamReader::new(
vec![
ranged(&format!("{base}/ranged"), 0, 3),
ranged(&format!("{base}/plain"), 8, 13),
ranged(&format!("{base}/ranged"), 14, 15),
],
None,
false,
)
.unwrap();
let mut bytes = Vec::new();
reader.read_to_end(&mut bytes).unwrap();
assert_eq!(bytes, b"INITBBBBBBcc");
}
struct CountingTransport {
gets: Mutex<Vec<String>>,
heads: Mutex<Vec<String>>,
gated: &'static [&'static str],
released: AtomicBool,
}
impl CountingTransport {
fn new(gated: &'static [&'static str]) -> Self {
Self {
gets: Mutex::new(Vec::new()),
heads: Mutex::new(Vec::new()),
gated,
released: AtomicBool::new(false),
}
}
fn gets(&self) -> Vec<String> {
self.gets.lock().unwrap().clone()
}
fn heads(&self) -> Vec<String> {
self.heads.lock().unwrap().clone()
}
}
impl Transport for CountingTransport {
fn head(&self, fragment: &Fragment) -> Result<u64, Error> {
self.heads.lock().unwrap().push(fragment.url.clone());
Ok(fragment.url.len() as u64)
}
fn get(&self, fragment: &Fragment, _size_hint: Option<u64>) -> Result<Arc<[u8]>, Error> {
self.gets.lock().unwrap().push(fragment.url.clone());
if self.gated.contains(&fragment.url.as_str()) {
while !self.released.load(Ordering::SeqCst) {
thread::sleep(Duration::from_millis(5));
}
}
Ok(fragment.url.as_bytes().into())
}
}
fn wait_for(mut ready: impl FnMut() -> bool) {
for _ in 0..400 {
if ready() {
return;
}
thread::sleep(Duration::from_millis(5));
}
panic!("condition not reached within 2s");
}
fn counting_cache(
gated: &'static [&'static str],
) -> (Arc<FragmentCache>, Arc<CountingTransport>) {
let transport = Arc::new(CountingTransport::new(gated));
(Arc::new(FragmentCache::new(transport.clone())), transport)
}
#[test]
fn fragment_cache_reuses_bodies_across_readers() {
let (cache, transport) = counting_cache(&[]);
let mut first = StreamReader::new_with_cache(
vec!["init".into(), "a".into(), "b".into()],
None,
false,
cache.clone(),
)
.unwrap();
let mut bytes = Vec::new();
first.read_to_end(&mut bytes).unwrap();
assert_eq!(bytes, b"initab");
let mut second = StreamReader::new_with_cache(
vec!["init".into(), "b".into()],
None,
true,
cache.clone(),
)
.unwrap();
bytes.clear();
second.read_to_end(&mut bytes).unwrap();
assert_eq!(bytes, b"initb");
assert_eq!(transport.gets(), ["init", "a", "b"]);
assert_eq!(cache.known_size(&"a".into()), Some(1));
assert!(transport.heads().is_empty());
}
#[test]
fn buffer_reports_cache_changes() {
let (cache, transport) = counting_cache(&["bb"]);
let reader = StreamReader::new_with_cache(
vec!["init".into(), "aa".into(), "bb".into()],
Some(MediaTimeline {
timescale: 10,
media_durations: vec![10, 10],
}),
false,
cache,
)
.unwrap();
let buffer = reader.buffer().unwrap();
let reported = Arc::new(Mutex::new(Vec::new()));
buffer.on_change({
let reported = reported.clone();
move |status: &BufferStatus| reported.lock().unwrap().push(status.clone())
});
wait_for(|| buffer.status().downloaded.fragments == 1);
let status = buffer.status();
assert_eq!(status.total.fragments, 2);
assert_eq!(status.total.time, Duration::from_secs(2));
assert_eq!(status.ranges, [Duration::ZERO..Duration::from_secs(1)]);
transport.released.store(true, Ordering::SeqCst);
wait_for(|| {
reported
.lock()
.unwrap()
.iter()
.any(BufferStatus::is_complete)
});
let status = buffer.status();
assert_eq!(status.ranges, [Duration::ZERO..Duration::from_secs(2)]);
assert_eq!(status.downloaded.bytes, 4);
}
#[test]
fn buffer_needs_a_timeline() {
let (cache, _transport) = counting_cache(&[]);
let reader =
StreamReader::new_with_cache(vec!["init".into(), "a".into()], None, false, cache)
.unwrap();
assert!(reader.buffer().is_none());
}
#[test]
fn fragment_cache_keeps_in_flight_bodies_after_reader_drop() {
let (cache, transport) = counting_cache(&["b"]);
let first = StreamReader::new_with_cache(
vec!["init".into(), "a".into(), "b".into()],
None,
false,
cache.clone(),
)
.unwrap();
wait_for(|| transport.gets().contains(&"b".to_owned()));
drop(first);
transport.released.store(true, Ordering::SeqCst);
wait_for(|| cache.cached(&"b".into()).is_some());
let mut second = StreamReader::new_with_cache(
vec!["init".into(), "b".into()],
None,
false,
cache.clone(),
)
.unwrap();
let mut bytes = Vec::new();
second.read_to_end(&mut bytes).unwrap();
assert_eq!(bytes, b"initb");
assert_eq!(transport.gets(), ["init", "a", "b"]);
}
#[test]
fn fragment_cache_joins_a_get_already_in_flight() {
let (cache, transport) = counting_cache(&["b"]);
let first = StreamReader::new_with_cache(
vec!["init".into(), "a".into(), "b".into()],
None,
false,
cache.clone(),
)
.unwrap();
wait_for(|| transport.gets().contains(&"b".to_owned()));
drop(first);
let mut second = StreamReader::new_with_cache(
vec!["init".into(), "b".into()],
None,
false,
cache.clone(),
)
.unwrap();
transport.released.store(true, Ordering::SeqCst);
let mut bytes = Vec::new();
second.read_to_end(&mut bytes).unwrap();
assert_eq!(bytes, b"initb");
assert_eq!(transport.gets(), ["init", "a", "b"]);
}
#[test]
fn fragment_cache_skips_wrong_length_ranged_bodies() {
let cache = FragmentCache::new(Arc::new(FnTransport {
get: |_fragment: &Fragment, _size_hint: Option<u64>| Ok(b"short".to_vec()),
head: |_fragment: &Fragment| Ok(0),
}));
let fragment = ranged("file", 0, 7);
assert!(matches!(
cache.get(&cache.data(fragment.clone())),
Err(Error::FragmentLength {
expected: 8,
actual: 5,
..
})
));
assert_eq!(cache.cached(&fragment), None);
let whole = Fragment::new("file");
cache.get(&cache.data(whole.clone())).unwrap();
assert_eq!(cache.cached(&whole).as_deref(), Some(&b"short"[..]));
}
fn spawn_keep_alive_server() -> (String, Arc<AtomicUsize>) {
use std::io::{BufRead, BufReader, Write};
use std::net::TcpListener;
let listener = TcpListener::bind("127.0.0.1:0").unwrap();
let base = format!("http://{}", listener.local_addr().unwrap());
let connections = Arc::new(AtomicUsize::new(0));
let accepted = connections.clone();
thread::spawn(move || {
for stream in listener.incoming() {
accepted.fetch_add(1, Ordering::SeqCst);
thread::spawn(move || {
let mut reader = BufReader::new(stream.unwrap());
loop {
let mut request_line = String::new();
if reader.read_line(&mut request_line).unwrap_or(0) == 0 {
return;
}
let mut header = String::new();
while reader.read_line(&mut header).unwrap() > 2 {
header.clear();
}
let path = request_line.split(' ').nth(1).unwrap();
let body = path.trim_start_matches('/');
let response = format!(
"HTTP/1.1 200 OK\r\nContent-Length: {}\r\n\r\n{body}",
body.len()
);
reader.get_mut().write_all(response.as_bytes()).unwrap();
}
});
}
});
(base, connections)
}
#[test]
fn fragment_cache_reuses_pooled_connections_across_readers() {
let (base, connections) = spawn_keep_alive_server();
let fragments = |paths: &[&str]| -> Vec<Fragment> {
paths
.iter()
.map(|path| format!("{base}/{path}").into())
.collect()
};
let read_all = |mut reader: StreamReader| {
let mut bytes = Vec::new();
reader.read_to_end(&mut bytes).unwrap();
bytes
};
let cache = Arc::new(FragmentCache::default());
let first =
StreamReader::new_with_cache(fragments(&["init", "a"]), None, false, cache.clone())
.unwrap();
assert_eq!(read_all(first), b"inita");
let second =
StreamReader::new_with_cache(fragments(&["init", "b"]), None, false, cache.clone())
.unwrap();
assert_eq!(read_all(second), b"initb");
assert_eq!(connections.load(Ordering::SeqCst), 1);
let third = StreamReader::new(fragments(&["init", "c"]), None, false).unwrap();
assert_eq!(read_all(third), b"initc");
assert_eq!(connections.load(Ordering::SeqCst), 2);
}
#[test]
fn fragment_cache_remembers_head_lengths_across_readers() {
let (cache, transport) = counting_cache(&["a", "bb"]);
let fragments = || vec!["init".into(), "a".into(), "bb".into()];
let first = StreamReader::new_with_cache(fragments(), None, true, cache.clone()).unwrap();
wait_for(|| cache.known_size(&"bb".into()).is_some());
drop(first);
let mut second =
StreamReader::new_with_cache(fragments(), None, true, cache.clone()).unwrap();
assert!(second.shared.lock().all_sizes_known());
assert_eq!(second.seek_bytes(SeekFrom::End(0)).unwrap().byte, Some(7));
assert_eq!(transport.heads(), ["a", "bb"]);
assert_eq!(cache.cached(&"bb".into()), None);
transport.released.store(true, Ordering::SeqCst);
wait_for(|| cache.cached(&"bb".into()).is_some());
assert_eq!(cache.known_size(&"bb".into()), Some(2));
}
}