use std::{
collections::HashMap,
fmt, io,
path::PathBuf,
sync::{
Arc,
atomic::{AtomicBool, AtomicU8, AtomicU64, AtomicUsize, Ordering},
},
time::Duration,
};
use parking_lot::{Mutex, RwLock};
use super::{
Source,
disk_cache::MediaDiskCache,
reader::{BytesReader, FileReader, Reader, read_all},
source::SourceKind,
thread_pool::ThreadPool,
};
use crate::time::Instant;
pub const DEFAULT_LOAD_TIMEOUT: Duration = Duration::from_secs(30);
pub const DEFAULT_MAX_LOAD_BYTES: u64 = 64 * 1024 * 1024;
const DEFAULT_FIRST_READ_BLOCK: u64 = 256 << 10;
const DEFAULT_READ_BLOCK: u64 = 1 << 20;
const LOADER_THREADS: usize = 4;
const KEPT_STARTS: usize = 8;
const IDLE_GRACE: Duration = Duration::from_millis(250);
const PROGRESS_STEP_BYTES: u64 = 256 * 1024;
pub type LoadWaker = Arc<dyn Fn() + Send + Sync>;
#[derive(Clone, Debug, PartialEq, Eq)]
pub enum LoadError {
NotFound,
Denied,
Status(u16),
TooLarge { limit: u64 },
TimedOut,
Cancelled,
Unsupported(&'static str),
Io(String),
}
impl fmt::Display for LoadError {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
match self {
Self::NotFound => f.write_str("it was not found"),
Self::Denied => f.write_str("the app does not allow this address"),
Self::Status(status) => write!(f, "the server answered {status}"),
Self::TooLarge { limit } => write!(f, "it is larger than {limit} bytes"),
Self::TimedOut => f.write_str("it took too long"),
Self::Cancelled => f.write_str("the load was stopped"),
Self::Unsupported(why) => f.write_str(why),
Self::Io(reason) => f.write_str(reason),
}
}
}
impl std::error::Error for LoadError {}
impl From<io::Error> for LoadError {
fn from(error: io::Error) -> Self {
if let Some(error) = error.get_ref().and_then(|inner| inner.downcast_ref::<LoadError>()) {
return error.clone();
}
match error.kind() {
io::ErrorKind::NotFound => Self::NotFound,
io::ErrorKind::TimedOut => Self::TimedOut,
io::ErrorKind::Interrupted => Self::Cancelled,
_ => Self::Io(error.to_string()),
}
}
}
#[derive(Clone, Debug)]
pub enum LoadState {
Loading {
loaded: u64,
total: Option<u64>,
},
Ready(Arc<[u8]>),
Failed(LoadError),
}
impl LoadState {
pub fn is_loading(&self) -> bool {
matches!(self, Self::Loading { .. })
}
}
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, PartialOrd, Ord, Hash)]
pub enum LoadPriority {
Low,
#[default]
Normal,
High,
}
impl LoadPriority {
fn of(level: u8) -> Self {
match level {
0 => Self::Low,
1 => Self::Normal,
_ => Self::High,
}
}
}
static ASKED: AtomicU64 = AtomicU64::new(1);
struct Entry {
state: Mutex<LoadState>,
cancelled: AtomicBool,
priority: AtomicU8,
asked: AtomicU64,
holders: AtomicUsize,
idle_since: Mutex<Option<Instant>>,
wakers: Mutex<Vec<LoadWaker>>,
told: AtomicU64,
}
impl Entry {
fn new(state: LoadState) -> Arc<Self> {
Arc::new(Self {
state: Mutex::new(state),
cancelled: AtomicBool::new(false),
priority: AtomicU8::new(LoadPriority::Normal as u8),
asked: AtomicU64::new(ASKED.fetch_add(1, Ordering::Relaxed)),
holders: AtomicUsize::new(0),
idle_since: Mutex::new(Some(Instant::now())),
wakers: Mutex::new(Vec::new()),
told: AtomicU64::new(0),
})
}
fn wake(&self) {
let wakers = self.wakers.lock().clone();
for wake in wakers {
wake();
}
}
fn wanted(&self) -> bool {
!self.cancelled.load(Ordering::Acquire)
}
fn progress(&self, loaded: u64, total: Option<u64>) -> bool {
*self.state.lock() = LoadState::Loading { loaded, total };
let step = total.map_or(PROGRESS_STEP_BYTES, |total| (total / 100).max(16 * 1024));
if loaded.saturating_sub(self.told.load(Ordering::Relaxed)) >= step {
self.told.store(loaded, Ordering::Relaxed);
self.wake();
}
!self.cancelled.load(Ordering::Acquire)
}
fn finish(&self, result: Result<Vec<u8>, LoadError>) {
*self.state.lock() = match result {
Ok(bytes) => LoadState::Ready(Arc::from(bytes)),
Err(error) => LoadState::Failed(error),
};
self.wake();
}
}
#[cfg(not(target_arch = "wasm32"))]
struct Waits {
turn: u64,
entry: Arc<Entry>,
read: Box<dyn FnOnce() + Send>,
}
#[cfg(not(target_arch = "wasm32"))]
#[derive(Default)]
struct Waiting {
reads: Mutex<Vec<Waits>>,
turns: AtomicU64,
}
#[cfg(not(target_arch = "wasm32"))]
impl Waiting {
fn push(self: &Arc<Self>, entry: Arc<Entry>, read: Box<dyn FnOnce() + Send>) -> Turn {
let turn = self.turns.fetch_add(1, Ordering::Relaxed);
self.reads.lock().push(Waits { turn, entry, read });
Turn {
waiting: self.clone(),
turn,
taken: false,
}
}
fn next(&self) -> Option<Box<dyn FnOnce() + Send>> {
let (next, gone) = {
let mut reads = self.reads.lock();
let (kept, gone): (Vec<Waits>, Vec<Waits>) = std::mem::take(&mut *reads)
.into_iter()
.partition(|waits| waits.entry.wanted());
*reads = kept;
let best = reads
.iter()
.enumerate()
.max_by_key(|(_, waits)| {
(
waits.entry.priority.load(Ordering::Relaxed),
waits.entry.asked.load(Ordering::Relaxed),
)
})
.map(|(at, _)| at);
(best.map(|at| reads.swap_remove(at)), gone)
};
drop(gone);
next.map(|waits| waits.read)
}
fn give_up(&self, turn: u64) {
let gone = {
let mut reads = self.reads.lock();
let at = reads.iter().position(|waits| waits.turn == turn).or_else(|| {
reads
.iter()
.enumerate()
.min_by_key(|(_, waits)| {
(
waits.entry.priority.load(Ordering::Relaxed),
waits.entry.asked.load(Ordering::Relaxed),
)
})
.map(|(at, _)| at)
});
at.map(|at| reads.swap_remove(at))
};
drop(gone);
}
}
#[cfg(not(target_arch = "wasm32"))]
struct Turn {
waiting: Arc<Waiting>,
turn: u64,
taken: bool,
}
#[cfg(not(target_arch = "wasm32"))]
impl Turn {
fn run(mut self) {
self.taken = true;
if let Some(read) = self.waiting.next() {
read();
}
}
}
#[cfg(not(target_arch = "wasm32"))]
impl Drop for Turn {
fn drop(&mut self) {
if !self.taken {
self.waiting.give_up(self.turn);
}
}
}
#[cfg(not(target_arch = "wasm32"))]
struct Unread(Arc<Entry>);
#[cfg(not(target_arch = "wasm32"))]
impl Drop for Unread {
fn drop(&mut self) {
if self.0.state.lock().is_loading() {
self.0.finish(Err(LoadError::Cancelled));
}
}
}
pub struct Load {
entry: Arc<Entry>,
}
impl Load {
fn of(entry: &Arc<Entry>) -> Self {
entry.holders.fetch_add(1, Ordering::AcqRel);
*entry.idle_since.lock() = None;
Self { entry: entry.clone() }
}
pub fn state(&self) -> LoadState {
self.entry.state.lock().clone()
}
pub fn set_priority(&self, priority: LoadPriority) {
self.entry.priority.store(priority as u8, Ordering::Relaxed);
}
pub fn with_priority(self, priority: LoadPriority) -> Self {
self.set_priority(priority);
self
}
pub fn priority(&self) -> LoadPriority {
LoadPriority::of(self.entry.priority.load(Ordering::Relaxed))
}
pub fn wake_with(&self, wake: LoadWaker) {
let mut wakers = self.entry.wakers.lock();
if !wakers.iter().any(|known| Arc::ptr_eq(known, &wake)) {
wakers.push(wake);
}
}
}
impl Clone for Load {
fn clone(&self) -> Self {
Self::of(&self.entry)
}
}
impl Drop for Load {
fn drop(&mut self) {
if self.entry.holders.fetch_sub(1, Ordering::AcqRel) == 1 {
*self.entry.idle_since.lock() = Some(Instant::now());
}
}
}
type Allow = Arc<dyn Fn(&str) -> bool + Send + Sync>;
type Proxy = Arc<dyn Fn(&str) -> String + Send + Sync>;
type HeaderRule = Arc<dyn Fn(&str) -> Vec<(String, String)> + Send + Sync>;
struct Settings {
root: Option<PathBuf>,
timeout: Duration,
max_bytes: u64,
allow: Option<Allow>,
proxy: Option<Proxy>,
pool_size: usize,
headers: Option<HeaderRule>,
#[cfg_attr(not(all(feature = "http", not(target_arch = "wasm32"))), allow(dead_code))]
first_block: u64,
#[cfg_attr(not(all(feature = "http", not(target_arch = "wasm32"))), allow(dead_code))]
block: u64,
}
struct Shared {
settings: RwLock<Settings>,
entries: Mutex<HashMap<u64, Arc<Entry>>>,
#[cfg(not(target_arch = "wasm32"))]
waiting: Arc<Waiting>,
pool: Mutex<Option<ThreadPool>>,
disk: MediaDiskCache,
agents: AgentPool,
#[cfg_attr(not(all(feature = "http", not(target_arch = "wasm32"))), allow(dead_code))]
starts: StartPool,
#[cfg(feature = "tokio")]
runtime: Mutex<Option<tokio::runtime::Handle>>,
#[cfg(feature = "tokio")]
room: Mutex<Arc<tokio::sync::Semaphore>>,
}
impl Drop for Shared {
fn drop(&mut self) {
for entry in self.entries.lock().values() {
entry.cancelled.store(true, Ordering::Release);
}
}
}
#[derive(Clone)]
pub struct Loader {
shared: Arc<Shared>,
}
impl Default for Loader {
fn default() -> Self {
Self::new()
}
}
impl Loader {
pub fn new() -> Self {
Self {
shared: Arc::new(Shared {
settings: RwLock::new(Settings {
root: None,
timeout: DEFAULT_LOAD_TIMEOUT,
max_bytes: DEFAULT_MAX_LOAD_BYTES,
allow: None,
proxy: None,
pool_size: LOADER_THREADS,
headers: None,
first_block: DEFAULT_FIRST_READ_BLOCK,
block: DEFAULT_READ_BLOCK,
}),
entries: Mutex::new(HashMap::new()),
#[cfg(not(target_arch = "wasm32"))]
waiting: Arc::default(),
pool: Mutex::new(None),
agents: agent_pool(LOADER_THREADS),
starts: start_pool(KEPT_STARTS),
disk: MediaDiskCache::new(),
#[cfg(feature = "tokio")]
runtime: Mutex::new(None),
#[cfg(feature = "tokio")]
room: Mutex::new(Arc::new(tokio::sync::Semaphore::new(LOADER_THREADS))),
}),
}
}
pub fn root(&self, root: impl Into<PathBuf>) -> &Self {
self.shared.settings.write().root = Some(root.into());
self
}
pub fn reset_root(&self) -> &Self {
self.shared.settings.write().root = None;
self
}
pub fn timeout(&self, timeout: Duration) -> &Self {
self.shared.settings.write().timeout = timeout;
self
}
pub fn max_bytes(&self, max_bytes: u64) -> &Self {
self.shared.settings.write().max_bytes = max_bytes;
self
}
pub fn pool_size(&self, size: usize) -> &Self {
let size = size.max(1);
{
let mut settings = self.shared.settings.write();
if settings.pool_size == size {
return self;
}
settings.pool_size = size;
}
#[cfg(all(feature = "http", not(target_arch = "wasm32")))]
self.shared.agents.resize(size);
#[cfg(feature = "tokio")]
{
*self.shared.room.lock() = Arc::new(tokio::sync::Semaphore::new(size));
}
if let Some(before) = self.shared.pool.lock().take() {
let _ = std::thread::Builder::new()
.name("lurq-loader-ending".to_owned())
.spawn(move || drop(before));
}
self
}
pub fn allow(&self, allow: impl Fn(&str) -> bool + Send + Sync + 'static) -> &Self {
self.shared.settings.write().allow = Some(Arc::new(allow));
self
}
pub fn proxy(&self, proxy: impl Fn(&str) -> String + Send + Sync + 'static) -> &Self {
self.shared.settings.write().proxy = Some(Arc::new(proxy));
self
}
fn fetched_from(&self, address: &str) -> String {
match &self.shared.settings.read().proxy {
Some(proxy) => proxy(address),
None => address.to_owned(),
}
}
pub fn headers(&self, rule: impl Fn(&str) -> Vec<(String, String)> + Send + Sync + 'static) -> &Self {
self.shared.settings.write().headers = Some(Arc::new(rule));
self
}
pub(crate) fn headers_of(&self, source: &Source) -> Vec<(Arc<str>, Arc<str>)> {
let rule = self.shared.settings.read().headers.clone();
let mut headers: Vec<(Arc<str>, Arc<str>)> = match (&source.kind, rule) {
(SourceKind::Url(address), Some(rule)) => rule(address)
.into_iter()
.map(|(name, value)| (Arc::from(name), Arc::from(value)))
.collect(),
_ => Vec::new(),
};
for (name, value) in &source.headers {
headers.retain(|(had, _)| !had.eq_ignore_ascii_case(name));
headers.push((name.clone(), value.clone()));
}
headers
}
#[doc(hidden)]
pub fn read_block(&self, bytes: u64) -> &Self {
self.read_blocks(bytes, bytes)
}
pub fn read_blocks(&self, first: u64, most: u64) -> &Self {
let mut settings = self.shared.settings.write();
settings.block = most.max(1);
settings.first_block = first.clamp(1, settings.block);
drop(settings);
self
}
pub fn keep_starts(&self, addresses: usize) -> &Self {
#[cfg(all(feature = "http", not(target_arch = "wasm32")))]
self.shared.starts.resize(addresses);
#[cfg(not(all(feature = "http", not(target_arch = "wasm32"))))]
let _ = addresses;
self
}
pub fn connect(&self, address: &str) -> &Self {
#[cfg(all(feature = "http", not(target_arch = "wasm32")))]
if self.allows(address) {
let timeout = self.shared.settings.read().timeout;
super::http::connect(
&self.shared.agents,
self.fetched_from(address),
self.headers_of(&Source::url(address)),
timeout,
);
}
#[cfg(not(all(feature = "http", not(target_arch = "wasm32"))))]
let _ = address;
self
}
#[cfg(feature = "tokio")]
pub fn tokio_handle(&self, handle: tokio::runtime::Handle) -> &Self {
*self.shared.runtime.lock() = Some(handle);
self
}
#[cfg(feature = "tokio")]
pub fn reset_tokio_handle(&self) -> &Self {
*self.shared.runtime.lock() = None;
self
}
#[cfg(all(test, feature = "tokio"))]
pub(crate) fn reads_on_a_runtime(&self) -> bool {
self.shared.runtime.lock().is_some()
}
pub fn enable_media_disk_cache(&self, folder: impl Into<PathBuf>) -> &Self {
self.shared.disk.turn_on(folder.into());
self
}
pub fn media_disk_cache(&self) -> &MediaDiskCache {
&self.shared.disk
}
#[cfg_attr(not(feature = "media"), allow(dead_code))]
pub(crate) fn allows(&self, address: &str) -> bool {
let allow = self.shared.settings.read().allow.clone();
allow.is_none_or(|allow| allow(address))
}
pub(crate) fn resource_path(&self, resource: &str) -> PathBuf {
match &self.shared.settings.read().root {
Some(root) => root.join(resource),
None => PathBuf::from(resource),
}
}
pub fn load(&self, source: &Source) -> Load {
let key = source.cache_key();
let mut entries = self.shared.entries.lock();
if let Some(entry) = entries.get(&key) {
entry
.asked
.store(ASKED.fetch_add(1, Ordering::Relaxed), Ordering::Relaxed);
return Load::of(entry);
}
let entry = match &source.kind {
SourceKind::Bytes(bytes) => Entry::new(LoadState::Ready(bytes.clone())),
_ => Entry::new(LoadState::Loading { loaded: 0, total: None }),
};
entries.insert(key, entry.clone());
drop(entries);
let load = Load::of(&entry);
if !matches!(source.kind, SourceKind::Bytes(_)) {
self.start(source, entry, None);
}
load
}
pub fn load_start(&self, source: &Source, bytes: u64) -> Load {
let key = {
let mut hasher = std::collections::hash_map::DefaultHasher::new();
std::hash::Hash::hash(&("start", source.cache_key(), bytes), &mut hasher);
std::hash::Hasher::finish(&hasher)
};
let mut entries = self.shared.entries.lock();
if let Some(entry) = entries.get(&key) {
return Load::of(entry);
}
let entry = match &source.kind {
SourceKind::Bytes(all) => Entry::new(LoadState::Ready(Arc::from(
&all[..all.len().min(usize::try_from(bytes).unwrap_or(usize::MAX))],
))),
_ => Entry::new(LoadState::Loading { loaded: 0, total: None }),
};
entries.insert(key, entry.clone());
drop(entries);
let load = Load::of(&entry);
if !matches!(source.kind, SourceKind::Bytes(_)) {
self.start(source, entry, Some(bytes));
}
load
}
fn start(&self, source: &Source, entry: Arc<Entry>, head: Option<u64>) {
#[cfg(target_arch = "wasm32")]
{
let (timeout, max_bytes, allow) = {
let settings = self.shared.settings.read();
(settings.timeout, settings.max_bytes, settings.allow.clone())
};
let address = match &source.kind {
SourceKind::Url(address) if allow.as_ref().is_some_and(|allow| !allow(address)) => {
entry.finish(Err(LoadError::Denied));
return;
}
SourceKind::Url(address) => self.fetched_from(address),
SourceKind::File(path) => super::web_fetch::address_of(path),
SourceKind::Resource(resource) => super::web_fetch::address_of(&self.resource_path(resource)),
SourceKind::Bytes(_) => return,
};
let headers = self.headers_of(source);
wasm_bindgen_futures::spawn_local(async move {
let told = |loaded, total| entry.progress(loaded, total);
let result = super::web_fetch::fetch(
&address,
&headers,
timeout,
head.unwrap_or(max_bytes),
head.is_some(),
told,
)
.await;
entry.finish(result);
});
}
#[cfg(not(target_arch = "wasm32"))]
{
let (timeout, max_bytes, allow) = {
let settings = self.shared.settings.read();
(settings.timeout, settings.max_bytes, settings.allow.clone())
};
let path = match &source.kind {
SourceKind::File(path) => Some(path.clone()),
SourceKind::Resource(resource) => Some(self.resource_path(resource)),
_ => None,
};
if let SourceKind::Url(address) = &source.kind
&& allow.as_ref().is_some_and(|allow| !allow(address))
{
entry.finish(Err(LoadError::Denied));
return;
}
let fetched = match &source.kind {
SourceKind::Url(address) => self.fetched_from(address),
_ => String::new(),
};
let headers = self.headers_of(source);
let source = source.clone();
let disk = self.shared.disk.clone();
let agents = self.shared.agents.clone();
let unread = Unread(entry.clone());
let waits = entry.clone();
let read = move || {
let _unread = unread;
if !entry.wanted() {
return;
}
let told = |loaded, total| entry.progress(loaded, total);
let result = match (path, &source.kind, head) {
(Some(path), _, Some(head)) => FileReader::open(&path)
.and_then(|reader| read_start(&reader, head))
.map_err(LoadError::from),
(None, SourceKind::Url(_), Some(head)) => fetch(&agents, &fetched, &headers, timeout, head, true, told),
(Some(path), _, None) => FileReader::open(&path)
.and_then(|reader| read_all(&reader, Some(max_bytes), told))
.map_err(|error| match error.kind() {
io::ErrorKind::FileTooLarge => LoadError::TooLarge { limit: max_bytes },
_ => LoadError::from(error),
}),
(None, SourceKind::Url(address), None) => match disk.read(source.cache_key(), max_bytes) {
Some(bytes) => Ok(bytes),
None => fetch(&agents, &fetched, &headers, timeout, max_bytes, false, told).inspect(|bytes| {
disk.put(source.cache_key(), address, bytes);
}),
},
(None, ..) => Err(LoadError::Unsupported("nothing reads this source")),
};
entry.finish(result);
};
let turn = self.shared.waiting.push(waits, Box::new(read));
#[cfg(feature = "tokio")]
if let Some(handle) = self.shared.runtime.lock().clone() {
let room = self.shared.room.lock().clone();
drop(handle.spawn(async move {
if let Ok(_room) = room.acquire().await {
let _ = tokio::task::spawn_blocking(move || turn.run()).await;
}
}));
return;
}
let size = self.shared.settings.read().pool_size;
self
.shared
.pool
.lock()
.get_or_insert_with(|| ThreadPool::new(size))
.execute(move || turn.run());
}
}
pub fn open(&self, source: &Source) -> Result<Arc<dyn Reader>, LoadError> {
self.open_reading(source, None)
}
#[cfg_attr(not(all(feature = "media", windows)), allow(dead_code))]
pub(crate) fn open_to_play(&self, source: &Source, read_ahead: bool) -> Result<Arc<dyn Reader>, LoadError> {
self.open_reading(source, Some(read_ahead))
}
fn open_reading(&self, source: &Source, read_ahead: Option<bool>) -> Result<Arc<dyn Reader>, LoadError> {
match &source.kind {
SourceKind::Bytes(bytes) => Ok(Arc::new(BytesReader::new(bytes.clone()))),
SourceKind::File(path) => Ok(Arc::new(FileReader::open(path)?)),
SourceKind::Resource(resource) => Ok(Arc::new(FileReader::open(self.resource_path(resource))?)),
SourceKind::Url(address) if !self.allows(address) => Err(LoadError::Denied),
SourceKind::Url(address) => self.open_address(address, source, read_ahead),
}
}
#[cfg(all(feature = "http", not(target_arch = "wasm32")))]
fn open_address(
&self,
address: &str,
source: &Source,
read_ahead: Option<bool>,
) -> Result<Arc<dyn Reader>, LoadError> {
if let Some(path) = self.shared.disk.file(source.cache_key())
&& let Ok(reader) = FileReader::open(path)
{
return Ok(Arc::new(reader));
}
let (timeout, cut) = {
let settings = self.shared.settings.read();
let cut = super::http::Cut {
first: settings.first_block,
most: settings.block,
};
(settings.timeout, cut)
};
Ok(Arc::new(super::http::HttpReader::open(
&self.shared.agents,
self.shared.starts.clone(),
super::http::Asked {
key: source.cache_key(),
address: self.fetched_from(address),
headers: self.headers_of(source),
timeout,
cut,
read_ahead,
},
)?))
}
#[cfg(not(all(feature = "http", not(target_arch = "wasm32"))))]
fn open_address(
&self,
_address: &str,
_source: &Source,
_read_ahead: Option<bool>,
) -> Result<Arc<dyn Reader>, LoadError> {
Err(LoadError::Unsupported(
"this build reads an address only whole: a reader of one needs the feature `http`, outside a browser",
))
}
pub fn forget(&self, source: &Source) -> bool {
let key = source.cache_key();
let kept = self.shared.disk.forget_key(key);
#[cfg(all(feature = "http", not(target_arch = "wasm32")))]
let kept = self.shared.starts.forget(key) || kept;
let mut entries = self.shared.entries.lock();
match entries.get(&key) {
Some(entry) if !entry.state.lock().is_loading() => {
entries.remove(&key);
true
}
_ => kept,
}
}
pub fn sweep(&self) {
let now = Instant::now();
self.shared.entries.lock().retain(|_, entry| {
if entry.holders.load(Ordering::Acquire) > 0 {
return true;
}
let idle = entry
.idle_since
.lock()
.is_some_and(|since| now.duration_since(since) >= IDLE_GRACE);
match &*entry.state.lock() {
LoadState::Failed(_) => true,
LoadState::Ready(_) => !idle,
LoadState::Loading { .. } => {
if idle {
entry.cancelled.store(true, Ordering::Release);
}
!idle
}
}
});
}
pub fn pending(&self) -> usize {
self
.shared
.entries
.lock()
.values()
.filter(|entry| entry.state.lock().is_loading())
.count()
}
}
#[cfg(not(target_arch = "wasm32"))]
fn read_start(reader: &dyn Reader, head: u64) -> io::Result<Vec<u8>> {
let mut bytes = vec![0u8; usize::try_from(head).unwrap_or(usize::MAX).min(16 * 1024 * 1024)];
let mut filled = 0;
while filled < bytes.len() {
let read = reader.read_at(filled as u64, &mut bytes[filled..])?;
if read == 0 {
break;
}
filled += read;
}
bytes.truncate(filled);
Ok(bytes)
}
#[cfg(all(feature = "http", not(target_arch = "wasm32")))]
type AgentPool = Arc<super::http::Agents>;
#[cfg(not(all(feature = "http", not(target_arch = "wasm32"))))]
type AgentPool = ();
#[cfg(all(feature = "http", not(target_arch = "wasm32")))]
fn agent_pool(size: usize) -> AgentPool {
super::http::Agents::new(size)
}
#[cfg(not(all(feature = "http", not(target_arch = "wasm32"))))]
fn agent_pool(_size: usize) -> AgentPool {}
#[cfg(all(feature = "http", not(target_arch = "wasm32")))]
type StartPool = Arc<super::http::Starts>;
#[cfg(not(all(feature = "http", not(target_arch = "wasm32"))))]
type StartPool = ();
#[cfg(all(feature = "http", not(target_arch = "wasm32")))]
fn start_pool(addresses: usize) -> StartPool {
super::http::Starts::new(addresses)
}
#[cfg(not(all(feature = "http", not(target_arch = "wasm32"))))]
fn start_pool(_addresses: usize) -> StartPool {}
#[cfg(all(feature = "http", not(target_arch = "wasm32")))]
fn fetch(
agents: &AgentPool,
address: &str,
headers: &[(Arc<str>, Arc<str>)],
timeout: Duration,
max_bytes: u64,
cut: bool,
told: impl FnMut(u64, Option<u64>) -> bool,
) -> Result<Vec<u8>, LoadError> {
super::http::fetch(agents, address, headers, timeout, max_bytes, cut, told)
}
#[cfg(all(not(feature = "http"), not(target_arch = "wasm32")))]
fn fetch(
_agents: &AgentPool,
_address: &str,
_headers: &[(Arc<str>, Arc<str>)],
_timeout: Duration,
_max_bytes: u64,
_cut: bool,
_told: impl FnMut(u64, Option<u64>) -> bool,
) -> Result<Vec<u8>, LoadError> {
static SAID: std::sync::Once = std::sync::Once::new();
SAID.call_once(|| tracing::warn!("lurq is built without the feature `http`: an address is not loaded"));
Err(LoadError::Unsupported(
"this build has no `http` feature: it loads no address",
))
}
#[cfg(all(test, not(target_arch = "wasm32")))]
mod tests {
use std::{path::Path, sync::atomic::AtomicUsize, time::Duration};
use super::*;
thread_local! {
static ON_A_RUNTIME: std::cell::Cell<bool> = const { std::cell::Cell::new(false) };
}
#[cfg(feature = "tokio")]
fn runtime() -> tokio::runtime::Handle {
static RUNTIME: std::sync::OnceLock<tokio::runtime::Runtime> = std::sync::OnceLock::new();
let runtime = RUNTIME.get_or_init(|| {
tokio::runtime::Builder::new_multi_thread()
.worker_threads(2)
.build()
.expect("a runtime for the tests")
});
runtime.handle().clone()
}
fn loader() -> Loader {
let loader = Loader::new();
#[cfg(feature = "tokio")]
if ON_A_RUNTIME.get() {
loader.tokio_handle(runtime());
}
loader
}
fn run() -> &'static str {
if ON_A_RUNTIME.get() { "-on-a-runtime" } else { "" }
}
fn ended(load: &Load) -> LoadState {
let deadline = std::time::Instant::now() + Duration::from_secs(10);
loop {
let state = load.state();
if !state.is_loading() {
return state;
}
assert!(std::time::Instant::now() < deadline, "the load does not end");
std::thread::sleep(Duration::from_millis(2));
}
}
fn file(name: &str, bytes: &[u8]) -> PathBuf {
let folder = std::env::temp_dir().join(format!("lurq-loader-{}{}", std::process::id(), run()));
std::fs::create_dir_all(&folder).unwrap();
let path = folder.join(name);
std::fs::write(&path, bytes).unwrap();
path
}
fn folder_of(path: &Path) -> PathBuf {
path.parent().unwrap().to_path_buf()
}
#[test]
fn bytes_are_ready_at_once_and_start_no_thread() {
let loader = loader();
let load = loader.load(&Source::bytes(vec![1, 2, 3]));
assert!(matches!(load.state(), LoadState::Ready(bytes) if bytes[..] == [1, 2, 3]));
assert!(loader.shared.pool.lock().is_none());
}
#[test]
fn a_file_and_a_resource_under_the_root_are_read() {
let path = file("read.bin", &[5, 6, 7]);
let loader = loader();
assert!(matches!(ended(&loader.load(&Source::file(&path))), LoadState::Ready(bytes) if bytes[..] == [5, 6, 7]));
loader.root(folder_of(&path));
let resource = loader.load(&Source::resource("read.bin"));
assert!(matches!(ended(&resource), LoadState::Ready(bytes) if bytes[..] == [5, 6, 7]));
}
#[test]
fn a_file_that_is_not_there_fails_and_stays_failed_until_forgotten() {
let path = std::env::temp_dir().join(format!("lurq-loader-none-{}{}.bin", std::process::id(), run()));
let source = Source::file(&path);
let loader = loader();
assert!(matches!(
ended(&loader.load(&source)),
LoadState::Failed(LoadError::NotFound)
));
std::fs::write(&path, [1]).unwrap();
assert!(matches!(
loader.load(&source).state(),
LoadState::Failed(LoadError::NotFound)
));
assert!(loader.forget(&source));
assert!(matches!(ended(&loader.load(&source)), LoadState::Ready(bytes) if bytes[..] == [1]));
std::fs::remove_file(&path).unwrap();
}
#[test]
fn a_file_over_the_limit_is_not_read_whole() {
let path = file("large.bin", &[0; 4096]);
let loader = loader();
loader.max_bytes(1024);
assert!(matches!(
ended(&loader.load(&Source::file(&path))),
LoadState::Failed(LoadError::TooLarge { limit: 1024 })
));
}
#[test]
fn the_start_of_a_source_is_read_and_no_more_and_is_no_error() {
let path = file("start.bin", &[1, 2, 3, 4, 5, 6]);
let loader = loader();
let source = Source::file(&path);
assert!(matches!(ended(&loader.load_start(&source, 4)), LoadState::Ready(bytes) if bytes[..] == [1, 2, 3, 4]));
assert!(
matches!(ended(&loader.load_start(&source, 100)), LoadState::Ready(bytes) if bytes[..] == [1, 2, 3, 4, 5, 6])
);
assert!(matches!(ended(&loader.load(&source)), LoadState::Ready(bytes) if bytes.len() == 6));
let bytes = Source::bytes(vec![7, 8, 9]);
assert!(matches!(loader.load_start(&bytes, 2).state(), LoadState::Ready(start) if start[..] == [7, 8]));
}
#[test]
fn the_same_source_asked_twice_is_one_load_and_wakes_who_waits() {
let path = file("twice.bin", &[9; 1000]);
let loader = loader();
let source = Source::file(&path);
let woken = Arc::new(AtomicUsize::new(0));
let first = loader.load(&source);
let count = woken.clone();
let wake: LoadWaker = Arc::new(move || {
count.fetch_add(1, Ordering::SeqCst);
});
first.wake_with(wake.clone());
first.wake_with(wake);
let second = loader.load(&source);
assert!(Arc::ptr_eq(&first.entry, &second.entry));
assert!(matches!(ended(&second), LoadState::Ready(_)));
assert!(woken.load(Ordering::SeqCst) <= 1);
}
#[test]
fn what_nobody_holds_is_let_go_after_a_moment_and_a_failure_is_kept() {
let path = file("sweep.bin", &[1, 2]);
let missing = Source::file(folder_of(&path).join("no-such-file.bin"));
let loader = loader();
let source = Source::file(&path);
let load = loader.load(&source);
let failed = loader.load(&missing);
ended(&load);
ended(&failed);
drop((load, failed));
loader.sweep();
assert_eq!(
loader.shared.entries.lock().len(),
2,
"a rebuild a moment later still finds both"
);
std::thread::sleep(IDLE_GRACE + Duration::from_millis(50));
loader.sweep();
let entries = loader.shared.entries.lock();
assert_eq!(entries.len(), 1, "the bytes are let go");
assert!(entries.contains_key(&missing.cache_key()), "the failure is kept");
}
#[test]
fn an_address_the_app_does_not_allow_is_denied_and_nothing_reads_it() {
let loader = loader();
loader.allow(|address| address.starts_with("https://cdn.example.com/"));
let load = loader.load(&Source::url("https://elsewhere.example.com/a.png"));
assert!(matches!(load.state(), LoadState::Failed(LoadError::Denied)));
assert!(loader.shared.pool.lock().is_none());
}
#[cfg(not(feature = "http"))]
#[test]
fn a_build_without_http_says_it_loads_no_address() {
let loader = loader();
let state = ended(&loader.load(&Source::url("https://example.com/a.png")));
assert!(matches!(state, LoadState::Failed(LoadError::Unsupported(_))));
}
#[test]
fn the_rules_headers_come_first_and_a_sources_own_take_the_place_of_one_of_the_same_name() {
let loader = loader();
loader.headers(|address| {
if address.starts_with("https://videos.example.com/") {
vec![
("Referer".to_owned(), "https://example.com/".to_owned()),
("X-Client".to_owned(), "lurq".to_owned()),
]
} else {
Vec::new()
}
});
let names = |source: &Source| -> Vec<(String, String)> {
loader
.headers_of(source)
.iter()
.map(|(name, value)| (name.to_string(), value.to_string()))
.collect()
};
let pair = |name: &str, value: &str| (name.to_owned(), value.to_owned());
assert_eq!(
names(&Source::url("https://videos.example.com/a.mp4").header("referer", "https://example.com/watch")),
vec![pair("X-Client", "lurq"), pair("referer", "https://example.com/watch")]
);
assert_eq!(
names(&Source::url("https://elsewhere.example.com/a.mp4").header("Authorization", "Bearer token")),
vec![pair("Authorization", "Bearer token")]
);
assert!(names(&Source::bytes(vec![1])).is_empty());
}
#[test]
fn a_reader_reads_a_part_of_a_file_and_of_bytes() {
let path = file("part.bin", &[1, 2, 3, 4, 5, 6]);
let loader = loader();
let mut buf = [0u8; 2];
let reader = loader.open(&Source::file(&path)).unwrap();
assert_eq!(reader.read_at(4, &mut buf).unwrap(), 2);
assert_eq!(buf, [5, 6]);
let reader = loader.open(&Source::bytes(vec![7, 8, 9])).unwrap();
assert_eq!(reader.read_at(1, &mut buf).unwrap(), 2);
assert_eq!(buf, [8, 9]);
assert!(matches!(
loader.open(&Source::file(folder_of(&path).join("none.bin"))),
Err(LoadError::NotFound)
));
#[cfg(not(feature = "http"))]
assert!(matches!(
loader.open(&Source::url("https://example.com/a.mp4")),
Err(LoadError::Unsupported(_))
));
loader.allow(|_| false);
assert!(matches!(
loader.open(&Source::url("https://example.com/a.mp4")),
Err(LoadError::Denied)
));
}
#[cfg(feature = "http")]
fn serve(body: &'static [u8]) -> (String, Arc<AtomicUsize>) {
use std::io::{Read, Write};
let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
let address = format!("http://{}/picture.png", listener.local_addr().unwrap());
let asked = Arc::new(AtomicUsize::new(0));
let counted = asked.clone();
std::thread::spawn(move || {
for stream in listener.incoming() {
let Ok(mut stream) = stream else {
return;
};
let (mut request, mut chunk) = (Vec::new(), [0_u8; 1024]);
while !request.windows(4).any(|window| window == b"\r\n\r\n") {
match stream.read(&mut chunk) {
Ok(read) if read > 0 => request.extend_from_slice(&chunk[..read]),
_ => break,
}
}
counted.fetch_add(1, Ordering::SeqCst);
let head = format!(
"HTTP/1.1 200 OK\r\nContent-Length: {}\r\nConnection: close\r\n\r\n",
body.len()
);
let _ = stream.write_all(head.as_bytes());
let _ = stream.write_all(body);
}
});
(address, asked)
}
#[cfg(feature = "http")]
#[derive(Default)]
struct Served {
connections: AtomicUsize,
now: AtomicUsize,
most: AtomicUsize,
}
#[cfg(feature = "http")]
fn serve_kept(body: &'static [u8], pause: Duration) -> (String, Arc<Served>) {
use std::io::{Read, Write};
let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
let address = format!("http://{}", listener.local_addr().unwrap());
let served = Arc::new(Served::default());
let counts = served.clone();
std::thread::spawn(move || {
for stream in listener.incoming() {
let Ok(mut stream) = stream else {
return;
};
counts.connections.fetch_add(1, Ordering::SeqCst);
let counts = counts.clone();
std::thread::spawn(move || {
let mut request = Vec::new();
let mut chunk = [0_u8; 1024];
loop {
while !request.windows(4).any(|window| window == b"\r\n\r\n") {
match stream.read(&mut chunk) {
Ok(read) if read > 0 => request.extend_from_slice(&chunk[..read]),
_ => return,
}
}
request.clear();
counts
.most
.fetch_max(counts.now.fetch_add(1, Ordering::SeqCst) + 1, Ordering::SeqCst);
std::thread::sleep(pause);
counts.now.fetch_sub(1, Ordering::SeqCst);
let head = format!("HTTP/1.1 200 OK\r\nContent-Length: {}\r\n\r\n", body.len());
if stream.write_all(head.as_bytes()).is_err() || stream.write_all(body).is_err() {
return;
}
}
});
}
});
(address, served)
}
#[cfg(feature = "http")]
fn serve_named(pause: Duration) -> (String, Arc<Mutex<Vec<String>>>) {
use std::io::{Read, Write};
let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
let address = format!("http://{}", listener.local_addr().unwrap());
let asked: Arc<Mutex<Vec<String>>> = Arc::default();
let paths = asked.clone();
std::thread::spawn(move || {
for stream in listener.incoming() {
let Ok(mut stream) = stream else {
return;
};
let paths = paths.clone();
std::thread::spawn(move || {
let mut request = Vec::new();
let mut chunk = [0_u8; 1024];
loop {
while !request.windows(4).any(|window| window == b"\r\n\r\n") {
match stream.read(&mut chunk) {
Ok(read) if read > 0 => request.extend_from_slice(&chunk[..read]),
_ => return,
}
}
let said = String::from_utf8_lossy(&request).into_owned();
let path = said.split_whitespace().nth(1).unwrap_or_default().to_owned();
request.clear();
paths.lock().push(path);
std::thread::sleep(pause);
if stream
.write_all(b"HTTP/1.1 200 OK\r\nContent-Length: 3\r\n\r\nabc")
.is_err()
{
return;
}
}
});
}
});
(address, asked)
}
#[cfg(feature = "http")]
fn asked_for(asked: &Mutex<Vec<String>>, count: usize) {
let until = Instant::now() + Duration::from_secs(5);
while asked.lock().len() < count {
assert!(Instant::now() < until, "asked for {:?} only", asked.lock());
std::thread::sleep(Duration::from_millis(2));
}
}
#[cfg(feature = "http")]
#[test]
fn the_loads_that_wait_are_read_by_their_priority_and_the_newest_first() {
let (address, asked) = serve_named(Duration::from_millis(60));
let loader = loader();
loader.pool_size(1);
let load = |name: &str| loader.load(&Source::url(&format!("{address}/{name}")));
let first = load("first");
asked_for(&asked, 1);
let (a, b, c) = (load("a"), load("b"), load("c"));
let later = load("later").with_priority(LoadPriority::Low);
assert_eq!(later.priority(), LoadPriority::Low);
a.set_priority(LoadPriority::High);
for load in [&first, &a, &b, &c, &later] {
assert!(matches!(ended(load), LoadState::Ready(bytes) if bytes[..] == *b"abc"));
}
assert_eq!(
*asked.lock(),
["/first", "/a", "/c", "/b", "/later"],
"the one in view, then the newest, then the one for later"
);
}
#[cfg(feature = "http")]
#[test]
fn a_load_nobody_waits_for_any_more_is_never_started() {
let (address, asked) = serve_named(Duration::from_millis(600));
let loader = loader();
loader.pool_size(1);
let load = |name: &str| loader.load(&Source::url(&format!("{address}/{name}")));
let first = load("first");
asked_for(&asked, 1);
let gone = load("gone");
let wake = Arc::new(AtomicUsize::new(0));
let woken = wake.clone();
gone.wake_with(Arc::new(move || {
woken.fetch_add(1, Ordering::SeqCst);
}));
drop(gone);
std::thread::sleep(IDLE_GRACE + Duration::from_millis(40));
loader.sweep();
let kept = load("kept");
for load in [&first, &kept] {
assert!(matches!(ended(load), LoadState::Ready(_)));
}
assert_eq!(*asked.lock(), ["/first", "/kept"], "the server was never asked for it");
assert_eq!(loader.pending(), 0);
assert!(matches!(ended(&load("gone")), LoadState::Ready(_)));
}
#[cfg(feature = "http")]
#[test]
fn the_loads_of_one_host_go_over_the_connection_the_first_opened() {
let (address, served) = serve_kept(&[1, 2, 3], Duration::ZERO);
let loader = loader();
for index in 0..5 {
let load = loader.load(&Source::url(&format!("{address}/picture-{index}.png")));
assert!(matches!(ended(&load), LoadState::Ready(bytes) if bytes[..] == [1, 2, 3]));
}
assert_eq!(
served.connections.load(Ordering::SeqCst),
1,
"one handshake for the five"
);
assert_eq!(loader.shared.agents.made(), 1, "and one agent");
}
#[cfg(feature = "http")]
#[test]
fn an_address_read_a_moment_ago_is_opened_again_with_no_request_until_its_start_is_forgotten() {
let (address, asked) = serve_named(Duration::ZERO);
let loader = Loader::new();
let source = Source::url(&format!("{address}/clip.mp4"));
let read = || {
let reader = loader.open(&source).unwrap();
let mut start = [0_u8; 8];
let count = reader.read_at(0, &mut start).unwrap();
start[..count].to_vec()
};
assert_eq!(read(), b"abc");
assert_eq!(asked.lock().len(), 1);
assert_eq!(read(), b"abc");
assert_eq!(asked.lock().len(), 1, "opened again with what the first read");
assert!(loader.forget(&source), "there was a start to forget");
assert_eq!(read(), b"abc");
assert_eq!(asked.lock().len(), 2, "forgotten, it is asked for again");
loader.keep_starts(0);
assert_eq!(read(), b"abc");
assert_eq!(read(), b"abc");
assert_eq!(asked.lock().len(), 4);
}
#[cfg(feature = "http")]
#[test]
fn no_connection_is_opened_ahead_to_an_address_the_app_does_not_allow() {
let (address, asked) = serve_named(Duration::ZERO);
let loader = Loader::new();
loader.allow(|_| false);
loader.connect(&format!("{address}/clip.mp4"));
std::thread::sleep(Duration::from_millis(150));
assert!(asked.lock().is_empty(), "nothing was sent");
}
#[cfg(feature = "http")]
#[test]
fn as_many_loads_are_read_at_once_as_the_pool_is_large_and_its_agents_are_kept() {
let (address, served) = serve_kept(&[9; 64], Duration::from_millis(60));
let loader = loader();
let read_at_once = |size: usize, batch: &str| {
loader.pool_size(size);
served.most.store(0, Ordering::SeqCst);
let loads: Vec<Load> = (0..size * 3)
.map(|index| loader.load(&Source::url(&format!("{address}/{batch}-{index}.png"))))
.collect();
for load in &loads {
assert!(matches!(ended(load), LoadState::Ready(bytes) if bytes.len() == 64));
}
served.most.load(Ordering::SeqCst)
};
assert_eq!(read_at_once(2, "two"), 2, "two at once");
assert!(
loader.shared.agents.made() <= 2,
"{} agents",
loader.shared.agents.made()
);
let most = read_at_once(3, "three");
assert!((2..=3).contains(&most), "{most} at once");
assert!(
loader.shared.agents.made() <= 3,
"{} agents",
loader.shared.agents.made()
);
assert!(served.connections.load(Ordering::SeqCst) <= 3);
}
#[cfg(feature = "http")]
#[test]
fn what_came_from_an_address_is_kept_in_the_folder_the_app_gave_and_read_from_there() {
let folder = std::env::temp_dir().join(format!("lurq-loader-cache-{}{}", std::process::id(), run()));
let _ = std::fs::remove_dir_all(&folder);
let (address, asked) = serve(&[4, 5, 6, 7]);
let source = Source::url(&address);
let first = loader();
first.enable_media_disk_cache(&folder);
assert!(matches!(ended(&first.load(&source)), LoadState::Ready(bytes) if bytes[..] == [4, 5, 6, 7]));
assert!(first.media_disk_cache().holds(&source));
assert_eq!(std::fs::read_dir(&folder).unwrap().count(), 1);
let next = loader();
next.enable_media_disk_cache(&folder);
assert!(matches!(ended(&next.load(&source)), LoadState::Ready(bytes) if bytes[..] == [4, 5, 6, 7]));
assert_eq!(asked.load(Ordering::SeqCst), 1);
assert!(next.forget(&source));
assert_eq!(std::fs::read_dir(&folder).unwrap().count(), 0);
assert!(matches!(ended(&next.load(&source)), LoadState::Ready(bytes) if bytes[..] == [4, 5, 6, 7]));
assert_eq!(asked.load(Ordering::SeqCst), 2);
let plain = loader();
assert!(matches!(ended(&plain.load(&source)), LoadState::Ready(_)));
assert_eq!(asked.load(Ordering::SeqCst), 3);
assert_eq!(std::fs::read_dir(&folder).unwrap().count(), 1);
let _ = std::fs::remove_dir_all(&folder);
}
#[cfg(feature = "tokio")]
mod on_a_runtime {
use super::*;
macro_rules! again {
($($test:ident),* $(,)?) => {
$(
#[test]
fn $test() {
ON_A_RUNTIME.set(true);
super::$test();
}
)*
};
}
again!(
bytes_are_ready_at_once_and_start_no_thread,
a_file_and_a_resource_under_the_root_are_read,
a_file_that_is_not_there_fails_and_stays_failed_until_forgotten,
a_file_over_the_limit_is_not_read_whole,
the_start_of_a_source_is_read_and_no_more_and_is_no_error,
the_same_source_asked_twice_is_one_load_and_wakes_who_waits,
what_nobody_holds_is_let_go_after_a_moment_and_a_failure_is_kept,
an_address_the_app_does_not_allow_is_denied_and_nothing_reads_it,
a_reader_reads_a_part_of_a_file_and_of_bytes,
);
#[cfg(feature = "http")]
again!(
what_came_from_an_address_is_kept_in_the_folder_the_app_gave_and_read_from_there,
the_loads_of_one_host_go_over_the_connection_the_first_opened,
as_many_loads_are_read_at_once_as_the_pool_is_large_and_its_agents_are_kept,
the_loads_that_wait_are_read_by_their_priority_and_the_newest_first,
a_load_nobody_waits_for_any_more_is_never_started,
);
fn read(loader: &Loader, test: &str, count: usize) {
for index in 0..count {
let path = file(&format!("{test}-{index}.bin"), &[index as u8; 8]);
let load = loader.load(&Source::file(&path));
assert!(matches!(ended(&load), LoadState::Ready(bytes) if bytes.len() == 8));
}
}
fn stopped() -> tokio::runtime::Handle {
let runtime = tokio::runtime::Builder::new_multi_thread()
.worker_threads(1)
.build()
.expect("a runtime to stop");
runtime.handle().clone()
}
#[test]
fn with_a_handle_given_a_load_ends_and_no_thread_of_the_loaders_own_is_started() {
let loader = Loader::new();
loader.tokio_handle(runtime());
read(&loader, "on-a-runtime", 6);
assert!(
loader.shared.pool.lock().is_none(),
"no pool of the loader's own was made"
);
assert_eq!(loader.pending(), 0);
}
#[test]
fn a_load_given_to_a_runtime_that_stopped_fails_and_does_not_wait_for_ever() {
let loader = Loader::new();
loader.tokio_handle(stopped());
let path = file("stopped.bin", &[1, 2, 3]);
assert!(matches!(
ended(&loader.load(&Source::file(&path))),
LoadState::Failed(LoadError::Cancelled)
));
assert!(loader.shared.pool.lock().is_none());
}
#[test]
fn a_handle_given_later_is_where_the_next_load_is_read_and_one_taken_away_gives_the_pool_back() {
let loader = Loader::new();
read(&loader, "before-a-handle", 1);
assert!(
loader.shared.pool.lock().is_some(),
"with no handle the loader reads on its own threads"
);
loader.tokio_handle(stopped());
let path = file("given-later.bin", &[1, 2, 3]);
assert!(matches!(
ended(&loader.load(&Source::file(&path))),
LoadState::Failed(LoadError::Cancelled)
));
loader.tokio_handle(runtime());
read(&loader, "after-a-handle", 1);
loader.tokio_handle(stopped());
loader.reset_tokio_handle();
read(&loader, "after-a-handle-was-taken", 1);
assert_eq!(loader.pending(), 0);
}
#[test]
fn no_more_loads_are_read_at_a_time_on_a_runtime_than_the_loader_has_threads() {
let loader = Loader::new();
loader.tokio_handle(runtime());
let (now, most) = (Arc::new(AtomicUsize::new(0)), Arc::new(AtomicUsize::new(0)));
let megabyte = vec![7_u8; 1 << 20];
let paths: Vec<PathBuf> = (0..LOADER_THREADS * 3)
.map(|index| file(&format!("room-{index}.bin"), &megabyte))
.collect();
let loads: Vec<Load> = paths
.iter()
.map(|path| {
let load = loader.load(&Source::file(path));
let (now, most) = (now.clone(), most.clone());
let wake: LoadWaker = Arc::new(move || {
most.fetch_max(now.fetch_add(1, Ordering::SeqCst) + 1, Ordering::SeqCst);
std::thread::sleep(Duration::from_millis(2));
now.fetch_sub(1, Ordering::SeqCst);
});
load.wake_with(wake);
load
})
.collect();
for load in &loads {
assert!(matches!(ended(load), LoadState::Ready(bytes) if bytes.len() == 1 << 20));
}
for path in paths {
let _ = std::fs::remove_file(path);
}
let most = most.load(Ordering::SeqCst);
assert!((1..=LOADER_THREADS).contains(&most), "{most} at a time");
assert!(loader.shared.pool.lock().is_none());
}
}
}