use std::{
collections::HashMap,
fmt, io,
path::PathBuf,
sync::{
Arc, OnceLock,
atomic::{AtomicBool, AtomicU64, AtomicUsize, Ordering},
},
time::Duration,
};
use parking_lot::{Mutex, RwLock};
use super::{
Source,
disk_cache::MediaDiskCache,
reader::{BytesReader, FileReader, Reader, read_all},
source::SourceKind,
thread_pool::ThreadPool,
};
use crate::time::Instant;
pub const DEFAULT_LOAD_TIMEOUT: Duration = Duration::from_secs(30);
pub const DEFAULT_MAX_LOAD_BYTES: u64 = 64 * 1024 * 1024;
const LOADER_THREADS: usize = 4;
const IDLE_GRACE: Duration = Duration::from_millis(250);
const PROGRESS_STEP_BYTES: u64 = 256 * 1024;
pub type LoadWaker = Arc<dyn Fn() + Send + Sync>;
#[derive(Clone, Debug, PartialEq, Eq)]
pub enum LoadError {
NotFound,
Denied,
Status(u16),
TooLarge { limit: u64 },
TimedOut,
Cancelled,
Unsupported(&'static str),
Io(String),
}
impl fmt::Display for LoadError {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
match self {
Self::NotFound => f.write_str("it was not found"),
Self::Denied => f.write_str("the app does not allow this address"),
Self::Status(status) => write!(f, "the server answered {status}"),
Self::TooLarge { limit } => write!(f, "it is larger than {limit} bytes"),
Self::TimedOut => f.write_str("it took too long"),
Self::Cancelled => f.write_str("the load was stopped"),
Self::Unsupported(why) => f.write_str(why),
Self::Io(reason) => f.write_str(reason),
}
}
}
impl std::error::Error for LoadError {}
impl From<io::Error> for LoadError {
fn from(error: io::Error) -> Self {
match error.kind() {
io::ErrorKind::NotFound => Self::NotFound,
io::ErrorKind::TimedOut => Self::TimedOut,
io::ErrorKind::Interrupted => Self::Cancelled,
_ => Self::Io(error.to_string()),
}
}
}
#[derive(Clone, Debug)]
pub enum LoadState {
Loading {
loaded: u64,
total: Option<u64>,
},
Ready(Arc<[u8]>),
Failed(LoadError),
}
impl LoadState {
pub fn is_loading(&self) -> bool {
matches!(self, Self::Loading { .. })
}
}
struct Entry {
state: Mutex<LoadState>,
cancelled: AtomicBool,
holders: AtomicUsize,
idle_since: Mutex<Option<Instant>>,
wakers: Mutex<Vec<LoadWaker>>,
told: AtomicU64,
}
impl Entry {
fn new(state: LoadState) -> Arc<Self> {
Arc::new(Self {
state: Mutex::new(state),
cancelled: AtomicBool::new(false),
holders: AtomicUsize::new(0),
idle_since: Mutex::new(Some(Instant::now())),
wakers: Mutex::new(Vec::new()),
told: AtomicU64::new(0),
})
}
fn wake(&self) {
let wakers = self.wakers.lock().clone();
for wake in wakers {
wake();
}
}
fn progress(&self, loaded: u64, total: Option<u64>) -> bool {
*self.state.lock() = LoadState::Loading { loaded, total };
let step = total.map_or(PROGRESS_STEP_BYTES, |total| (total / 100).max(16 * 1024));
if loaded.saturating_sub(self.told.load(Ordering::Relaxed)) >= step {
self.told.store(loaded, Ordering::Relaxed);
self.wake();
}
!self.cancelled.load(Ordering::Acquire)
}
fn finish(&self, result: Result<Vec<u8>, LoadError>) {
*self.state.lock() = match result {
Ok(bytes) => LoadState::Ready(Arc::from(bytes)),
Err(error) => LoadState::Failed(error),
};
self.wake();
}
}
#[cfg(not(target_arch = "wasm32"))]
struct Unread(Arc<Entry>);
#[cfg(not(target_arch = "wasm32"))]
impl Drop for Unread {
fn drop(&mut self) {
if self.0.state.lock().is_loading() {
self.0.finish(Err(LoadError::Cancelled));
}
}
}
pub struct Load {
entry: Arc<Entry>,
}
impl Load {
fn of(entry: &Arc<Entry>) -> Self {
entry.holders.fetch_add(1, Ordering::AcqRel);
*entry.idle_since.lock() = None;
Self { entry: entry.clone() }
}
pub fn state(&self) -> LoadState {
self.entry.state.lock().clone()
}
pub fn wake_with(&self, wake: LoadWaker) {
let mut wakers = self.entry.wakers.lock();
if !wakers.iter().any(|known| Arc::ptr_eq(known, &wake)) {
wakers.push(wake);
}
}
}
impl Clone for Load {
fn clone(&self) -> Self {
Self::of(&self.entry)
}
}
impl Drop for Load {
fn drop(&mut self) {
if self.entry.holders.fetch_sub(1, Ordering::AcqRel) == 1 {
*self.entry.idle_since.lock() = Some(Instant::now());
}
}
}
type Allow = Arc<dyn Fn(&str) -> bool + Send + Sync>;
type Proxy = Arc<dyn Fn(&str) -> String + Send + Sync>;
struct Settings {
root: Option<PathBuf>,
timeout: Duration,
max_bytes: u64,
allow: Option<Allow>,
proxy: Option<Proxy>,
}
struct Shared {
settings: RwLock<Settings>,
entries: Mutex<HashMap<u64, Arc<Entry>>>,
pool: OnceLock<ThreadPool>,
disk: MediaDiskCache,
#[cfg(feature = "tokio")]
runtime: Mutex<Option<tokio::runtime::Handle>>,
#[cfg(feature = "tokio")]
room: Arc<tokio::sync::Semaphore>,
}
impl Drop for Shared {
fn drop(&mut self) {
for entry in self.entries.lock().values() {
entry.cancelled.store(true, Ordering::Release);
}
}
}
#[derive(Clone)]
pub struct Loader {
shared: Arc<Shared>,
}
impl Default for Loader {
fn default() -> Self {
Self::new()
}
}
impl Loader {
pub fn new() -> Self {
Self {
shared: Arc::new(Shared {
settings: RwLock::new(Settings {
root: None,
timeout: DEFAULT_LOAD_TIMEOUT,
max_bytes: DEFAULT_MAX_LOAD_BYTES,
allow: None,
proxy: None,
}),
entries: Mutex::new(HashMap::new()),
pool: OnceLock::new(),
disk: MediaDiskCache::new(),
#[cfg(feature = "tokio")]
runtime: Mutex::new(None),
#[cfg(feature = "tokio")]
room: Arc::new(tokio::sync::Semaphore::new(LOADER_THREADS)),
}),
}
}
pub fn root(&self, root: impl Into<PathBuf>) -> &Self {
self.shared.settings.write().root = Some(root.into());
self
}
pub fn reset_root(&self) -> &Self {
self.shared.settings.write().root = None;
self
}
pub fn timeout(&self, timeout: Duration) -> &Self {
self.shared.settings.write().timeout = timeout;
self
}
pub fn max_bytes(&self, max_bytes: u64) -> &Self {
self.shared.settings.write().max_bytes = max_bytes;
self
}
pub fn allow(&self, allow: impl Fn(&str) -> bool + Send + Sync + 'static) -> &Self {
self.shared.settings.write().allow = Some(Arc::new(allow));
self
}
pub fn proxy(&self, proxy: impl Fn(&str) -> String + Send + Sync + 'static) -> &Self {
self.shared.settings.write().proxy = Some(Arc::new(proxy));
self
}
fn fetched_from(&self, address: &str) -> String {
match &self.shared.settings.read().proxy {
Some(proxy) => proxy(address),
None => address.to_owned(),
}
}
#[cfg(feature = "tokio")]
pub fn tokio_handle(&self, handle: tokio::runtime::Handle) -> &Self {
*self.shared.runtime.lock() = Some(handle);
self
}
#[cfg(feature = "tokio")]
pub fn reset_tokio_handle(&self) -> &Self {
*self.shared.runtime.lock() = None;
self
}
#[cfg(all(test, feature = "tokio"))]
pub(crate) fn reads_on_a_runtime(&self) -> bool {
self.shared.runtime.lock().is_some()
}
pub fn enable_media_disk_cache(&self, folder: impl Into<PathBuf>) -> &Self {
self.shared.disk.turn_on(folder.into());
self
}
pub fn media_disk_cache(&self) -> &MediaDiskCache {
&self.shared.disk
}
#[cfg_attr(not(feature = "media"), allow(dead_code))]
pub(crate) fn allows(&self, address: &str) -> bool {
let allow = self.shared.settings.read().allow.clone();
allow.is_none_or(|allow| allow(address))
}
pub(crate) fn resource_path(&self, resource: &str) -> PathBuf {
match &self.shared.settings.read().root {
Some(root) => root.join(resource),
None => PathBuf::from(resource),
}
}
pub fn load(&self, source: &Source) -> Load {
let key = source.cache_key();
let mut entries = self.shared.entries.lock();
if let Some(entry) = entries.get(&key) {
return Load::of(entry);
}
let entry = match &source.kind {
SourceKind::Bytes(bytes) => Entry::new(LoadState::Ready(bytes.clone())),
_ => Entry::new(LoadState::Loading { loaded: 0, total: None }),
};
entries.insert(key, entry.clone());
drop(entries);
let load = Load::of(&entry);
if !matches!(source.kind, SourceKind::Bytes(_)) {
self.start(source, entry, None);
}
load
}
pub fn load_start(&self, source: &Source, bytes: u64) -> Load {
let key = {
let mut hasher = std::collections::hash_map::DefaultHasher::new();
std::hash::Hash::hash(&("start", source.cache_key(), bytes), &mut hasher);
std::hash::Hasher::finish(&hasher)
};
let mut entries = self.shared.entries.lock();
if let Some(entry) = entries.get(&key) {
return Load::of(entry);
}
let entry = match &source.kind {
SourceKind::Bytes(all) => Entry::new(LoadState::Ready(Arc::from(
&all[..all.len().min(usize::try_from(bytes).unwrap_or(usize::MAX))],
))),
_ => Entry::new(LoadState::Loading { loaded: 0, total: None }),
};
entries.insert(key, entry.clone());
drop(entries);
let load = Load::of(&entry);
if !matches!(source.kind, SourceKind::Bytes(_)) {
self.start(source, entry, Some(bytes));
}
load
}
fn start(&self, source: &Source, entry: Arc<Entry>, head: Option<u64>) {
#[cfg(target_arch = "wasm32")]
{
let (timeout, max_bytes, allow) = {
let settings = self.shared.settings.read();
(settings.timeout, settings.max_bytes, settings.allow.clone())
};
let address = match &source.kind {
SourceKind::Url(address) if allow.as_ref().is_some_and(|allow| !allow(address)) => {
entry.finish(Err(LoadError::Denied));
return;
}
SourceKind::Url(address) => self.fetched_from(address),
SourceKind::File(path) => super::web_fetch::address_of(path),
SourceKind::Resource(resource) => super::web_fetch::address_of(&self.resource_path(resource)),
SourceKind::Bytes(_) => return,
};
let headers = source.headers.clone();
wasm_bindgen_futures::spawn_local(async move {
let told = |loaded, total| entry.progress(loaded, total);
let result = super::web_fetch::fetch(
&address,
&headers,
timeout,
head.unwrap_or(max_bytes),
head.is_some(),
told,
)
.await;
entry.finish(result);
});
}
#[cfg(not(target_arch = "wasm32"))]
{
let (timeout, max_bytes, allow) = {
let settings = self.shared.settings.read();
(settings.timeout, settings.max_bytes, settings.allow.clone())
};
let path = match &source.kind {
SourceKind::File(path) => Some(path.clone()),
SourceKind::Resource(resource) => Some(self.resource_path(resource)),
_ => None,
};
if let SourceKind::Url(address) = &source.kind
&& allow.as_ref().is_some_and(|allow| !allow(address))
{
entry.finish(Err(LoadError::Denied));
return;
}
let fetched = match &source.kind {
SourceKind::Url(address) => self.fetched_from(address),
_ => String::new(),
};
let source = source.clone();
let disk = self.shared.disk.clone();
let unread = Unread(entry.clone());
let read = move || {
let _unread = unread;
let told = |loaded, total| entry.progress(loaded, total);
let result = match (path, &source.kind, head) {
(Some(path), _, Some(head)) => FileReader::open(&path)
.and_then(|reader| read_start(&reader, head))
.map_err(LoadError::from),
(None, SourceKind::Url(_), Some(head)) => fetch(&fetched, &source, timeout, head, true, told),
(Some(path), _, None) => FileReader::open(&path)
.and_then(|reader| read_all(&reader, Some(max_bytes), told))
.map_err(|error| match error.kind() {
io::ErrorKind::FileTooLarge => LoadError::TooLarge { limit: max_bytes },
_ => LoadError::from(error),
}),
(None, SourceKind::Url(address), None) => match disk.read(source.cache_key(), max_bytes) {
Some(bytes) => Ok(bytes),
None => fetch(&fetched, &source, timeout, max_bytes, false, told).inspect(|bytes| {
disk.put(source.cache_key(), address, bytes);
}),
},
(None, ..) => Err(LoadError::Unsupported("nothing reads this source")),
};
entry.finish(result);
};
#[cfg(feature = "tokio")]
if let Some(handle) = self.shared.runtime.lock().clone() {
let room = self.shared.room.clone();
drop(handle.spawn(async move {
if let Ok(_turn) = room.acquire().await {
let _ = tokio::task::spawn_blocking(read).await;
}
}));
return;
}
let pool = self.shared.pool.get_or_init(|| ThreadPool::new(LOADER_THREADS));
pool.execute(read);
}
}
pub fn open(&self, source: &Source) -> Result<Arc<dyn Reader>, LoadError> {
match &source.kind {
SourceKind::Bytes(bytes) => Ok(Arc::new(BytesReader::new(bytes.clone()))),
SourceKind::File(path) => Ok(Arc::new(FileReader::open(path)?)),
SourceKind::Resource(resource) => Ok(Arc::new(FileReader::open(self.resource_path(resource))?)),
SourceKind::Url(_) => Err(LoadError::Unsupported(
"a reader of an address is not built yet: only a whole load of one",
)),
}
}
pub fn forget(&self, source: &Source) -> bool {
let key = source.cache_key();
let kept = self.shared.disk.forget_key(key);
let mut entries = self.shared.entries.lock();
match entries.get(&key) {
Some(entry) if !entry.state.lock().is_loading() => {
entries.remove(&key);
true
}
_ => kept,
}
}
pub fn sweep(&self) {
let now = Instant::now();
self.shared.entries.lock().retain(|_, entry| {
if entry.holders.load(Ordering::Acquire) > 0 {
return true;
}
let idle = entry
.idle_since
.lock()
.is_some_and(|since| now.duration_since(since) >= IDLE_GRACE);
match &*entry.state.lock() {
LoadState::Failed(_) => true,
LoadState::Ready(_) => !idle,
LoadState::Loading { .. } => {
if idle {
entry.cancelled.store(true, Ordering::Release);
}
!idle
}
}
});
}
pub fn pending(&self) -> usize {
self
.shared
.entries
.lock()
.values()
.filter(|entry| entry.state.lock().is_loading())
.count()
}
}
#[cfg(not(target_arch = "wasm32"))]
fn read_start(reader: &dyn Reader, head: u64) -> io::Result<Vec<u8>> {
let mut bytes = vec![0u8; usize::try_from(head).unwrap_or(usize::MAX).min(16 * 1024 * 1024)];
let mut filled = 0;
while filled < bytes.len() {
let read = reader.read_at(filled as u64, &mut bytes[filled..])?;
if read == 0 {
break;
}
filled += read;
}
bytes.truncate(filled);
Ok(bytes)
}
#[cfg(all(feature = "http", not(target_arch = "wasm32")))]
fn fetch(
address: &str,
source: &Source,
timeout: Duration,
max_bytes: u64,
cut: bool,
told: impl FnMut(u64, Option<u64>) -> bool,
) -> Result<Vec<u8>, LoadError> {
super::http::fetch(address, &source.headers, timeout, max_bytes, cut, told)
}
#[cfg(all(not(feature = "http"), not(target_arch = "wasm32")))]
fn fetch(
_address: &str,
_source: &Source,
_timeout: Duration,
_max_bytes: u64,
_cut: bool,
_told: impl FnMut(u64, Option<u64>) -> bool,
) -> Result<Vec<u8>, LoadError> {
static SAID: std::sync::Once = std::sync::Once::new();
SAID.call_once(|| tracing::warn!("lurq is built without the feature `http`: an address is not loaded"));
Err(LoadError::Unsupported(
"this build has no `http` feature: it loads no address",
))
}
#[cfg(all(test, not(target_arch = "wasm32")))]
mod tests {
use std::{path::Path, sync::atomic::AtomicUsize, time::Duration};
use super::*;
thread_local! {
static ON_A_RUNTIME: std::cell::Cell<bool> = const { std::cell::Cell::new(false) };
}
#[cfg(feature = "tokio")]
fn runtime() -> tokio::runtime::Handle {
static RUNTIME: OnceLock<tokio::runtime::Runtime> = OnceLock::new();
let runtime = RUNTIME.get_or_init(|| {
tokio::runtime::Builder::new_multi_thread()
.worker_threads(2)
.build()
.expect("a runtime for the tests")
});
runtime.handle().clone()
}
fn loader() -> Loader {
let loader = Loader::new();
#[cfg(feature = "tokio")]
if ON_A_RUNTIME.get() {
loader.tokio_handle(runtime());
}
loader
}
fn run() -> &'static str {
if ON_A_RUNTIME.get() { "-on-a-runtime" } else { "" }
}
fn ended(load: &Load) -> LoadState {
let deadline = std::time::Instant::now() + Duration::from_secs(10);
loop {
let state = load.state();
if !state.is_loading() {
return state;
}
assert!(std::time::Instant::now() < deadline, "the load does not end");
std::thread::sleep(Duration::from_millis(2));
}
}
fn file(name: &str, bytes: &[u8]) -> PathBuf {
let folder = std::env::temp_dir().join(format!("lurq-loader-{}{}", std::process::id(), run()));
std::fs::create_dir_all(&folder).unwrap();
let path = folder.join(name);
std::fs::write(&path, bytes).unwrap();
path
}
fn folder_of(path: &Path) -> PathBuf {
path.parent().unwrap().to_path_buf()
}
#[test]
fn bytes_are_ready_at_once_and_start_no_thread() {
let loader = loader();
let load = loader.load(&Source::bytes(vec![1, 2, 3]));
assert!(matches!(load.state(), LoadState::Ready(bytes) if bytes[..] == [1, 2, 3]));
assert!(loader.shared.pool.get().is_none());
}
#[test]
fn a_file_and_a_resource_under_the_root_are_read() {
let path = file("read.bin", &[5, 6, 7]);
let loader = loader();
assert!(matches!(ended(&loader.load(&Source::file(&path))), LoadState::Ready(bytes) if bytes[..] == [5, 6, 7]));
loader.root(folder_of(&path));
let resource = loader.load(&Source::resource("read.bin"));
assert!(matches!(ended(&resource), LoadState::Ready(bytes) if bytes[..] == [5, 6, 7]));
}
#[test]
fn a_file_that_is_not_there_fails_and_stays_failed_until_forgotten() {
let path = std::env::temp_dir().join(format!("lurq-loader-none-{}{}.bin", std::process::id(), run()));
let source = Source::file(&path);
let loader = loader();
assert!(matches!(
ended(&loader.load(&source)),
LoadState::Failed(LoadError::NotFound)
));
std::fs::write(&path, [1]).unwrap();
assert!(matches!(
loader.load(&source).state(),
LoadState::Failed(LoadError::NotFound)
));
assert!(loader.forget(&source));
assert!(matches!(ended(&loader.load(&source)), LoadState::Ready(bytes) if bytes[..] == [1]));
std::fs::remove_file(&path).unwrap();
}
#[test]
fn a_file_over_the_limit_is_not_read_whole() {
let path = file("large.bin", &[0; 4096]);
let loader = loader();
loader.max_bytes(1024);
assert!(matches!(
ended(&loader.load(&Source::file(&path))),
LoadState::Failed(LoadError::TooLarge { limit: 1024 })
));
}
#[test]
fn the_start_of_a_source_is_read_and_no_more_and_is_no_error() {
let path = file("start.bin", &[1, 2, 3, 4, 5, 6]);
let loader = loader();
let source = Source::file(&path);
assert!(matches!(ended(&loader.load_start(&source, 4)), LoadState::Ready(bytes) if bytes[..] == [1, 2, 3, 4]));
assert!(
matches!(ended(&loader.load_start(&source, 100)), LoadState::Ready(bytes) if bytes[..] == [1, 2, 3, 4, 5, 6])
);
assert!(matches!(ended(&loader.load(&source)), LoadState::Ready(bytes) if bytes.len() == 6));
let bytes = Source::bytes(vec![7, 8, 9]);
assert!(matches!(loader.load_start(&bytes, 2).state(), LoadState::Ready(start) if start[..] == [7, 8]));
}
#[test]
fn the_same_source_asked_twice_is_one_load_and_wakes_who_waits() {
let path = file("twice.bin", &[9; 1000]);
let loader = loader();
let source = Source::file(&path);
let woken = Arc::new(AtomicUsize::new(0));
let first = loader.load(&source);
let count = woken.clone();
let wake: LoadWaker = Arc::new(move || {
count.fetch_add(1, Ordering::SeqCst);
});
first.wake_with(wake.clone());
first.wake_with(wake);
let second = loader.load(&source);
assert!(Arc::ptr_eq(&first.entry, &second.entry));
assert!(matches!(ended(&second), LoadState::Ready(_)));
assert!(woken.load(Ordering::SeqCst) <= 1);
}
#[test]
fn what_nobody_holds_is_let_go_after_a_moment_and_a_failure_is_kept() {
let path = file("sweep.bin", &[1, 2]);
let missing = Source::file(folder_of(&path).join("no-such-file.bin"));
let loader = loader();
let source = Source::file(&path);
let load = loader.load(&source);
let failed = loader.load(&missing);
ended(&load);
ended(&failed);
drop((load, failed));
loader.sweep();
assert_eq!(
loader.shared.entries.lock().len(),
2,
"a rebuild a moment later still finds both"
);
std::thread::sleep(IDLE_GRACE + Duration::from_millis(50));
loader.sweep();
let entries = loader.shared.entries.lock();
assert_eq!(entries.len(), 1, "the bytes are let go");
assert!(entries.contains_key(&missing.cache_key()), "the failure is kept");
}
#[test]
fn an_address_the_app_does_not_allow_is_denied_and_nothing_reads_it() {
let loader = loader();
loader.allow(|address| address.starts_with("https://cdn.example.com/"));
let load = loader.load(&Source::url("https://elsewhere.example.com/a.png"));
assert!(matches!(load.state(), LoadState::Failed(LoadError::Denied)));
assert!(loader.shared.pool.get().is_none());
}
#[cfg(not(feature = "http"))]
#[test]
fn a_build_without_http_says_it_loads_no_address() {
let loader = loader();
let state = ended(&loader.load(&Source::url("https://example.com/a.png")));
assert!(matches!(state, LoadState::Failed(LoadError::Unsupported(_))));
}
#[test]
fn a_reader_reads_a_part_of_a_file_and_of_bytes() {
let path = file("part.bin", &[1, 2, 3, 4, 5, 6]);
let loader = loader();
let mut buf = [0u8; 2];
let reader = loader.open(&Source::file(&path)).unwrap();
assert_eq!(reader.read_at(4, &mut buf).unwrap(), 2);
assert_eq!(buf, [5, 6]);
let reader = loader.open(&Source::bytes(vec![7, 8, 9])).unwrap();
assert_eq!(reader.read_at(1, &mut buf).unwrap(), 2);
assert_eq!(buf, [8, 9]);
assert!(matches!(
loader.open(&Source::file(folder_of(&path).join("none.bin"))),
Err(LoadError::NotFound)
));
assert!(matches!(
loader.open(&Source::url("https://example.com/a.mp4")),
Err(LoadError::Unsupported(_))
));
}
#[cfg(feature = "http")]
fn serve(body: &'static [u8]) -> (String, Arc<AtomicUsize>) {
use std::io::{Read, Write};
let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
let address = format!("http://{}/picture.png", listener.local_addr().unwrap());
let asked = Arc::new(AtomicUsize::new(0));
let counted = asked.clone();
std::thread::spawn(move || {
for stream in listener.incoming() {
let Ok(mut stream) = stream else {
return;
};
let (mut request, mut chunk) = (Vec::new(), [0_u8; 1024]);
while !request.windows(4).any(|window| window == b"\r\n\r\n") {
match stream.read(&mut chunk) {
Ok(read) if read > 0 => request.extend_from_slice(&chunk[..read]),
_ => break,
}
}
counted.fetch_add(1, Ordering::SeqCst);
let head = format!(
"HTTP/1.1 200 OK\r\nContent-Length: {}\r\nConnection: close\r\n\r\n",
body.len()
);
let _ = stream.write_all(head.as_bytes());
let _ = stream.write_all(body);
}
});
(address, asked)
}
#[cfg(feature = "http")]
#[test]
fn what_came_from_an_address_is_kept_in_the_folder_the_app_gave_and_read_from_there() {
let folder = std::env::temp_dir().join(format!("lurq-loader-cache-{}{}", std::process::id(), run()));
let _ = std::fs::remove_dir_all(&folder);
let (address, asked) = serve(&[4, 5, 6, 7]);
let source = Source::url(&address);
let first = loader();
first.enable_media_disk_cache(&folder);
assert!(matches!(ended(&first.load(&source)), LoadState::Ready(bytes) if bytes[..] == [4, 5, 6, 7]));
assert!(first.media_disk_cache().holds(&source));
assert_eq!(std::fs::read_dir(&folder).unwrap().count(), 1);
let next = loader();
next.enable_media_disk_cache(&folder);
assert!(matches!(ended(&next.load(&source)), LoadState::Ready(bytes) if bytes[..] == [4, 5, 6, 7]));
assert_eq!(asked.load(Ordering::SeqCst), 1);
assert!(next.forget(&source));
assert_eq!(std::fs::read_dir(&folder).unwrap().count(), 0);
assert!(matches!(ended(&next.load(&source)), LoadState::Ready(bytes) if bytes[..] == [4, 5, 6, 7]));
assert_eq!(asked.load(Ordering::SeqCst), 2);
let plain = loader();
assert!(matches!(ended(&plain.load(&source)), LoadState::Ready(_)));
assert_eq!(asked.load(Ordering::SeqCst), 3);
assert_eq!(std::fs::read_dir(&folder).unwrap().count(), 1);
let _ = std::fs::remove_dir_all(&folder);
}
#[cfg(feature = "tokio")]
mod on_a_runtime {
use super::*;
macro_rules! again {
($($test:ident),* $(,)?) => {
$(
#[test]
fn $test() {
ON_A_RUNTIME.set(true);
super::$test();
}
)*
};
}
again!(
bytes_are_ready_at_once_and_start_no_thread,
a_file_and_a_resource_under_the_root_are_read,
a_file_that_is_not_there_fails_and_stays_failed_until_forgotten,
a_file_over_the_limit_is_not_read_whole,
the_start_of_a_source_is_read_and_no_more_and_is_no_error,
the_same_source_asked_twice_is_one_load_and_wakes_who_waits,
what_nobody_holds_is_let_go_after_a_moment_and_a_failure_is_kept,
an_address_the_app_does_not_allow_is_denied_and_nothing_reads_it,
a_reader_reads_a_part_of_a_file_and_of_bytes,
);
#[cfg(feature = "http")]
again!(what_came_from_an_address_is_kept_in_the_folder_the_app_gave_and_read_from_there);
fn read(loader: &Loader, test: &str, count: usize) {
for index in 0..count {
let path = file(&format!("{test}-{index}.bin"), &[index as u8; 8]);
let load = loader.load(&Source::file(&path));
assert!(matches!(ended(&load), LoadState::Ready(bytes) if bytes.len() == 8));
}
}
fn stopped() -> tokio::runtime::Handle {
let runtime = tokio::runtime::Builder::new_multi_thread()
.worker_threads(1)
.build()
.expect("a runtime to stop");
runtime.handle().clone()
}
#[test]
fn with_a_handle_given_a_load_ends_and_no_thread_of_the_loaders_own_is_started() {
let loader = Loader::new();
loader.tokio_handle(runtime());
read(&loader, "on-a-runtime", 6);
assert!(
loader.shared.pool.get().is_none(),
"no pool of the loader's own was made"
);
assert_eq!(loader.pending(), 0);
}
#[test]
fn a_load_given_to_a_runtime_that_stopped_fails_and_does_not_wait_for_ever() {
let loader = Loader::new();
loader.tokio_handle(stopped());
let path = file("stopped.bin", &[1, 2, 3]);
assert!(matches!(
ended(&loader.load(&Source::file(&path))),
LoadState::Failed(LoadError::Cancelled)
));
assert!(loader.shared.pool.get().is_none());
}
#[test]
fn a_handle_given_later_is_where_the_next_load_is_read_and_one_taken_away_gives_the_pool_back() {
let loader = Loader::new();
read(&loader, "before-a-handle", 1);
assert!(
loader.shared.pool.get().is_some(),
"with no handle the loader reads on its own threads"
);
loader.tokio_handle(stopped());
let path = file("given-later.bin", &[1, 2, 3]);
assert!(matches!(
ended(&loader.load(&Source::file(&path))),
LoadState::Failed(LoadError::Cancelled)
));
loader.tokio_handle(runtime());
read(&loader, "after-a-handle", 1);
loader.tokio_handle(stopped());
loader.reset_tokio_handle();
read(&loader, "after-a-handle-was-taken", 1);
assert_eq!(loader.pending(), 0);
}
#[test]
fn no_more_loads_are_read_at_a_time_on_a_runtime_than_the_loader_has_threads() {
let loader = Loader::new();
loader.tokio_handle(runtime());
let (now, most) = (Arc::new(AtomicUsize::new(0)), Arc::new(AtomicUsize::new(0)));
let megabyte = vec![7_u8; 1 << 20];
let paths: Vec<PathBuf> = (0..LOADER_THREADS * 3)
.map(|index| file(&format!("room-{index}.bin"), &megabyte))
.collect();
let loads: Vec<Load> = paths
.iter()
.map(|path| {
let load = loader.load(&Source::file(path));
let (now, most) = (now.clone(), most.clone());
let wake: LoadWaker = Arc::new(move || {
most.fetch_max(now.fetch_add(1, Ordering::SeqCst) + 1, Ordering::SeqCst);
std::thread::sleep(Duration::from_millis(2));
now.fetch_sub(1, Ordering::SeqCst);
});
load.wake_with(wake);
load
})
.collect();
for load in &loads {
assert!(matches!(ended(load), LoadState::Ready(bytes) if bytes.len() == 1 << 20));
}
for path in paths {
let _ = std::fs::remove_file(path);
}
let most = most.load(Ordering::SeqCst);
assert!((1..=LOADER_THREADS).contains(&most), "{most} at a time");
assert!(loader.shared.pool.get().is_none());
}
}
}