use std::{
collections::VecDeque,
io::{self, Read},
ops::Range,
sync::{
Arc,
atomic::{AtomicBool, AtomicU64, AtomicUsize, Ordering},
},
time::{Duration, Instant},
};
use parking_lot::{Condvar, Mutex};
use super::{LoadError, Reader};
const BUSY_RETRIES: u32 = 2;
const BUSY_PAUSE: Duration = Duration::from_secs(2);
pub(super) fn fetch(
agents: &Agents,
address: &str,
headers: &[(Arc<str>, Arc<str>)],
timeout: Duration,
max_bytes: u64,
cut: bool,
mut told: impl FnMut(u64, Option<u64>) -> bool,
) -> Result<Vec<u8>, LoadError> {
let agent = agents.lend(address);
let response = ask(&agent, address, headers, None, timeout, &mut told)?;
let body = response.into_body();
let total = body.content_length();
if !cut && total.is_some_and(|total| total > max_bytes) {
return Err(LoadError::TooLarge { limit: max_bytes });
}
let mut reader = body.into_reader();
let mut bytes = Vec::with_capacity(total.unwrap_or(0).min(max_bytes) as usize);
let mut chunk = vec![0u8; 64 * 1024];
loop {
let read = reader.read(&mut chunk).map_err(read_failed)?;
if read == 0 {
return Ok(bytes);
}
bytes.extend_from_slice(&chunk[..read]);
if bytes.len() as u64 >= max_bytes && cut {
bytes.truncate(usize::try_from(max_bytes).unwrap_or(usize::MAX));
return Ok(bytes);
}
if bytes.len() as u64 > max_bytes {
return Err(LoadError::TooLarge { limit: max_bytes });
}
if !told(bytes.len() as u64, total) {
return Err(LoadError::Cancelled);
}
}
}
pub(super) struct Agents {
idle: Mutex<Vec<(ureq::Agent, Box<str>)>>,
size: AtomicUsize,
made: AtomicUsize,
connected: Mutex<Vec<(Box<str>, Instant)>>,
}
impl Agents {
pub(super) fn new(size: usize) -> Arc<Self> {
Arc::new(Self {
idle: Mutex::new(Vec::new()),
size: AtomicUsize::new(size),
made: AtomicUsize::new(0),
connected: Mutex::new(Vec::new()),
})
}
pub(super) fn resize(&self, size: usize) {
self.size.store(size, Ordering::Release);
self.idle.lock().truncate(size);
}
pub(super) fn made(&self) -> usize {
self.made.load(Ordering::Acquire)
}
pub(super) fn lend(&self, address: &str) -> Lent<'_> {
let host = host_of(address);
let idle = {
let mut idle = self.idle.lock();
let at = idle
.iter()
.rposition(|(_, asked)| **asked == *host)
.or_else(|| idle.len().checked_sub(1));
at.map(|at| idle.remove(at).0)
};
let agent = idle.unwrap_or_else(|| {
self.made.fetch_add(1, Ordering::AcqRel);
ureq::Agent::config_builder().http_status_as_error(false).build().into()
});
Lent {
agents: self,
agent: Some(agent),
host: host.into(),
}
}
}
fn host_of(address: &str) -> &str {
let after = address.find("://").map_or(0, |at| at + 3);
let end = address[after..]
.find(['/', '?', '#'])
.map_or(address.len(), |at| after + at);
&address[..end]
}
pub(super) struct Lent<'a> {
agents: &'a Agents,
agent: Option<ureq::Agent>,
host: Box<str>,
}
impl std::ops::Deref for Lent<'_> {
type Target = ureq::Agent;
fn deref(&self) -> &ureq::Agent {
self.agent.as_ref().expect("an agent until it is given back")
}
}
impl Drop for Lent<'_> {
fn drop(&mut self) {
let mut idle = self.agents.idle.lock();
if idle.len() < self.agents.size.load(Ordering::Acquire)
&& let Some(agent) = self.agent.take()
{
idle.push((agent, std::mem::take(&mut self.host)));
}
}
}
pub(super) fn connect(agents: &Arc<Agents>, address: String, headers: Vec<(Arc<str>, Arc<str>)>, timeout: Duration) {
{
let host = host_of(&address);
let mut connected = agents.connected.lock();
connected.retain(|(_, at)| at.elapsed() < CONNECTED_FOR);
if connected.iter().any(|(had, _)| **had == *host) {
return;
}
connected.push((host.into(), Instant::now()));
}
let agents = agents.clone();
let _ = std::thread::Builder::new()
.name("lurq-connect".to_owned())
.spawn(move || {
let asked = Instant::now();
let agent = agents.lend(&address);
let mut request = agent.head(&address);
for (name, value) in &headers {
request = request.header(name.as_ref(), value.as_ref());
}
let answered = request
.config()
.timeout_global(Some(timeout.min(CONNECT_WITHIN)))
.build()
.call();
let ms = asked.elapsed().as_millis() as u64;
match answered {
Ok(response) => {
let status = response.status().as_u16();
let _ = io::copy(&mut response.into_body().into_reader(), &mut io::sink());
tracing::debug!(target: "lurq::resources", address = %address, status, ms, "connected ahead");
}
Err(error) => {
tracing::debug!(target: "lurq::resources", address = %address, ms, "no connection ahead: {error}");
}
}
});
}
fn ask(
agent: &ureq::Agent,
address: &str,
headers: &[(Arc<str>, Arc<str>)],
range: Option<&str>,
timeout: Duration,
told: &mut dyn FnMut(u64, Option<u64>) -> bool,
) -> Result<ureq::http::Response<ureq::Body>, LoadError> {
let started = Instant::now();
let mut busy = 0;
loop {
let left = timeout.saturating_sub(started.elapsed());
if left.is_zero() {
return Err(LoadError::TimedOut);
}
let mut request = agent.get(address);
for (name, value) in headers {
request = request.header(name.as_ref(), value.as_ref());
}
if let Some(range) = range {
request = request.header("Range", range);
}
let response = request
.config()
.timeout_global(Some(left))
.build()
.call()
.map_err(error_of)?;
let status = response.status().as_u16();
if status < 400 {
return Ok(response);
}
if matches!(status, 429 | 503) && busy < BUSY_RETRIES {
busy += 1;
let said = response
.headers()
.get("retry-after")
.and_then(|value| value.to_str().ok())
.and_then(|value| value.trim().parse::<u64>().ok())
.map(Duration::from_secs);
let pause = said
.unwrap_or(Duration::from_secs(u64::from(busy)))
.min(BUSY_PAUSE)
.min(timeout.saturating_sub(started.elapsed()));
tracing::debug!(target: "lurq::resources", "{address} answered {status}: asked again in {pause:?}");
if !told(0, None) {
return Err(LoadError::Cancelled);
}
std::thread::sleep(pause);
continue;
}
return Err(match status {
404 => LoadError::NotFound,
status => LoadError::Status(status),
});
}
}
fn read_failed(error: io::Error) -> LoadError {
match error.kind() {
io::ErrorKind::TimedOut => LoadError::TimedOut,
_ => LoadError::Io(error.to_string()),
}
}
const HELD_BLOCKS: usize = 16;
const READ_AHEAD: u64 = 2;
const REQUESTS: usize = 3;
const OPENED_WITH: usize = 1024;
const CHUNK: usize = 32 * 1024;
const START_KEPT: u64 = 2 << 20;
const START_KEPT_FOR: Duration = Duration::from_secs(5 * 60);
const CONNECTED_FOR: Duration = Duration::from_secs(10);
const CONNECT_WITHIN: Duration = Duration::from_secs(10);
pub(super) struct Starts {
kept: Mutex<VecDeque<Start>>,
most: AtomicUsize,
}
#[derive(Clone)]
struct Start {
key: u64,
cut: Cut,
read: Instant,
len: Option<u64>,
content_type: Option<Arc<str>>,
blocks: Vec<Arc<[u8]>>,
}
impl Starts {
pub(super) fn new(most: usize) -> Arc<Self> {
Arc::new(Self {
kept: Mutex::new(VecDeque::new()),
most: AtomicUsize::new(most),
})
}
pub(super) fn resize(&self, most: usize) {
self.most.store(most, Ordering::Release);
let mut kept = self.kept.lock();
while kept.len() > most {
kept.pop_front();
}
}
pub(super) fn forget(&self, key: u64) -> bool {
let mut kept = self.kept.lock();
let before = kept.len();
kept.retain(|start| start.key != key);
kept.len() != before
}
fn keep(&self, start: Start) {
let mut kept = self.kept.lock();
kept.retain(|had| had.key != start.key);
kept.push_back(start);
while kept.len() > self.most.load(Ordering::Acquire) {
kept.pop_front();
}
}
fn of(&self, key: u64, cut: Cut) -> Option<Start> {
let mut kept = self.kept.lock();
kept.retain(|start| start.read.elapsed() < START_KEPT_FOR);
kept.iter().find(|start| start.key == key && start.cut == cut).cloned()
}
}
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub(super) struct Cut {
pub first: u64,
pub most: u64,
}
impl Cut {
#[cfg(test)]
pub(super) fn even(size: u64) -> Self {
Self {
first: size,
most: size,
}
}
fn sizes(self) -> (u64, u64) {
let most = self.most.max(1);
(self.first.clamp(1, most), most)
}
fn block(self, index: u64) -> (u64, u64) {
let (first, most) = self.sizes();
let (mut start, mut size) = (0, first);
for at in 0..index {
if size == most {
return (start + (index - at) * most, most);
}
start += size;
if at > 0 {
size = (size * 2).min(most);
}
}
(start, size)
}
fn index_of(self, offset: u64) -> u64 {
let (first, most) = self.sizes();
let (mut index, mut start, mut size) = (0, 0, first);
while offset >= start + size {
if size == most {
return index + (offset - start) / most;
}
start += size;
if index > 0 {
size = (size * 2).min(most);
}
index += 1;
}
index
}
}
fn index_at_the_end(start: &[u8]) -> Option<u64> {
let mut at = 0usize;
let mut seen_type = false;
for _ in 0..8 {
let header = start.get(at..at + 8)?;
let short = u64::from(u32::from_be_bytes(header[..4].try_into().ok()?));
let kind = &header[4..8];
let size = match short {
0 => return None,
1 => u64::from_be_bytes(start.get(at + 8..at + 16)?.try_into().ok()?),
short => short,
};
match kind {
b"ftyp" | b"styp" => seen_type = true,
b"moov" => return None,
b"mdat" if seen_type => return (at as u64).checked_add(size),
b"free" | b"skip" | b"wide" | b"mdat" => {}
_ if !seen_type => return None,
_ => {}
}
at = usize::try_from((at as u64).checked_add(size)?).ok()?;
}
None
}
pub(super) struct HttpReader(Arc<Ranges>);
struct Ranges {
address: String,
headers: Vec<(Arc<str>, Arc<str>)>,
timeout: Duration,
cut: Cut,
agent: ureq::Agent,
blocks: Mutex<Blocks>,
came: Condvar,
eager: AtomicBool,
last: AtomicU64,
starts: Arc<Starts>,
key: u64,
since: Instant,
}
#[derive(Default)]
struct Blocks {
len: Option<u64>,
content_type: Option<Arc<str>>,
answered: bool,
held: VecDeque<Held>,
coming: Vec<u64>,
waited: Vec<u64>,
failed: Vec<(u64, LoadError, bool)>,
wanted: Vec<u64>,
working: usize,
ahead: bool,
began: bool,
closed: bool,
}
struct Held {
index: u64,
bytes: Bytes,
}
enum Bytes {
Coming(Vec<u8>),
Whole(Arc<[u8]>),
}
impl Bytes {
fn len(&self) -> usize {
match self {
Self::Coming(bytes) => bytes.len(),
Self::Whole(bytes) => bytes.len(),
}
}
}
impl Blocks {
fn held_at(&self, index: u64) -> Option<usize> {
self.held.iter().position(|held| held.index == index)
}
fn whole(&self, index: u64) -> bool {
self
.held
.iter()
.any(|held| held.index == index && matches!(held.bytes, Bytes::Whole(_)))
}
fn filled(&self, index: u64) -> u64 {
self.held_at(index).map_or(0, |at| self.held[at].bytes.len() as u64)
}
fn failed_of(&self, index: u64) -> Option<(LoadError, bool)> {
self
.failed
.iter()
.find(|(failed, ..)| *failed == index)
.map(|(_, error, ahead)| (error.clone(), *ahead))
}
fn waits_no_more(&mut self, index: u64) {
if let Some(at) = self.waited.iter().position(|waited| *waited == index) {
self.waited.swap_remove(at);
}
}
}
pub(super) struct Asked {
pub key: u64,
pub address: String,
pub headers: Vec<(Arc<str>, Arc<str>)>,
pub timeout: Duration,
pub cut: Cut,
pub read_ahead: Option<bool>,
}
struct Run {
first: u64,
last: u64,
ahead: bool,
}
impl Run {
fn holds(&self, index: u64) -> bool {
(self.first..=self.last).contains(&index)
}
}
impl HttpReader {
pub(super) fn open(agents: &Agents, starts: Arc<Starts>, asked: Asked) -> Result<Self, LoadError> {
let Asked {
key,
address,
headers,
timeout,
cut,
read_ahead,
} = asked;
let agent = (*agents.lend(&address)).clone();
let kept = starts.of(key, cut);
let mut blocks = Blocks {
began: read_ahead == Some(true),
..Blocks::default()
};
let since = kept.as_ref().map_or_else(Instant::now, |start| start.read);
if let Some(start) = kept {
tracing::debug!(target: "lurq::resources", address = %address, blocks = start.blocks.len(), "its start was kept");
blocks.len = start.len;
blocks.content_type = start.content_type;
blocks.answered = true;
blocks.held = start
.blocks
.into_iter()
.enumerate()
.map(|(index, bytes)| Held {
index: index as u64,
bytes: Bytes::Whole(bytes),
})
.collect();
}
let ranges = Arc::new(Ranges {
address,
headers,
timeout,
cut,
agent,
blocks: Mutex::new(blocks),
came: Condvar::new(),
eager: AtomicBool::new(read_ahead != Some(false)),
last: AtomicU64::new(0),
starts,
key,
since,
});
let reader = Self(ranges.clone());
let mut blocks = ranges.blocks.lock();
blocks.waited.push(0);
let start = loop {
let there = blocks.held_at(0).map(|at| &blocks.held[at].bytes);
match there {
Some(Bytes::Whole(bytes)) => break bytes.to_vec(),
Some(Bytes::Coming(bytes)) if blocks.answered && bytes.len() >= OPENED_WITH => break bytes.clone(),
_ => {}
}
if blocks.answered && blocks.len == Some(0) {
break Vec::new();
}
if !blocks.coming.contains(&0) {
if let Some((error, _)) = blocks.failed_of(0) {
return Err(error);
}
ranges.rouse(&mut blocks);
if !blocks.coming.contains(&0) {
return Err(blocks.failed_of(0).map_or(LoadError::Cancelled, |(error, _)| error));
}
}
ranges.came.wait(&mut blocks);
};
blocks.waits_no_more(0);
if let Some(index) = index_at_the_end(&start)
&& blocks.len.is_some_and(|len| index < len)
{
tracing::debug!(target: "lurq::resources", address = %ranges.address, at = index, "the index is at the end: read ahead");
let first = ranges.cut.index_of(index);
blocks.wanted = (first..=first + READ_AHEAD).collect();
}
ranges.rouse(&mut blocks);
drop(blocks);
ranges.came.notify_all();
Ok(reader)
}
}
impl Drop for HttpReader {
fn drop(&mut self) {
let ranges = &self.0;
let mut blocks = ranges.blocks.lock();
blocks.closed = true;
let mut start = Vec::new();
let mut bytes = 0;
for index in 0.. {
let Some(Bytes::Whole(block)) = blocks.held_at(index).map(|at| &blocks.held[at].bytes) else {
break;
};
bytes += block.len() as u64;
if bytes > START_KEPT {
break;
}
start.push(block.clone());
}
let (len, content_type) = (blocks.len, blocks.content_type.clone());
drop(blocks);
ranges.came.notify_all();
if !start.is_empty() {
ranges.starts.keep(Start {
key: ranges.key,
cut: ranges.cut,
read: ranges.since,
len,
content_type,
blocks: start,
});
}
}
}
impl Reader for HttpReader {
fn len(&self) -> Option<u64> {
self.0.blocks.lock().len
}
fn content_type(&self) -> Option<Arc<str>> {
self.0.blocks.lock().content_type.clone()
}
fn read_at(&self, offset: u64, buf: &mut [u8]) -> io::Result<usize> {
if buf.is_empty() {
return Ok(0);
}
let ranges = &self.0;
let index = ranges.cut.index_of(offset);
let within = (offset - ranges.cut.block(index).0) as usize;
let mut blocks = ranges.blocks.lock();
blocks.began = true;
let mut waited = false;
loop {
if blocks.len.is_some_and(|len| offset >= len) {
return Ok(0);
}
if let Some(at) = blocks.held_at(index) {
if blocks.held[at].bytes.len() > within {
let held = blocks.held.remove(at).expect("the block just found");
let count = (held.bytes.len() - within).min(buf.len());
let whole = match &held.bytes {
Bytes::Whole(bytes) => Some(bytes.clone()),
Bytes::Coming(bytes) => {
buf[..count].copy_from_slice(&bytes[within..within + count]);
None
}
};
blocks.held.push_back(held);
ranges.last.store(index, Ordering::Relaxed);
ranges.rouse(&mut blocks);
drop(blocks);
if let Some(bytes) = whole {
buf[..count].copy_from_slice(&bytes[within..within + count]);
}
return Ok(count);
}
if matches!(blocks.held[at].bytes, Bytes::Whole(_)) {
return Ok(0);
}
}
if !blocks.coming.contains(&index) {
match blocks.failed_of(index) {
Some((error, false)) if waited => return Err(io::Error::other(error)),
Some(_) => blocks.failed.retain(|(failed, ..)| *failed != index),
None => {}
}
}
blocks.waited.push(index);
ranges.rouse(&mut blocks);
if !blocks.coming.contains(&index)
&& let Some((error, _)) = blocks.failed_of(index)
{
blocks.waits_no_more(index);
return Err(io::Error::other(error));
}
ranges.came.notify_all();
ranges.came.wait(&mut blocks);
blocks.waits_no_more(index);
waited = true;
}
}
fn buffered(&self) -> Vec<Range<u64>> {
let cut = self.0.cut;
let mut held: Vec<Range<u64>> = self
.0
.blocks
.lock()
.held
.iter()
.filter(|held| held.bytes.len() > 0)
.map(|held| {
let (start, _) = cut.block(held.index);
start..start + held.bytes.len() as u64
})
.collect();
held.sort_by_key(|range| range.start);
let mut joined: Vec<Range<u64>> = Vec::new();
for range in held {
match joined.last_mut() {
Some(last) if last.end >= range.start => last.end = last.end.max(range.end),
_ => joined.push(range),
}
}
joined
}
fn read_ahead(&self, ahead: bool) {
if self.0.eager.swap(ahead, Ordering::Relaxed) == ahead || !ahead {
return;
}
let mut blocks = self.0.blocks.lock();
self.0.rouse(&mut blocks);
}
}
impl Ranges {
fn missing(&self, blocks: &Blocks, index: u64) -> bool {
!blocks.whole(index)
&& !blocks.coming.contains(&index)
&& blocks.len.is_none_or(|len| self.cut.block(index).0 < len)
}
fn job(&self, blocks: &mut Blocks, in_turn: bool) -> Option<Run> {
if blocks.closed {
return None;
}
let eager = self.eager.load(Ordering::Relaxed);
let last = self.last.load(Ordering::Relaxed);
let fresh = |index: &u64| self.missing(blocks, *index) && blocks.failed_of(*index).is_none();
let waited = blocks.waited.iter().copied().find(fresh);
let (first, ahead) = match waited {
Some(index) => (index, false),
None if !blocks.ahead && blocks.waited.iter().all(|index| blocks.whole(*index)) => {
let wanted = blocks.wanted.iter().copied().find(fresh);
let next = || (1..=READ_AHEAD).map(|ahead| last + ahead).find(fresh);
(
wanted.or_else(|| (in_turn && eager && blocks.began).then(next).flatten())?,
true,
)
}
None => return None,
};
let reach = last.max(first) + READ_AHEAD;
let (_, most) = self.cut.sizes();
let mut run = Run {
first,
last: first,
ahead,
};
let mut bytes = self.cut.block(first).1;
loop {
let next = run.last + 1;
let size = self.cut.block(next).1;
let asked_for = (eager && blocks.began && next <= reach) || blocks.wanted.contains(&next);
if !asked_for
|| bytes + size > most
|| !self.missing(blocks, next)
|| blocks.held_at(next).is_some()
|| blocks.failed_of(next).is_some()
{
break;
}
bytes += size;
run.last = next;
}
blocks.coming.extend(run.first..=run.last);
blocks.working += 1;
blocks.ahead |= run.ahead;
Some(run)
}
fn rouse(self: &Arc<Self>, blocks: &mut Blocks) {
while blocks.working < REQUESTS {
let idle = blocks.working == 0;
let Some(run) = self.job(blocks, idle) else {
return;
};
let (ranges, first, last, ahead) = (self.clone(), run.first, run.last, run.ahead);
let spawned = std::thread::Builder::new()
.name("lurq-read".to_owned())
.spawn(move || ranges.work(run));
if let Err(error) = spawned {
blocks.coming.retain(|index| !(first..=last).contains(index));
blocks.working -= 1;
blocks.ahead &= !ahead;
blocks
.failed
.extend((first..=last).map(|index| (index, LoadError::Io(error.to_string()), false)));
return;
}
}
}
fn work(self: Arc<Self>, mut run: Run) {
loop {
let brought = self.bring(&run);
let mut blocks = self.blocks.lock();
blocks.coming.retain(|index| !run.holds(*index));
blocks.working -= 1;
blocks.ahead &= !run.ahead;
let went_well = brought.is_ok();
if let Err(error) = brought {
tracing::debug!(target: "lurq::resources", address = %self.address, "a range failed: {error}");
for index in run.first..=run.last {
if !blocks.whole(index) {
blocks.failed.push((index, error.clone(), run.ahead));
}
}
}
let next = self.job(&mut blocks, went_well);
drop(blocks);
self.came.notify_all();
match next {
Some(next) => run = next,
None => return,
}
}
}
fn goes_on(&self, run: &Run, asked: Instant) -> bool {
let mut blocks = self.blocks.lock();
loop {
if blocks.closed {
return false;
}
let gives_way = !blocks.waited.iter().any(|index| run.holds(*index))
&& blocks
.waited
.iter()
.any(|index| blocks.coming.contains(index) && !blocks.whole(*index))
&& asked.elapsed() < self.timeout / 2;
if !gives_way {
return true;
}
self.came.wait_for(&mut blocks, Duration::from_millis(20));
}
}
fn put(&self, mut at: u64, mut bytes: &[u8]) {
let mut blocks = self.blocks.lock();
while !bytes.is_empty() {
let index = self.cut.index_of(at);
let (start, size) = self.cut.block(index);
let place = match blocks.held_at(index) {
Some(place) => place,
None => {
let (_, most) = self.cut.sizes();
if blocks.held.len() >= HELD_BLOCKS
&& let Some(oldest) = blocks.held.iter().position(|held| {
self.cut.block(held.index).0 >= most
&& !blocks.coming.contains(&held.index)
&& !blocks.waited.contains(&held.index)
})
{
blocks.held.remove(oldest);
}
blocks.held.push_back(Held {
index,
bytes: Bytes::Coming(Vec::with_capacity(size as usize)),
});
blocks.held.len() - 1
}
};
let room = (start + size - at) as usize;
let (now, rest) = bytes.split_at(bytes.len().min(room));
let len = blocks.len;
let held = &mut blocks.held[place];
if let Bytes::Coming(there) = &mut held.bytes {
if start + there.len() as u64 == at {
there.extend_from_slice(now);
}
let end = start + there.len() as u64;
if there.len() as u64 >= size || len.is_some_and(|len| end >= len) {
held.bytes = Bytes::Whole(Arc::from(std::mem::take(there)));
}
}
at += now.len() as u64;
bytes = rest;
}
drop(blocks);
self.came.notify_all();
}
fn ends_at(&self, at: u64) {
let mut blocks = self.blocks.lock();
let len = blocks.len.map_or(at, |len| len.min(at));
blocks.len = Some(len);
blocks.answered = true;
for held in &mut blocks.held {
let (start, _) = self.cut.block(held.index);
if let Bytes::Coming(there) = &mut held.bytes
&& start + there.len() as u64 >= len
{
held.bytes = Bytes::Whole(Arc::from(std::mem::take(there)));
}
}
drop(blocks);
self.came.notify_all();
}
fn bring(&self, run: &Run) -> Result<(), LoadError> {
let end = {
let (start, size) = self.cut.block(run.last);
start + size
};
loop {
let from = {
let blocks = self.blocks.lock();
if blocks.closed {
return Ok(());
}
let mut index = run.first;
while index < run.last && blocks.whole(index) {
index += 1;
}
let at = self.cut.block(index).0 + blocks.filled(index);
if at >= end || blocks.whole(run.last) || blocks.len.is_some_and(|len| at >= len) {
return Ok(());
}
at
};
let range = format!("bytes={from}-{}", end - 1);
let asked = Instant::now();
let response = match ask(
&self.agent,
&self.address,
&self.headers,
Some(&range),
self.timeout,
&mut |_, _| !self.blocks.lock().closed,
) {
Ok(response) => response,
Err(LoadError::Status(416)) => {
self.ends_at(from);
return Ok(());
}
Err(error) => return Err(error),
};
let ranged = response.status().as_u16() == 206;
let header = |name: &str| response.headers().get(name).and_then(|value| value.to_str().ok());
let said = if ranged {
total_of(header("content-range"))
} else {
response.body().content_length()
};
let content_type =
header("content-type").map(|kind| Arc::<str>::from(kind.split(';').next().unwrap_or(kind).trim()));
{
let mut blocks = self.blocks.lock();
blocks.len = said.or(blocks.len);
blocks.content_type = content_type.or(blocks.content_type.take());
blocks.answered = true;
}
self.came.notify_all();
let mut body = response.into_body().into_reader();
if !ranged {
io::copy(&mut (&mut body).take(from), &mut io::sink()).map_err(read_failed)?;
}
let mut at = from;
let mut chunk = vec![0u8; CHUNK];
while at < end {
if !self.goes_on(run, asked) {
return Ok(());
}
let most = (end - at).min(CHUNK as u64) as usize;
let read = body.read(&mut chunk[..most]).map_err(read_failed)?;
if read == 0 {
break;
}
self.put(at, &chunk[..read]);
at += read as u64;
}
tracing::debug!(
target: "lurq::resources",
address = %self.address,
range = %range,
bytes = at - from,
ms = asked.elapsed().as_millis() as u64,
"range"
);
if at >= end {
if ranged {
io::copy(&mut body, &mut io::sink()).map_err(read_failed)?;
}
return Ok(());
}
if !ranged || at == from {
self.ends_at(at);
return Ok(());
}
}
}
}
fn total_of(content_range: Option<&str>) -> Option<u64> {
content_range?.rsplit('/').next()?.trim().parse().ok()
}
fn error_of(error: ureq::Error) -> LoadError {
match error {
ureq::Error::StatusCode(404) => LoadError::NotFound,
ureq::Error::StatusCode(status) => LoadError::Status(status),
ureq::Error::Timeout(_) => LoadError::TimedOut,
other => LoadError::Io(other.to_string()),
}
}
#[cfg(test)]
mod tests {
use std::{
io::{Read, Write},
net::TcpListener,
sync::atomic::{AtomicBool, AtomicUsize, Ordering},
thread,
};
use super::*;
fn serve_once(response: Vec<u8>) -> (String, thread::JoinHandle<String>) {
let (address, served) = serve(vec![response]);
(address, thread::spawn(move || served.join().unwrap().remove(0)))
}
fn serve(responses: Vec<Vec<u8>>) -> (String, thread::JoinHandle<Vec<String>>) {
let listener = TcpListener::bind("127.0.0.1:0").unwrap();
let address = format!("http://{}/picture.png", listener.local_addr().unwrap());
let served = thread::spawn(move || {
let mut requests = Vec::new();
for response in responses {
let (mut stream, _) = listener.accept().unwrap();
let mut request = Vec::new();
let mut chunk = [0u8; 1024];
while !request.windows(4).any(|window| window == b"\r\n\r\n") {
let read = stream.read(&mut chunk).unwrap();
if read == 0 {
break;
}
request.extend_from_slice(&chunk[..read]);
}
stream.write_all(&response).unwrap();
requests.push(String::from_utf8_lossy(&request).into_owned());
}
requests
});
(address, served)
}
fn busy(status: &str, retry_after: &str) -> Vec<u8> {
format!("HTTP/1.1 {status}\r\nRetry-After: {retry_after}\r\nContent-Length: 0\r\nConnection: close\r\n\r\n")
.into_bytes()
}
fn ok(body: &[u8]) -> Vec<u8> {
let mut response = format!(
"HTTP/1.1 200 OK\r\nContent-Length: {}\r\nConnection: close\r\n\r\n",
body.len()
)
.into_bytes();
response.extend_from_slice(body);
response
}
const SECOND: Duration = Duration::from_secs(5);
#[test]
fn what_is_at_an_address_is_read_with_its_headers_and_its_progress() {
let body = vec![7u8; 200_000];
let (address, served) = serve_once(ok(&body));
let headers = [(Arc::from("X-Asked-By"), Arc::from("lurq"))];
let mut steps = Vec::new();
let bytes = fetch(
&Agents::new(1),
&address,
&headers,
SECOND,
1 << 20,
false,
|loaded, total| {
steps.push((loaded, total));
true
},
)
.unwrap();
assert_eq!(bytes, body);
assert_eq!(steps.last(), Some(&(200_000, Some(200_000))));
assert!(served.join().unwrap().to_lowercase().contains("x-asked-by: lurq"));
}
#[test]
fn an_answer_that_is_an_error_is_the_error() {
let (address, served) =
serve_once(b"HTTP/1.1 404 Not Found\r\nContent-Length: 0\r\nConnection: close\r\n\r\n".to_vec());
assert_eq!(
fetch(&Agents::new(1), &address, &[], SECOND, 1 << 20, false, |_, _| true),
Err(LoadError::NotFound)
);
served.join().unwrap();
let (address, served) =
serve_once(b"HTTP/1.1 500 Broken\r\nContent-Length: 0\r\nConnection: close\r\n\r\n".to_vec());
assert_eq!(
fetch(&Agents::new(1), &address, &[], SECOND, 1 << 20, false, |_, _| true),
Err(LoadError::Status(500))
);
served.join().unwrap();
}
#[test]
fn a_server_that_says_it_is_busy_is_asked_again_and_its_answer_is_read() {
let (address, served) = serve(vec![
busy("429 Too Many Requests", "556"),
busy("503 Service Unavailable", "0"),
ok(b"the picture"),
]);
let started = std::time::Instant::now();
assert_eq!(
fetch(&Agents::new(1), &address, &[], SECOND, 1 << 20, false, |_, _| true),
Ok(b"the picture".to_vec())
);
assert_eq!(served.join().unwrap().len(), 3);
let waited = started.elapsed();
assert!(
waited < Duration::from_secs(4),
"a long Retry-After is not waited for: {waited:?}"
);
}
#[test]
fn a_server_that_stays_busy_is_its_status_after_two_more_asks() {
let (address, served) = serve(vec![
busy("503 Busy", "0"),
busy("503 Busy", "0"),
busy("503 Busy", "0"),
]);
assert_eq!(
fetch(&Agents::new(1), &address, &[], SECOND, 1 << 20, false, |_, _| true),
Err(LoadError::Status(503))
);
assert_eq!(served.join().unwrap().len(), 3, "asked three times in all");
}
#[test]
fn more_than_the_limit_is_not_read() {
let (address, served) = serve_once(ok(&[1u8; 4096]));
assert_eq!(
fetch(&Agents::new(1), &address, &[], SECOND, 1024, false, |_, _| true),
Err(LoadError::TooLarge { limit: 1024 })
);
served.join().unwrap();
}
#[test]
fn a_read_of_the_start_ends_with_the_start_and_no_error() {
let body: Vec<u8> = (0..4096u32).map(|n| n as u8).collect();
let (address, served) = serve_once(ok(&body));
assert_eq!(
fetch(&Agents::new(1), &address, &[], SECOND, 1000, true, |_, _| true),
Ok(body[..1000].to_vec())
);
served.join().unwrap();
let (address, served) = serve_once(ok(&body[..10]));
assert_eq!(
fetch(&Agents::new(1), &address, &[], SECOND, 1000, true, |_, _| true),
Ok(body[..10].to_vec())
);
served.join().unwrap();
}
#[test]
fn a_load_that_is_stopped_ends() {
let (address, served) = serve_once(ok(&[1u8; 300_000]));
assert_eq!(
fetch(&Agents::new(1), &address, &[], SECOND, 1 << 20, false, |_, _| false),
Err(LoadError::Cancelled)
);
served.join().unwrap();
}
fn serve_file(body: Vec<u8>, ranges: bool) -> (String, Arc<Mutex<Vec<String>>>, Arc<AtomicBool>) {
let listener = TcpListener::bind("127.0.0.1:0").unwrap();
let address = format!("http://{}/clip.mp4", listener.local_addr().unwrap());
let (body, asked) = (Arc::new(body), Arc::new(Mutex::new(Vec::new())));
let refusing = Arc::new(AtomicBool::new(false));
let (told, refuses) = (asked.clone(), refusing.clone());
thread::spawn(move || {
for stream in listener.incoming() {
let (Ok(mut stream), body, told, refuses) = (stream, body.clone(), told.clone(), refuses.clone()) else {
return;
};
thread::spawn(move || {
let mut request = Vec::new();
let mut chunk = [0u8; 1024];
while !request.windows(4).any(|window| window == b"\r\n\r\n") {
match stream.read(&mut chunk) {
Ok(read) if read > 0 => request.extend_from_slice(&chunk[..read]),
_ => return,
}
}
let request = String::from_utf8_lossy(&request).to_lowercase();
if !request.contains("x-asked-by: lurq") || refuses.load(Ordering::SeqCst) {
let _ = stream.write_all(b"HTTP/1.1 403 Forbidden\r\nContent-Length: 0\r\nConnection: close\r\n\r\n");
return;
}
let range = request
.lines()
.find_map(|line| line.strip_prefix("range: bytes="))
.map(str::to_owned);
told.lock().push(range.clone().unwrap_or_default());
let len = body.len();
let part = range.filter(|_| ranges).and_then(|range| {
let (from, to) = range.split_once('-')?;
let from: usize = from.parse().ok()?;
let to = to.parse::<usize>().map_or(len - 1, |to| to.min(len - 1));
Some((from, to))
});
let head = match part {
Some((from, _)) if from >= len => {
format!(
"HTTP/1.1 416 Range Not Satisfiable\r\nContent-Range: bytes */{len}\r\nContent-Length: 0\r\nConnection: close\r\n\r\n"
)
}
Some((from, to)) => format!(
"HTTP/1.1 206 Partial Content\r\nContent-Type: video/mp4; codecs=avc1\r\nContent-Range: bytes {from}-{to}/{len}\r\nContent-Length: {}\r\nConnection: close\r\n\r\n",
to + 1 - from
),
None => format!(
"HTTP/1.1 200 OK\r\nContent-Type: video/mp4\r\nContent-Length: {len}\r\nConnection: close\r\n\r\n"
),
};
let _ = stream.write_all(head.as_bytes());
let sent = match part {
Some((from, _)) if from >= len => &body[..0],
Some((from, to)) => &body[from..=to],
None => &body[..],
};
let _ = stream.write_all(sent);
});
}
});
(address, asked, refusing)
}
struct Capped {
address: String,
connections: Arc<AtomicUsize>,
asked: Arc<Mutex<Vec<String>>>,
}
fn serve_capped(body: Vec<u8>, rate: u64, pause: Duration) -> Capped {
let listener = TcpListener::bind("127.0.0.1:0").unwrap();
let address = format!("http://{}/clip.mp4", listener.local_addr().unwrap());
let host = Capped {
address,
connections: Arc::default(),
asked: Arc::default(),
};
let (body, connections, asked) = (Arc::new(body), host.connections.clone(), host.asked.clone());
let through = Arc::new(Mutex::new(Instant::now()));
thread::spawn(move || {
for stream in listener.incoming() {
let (Ok(mut stream), body, asked, through) = (stream, body.clone(), asked.clone(), through.clone()) else {
return;
};
connections.fetch_add(1, Ordering::SeqCst);
thread::spawn(move || {
thread::sleep(pause);
loop {
let mut request = Vec::new();
let mut chunk = [0u8; 1024];
while !request.windows(4).any(|window| window == b"\r\n\r\n") {
match stream.read(&mut chunk) {
Ok(read) if read > 0 => request.extend_from_slice(&chunk[..read]),
_ => return,
}
}
let request = String::from_utf8_lossy(&request).to_lowercase();
let len = body.len();
if request.starts_with("head ") {
asked.lock().push("HEAD".to_owned());
let head = format!("HTTP/1.1 200 OK\r\nContent-Type: video/mp4\r\nContent-Length: {len}\r\n\r\n");
if stream.write_all(head.as_bytes()).is_err() {
return;
}
continue;
}
let range = request
.lines()
.find_map(|line| line.strip_prefix("range: bytes="))
.unwrap_or("0-")
.to_owned();
asked.lock().push(range.clone());
let (from, to) = range.split_once('-').unwrap();
let from: usize = from.parse().unwrap();
let to = to.parse::<usize>().map_or(len - 1, |to| to.min(len - 1));
if from >= len {
let head = format!(
"HTTP/1.1 416 Range Not Satisfiable\r\nContent-Range: bytes */{len}\r\nContent-Length: 0\r\n\r\n"
);
if stream.write_all(head.as_bytes()).is_err() {
return;
}
continue;
}
let head = format!(
"HTTP/1.1 206 Partial Content\r\nContent-Type: video/mp4\r\nContent-Range: bytes {from}-{to}/{len}\r\nContent-Length: {}\r\n\r\n",
to + 1 - from
);
if stream.write_all(head.as_bytes()).is_err() {
return;
}
for piece in body[from..=to].chunks(2048) {
let at = {
let mut through = through.lock();
*through = (*through).max(Instant::now()) + Duration::from_secs_f64(piece.len() as f64 / rate as f64);
*through
};
thread::sleep(at.saturating_duration_since(Instant::now()));
if stream.write_all(piece).is_err() {
return;
}
}
}
});
}
});
host
}
fn numbered(len: usize) -> Vec<u8> {
(0..len).map(|at| (at % 251) as u8).collect()
}
const SMALL: u64 = 4096;
fn asked(address: &str, headers: Vec<(Arc<str>, Arc<str>)>, cut: Cut) -> Asked {
Asked {
key: 0,
address: address.to_owned(),
headers,
timeout: SECOND,
cut,
read_ahead: None,
}
}
fn opening(address: &str, headers: Vec<(Arc<str>, Arc<str>)>, cut: Cut) -> Result<HttpReader, LoadError> {
HttpReader::open(&Agents::new(4), Starts::new(0), asked(address, headers, cut))
}
fn asked_by_lurq() -> Vec<(Arc<str>, Arc<str>)> {
vec![(Arc::from("X-Asked-By"), Arc::from("lurq"))]
}
fn asking(address: &str) -> Result<HttpReader, LoadError> {
opening(address, asked_by_lurq(), Cut::even(SMALL))
}
fn read(reader: &HttpReader, at: u64, len: usize) -> io::Result<Vec<u8>> {
let mut buf = vec![0u8; len];
let mut filled = 0;
while filled < len {
let read = reader.read_at(at + filled as u64, &mut buf[filled..])?;
if read == 0 {
break;
}
filled += read;
}
buf.truncate(filled);
Ok(buf)
}
#[test]
fn an_address_is_read_by_ranges_with_its_headers_from_any_place() {
let body = numbered(3 * SMALL as usize + 1234);
let (address, asked, _) = serve_file(body.clone(), true);
let reader = asking(&address).unwrap();
assert_eq!(asked.lock().len(), 1, "opened, it asked for its first block");
assert_eq!(reader.len(), Some(body.len() as u64));
assert_eq!(reader.content_type().as_deref(), Some("video/mp4"));
for at in [0, SMALL - 3, 2 * SMALL + 17, body.len() as u64 - 5] {
let at_ = at as usize;
assert_eq!(
read(&reader, at, 8).unwrap(),
&body[at_..(at_ + 8).min(body.len())],
"at {at}"
);
}
assert_eq!(reader.read_at(body.len() as u64, &mut [0u8; 4]).unwrap(), 0, "the end");
assert!(!reader.buffered().is_empty());
let asked = asked.lock().clone();
let blocks: Vec<u64> = asked
.iter()
.map(|range| range.split_once('-').and_then(|(from, _)| from.parse().ok()).unwrap())
.collect();
assert!(
blocks.iter().all(|from| from % SMALL == 0),
"every request asks for a block from its start: {asked:?}"
);
assert_eq!(
blocks.iter().filter(|from| **from == 0).count(),
1,
"a block that is held is not asked for again: {asked:?}"
);
}
#[test]
fn the_first_blocks_are_small_and_the_ones_after_them_grow_to_the_largest() {
let cut = Cut { first: 256, most: 1024 };
let blocks: Vec<(u64, u64)> = (0..6).map(|index| cut.block(index)).collect();
assert_eq!(
blocks,
[
(0, 256),
(256, 256),
(512, 512),
(1024, 1024),
(2048, 1024),
(3072, 1024)
]
);
for (index, (start, size)) in blocks.iter().enumerate() {
for offset in [*start, start + size / 2, start + size - 1] {
assert_eq!(cut.index_of(offset), index as u64, "the byte at {offset}");
}
}
assert_eq!(
cut.index_of(1024 * 100 + 5),
3 + 99,
"far in, where every block is the largest"
);
assert_eq!(cut.block(3 + 99), (1024 * 100, 1024));
let even = Cut::even(4096);
assert_eq!((even.block(0), even.block(3)), ((0, 4096), (12288, 4096)));
assert_eq!(even.index_of(12288 + 17), 3);
let wide = Cut {
first: 4096,
most: 1024,
};
assert_eq!((wide.block(0), wide.block(2)), ((0, 1024), (2048, 1024)));
}
#[test]
fn an_address_cut_into_growing_blocks_is_read_whole_and_asks_for_its_start_first_in_a_small_block() {
let body = numbered(5000);
let (address, asked, _) = serve_file(body.clone(), true);
let cut = Cut { first: 256, most: 1024 };
let reader = opening(&address, asked_by_lurq(), cut).unwrap();
assert_eq!(*asked.lock(), ["0-255"], "its start, in the small first block");
assert_eq!(read(&reader, 0, body.len()).unwrap(), body);
for at in [250, 500, 1020, 2040, 4990] {
assert_eq!(
read(&reader, at, 16).unwrap(),
&body[at as usize..(at as usize + 16).min(body.len())],
"at {at}"
);
}
let asked = asked.lock().clone();
for range in ["256-1023", "1024-2047", "2048-3071"] {
assert!(asked.iter().any(|asked| asked == range), "{range} of {asked:?}");
}
}
fn mp4_start(boxes: &[(&[u8; 4], u32)]) -> Vec<u8> {
let mut start = Vec::new();
start.extend_from_slice(&24u32.to_be_bytes());
start.extend_from_slice(b"ftypisom");
start.extend_from_slice(&[0; 12]);
for (name, size) in boxes {
start.extend_from_slice(&size.to_be_bytes());
start.extend_from_slice(*name);
}
start
}
#[test]
fn where_the_index_of_an_mp4_is_after_its_pictures_is_read_from_its_start() {
assert_eq!(index_at_the_end(&mp4_start(&[(b"moov", 900), (b"mdat", 50_000)])), None);
assert_eq!(index_at_the_end(&mp4_start(&[(b"mdat", 50_000)])), Some(24 + 50_000));
let mut after_free = mp4_start(&[(b"free", 8)]);
after_free.extend_from_slice(&50_000u32.to_be_bytes());
after_free.extend_from_slice(b"mdat");
assert_eq!(index_at_the_end(&after_free), Some(24 + 8 + 50_000));
let mut long = mp4_start(&[(b"mdat", 1)]);
long.extend_from_slice(&(6_000_000_000u64).to_be_bytes());
assert_eq!(index_at_the_end(&long), Some(24 + 6_000_000_000));
assert_eq!(index_at_the_end(b"GIF89a, and then what a picture holds"), None);
assert_eq!(index_at_the_end(&mp4_start(&[])[..20]), None);
}
#[test]
fn an_mp4_with_its_index_at_its_end_has_its_end_asked_for_as_it_opens() {
let mut body = mp4_start(&[(b"mdat", 9000 - 24)]);
body.resize(10_000, 7);
let (address, asked, _) = serve_file(body, true);
let cut = Cut { first: 256, most: 1024 };
let reader = opening(&address, asked_by_lurq(), cut).unwrap();
let until = Instant::now() + SECOND;
while !asked.lock().iter().any(|range| range == "8192-9215") {
assert!(Instant::now() < until, "the end was not asked for: {:?}", asked.lock());
thread::sleep(Duration::from_millis(5));
}
drop(reader);
}
#[test]
fn told_not_to_read_ahead_it_asks_for_what_is_read_and_no_more_until_it_is_told_to() {
let body = numbered(10 * SMALL as usize);
let (address, asked, _) = serve_file(body.clone(), true);
let reader = asking(&address).unwrap();
reader.read_ahead(false);
assert_eq!(read(&reader, 100, 8).unwrap(), &body[100..108]);
thread::sleep(Duration::from_millis(150));
assert_eq!(*asked.lock(), ["0-4095"], "its first block, and nothing after it");
reader.read_ahead(true);
let until = Instant::now() + SECOND;
while asked.lock().len() < 3 {
assert!(Instant::now() < until, "nothing was read ahead: {:?}", asked.lock());
thread::sleep(Duration::from_millis(5));
}
let mut ahead = asked.lock().clone();
ahead.sort();
assert_eq!(ahead, ["0-4095", "4096-8191", "8192-12287"]);
}
#[test]
fn an_address_opened_without_the_header_it_needs_fails_with_what_the_server_answered() {
let (address, ..) = serve_file(numbered(1000), true);
let refused = opening(&address, Vec::new(), Cut::even(SMALL));
assert_eq!(refused.err(), Some(LoadError::Status(403)));
}
#[test]
fn a_read_the_server_refuses_on_the_way_says_what_it_answered() {
let body = numbered(40 * SMALL as usize);
let (address, _, refusing) = serve_file(body.clone(), true);
let reader = asking(&address).unwrap();
refusing.store(true, Ordering::SeqCst);
let error = read(&reader, 30 * SMALL, 8).unwrap_err();
assert_eq!(LoadError::from(error), LoadError::Status(403));
refusing.store(false, Ordering::SeqCst);
let at = 30 * SMALL as usize;
assert_eq!(read(&reader, at as u64, 8).unwrap(), &body[at..at + 8]);
}
#[test]
fn a_server_that_reads_no_ranges_is_read_from_its_start_to_the_place() {
let body = numbered(2 * SMALL as usize + 99);
let (address, ..) = serve_file(body.clone(), false);
let reader = asking(&address).unwrap();
let at = 2 * SMALL as usize + 10;
assert_eq!(read(&reader, at as u64, 6).unwrap(), &body[at..at + 6]);
assert_eq!(reader.len(), Some(body.len() as u64));
}
fn span(range: &str) -> (u64, u64) {
let (from, to) = range.split_once('-').unwrap();
(from.parse().unwrap(), to.parse().unwrap())
}
#[test]
fn a_read_has_its_bytes_as_they_come_and_does_not_wait_for_its_block() {
let block = 256 << 10;
let body = numbered(300 << 10);
let host = serve_capped(body.clone(), block, Duration::ZERO);
let started = Instant::now();
let reader = opening(&host.address, Vec::new(), Cut::even(block)).unwrap();
assert_eq!(reader.len(), Some(body.len() as u64), "opened, it knows how long it is");
assert_eq!(read(&reader, 0, 16 << 10).unwrap(), &body[..16 << 10]);
let there = reader.buffered();
assert!(
there.len() == 1 && there[0].start == 0 && there[0].end < block,
"the read had its bytes with its block still on its way: {there:?}"
);
assert!(
started.elapsed() < Duration::from_millis(600),
"opened and read in {:?}",
started.elapsed()
);
let at = 100 << 10;
assert_eq!(read(&reader, at as u64, 8).unwrap(), &body[at..at + 8]);
assert!(reader.buffered()[0].end < block);
assert_eq!(read(&reader, 0, body.len()).unwrap(), body);
}
#[test]
fn a_reader_that_plays_asks_in_order_over_one_connection() {
let body = numbered(600 << 10);
let host = serve_capped(body.clone(), 1 << 20, Duration::from_millis(50));
let cut = Cut {
first: 64 << 10,
most: 256 << 10,
};
let reader = HttpReader::open(
&Agents::new(4),
Starts::new(0),
Asked {
read_ahead: Some(true),
..asked(&host.address, Vec::new(), cut)
},
)
.unwrap();
assert_eq!(read(&reader, 0, body.len()).unwrap(), body);
let asked = host.asked.lock().clone();
assert_eq!(
asked[0], "0-262143",
"the small first blocks are one request, as long as the largest block"
);
let mut at = 0;
for range in &asked {
let (from, to) = span(range);
assert_eq!(
from, at,
"each request goes on from where the one before ended: {asked:?}"
);
at = to + 1;
}
assert_eq!(
host.connections.load(Ordering::SeqCst),
1,
"all of them over the connection the first opened: {asked:?}"
);
}
#[test]
fn opened_to_be_played_later_it_asks_for_what_is_read_and_no_more() {
let body = numbered(600 << 10);
let host = serve_capped(body.clone(), 8 << 20, Duration::ZERO);
let cut = Cut {
first: 64 << 10,
most: 256 << 10,
};
let reader = HttpReader::open(
&Agents::new(4),
Starts::new(0),
Asked {
read_ahead: Some(false),
..asked(&host.address, Vec::new(), cut)
},
)
.unwrap();
assert_eq!(read(&reader, 100, 8).unwrap(), &body[100..108]);
thread::sleep(Duration::from_millis(150));
assert_eq!(*host.asked.lock(), ["0-65535"], "its first block, alone");
let reader = opening(&host.address, Vec::new(), cut).unwrap();
thread::sleep(Duration::from_millis(150));
assert_eq!(*host.asked.lock(), ["0-65535", "0-65535"]);
assert_eq!(read(&reader, 100, 8).unwrap(), &body[100..108]);
let until = Instant::now() + SECOND;
while host.asked.lock().len() < 3 {
assert!(
Instant::now() < until,
"nothing was read ahead: {:?}",
host.asked.lock()
);
thread::sleep(Duration::from_millis(5));
}
assert_eq!(host.asked.lock()[2], "65536-262143");
}
#[test]
fn a_read_sent_to_another_place_is_asked_for_beside_the_block_on_its_way() {
let block: u64 = 256 << 10;
let body = numbered(2 << 20);
let host = serve_capped(body.clone(), 2 * block, Duration::ZERO);
let reader = opening(&host.address, Vec::new(), Cut::even(block)).unwrap();
assert_eq!(read(&reader, 0, 1024).unwrap(), &body[..1024]);
let far = 6 * block as usize + 100;
let started = Instant::now();
assert_eq!(read(&reader, far as u64, 1024).unwrap(), &body[far..far + 1024]);
assert!(
started.elapsed() < Duration::from_millis(300),
"it did not wait for the block on its way: {:?}",
started.elapsed()
);
assert!(reader.buffered()[0].end < block, "which is still on its way");
assert_eq!(host.connections.load(Ordering::SeqCst), 2);
assert_eq!(host.asked.lock()[1], format!("{}-{}", 6 * block, 7 * block - 1));
}
#[test]
fn a_request_nobody_waits_for_stands_still_while_a_read_waits_for_another() {
let (address, ..) = serve_file(numbered(100), true);
let reader = asking(&address).unwrap();
let ranges = reader.0.clone();
let ahead = || Run {
first: 3,
last: 3,
ahead: true,
};
assert!(
ranges.goes_on(&ahead(), Instant::now()),
"nothing is waited for: it reads on"
);
{
let mut blocks = ranges.blocks.lock();
blocks.waited.push(7);
blocks.coming.push(7);
}
let went = Arc::new(AtomicBool::new(false));
let standing = {
let (ranges, went) = (ranges.clone(), went.clone());
thread::spawn(move || {
let on = ranges.goes_on(&ahead(), Instant::now());
went.store(true, Ordering::SeqCst);
on
})
};
thread::sleep(Duration::from_millis(120));
assert!(!went.load(Ordering::SeqCst), "it stands still");
let waited_for = Run {
first: 7,
last: 7,
ahead: false,
};
assert!(
ranges.goes_on(&waited_for, Instant::now()),
"the request that brings what is waited for reads on"
);
let long_ago = Instant::now().checked_sub(SECOND * 3 / 5).unwrap();
assert!(
ranges.goes_on(&ahead(), long_ago),
"past half of its time it reads on whatever is waited for"
);
ranges.blocks.lock().waited.clear();
assert!(standing.join().unwrap(), "and it reads on");
drop(reader);
assert!(
!ranges.goes_on(&ahead(), Instant::now()),
"the reader is dropped: its requests end"
);
}
#[test]
fn a_read_that_waited_for_what_was_read_ahead_asks_again_itself_where_that_failed() {
let body = numbered(40 * SMALL as usize);
let (address, ..) = serve_file(body.clone(), true);
let reader = Arc::new(asking(&address).unwrap());
for (index, ahead) in [(30_u64, true), (35, false)] {
let at = (index * SMALL) as usize;
reader.0.blocks.lock().coming.push(index);
let waiting = {
let reader = reader.clone();
thread::spawn(move || read(&reader, at as u64, 8))
};
let until = Instant::now() + SECOND;
while !reader.0.blocks.lock().waited.contains(&index) {
assert!(Instant::now() < until, "the read does not wait");
thread::sleep(Duration::from_millis(2));
}
{
let mut blocks = reader.0.blocks.lock();
blocks.coming.retain(|coming| *coming != index);
blocks.failed.push((index, LoadError::TimedOut, ahead));
}
reader.0.came.notify_all();
let read = waiting.join().unwrap();
if ahead {
assert_eq!(read.unwrap(), &body[at..at + 8], "asked again, by the read");
} else {
assert_eq!(LoadError::from(read.unwrap_err()), LoadError::TimedOut);
}
}
}
#[test]
fn the_start_of_an_address_is_kept_for_the_reader_that_opens_it_next() {
let block = SMALL as usize;
let body = numbered(40 * block);
let (address, ranges, _) = serve_file(body.clone(), true);
let (agents, starts) = (Agents::new(4), Starts::new(2));
let open = |key: u64, cut: Cut| {
HttpReader::open(
&agents,
starts.clone(),
Asked {
key,
read_ahead: Some(false),
..asked(&address, asked_by_lurq(), cut)
},
)
.unwrap()
};
let cut = Cut::even(SMALL);
let reader = open(1, cut);
assert_eq!(read(&reader, 0, 3 * block).unwrap(), &body[..3 * block]);
assert_eq!(read(&reader, 10 * SMALL, 8).unwrap(), &body[10 * block..10 * block + 8]);
drop(reader);
let before = ranges.lock().len();
assert_eq!(before, 4);
let again = open(1, cut);
assert_eq!(again.len(), Some(body.len() as u64));
assert_eq!(again.content_type().as_deref(), Some("video/mp4"));
assert_eq!(
again.buffered(),
vec![0..3 * SMALL],
"the blocks from its start, and not one further in"
);
assert_eq!(read(&again, 0, 3 * block).unwrap(), &body[..3 * block]);
assert_eq!(ranges.lock().len(), before, "opened and read with no request");
assert_eq!(read(&again, 3 * SMALL, 8).unwrap(), &body[3 * block..3 * block + 8]);
assert_eq!(ranges.lock().last().unwrap(), "12288-16383");
drop(again);
drop(open(2, cut));
assert_eq!(ranges.lock().last().unwrap(), "0-4095");
drop(open(1, Cut::even(2 * SMALL)));
assert_eq!(ranges.lock().last().unwrap(), "0-8191");
let asked_so_far = ranges.lock().len();
drop(open(2, cut));
assert_eq!(ranges.lock().len(), asked_so_far, "kept until it is forgotten");
assert!(starts.forget(2));
assert!(!starts.forget(2));
drop(open(2, cut));
assert_eq!(ranges.lock().len(), asked_so_far + 1);
drop(open(3, cut));
drop(open(4, cut));
let asked_so_far = ranges.lock().len();
drop(open(2, cut));
assert_eq!(ranges.lock().len(), asked_so_far + 1, "two were kept after it");
starts.resize(0);
let asked_so_far = ranges.lock().len();
drop(open(2, cut));
drop(open(2, cut));
assert_eq!(ranges.lock().len(), asked_so_far + 2);
}
#[test]
fn a_request_is_lent_the_agent_that_asked_its_host_last() {
assert_eq!(host_of("https://one.example/a/b.mp4?x=1"), "https://one.example");
assert_eq!(host_of("http://127.0.0.1:8080"), "http://127.0.0.1:8080");
assert_eq!(host_of("https://one.example?x=/y"), "https://one.example");
let agents = Agents::new(4);
let (one, two) = (
agents.lend("https://one.example/a"),
agents.lend("https://two.example/b"),
);
drop(one);
drop(two);
assert_eq!(agents.made(), 2);
let idle =
|agents: &Agents| -> Vec<String> { agents.idle.lock().iter().map(|(_, host)| host.to_string()).collect() };
let again = agents.lend("https://one.example/another");
assert_eq!(idle(&agents), ["https://two.example"]);
drop(again);
let third = agents.lend("https://three.example/c");
assert_eq!(idle(&agents), ["https://two.example"]);
drop(third);
assert_eq!(idle(&agents), ["https://two.example", "https://three.example"]);
assert_eq!(agents.made(), 2);
}
#[test]
fn a_connection_opened_ahead_is_the_one_the_first_request_goes_over() {
let body = numbered(100 << 10);
let host = serve_capped(body.clone(), 8 << 20, Duration::from_millis(200));
let agents = Agents::new(4);
connect(&agents, host.address.clone(), Vec::new(), SECOND);
connect(&agents, host.address.clone(), Vec::new(), SECOND);
let until = Instant::now() + SECOND;
while agents.idle.lock().is_empty() {
assert!(Instant::now() < until, "no connection was opened ahead");
thread::sleep(Duration::from_millis(5));
}
assert_eq!(*host.asked.lock(), ["HEAD"]);
let started = Instant::now();
let reader = HttpReader::open(
&agents,
Starts::new(0),
asked(&host.address, Vec::new(), Cut::even(64 << 10)),
)
.unwrap();
assert_eq!(read(&reader, 0, 1024).unwrap(), &body[..1024]);
assert!(
started.elapsed() < Duration::from_millis(150),
"opened with no new connection: {:?}",
started.elapsed()
);
assert_eq!(host.connections.load(Ordering::SeqCst), 1);
assert_eq!(*host.asked.lock(), ["HEAD", "0-65535"]);
}
}