use std::collections::{HashMap, HashSet};
use std::path::PathBuf;
use std::sync::Arc;
use std::sync::atomic::{AtomicU64, AtomicUsize, Ordering};
use std::time::{Duration, Instant};
use crate::player::commands::PlayerCommand;
use crate::player::state::{ItemState, QueueItemId, SharedPlayerState};
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum DownloadState {
Queued,
Running,
Done,
Failed(String),
}
impl DownloadState {
pub fn is_settled(&self) -> bool {
matches!(self, Self::Done | Self::Failed(_))
}
}
#[derive(Debug, Clone)]
pub struct Download {
pub track_id: i64,
pub title: String,
pub artist: String,
pub source: PathBuf,
pub dest: PathBuf,
pub total: u64,
pub written: Arc<ByteFeed>,
pub state: DownloadState,
pub bytes_per_second: u64,
}
impl Download {
pub fn fraction(&self) -> Option<f64> {
(self.total > 0)
.then(|| self.written.load(Ordering::Relaxed) as f64 / self.total as f64)
.map(|f| f.clamp(0.0, 1.0))
}
pub fn bytes_written(&self) -> u64 {
self.written.load(Ordering::Relaxed)
}
}
#[derive(Debug, Clone)]
pub struct Live {
pub source: PathBuf,
pub total: u64,
pub written: Arc<ByteFeed>,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub struct Reading {
pub track_id: i64,
pub written: u64,
pub total: u64,
pub bytes_per_second: u64,
}
impl Reading {
pub fn fraction(&self) -> Option<f64> {
(self.total > 0).then(|| (self.written as f64 / self.total as f64).clamp(0.0, 1.0))
}
}
#[derive(Debug, Default)]
pub struct ByteFeed {
written: AtomicU64,
parked: AtomicUsize,
at: parking_lot::Mutex<()>,
more: parking_lot::Condvar,
}
impl ByteFeed {
pub fn new() -> Arc<Self> {
Arc::new(Self::default())
}
pub fn load(&self, order: Ordering) -> u64 {
self.written.load(order)
}
pub fn set(&self, bytes: u64) {
self.written.store(bytes, Ordering::SeqCst);
self.wake_parked();
}
pub fn advance(&self, bytes: u64) {
self.written.fetch_add(bytes, Ordering::SeqCst);
self.wake_parked();
}
fn wake_parked(&self) {
if self.parked.load(Ordering::SeqCst) > 0 {
let _at = self.at.lock();
self.more.notify_all();
}
}
pub fn done(&self) {
let _at = self.at.lock();
self.more.notify_all();
}
pub fn wait_past(&self, seen: u64, deadline: Instant) -> u64 {
let mut at = self.at.lock();
self.parked.fetch_add(1, Ordering::SeqCst);
let written = self.written.load(Ordering::SeqCst);
if written <= seen
&& let Some(left) = deadline.checked_duration_since(Instant::now())
{
self.more.wait_for(&mut at, left);
}
self.parked.fetch_sub(1, Ordering::SeqCst);
self.written.load(Ordering::Acquire)
}
}
#[derive(Debug)]
struct Entry {
download: Download,
listed: bool,
waiters: HashSet<QueueItemId>,
keep: bool,
}
impl Entry {
fn is_live(&self) -> bool {
!self.download.state.is_settled()
}
fn wanted(&self) -> bool {
self.keep || !self.waiters.is_empty()
}
}
fn join_in(entries: &mut [Entry], track_id: i64, waiter: Option<QueueItemId>) -> bool {
let Some(entry) = entries
.iter_mut()
.find(|e| e.download.track_id == track_id && e.is_live())
else {
return false;
};
match waiter {
Some(id) => {
entry.waiters.insert(id);
}
None => entry.keep = true,
}
true
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum Outcome {
Done,
Failed(String),
Withdrawn,
}
#[derive(Debug, Default)]
pub struct DownloadStore {
entries: parking_lot::RwLock<Vec<Entry>>,
version: AtomicU64,
samples: parking_lot::Mutex<HashMap<i64, Sample>>,
last_sample: AtomicU64,
figures: AtomicU64,
moved: crate::signal::Wake,
}
fn clock_ns() -> u64 {
static START: std::sync::OnceLock<Instant> = std::sync::OnceLock::new();
START.get_or_init(Instant::now).elapsed().as_nanos() as u64 + 1
}
const SETTLED_LIMIT: usize = 50;
impl DownloadStore {
pub fn new() -> Arc<Self> {
Arc::new(Self::default())
}
pub fn version(&self) -> u64 {
self.version.load(Ordering::Acquire)
}
pub fn figures(&self) -> u64 {
self.figures.load(Ordering::Acquire)
}
pub fn moved(&self) -> &crate::signal::Wake {
&self.moved
}
pub fn join(&self, track_id: i64, waiter: Option<QueueItemId>) -> bool {
join_in(&mut self.entries.write(), track_id, waiter)
}
pub fn claim(&self, track_id: i64, waiter: Option<QueueItemId>) -> bool {
let mut entries = self.entries.write();
if join_in(&mut entries, track_id, waiter) {
return false;
}
let had_row = entries
.iter()
.any(|e| e.download.track_id == track_id && e.listed);
entries.retain(|e| e.download.track_id != track_id);
entries.insert(
0,
Entry {
download: Download {
track_id,
title: String::new(),
artist: String::new(),
source: PathBuf::new(),
dest: PathBuf::new(),
total: 0,
written: ByteFeed::new(),
state: DownloadState::Queued,
bytes_per_second: 0,
},
listed: false,
waiters: waiter.into_iter().collect(),
keep: waiter.is_none(),
},
);
drop(entries);
if had_row {
self.bump();
}
true
}
pub fn resync(&self, wanted: &[(i64, QueueItemId)]) -> Vec<(i64, QueueItemId)> {
let mut entries = self.entries.write();
let mut live: HashMap<i64, usize> = HashMap::new();
for (ix, entry) in entries.iter_mut().enumerate().filter(|(_, e)| e.is_live()) {
entry.waiters.clear();
live.insert(entry.download.track_id, ix);
}
let mut unfetched = Vec::new();
for &(track_id, id) in wanted {
match live.get(&track_id) {
Some(&ix) => {
entries[ix].waiters.insert(id);
}
None => unfetched.push((track_id, id)),
}
}
unfetched
}
pub fn in_flight(&self, track_id: i64) -> bool {
self.entries
.read()
.iter()
.any(|e| e.download.track_id == track_id && e.is_live())
}
pub fn waiters(&self, track_id: i64) -> Vec<QueueItemId> {
self.entries
.read()
.iter()
.find(|e| e.download.track_id == track_id && e.is_live())
.map(|e| e.waiters.iter().copied().collect())
.unwrap_or_default()
}
pub fn kept(&self, track_id: i64) -> bool {
self.entries
.read()
.iter()
.any(|e| e.download.track_id == track_id && e.is_live() && e.keep)
}
pub fn abandoned(&self, track_id: i64) -> bool {
self.entries
.read()
.iter()
.find(|e| e.download.track_id == track_id && e.is_live())
.is_none_or(|e| !e.wanted())
}
pub fn announce(
&self,
track_id: i64,
title: String,
artist: String,
source: PathBuf,
dest: PathBuf,
) -> Arc<ByteFeed> {
let mut entries = self.entries.write();
let feed = match entries
.iter_mut()
.find(|e| e.download.track_id == track_id && e.is_live())
{
Some(entry) => {
let d = &mut entry.download;
(d.title, d.artist, d.source, d.dest) = (title, artist, source, dest);
entry.listed = true;
d.written.clone()
}
None => ByteFeed::new(),
};
drop(entries);
self.bump();
feed
}
pub fn started(&self, track_id: i64, total: u64) {
let mut entries = self.entries.write();
if let Some(entry) = entries
.iter_mut()
.find(|e| e.download.track_id == track_id && e.is_live())
{
entry.download.total = total;
entry.download.state = DownloadState::Running;
}
drop(entries);
self.bump();
}
fn end(&self, track_id: i64, outcome: Outcome) -> Option<(Vec<QueueItemId>, Arc<ByteFeed>)> {
let mut entries = self.entries.write();
let ix = entries
.iter()
.position(|e| e.download.track_id == track_id && e.is_live())?;
let entry = &mut entries[ix];
let waiters = std::mem::take(&mut entry.waiters).into_iter().collect();
let feed = entry.download.written.clone();
let listed = entry.listed;
match outcome {
Outcome::Done if listed => entry.download.state = DownloadState::Done,
Outcome::Failed(reason) if listed => {
entry.download.state = DownloadState::Failed(reason)
}
_ => {
entries.remove(ix);
}
}
if let Some(entry) = entries
.get_mut(ix)
.filter(|e| e.download.track_id == track_id)
{
entry.download.bytes_per_second = 0;
}
let (mut live, settled): (Vec<_>, Vec<_>) = entries.drain(..).partition(Entry::is_live);
let (now, older): (Vec<_>, Vec<_>) = settled
.into_iter()
.partition(|e| e.download.track_id == track_id);
live.extend(now);
live.extend(older.into_iter().take(SETTLED_LIMIT.saturating_sub(1)));
*entries = live;
drop(entries);
self.samples.lock().remove(&track_id);
if listed {
self.figures.fetch_add(1, Ordering::Release);
self.moved.bump();
self.bump();
}
Some((waiters, feed))
}
pub fn all(&self) -> Vec<Download> {
self.entries
.read()
.iter()
.filter(|e| e.listed)
.map(|e| e.download.clone())
.collect()
}
pub fn live(&self, track_id: i64) -> Option<Live> {
self.entries
.read()
.iter()
.find(|e| e.download.track_id == track_id && e.listed && e.is_live())
.map(|e| Live {
source: e.download.source.clone(),
total: e.download.total,
written: e.download.written.clone(),
})
}
pub fn readings(&self) -> Vec<Reading> {
self.entries
.read()
.iter()
.filter(|e| e.listed && e.is_live())
.map(|e| Reading {
track_id: e.download.track_id,
written: e.download.bytes_written(),
total: e.download.total,
bytes_per_second: e.download.bytes_per_second,
})
.collect()
}
pub fn active(&self) -> usize {
self.entries
.read()
.iter()
.filter(|e| e.listed && e.is_live())
.count()
}
pub fn clear_settled(&self) {
let mut entries = self.entries.write();
let before = entries.len();
entries.retain(Entry::is_live);
let changed = entries.len() != before;
drop(entries);
if changed {
self.bump();
}
}
fn bump(&self) {
self.version.fetch_add(1, Ordering::Release);
crate::signal::engine_changed().bump();
}
}
#[must_use = "announce wakes a decoder parked on the transfer and tells the player"]
pub struct Settled {
feed: Option<Arc<ByteFeed>>,
tell: Vec<PlayerCommand>,
}
impl Settled {
pub fn announce(self, tx: &crossbeam_channel::Sender<PlayerCommand>) {
if let Some(feed) = self.feed {
feed.done();
}
for cmd in self.tell {
tx.send(cmd).ok();
}
}
}
pub fn settle(
state: &SharedPlayerState,
track_id: i64,
result: &Result<PathBuf, String>,
) -> Settled {
let store = state.downloads();
let waiters = store.waiters(track_id);
for &id in &waiters {
match result {
Ok(path) => {
state.update_paths(&[(id, path.clone())]);
state.update_item_state(id, ItemState::Ready);
}
Err(reason) => state.update_item_state(id, ItemState::Failed(reason.clone())),
}
}
let outcome = match result {
Ok(_) => Outcome::Done,
Err(reason) => Outcome::Failed(reason.clone()),
};
let feed = store.end(track_id, outcome).map(|(_, feed)| feed);
let tell = waiters
.into_iter()
.filter(|id| state.is_cursor(*id))
.map(|id| match result {
Ok(_) => PlayerCommand::TrackReady(id),
Err(_) => PlayerCommand::TrackFailed(id),
})
.collect();
Settled { feed, tell }
}
pub fn withdraw(store: &DownloadStore, track_id: i64) -> Vec<QueueItemId> {
match store.end(track_id, Outcome::Withdrawn) {
Some((waiters, feed)) => {
feed.done();
waiters
}
None => Vec::new(),
}
}
#[derive(Debug)]
struct Sample {
at: Instant,
bytes: u64,
bps: f64,
}
const RATE_SMOOTHING: f64 = 0.3;
const MIN_SAMPLE_GAP: Duration = Duration::from_millis(250);
impl DownloadStore {
pub fn progressed(&self) {
let now = clock_ns();
let last = self.last_sample.load(Ordering::Relaxed);
if last != 0 && now.saturating_sub(last) < MIN_SAMPLE_GAP.as_nanos() as u64 {
return;
}
if self
.last_sample
.compare_exchange(last, now, Ordering::Relaxed, Ordering::Relaxed)
.is_ok()
{
self.sample_rates_at(Instant::now());
}
}
fn sample_rates_at(&self, now: Instant) {
let mut entries = self.entries.write();
let mut samples = self.samples.lock();
for entry in entries.iter_mut() {
let track_id = entry.download.track_id;
let entry = &mut entry.download;
if entry.state.is_settled() {
entry.bytes_per_second = 0;
samples.remove(&track_id);
continue;
}
let bytes = entry.written.load(Ordering::Relaxed);
match samples.get_mut(&track_id) {
Some(previous) => {
let elapsed = now.saturating_duration_since(previous.at);
if elapsed < MIN_SAMPLE_GAP {
entry.bytes_per_second = previous.bps as u64;
continue;
}
let moved = bytes.saturating_sub(previous.bytes) as f64;
let instant = moved / elapsed.as_secs_f64();
previous.bps = previous.bps * (1.0 - RATE_SMOOTHING) + instant * RATE_SMOOTHING;
previous.at = now;
previous.bytes = bytes;
entry.bytes_per_second = previous.bps as u64;
}
None => {
samples.insert(
track_id,
Sample {
at: now,
bytes,
bps: 0.0,
},
);
entry.bytes_per_second = 0;
}
}
}
let live: HashSet<i64> = entries.iter().map(|e| e.download.track_id).collect();
samples.retain(|track_id, _| live.contains(track_id));
self.figures.fetch_add(1, Ordering::Release);
self.moved.bump();
}
}
#[cfg(test)]
mod tests {
use super::*;
fn running(store: &DownloadStore, id: i64, title: &str) -> (i64, QueueItemId, Arc<ByteFeed>) {
let track_id = id;
let waiter = QueueItemId::new();
assert!(store.claim(track_id, Some(waiter)));
let feed = store.announce(
track_id,
title.into(),
"Artist".into(),
PathBuf::from(format!("/cache/{title}.opus.part")),
PathBuf::from(format!("/cache/{title}.opus")),
);
(track_id, waiter, feed)
}
#[test]
fn a_track_wanted_twice_is_one_transfer_with_two_waiters() {
let store = DownloadStore::new();
let track_id = 7;
let (first, again) = (QueueItemId::new(), QueueItemId::new());
assert!(
store.claim(track_id, Some(first)),
"the first claim starts it"
);
assert!(!store.claim(track_id, Some(again)), "the second joins it");
let waiters: HashSet<_> = store.waiters(track_id).into_iter().collect();
assert_eq!(waiters, HashSet::from([first, again]));
}
#[test]
fn a_claim_is_not_listed_until_bytes_are_to_be_fetched() {
let store = DownloadStore::new();
let track_id = 1;
store.claim(track_id, Some(QueueItemId::new()));
assert!(store.all().is_empty());
assert!(store.live(track_id).is_none());
assert!(
store.in_flight(track_id),
"but it is in flight: nothing fetches it twice"
);
let version = store.version();
let _ = withdraw(&store, track_id);
assert_eq!(store.version(), version, "and it leaves without a trace");
assert!(!store.in_flight(track_id));
}
#[test]
fn a_transfer_runs_then_settles() {
let store = DownloadStore::new();
let (track_id, _, feed) = running(&store, 1, "train");
assert_eq!(store.active(), 1);
store.started(track_id, 400);
feed.set(100);
assert_eq!(store.all()[0].fraction(), Some(0.25));
assert_eq!(store.live(track_id).map(|l| l.total), Some(400));
store.end(track_id, Outcome::Done);
assert_eq!(store.active(), 0);
assert_eq!(store.all()[0].state, DownloadState::Done);
assert!(store.live(track_id).is_none());
}
#[test]
fn settling_hands_back_every_waiter_and_the_feed() {
let store = DownloadStore::new();
let (track_id, first, feed) = running(&store, 1, "train");
let again = QueueItemId::new();
store.join(track_id, Some(again));
let (waiters, settled_feed) = store.end(track_id, Outcome::Failed("404".into())).unwrap();
assert_eq!(
waiters.into_iter().collect::<HashSet<_>>(),
HashSet::from([first, again])
);
assert!(Arc::ptr_eq(&feed, &settled_feed));
assert!(
!store.join(track_id, Some(QueueItemId::new())),
"nothing to join once settled"
);
}
#[test]
fn readings_are_live_and_leave_settled_transfers_out() {
let store = DownloadStore::new();
let (going, _, feed) = running(&store, 1, "going");
let (landed, _, _) = running(&store, 2, "landed");
store.started(going, 400);
store.end(landed, Outcome::Done);
feed.set(100);
let readings = store.readings();
assert_eq!(readings.len(), 1);
assert_eq!(readings[0].track_id, going);
assert_eq!(readings[0].fraction(), Some(0.25));
feed.set(300);
assert_eq!(store.readings()[0].fraction(), Some(0.75));
}
#[test]
fn progress_does_not_move_the_version() {
let store = DownloadStore::new();
let (track_id, _, feed) = running(&store, 1, "train");
store.started(track_id, 1000);
let before = store.version();
feed.set(500);
assert_eq!(store.version(), before);
assert_eq!(store.all()[0].bytes_written(), 500);
}
#[test]
fn no_content_length_means_no_fraction() {
let store = DownloadStore::new();
let (track_id, _, feed) = running(&store, 1, "chunked");
store.started(track_id, 0);
feed.set(9000);
assert_eq!(store.all()[0].fraction(), None);
assert_eq!(store.all()[0].bytes_written(), 9000);
}
#[test]
fn fetching_the_same_track_again_restarts_its_row() {
let store = DownloadStore::new();
let (track_id, _, _) = running(&store, 1, "train");
store.end(track_id, Outcome::Done);
running(&store, 1, "train");
assert_eq!(store.all().len(), 1);
assert_eq!(store.all()[0].state, DownloadState::Queued);
}
#[test]
fn running_transfers_sort_above_settled_ones() {
let store = DownloadStore::new();
let (done, _, _) = running(&store, 1, "done");
running(&store, 2, "running");
store.end(done, Outcome::Done);
let all = store.all();
assert_eq!(all[0].title, "running");
assert_eq!(all[1].title, "done");
}
#[test]
fn a_failure_keeps_its_reason() {
let store = DownloadStore::new();
let (track_id, _, _) = running(&store, 1, "gone");
store.end(track_id, Outcome::Failed("server returned 404".into()));
assert_eq!(
store.all()[0].state,
DownloadState::Failed("server returned 404".into())
);
}
#[test]
fn a_withdrawn_transfer_leaves_no_row_and_returns_late_waiters() {
let store = DownloadStore::new();
let (track_id, first, _) = running(&store, 1, "train");
assert_eq!(withdraw(&store, track_id), vec![first]);
assert!(store.all().is_empty());
}
#[test]
fn a_resync_hands_a_transfer_from_its_old_entry_to_its_new_in_one_step() {
let store = DownloadStore::new();
let (track_id, old, _) = running(&store, 7, "train");
let new = QueueItemId::new();
let other = QueueItemId::new();
let unfetched = store.resync(&[(7, new), (8, other)]);
assert_eq!(store.waiters(track_id), vec![new]);
assert!(!store.waiters(track_id).contains(&old));
assert!(!store.abandoned(track_id));
assert_eq!(
unfetched,
vec![(8, other)],
"the rest, in order, to be fetched"
);
}
#[test]
fn a_transfer_nothing_waits_on_is_abandoned() {
let store = DownloadStore::new();
let (track_id, waiter, _) = running(&store, 1, "train");
assert!(!store.abandoned(track_id));
store.resync(&[]);
assert!(store.abandoned(track_id));
store.join(track_id, Some(waiter));
assert!(!store.abandoned(track_id), "wanted again");
}
#[test]
fn a_transfer_kept_for_the_cache_is_never_abandoned() {
let store = DownloadStore::new();
let track_id = 5;
assert!(store.claim(track_id, None));
store.resync(&[]);
assert!(!store.abandoned(track_id));
}
#[test]
fn a_rate_needs_two_readings_and_a_gap_between_them() {
let store = DownloadStore::new();
let (track_id, _, feed) = running(&store, 1, "train");
store.started(track_id, 1_000_000);
let start = Instant::now();
store.sample_rates_at(start);
assert_eq!(
store.all()[0].bytes_per_second,
0,
"one reading is not a rate"
);
feed.set(100_000);
store.sample_rates_at(start + Duration::from_millis(50));
assert_eq!(store.all()[0].bytes_per_second, 0);
store.sample_rates_at(start + Duration::from_secs(1));
let bps = store.all()[0].bytes_per_second;
assert!(bps > 0, "a rate should have been worked out, got {bps}");
assert!(bps < 100_000, "and smoothed rather than taken whole: {bps}");
}
#[test]
fn a_settled_transfer_has_no_rate() {
let store = DownloadStore::new();
let (track_id, _, feed) = running(&store, 1, "train");
store.started(track_id, 1000);
let start = Instant::now();
store.sample_rates_at(start);
feed.set(500);
store.sample_rates_at(start + Duration::from_secs(1));
assert!(store.all()[0].bytes_per_second > 0);
store.end(track_id, Outcome::Done);
store.sample_rates_at(start + Duration::from_secs(2));
assert_eq!(store.all()[0].bytes_per_second, 0);
}
#[test]
fn clearing_settled_leaves_the_running_alone() {
let store = DownloadStore::new();
let (done, _, _) = running(&store, 1, "done");
running(&store, 2, "running");
store.end(done, Outcome::Done);
store.clear_settled();
let all = store.all();
assert_eq!(all.len(), 1);
assert_eq!(all[0].title, "running");
}
#[test]
fn the_settled_tail_is_bounded() {
let store = DownloadStore::new();
for id in 0..(SETTLED_LIMIT as i64 + 10) {
let (track_id, _, _) = running(&store, id, "t");
store.end(track_id, Outcome::Done);
}
assert_eq!(store.all().len(), SETTLED_LIMIT);
}
#[test]
fn a_parked_reader_is_woken_by_every_write_it_waits_for() {
let feed = ByteFeed::new();
let reader = {
let feed = feed.clone();
std::thread::spawn(move || {
let deadline = Instant::now() + Duration::from_secs(10);
let mut seen = 0;
while seen < 20_000 {
seen = feed.wait_past(seen, deadline);
assert!(
Instant::now() < deadline,
"a write was slept through at {seen}"
);
}
})
};
for n in 1..=20_000 {
feed.set(n);
}
reader.join().unwrap();
}
#[test]
fn a_reading_rings_the_store_and_not_the_engine() {
let store = DownloadStore::new();
let (track_id, _, feed) = running(&store, 1, "train");
store.started(track_id, 1000);
let before = store.moved().generation();
feed.set(500);
store.progressed();
assert_ne!(store.moved().generation(), before);
}
}