use std::collections::{HashMap, VecDeque};
use std::sync::Arc;
use wasm_bindgen::JsCast;
use wasm_bindgen::closure::Closure;
use web_sys::{
AudioBuffer, AudioBufferSourceNode, AudioContext, AudioContextState, AudioScheduledSourceNode,
GainNode, StereoPannerNode, WaveShaperNode,
};
use crate::assets::Stream;
use crate::sound::Encoded;
use crate::sound::data::{Body, Clip, ClipFrame};
use crate::sound::mixer::{
Audible, Command, Fade, GLIDE, Levels, LiveKnobs, Playing, STREAM_AHEAD, SoundId, Sustained,
Sustaining, Sustains, Voicing, Window, limit,
};
use crate::sound::output::MixRate;
use crate::sound::schedule::{Cut, Scheduled, split};
const CHUNK: f64 = 0.25;
const LIMITER_POINTS: usize = 1 << 10;
pub(crate) struct Backend {
context: AudioContext,
master: GainNode,
sustained: Sustains<Sustain>,
shots: Vec<Playing<Voice>>,
starting: Vec<Voicing>,
buffers: Buffers,
volume: f32,
unlocked: bool,
_hidden: Closure<dyn FnMut()>,
}
impl Backend {
pub(crate) fn rate(&self) -> Option<MixRate> {
None
}
pub(crate) fn frame(&mut self, commands: impl Iterator<Item = Command>) {
let now = self.context.current_time();
for command in commands {
self.apply(command, now);
}
self.allocate(now);
for sustain in self.sustained.iter_mut() {
sustain.top_up(now);
}
for shot in self.shots.iter_mut().filter_map(Playing::voice_mut) {
shot.top_up(now);
}
self.shots.retain(|shot| !shot.spent(now));
self.sustained.settle(now);
}
pub(crate) fn unlocked(&self) -> bool {
self.unlocked
}
pub(crate) fn unlock(&mut self) {
if !self.unlocked {
let _ = self.context.resume();
self.unlocked = true;
}
}
fn apply(&mut self, command: Command, now: f64) {
match command {
Command::Play(voicing) => match self.unlocked {
true => self.starting.push(voicing),
false => log::debug!("a sound played before the page could; ignoring it"),
},
Command::Sustained(declared) => self.sustained.declare(declared, now),
Command::Volume(volume) => self.set_volume(volume.max(0.0), now),
}
}
fn set_volume(&mut self, volume: f32, now: f64) {
if volume == self.volume {
return;
}
self.volume = volume;
let gain = self.master.gain();
let _ = gain.set_value_at_time(gain.value(), now);
let _ = gain.linear_ramp_to_value_at_time(volume, now + GLIDE.as_secs_f64());
}
fn allocate(&mut self, now: f64) {
let mut graph = Graph {
context: &self.context,
master: &self.master,
buffers: &mut self.buffers,
};
self.sustained
.allocate(&mut self.shots, &mut self.starting, &mut graph, now);
for voicing in self.starting.drain(..) {
let levels = Sliding::held(voicing.live.levels);
if let Some(voice) = Voice::start(&mut graph, voicing, ClipFrame::ZERO, levels, now) {
self.shots.push(Playing::Voiced(voice));
}
}
}
}
struct Graph<'a> {
context: &'a AudioContext,
master: &'a GainNode,
buffers: &'a mut Buffers,
}
struct Sustain {
sustained: Sustained,
clip: Arc<Clip>,
window: Window,
live: LiveKnobs,
levels: Sliding,
taken: f64,
since: f64,
playing: Playing<Voice>,
}
impl Sustaining for Sustain {
type Pace = f64;
type Voices<'a> = Graph<'a>;
fn started(sustained: Sustained, voicing: Voicing, now: f64) -> Self {
Self {
sustained,
clip: voicing.clip,
window: voicing.window,
levels: Sliding::held(voicing.live.levels),
live: voicing.live,
taken: 0.0,
since: now,
playing: Playing::Virtual,
}
}
fn sustained(&self) -> Sustained {
self.sustained
}
fn declare(&mut self, voicing: Voicing, now: f64) {
if voicing.window != self.window {
log::debug!(
"a sustained value declared a different window while sounding; keeping the one it started on"
);
}
self.live = voicing.live;
match self.playing.voice_mut() {
Some(voice) => voice.retarget(&self.live, now),
None => {
self.levels
.aim(self.live.levels, self.live.glide.as_secs_f64(), now);
}
}
self.playing.declared(now);
}
fn stop(&mut self, now: f64) -> bool {
self.playing.stopped(now)
}
fn voiced(&mut self, wins: bool, graph: &mut Graph<'_>, now: f64) {
let at = ClipFrame::new(self.played(now) as u64);
let started = || {
let voicing = Voicing {
sound: self.sustained.sound,
clip: Arc::clone(&self.clip),
window: self.window,
live: self.live,
};
Voice::start(graph, voicing, at, self.levels, now)
};
self.playing.voiced(wins, now, started);
}
fn audibility(&self, now: f64) -> f32 {
match self.playing.voice() {
Some(voice) => voice.audibility(now),
None => self.levels.loudest(now),
}
}
fn settle(&mut self, now: f64) -> bool {
match self.playing.landed(now) {
Some((voice, Fade::Cut)) => {
self.taken = voice.played(now);
self.since = now;
self.levels = voice.levels;
true
}
Some((_, Fade::Stop)) => false,
None => true,
}
}
}
impl Sustain {
fn played(&self, now: f64) -> f64 {
match self.playing.voice() {
Some(voice) => voice.played(now),
None => {
let rate = f64::from(self.clip.rate) * f64::from(self.live.pitch);
self.taken + (now - self.since) * rate
}
}
}
fn top_up(&mut self, now: f64) {
if let Some(voice) = self.playing.voice_mut() {
voice.top_up(now);
}
}
}
#[derive(Clone, Copy)]
struct Sliding {
from: Levels,
to: Levels,
since: f64,
span: f64,
}
impl Sliding {
fn held(levels: Levels) -> Self {
Self {
from: levels,
to: levels,
since: 0.0,
span: 0.0,
}
}
fn aim(&mut self, levels: Levels, span: f64, now: f64) -> bool {
if levels == self.to {
return false;
}
self.from = self.at(now);
self.to = levels;
self.since = now;
self.span = span;
true
}
fn at(&self, now: f64) -> Levels {
let over = match self.span > 0.0 {
true => (((now - self.since) / self.span).clamp(0.0, 1.0)) as f32,
false => 1.0,
};
Levels {
left: self.from.left + (self.to.left - self.from.left) * over,
right: self.from.right + (self.to.right - self.from.right) * over,
}
}
fn until(&self) -> f64 {
self.since + self.span
}
fn loudest(&self, now: f64) -> f32 {
self.at(now).loudest().max(self.to.loudest())
}
}
#[derive(Default)]
struct Buffers(HashMap<SoundId, AudioBuffer>);
impl Buffers {
fn of(&mut self, context: &AudioContext, sound: SoundId, clip: &Clip) -> Option<AudioBuffer> {
if let Some(held) = self.0.get(&sound) {
return Some(held.clone());
}
let Body::Samples(samples) = &clip.body else {
return None;
};
let buffer = upload(context, samples, clip)?;
self.0.insert(sound, buffer.clone());
Some(buffer)
}
}
#[derive(Clone, Copy)]
struct Release {
lands: f64,
span: f64,
}
impl Release {
fn left(self, now: f64) -> f32 {
match self.span > 0.0 {
true => (((self.lands - now) / self.span) as f32).clamp(0.0, 1.0),
false => 0.0,
}
}
}
struct Voice {
context: AudioContext,
clip: Arc<Clip>,
window: Window,
gain: GainNode,
pan: StereoPannerNode,
scheduled: Vec<(AudioBufferSourceNode, f64)>,
chain: Option<Chain>,
levels: Sliding,
pitch: f32,
fade: f64,
glide: f64,
taken: f64,
since: f64,
ends_at: Option<f64>,
releasing: Option<Release>,
}
impl Voice {
fn start(
graph: &mut Graph<'_>,
voicing: Voicing,
at: ClipFrame,
levels: Sliding,
now: f64,
) -> Option<Self> {
let held = graph
.buffers
.of(graph.context, voicing.sound, &voicing.clip);
let mut voice = Self::new(graph, voicing, at, levels, now)?;
match (held, &voice.clip.body) {
(Some(buffer), _) => voice.play_buffer(&buffer, now),
(None, Body::Encoded(encoded)) => {
let encoded = Arc::clone(encoded);
voice.open_chain(encoded, now);
}
(None, Body::Samples(_)) => return None,
}
Some(voice)
}
fn new(
graph: &Graph<'_>,
voicing: Voicing,
played: ClipFrame,
levels: Sliding,
now: f64,
) -> Option<Self> {
let gain = graph.context.create_gain().ok()?;
let pan = graph.context.create_stereo_panner().ok()?;
gain.connect_with_audio_node(&pan).ok()?;
pan.connect_with_audio_node(graph.master).ok()?;
let mut voice = Self {
context: graph.context.clone(),
clip: voicing.clip,
window: voicing.window,
gain,
pan,
scheduled: Vec::new(),
chain: None,
levels,
pitch: voicing.live.pitch,
fade: voicing.live.fade.as_secs_f64(),
glide: voicing.live.glide.as_secs_f64(),
taken: played.get() as f64,
since: now,
ends_at: None,
releasing: None,
};
let (level, panned) = split(&levels.at(now));
voice.pan.pan().set_value(panned);
voice.gain.gain().set_value_at_time(0.0, now).ok()?;
voice
.gain
.gain()
.linear_ramp_to_value_at_time(level, now + voice.fade)
.ok()?;
voice.aim(now);
Some(voice)
}
fn retarget(&mut self, live: &LiveKnobs, now: f64) {
self.fade = live.fade.as_secs_f64();
self.glide = live.glide.as_secs_f64();
if self.levels.aim(live.levels, self.glide, now) && self.releasing.is_none() {
self.aim(now);
}
self.repitch(live.pitch, now);
}
fn repitch(&mut self, pitch: f32, now: f64) {
if pitch == self.pitch {
return;
}
match self.recut(now) {
Some(cut) => {
self.taken = cut.produced.get() as f64;
self.since = cut.at;
}
None => {
self.taken = self.played(now);
self.since = now;
for (source, _) in &self.scheduled {
source.playback_rate().set_value(pitch);
}
}
}
self.pitch = pitch;
}
fn recut(&mut self, now: f64) -> Option<Cut> {
let ends: Vec<f64> = self.scheduled.iter().map(|(_, until)| *until).collect();
let rate = f64::from(self.clip.rate) * f64::from(self.pitch);
let chain = self.chain.as_mut()?;
let cut = Scheduled {
ends: &ends,
until: chain.next_at,
produced: chain.produced,
rate,
}
.cut(now);
chain.next_at = cut.at;
chain.produced = cut.produced;
chain.decoded.clear();
for (source, until) in &mut self.scheduled {
let scheduled: &AudioScheduledSourceNode = source;
let _ = scheduled.stop_with_when(cut.at);
*until = until.min(cut.at);
}
Some(cut)
}
fn aim(&mut self, now: f64) {
let (level, panned) = split(&self.levels.to);
let until = self.levels.until().max(now + self.fade);
let _ = self.gain.gain().linear_ramp_to_value_at_time(level, until);
let _ = self.pan.pan().linear_ramp_to_value_at_time(panned, until);
}
fn release(&mut self, span: f64, now: f64) {
let lands = now + span;
if self.releasing.is_some_and(|release| release.lands <= lands) {
return;
}
self.releasing = Some(Release { lands, span });
let gain = self.gain.gain();
let _ = gain.cancel_scheduled_values(now);
let _ = gain.set_value_at_time(gain.value(), now);
let _ = gain.linear_ramp_to_value_at_time(0.0, lands);
}
fn revive(&mut self, span: f64, now: f64) {
self.releasing = None;
let (level, panned) = split(&self.levels.to);
let gain = self.gain.gain();
let _ = gain.cancel_scheduled_values(now);
let _ = gain.set_value_at_time(gain.value(), now);
let _ = gain.linear_ramp_to_value_at_time(level, now + span);
let _ = self
.pan
.pan()
.linear_ramp_to_value_at_time(panned, now + span);
}
fn played(&self, now: f64) -> f64 {
let rate = f64::from(self.clip.rate) * f64::from(self.pitch);
self.taken + (now - self.since) * rate
}
fn play_buffer(&mut self, buffer: &AudioBuffer, now: f64) {
let Some(source) = self.source() else {
return;
};
source.set_buffer(Some(buffer));
let rate = f64::from(self.clip.rate);
let at = self
.window
.at(ClipFrame::new(self.taken as u64))
.unwrap_or(self.window.start);
let until = match self.window.looping {
true => {
source.set_loop(true);
source.set_loop_start(self.window.wrap.get() as f64 / rate);
source.set_loop_end(self.window.end.get() as f64 / rate);
f64::INFINITY
}
false => {
let left = (self.window.end - at).get() as f64 / rate / f64::from(self.pitch);
self.ends_at = Some(now + left);
now + left
}
};
let _ = source.start_with_when_and_grain_offset(now, at.get() as f64 / rate);
self.scheduled.push((source, until));
}
fn open_chain(&mut self, encoded: Arc<Encoded>, now: f64) {
self.chain = Some(Chain {
stream: Stream::open(encoded, self.clip.rate)
.inspect_err(|error| log::debug!("{error}"))
.map(Box::new)
.ok(),
produced: ClipFrame::new(self.taken as u64),
next_at: now,
decoded: VecDeque::new(),
});
}
fn top_up(&mut self, now: f64) {
self.scheduled.retain(|(_, until)| *until > now);
let channels = self.clip.channels.count();
let rate = f64::from(self.clip.rate);
let pitch = f64::from(self.pitch);
let window = self.window;
let Some(chain) = &mut self.chain else {
return;
};
if chain.next_at > now + STREAM_AHEAD.as_secs_f64() {
return;
}
let Some(spliced) = chain.decode(&window, channels, (CHUNK * rate) as usize) else {
let spent = chain.next_at;
self.ends_at = Some(spent);
return;
};
let at = chain.next_at.max(now);
let until = at + (spliced.len() / channels) as f64 / rate / pitch;
chain.next_at = until;
let Some(buffer) = upload(&self.context, &spliced, &self.clip) else {
return;
};
let Some(source) = self.source() else {
return;
};
source.set_buffer(Some(&buffer));
let _ = source.start_with_when(at);
self.scheduled.push((source, until));
}
fn source(&self) -> Option<AudioBufferSourceNode> {
let source = self.context.create_buffer_source().ok()?;
source.playback_rate().set_value(self.pitch);
source.connect_with_audio_node(&self.gain).ok()?;
Some(source)
}
}
impl Audible for Voice {
type Pace = f64;
fn audibility(&self, now: f64) -> f32 {
self.levels.loudest(now) * self.releasing.map_or(1.0, |release| release.left(now))
}
fn stop(&mut self, now: f64) {
self.release(self.fade, now);
}
fn rise(&mut self, now: f64) {
self.revive(self.fade, now);
}
fn cut(&mut self, now: f64) {
self.release(self.glide, now);
}
fn recover(&mut self, now: f64) {
self.revive(self.glide, now);
}
fn spent(&self, now: f64) -> bool {
let landed = |at: f64| at <= now;
self.releasing.is_some_and(|release| landed(release.lands))
|| self.ends_at.is_some_and(landed)
}
}
impl Drop for Voice {
fn drop(&mut self) {
for (source, _) in &self.scheduled {
let scheduled: &AudioScheduledSourceNode = source;
let _ = scheduled.stop();
}
let _ = self.gain.disconnect();
let _ = self.pan.disconnect();
}
}
struct Chain {
stream: Option<Box<Stream>>,
produced: ClipFrame,
next_at: f64,
decoded: VecDeque<f32>,
}
impl Chain {
fn decode(&mut self, window: &Window, channels: usize, frames: usize) -> Option<Vec<f32>> {
let mut stream = self.stream.take()?;
while self.decoded.len() / channels < frames {
let Some(at) = window.at(self.produced) else {
break;
};
if stream.position() != at
&& let Err(error) = stream.seek(at)
{
log::debug!("{error}");
break;
}
match stream.read(
frames.min((window.end - at).get() as usize),
&mut self.decoded,
) {
Ok(0) => break,
Ok(read) => self.produced += read as u64,
Err(error) => {
log::debug!("{error}");
break;
}
}
}
let taken: Vec<f32> = self.decoded.drain(..).collect();
self.stream = Some(stream);
(!taken.is_empty()).then_some(taken)
}
}
pub(crate) fn open() -> Option<Backend> {
let context = AudioContext::new()
.inspect_err(|_| log::debug!("this browser has no audio; playing nothing"))
.ok()?;
let master = context.create_gain().ok()?;
let limiter = limiter(&context)?;
master.connect_with_audio_node(&limiter).ok()?;
limiter
.connect_with_audio_node(&context.destination())
.ok()?;
Some(Backend {
_hidden: watch_visibility(&context),
unlocked: matches!(context.state(), AudioContextState::Running),
context,
master,
sustained: Sustains::default(),
shots: Vec::new(),
starting: Vec::new(),
buffers: Buffers::default(),
volume: 1.0,
})
}
fn limiter(context: &AudioContext) -> Option<WaveShaperNode> {
let mut curve: Vec<f32> = (0..LIMITER_POINTS)
.map(|at| limit(at as f32 / (LIMITER_POINTS - 1) as f32 * 2.0 - 1.0))
.collect();
let shaper = context.create_wave_shaper().ok()?;
shaper.set_curve_opt_f32_slice(Some(&mut curve));
Some(shaper)
}
fn watch_visibility(context: &AudioContext) -> Closure<dyn FnMut()> {
let watched = context.clone();
let hidden = Closure::<dyn FnMut()>::new(move || {
let Some(document) = web_sys::window().and_then(|window| window.document()) else {
return;
};
let _ = match document.hidden() {
true => watched.suspend(),
false => watched.resume(),
};
});
if let Some(document) = web_sys::window().and_then(|window| window.document()) {
document.set_onvisibilitychange(Some(hidden.as_ref().unchecked_ref()));
}
hidden
}
fn upload(context: &AudioContext, samples: &[f32], clip: &Clip) -> Option<AudioBuffer> {
let channels = clip.channels.count();
let frames = samples.len() / channels;
let buffer = context
.create_buffer(channels as u32, frames as u32, f32::from(clip.rate))
.ok()?;
let mut channel = Vec::with_capacity(frames);
for at in 0..channels {
channel.clear();
channel.extend(samples.iter().skip(at).step_by(channels));
buffer.copy_to_channel(&channel, at as i32).ok()?;
}
Some(buffer)
}