use anyhow::Result;
use base64::Engine as _;
use futures::future::FutureExt;
use futures::stream::{FuturesUnordered, StreamExt};
use wasm_bindgen::JsCast;
use wasm_bindgen_futures::JsFuture;
use awsm_audio_schema::{
AssetId, AudioSource, ConnectionSink, ConnectionSource, Graph, NodeKind, SampleId, SampleKind,
SampleLibrary, WasmSource,
};
use crate::{bounce, AudioClipPart, ControlLanePart, Player, SongVoiceSpec, TriggerPart};
const RELEASE_TAIL: f64 = 3.0;
const DEFAULT_SOUND_SECS: f64 = 6.0;
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum PlayKind {
Sound,
Sequence,
Arrangement,
}
#[derive(Debug, Clone, Copy, PartialEq)]
pub enum SoundShape {
OneShot { secs: f64 },
Sustaining,
}
impl SoundShape {
pub fn secs(self) -> Option<f64> {
match self {
SoundShape::OneShot { secs } => Some(secs),
SoundShape::Sustaining => None,
}
}
}
#[derive(Debug, Clone, Copy)]
pub struct PlayOptions {
pub looping: bool,
pub duration_secs: Option<f64>,
pub seek_secs: f64,
}
impl Default for PlayOptions {
fn default() -> Self {
Self {
looping: false,
duration_secs: None,
seek_secs: 0.0,
}
}
}
pub struct Playback {
pub kind: PlayKind,
pub looping: bool,
pub started_at: f64,
content_secs: Option<f64>,
next_at: f64,
sequence: Option<SequenceParts>,
clips: Vec<AudioClipPart>,
}
impl Playback {
pub fn content_secs(&self) -> Option<f64> {
self.content_secs
}
pub fn next_loop_at(&self) -> f64 {
self.next_at
}
pub fn ended(&self, now: f64) -> bool {
if self.looping {
return false;
}
match self.content_secs {
Some(secs) => now >= self.started_at + secs + 0.25,
None => false,
}
}
}
pub fn classify(lib: &SampleLibrary, target: SampleId) -> PlayKind {
match lib.sample(target) {
Some(s) if s.kind == SampleKind::Arrangement => PlayKind::Arrangement,
Some(s) if is_sequence(&s.graph) => PlayKind::Sequence,
_ => PlayKind::Sound,
}
}
pub fn is_sequence(graph: &Graph) -> bool {
graph
.connections
.iter()
.any(|c| matches!(c.to, ConnectionSink::Trigger { .. }))
|| graph
.nodes
.iter()
.any(|n| matches!(n.kind, NodeKind::Output(_) | NodeKind::SpatialOutput(_)))
}
pub struct SequenceParts {
pub graph: Graph,
pub triggers: Vec<TriggerPart>,
pub control: Vec<ControlLanePart>,
pub loop_secs: f64,
}
pub fn sequence_parts(lib: &SampleLibrary, id: SampleId) -> SequenceParts {
let Some(sample) = lib.sample(id) else {
return SequenceParts {
graph: Graph::default(),
triggers: Vec::new(),
control: Vec::new(),
loop_secs: 0.0,
};
};
let g = sample.graph.clone();
let mut triggers = Vec::new();
let mut control = Vec::new();
let mut content_end = 0.0f64;
let mut window_secs = 0.0f64;
for c in &g.connections {
match (&c.from, &c.to) {
(ConnectionSource::SeqOut { node: seqn, key }, ConnectionSink::Trigger { node }) => {
let Some(NodeKind::NoteSequencer(ms)) = g.node(*seqn).map(|n| &n.kind) else {
continue;
};
let Some(outp) = ms.outputs.iter().find(|o| &o.key == key) else {
continue;
};
let Some(track) = ms.song.tracks.get(outp.track) else {
continue;
};
let Some(NodeKind::Sample(sref)) = g.node(*node).map(|n| &n.kind) else {
continue;
};
let instrument = awsm_audio_schema::flatten(lib, sref.sample);
if instrument.nodes.is_empty() {
continue;
}
let base = ms.song.beats_to_secs(ms.start);
let mut notes = Vec::new();
for ev in &track.events {
if let Some(n) = outp.note {
if ev.note != n {
continue;
}
}
if ev.start < ms.start {
continue;
}
if let Some(end) = ms.end {
if ev.start >= end {
continue;
}
}
let semitones = if outp.note.is_some() {
outp.transpose
} else {
ev.note as i32 - 60 + outp.transpose
};
let end = ms.song.beats_to_secs(ev.start + ev.length) - base;
content_end = content_end.max(end);
notes.push(SongVoiceSpec {
start: ms.song.beats_to_secs(ev.start) - base,
end,
semitones,
velocity: ((ev.velocity as f32 / 127.0) * outp.gain).clamp(0.0, 1.0),
});
}
let win_beats = ms.end.or((ms.length > 0.0).then_some(ms.length));
if let Some(win_end) = win_beats {
let win = ms.song.beats_to_secs(win_end) - base;
window_secs = window_secs.max(win.max(0.05));
}
if !notes.is_empty() {
triggers.push(TriggerPart {
target: *node,
instrument,
notes,
});
}
}
(
ConnectionSource::SeqOut { node: seqn, key },
ConnectionSink::NodeParam { node, param },
) => {
let Some(NodeKind::ControlSequencer(cs)) = g.node(*seqn).map(|n| &n.kind) else {
continue;
};
let Some(lane) = cs.lanes.iter().find(|l| &l.key == key) else {
continue;
};
let bpm = if cs.bpm > 0.0 { cs.bpm } else { 120.0 };
let points = lane
.points
.iter()
.map(|p| (p.beat * 60.0 / bpm, p.value, p.curve))
.collect();
control.push(ControlLanePart {
target: *node,
param: param.0.clone(),
points,
});
}
_ => {}
}
}
let loop_secs = if window_secs > 0.0 {
window_secs
} else {
content_end
};
SequenceParts {
graph: g,
triggers,
control,
loop_secs,
}
}
pub fn audio_clip_parts(lib: &SampleLibrary, id: SampleId, seek: f64) -> Vec<AudioClipPart> {
let Some(sample) = lib.sample(id) else {
return Vec::new();
};
if sample.kind != SampleKind::Arrangement {
return Vec::new();
}
let arr = &sample.arrangement;
let seek = seek.max(0.0);
let any_solo = arr.tracks.iter().any(|t| t.solo);
let mut out = Vec::new();
for track in &arr.tracks {
if track.mute || (any_solo && !track.solo) {
continue;
}
for clip in &track.clips {
let Some(buffer) = lib
.sample(clip.source)
.and_then(|s| s.bounce.as_ref().map(|b| b.asset))
else {
continue; };
let clip_end = clip.start + clip.length;
if clip_end <= seek {
continue; }
let lead = (seek - clip.start).max(0.0); let speed = if clip.speed > 0.0 {
clip.speed as f64
} else {
1.0
};
out.push(AudioClipPart {
buffer,
start: (clip.start - seek).max(0.0),
offset: clip.offset + lead * speed,
length: (clip.length - lead).max(0.0),
gain: clip.gain * track.gain,
gain_curve: clip_gain_curve(track, clip, seek),
looping: clip.looping,
speed,
});
}
}
out
}
fn clip_gain_curve(
track: &awsm_audio_schema::ArrTrack,
clip: &awsm_audio_schema::Clip,
seek: f64,
) -> Vec<(f64, f32)> {
if track.gain_automation.is_empty() {
return Vec::new();
}
let abs_start = clip.start.max(seek);
let abs_end = clip.start + clip.length;
if abs_end <= abs_start {
return Vec::new();
}
let base = clip.gain * track.gain;
let mut out = vec![(0.0, base * track.gain_at(abs_start))];
for p in &track.gain_automation {
if p.time > abs_start && p.time < abs_end {
out.push((p.time - abs_start, base * p.gain));
}
}
out.push((abs_end - abs_start, base * track.gain_at(abs_end)));
out
}
enum Loaded {
Buffer(AssetId, web_sys::AudioBuffer),
Module(AssetId, js_sys::WebAssembly::Module),
}
impl Player {
pub async fn register(&mut self, lib: &SampleLibrary) -> Result<()> {
if !self.worklet_ready {
if let Ok(promise) = self.add_worklet_shim() {
let _ = JsFuture::from(promise).await;
self.mark_worklet_ready();
}
}
let b64 = base64::engine::general_purpose::STANDARD;
let ctx = self.ctx.clone();
let mut pool = FuturesUnordered::new();
for asset in &lib.assets.buffers {
if self.has_buffer(asset.id) {
continue;
}
match &asset.source {
AudioSource::Pcm {
sample_rate,
channels,
} => {
self.store_pcm(asset.id, *sample_rate, channels)?;
}
AudioSource::Encoded(data) => {
let id = asset.id;
let ctx = ctx.clone();
let bytes = b64.decode(data)?;
pool.push(
async move {
let buf = decode_audio(&ctx, &bytes).await?;
Ok::<_, anyhow::Error>(Loaded::Buffer(id, buf))
}
.boxed_local(),
);
}
AudioSource::Url(_) | AudioSource::Path(_) => {
tracing::warn!(
"register: skipping non-inline audio asset {} (Url/Path — \
rehydrate to inline bytes first)",
asset.id
);
}
}
}
for asset in &lib.assets.wasm_modules {
if self.has_module(&asset.id) {
continue;
}
match &asset.source {
WasmSource::Base64(data) => {
let id = asset.id;
let bytes = b64.decode(data)?;
pool.push(
async move {
let module = compile_wasm(&bytes).await?;
Ok::<_, anyhow::Error>(Loaded::Module(id, module))
}
.boxed_local(),
);
}
WasmSource::Url(_) | WasmSource::Path(_) => {
tracing::warn!(
"register: skipping non-inline wasm asset {} (Url/Path — \
rehydrate to inline bytes first)",
asset.id
);
}
}
}
while let Some(loaded) = pool.next().await {
match loaded? {
Loaded::Buffer(id, buf) => self.store_buffer(id, buf),
Loaded::Module(id, m) => self.store_module(id, m),
}
}
let mut bounces = FuturesUnordered::new();
for sample in &lib.samples {
if sample.kind != SampleKind::Sound {
continue;
}
let Some(b) = &sample.bounce else { continue };
if self.has_buffer(b.asset) {
continue; }
let Some(job) = self.bounce_job_for_document(lib, sample.id) else {
continue;
};
let asset = b.asset;
bounces.push(async move { (asset, bounce::render(job).await) });
}
while let Some((asset, result)) = bounces.next().await {
match result {
Ok((channels, sr)) => self.store_pcm(asset, sr as f32, &channels)?,
Err(e) => tracing::error!("register: bounce of asset {asset} failed: {e}"),
}
}
Ok(())
}
fn bounce_job_for_document(
&self,
lib: &SampleLibrary,
id: SampleId,
) -> Option<bounce::BounceJob> {
let sample = lib.sample(id)?;
if sample.kind != SampleKind::Sound {
return None;
}
let (graph, parts, control, duration, loop_secs) = if is_sequence(&sample.graph) {
let sp = sequence_parts(lib, id);
let loop_len = sp.loop_secs.max(0.05);
(
sp.graph,
sp.triggers,
sp.control,
loop_len + RELEASE_TAIL,
Some(loop_len),
)
} else {
(
awsm_audio_schema::flatten(lib, id),
Vec::new(),
Vec::new(),
DEFAULT_SOUND_SECS,
None,
)
};
Some(self.bounce_job(graph, parts, control, duration, loop_secs))
}
pub async fn measure_sound(
&self,
lib: &SampleLibrary,
target: SampleId,
max_secs: f64,
) -> Result<SoundShape> {
let window = max_secs.max(0.2);
let graph = awsm_audio_schema::flatten(lib, target);
let job = self.bounce_job(graph, Vec::new(), Vec::new(), window, None);
let (channels, sr) = bounce::render(job).await?;
let frames = channels.iter().map(Vec::len).max().unwrap_or(0);
let secs = frames as f64 / (sr.max(1) as f64);
Ok(if secs + 0.05 < window {
SoundShape::OneShot { secs }
} else {
SoundShape::Sustaining
})
}
pub fn play_document(
&mut self,
lib: &SampleLibrary,
target: SampleId,
opts: PlayOptions,
) -> Result<Playback> {
self.set_master_gain(1.0);
let kind = classify(lib, target);
match kind {
PlayKind::Sound => {
let graph = awsm_audio_schema::flatten(lib, target);
self.play(&graph, false)?;
Ok(Playback {
kind,
looping: false,
started_at: self.current_time(),
content_secs: opts.duration_secs,
next_at: f64::INFINITY,
sequence: None,
clips: Vec::new(),
})
}
PlayKind::Sequence => {
let sp = sequence_parts(lib, target);
self.play_arrangement(&sp.graph, opts.looping)?;
let at = self.current_time() + 0.1;
self.schedule_triggers(&sp.triggers, at)?;
self.schedule_control(&sp.control, at);
let natural = (sp.loop_secs > 0.0).then(|| sp.loop_secs.max(0.05));
let content_secs = opts.duration_secs.or(natural);
let next_at = match (opts.looping, content_secs) {
(true, Some(secs)) => at + secs,
_ => f64::INFINITY,
};
Ok(Playback {
kind,
looping: opts.looping,
started_at: at,
content_secs,
next_at,
sequence: Some(sp),
clips: Vec::new(),
})
}
PlayKind::Arrangement => {
let seek = opts.seek_secs.max(0.0);
let clips = audio_clip_parts(lib, target, seek);
self.arrange_audio_begin();
let at = self.current_time() + 0.1;
self.schedule_audio_clips(&clips, at)?;
let natural = lib
.sample(target)
.map(|s| (s.arrangement.length_secs - seek).max(0.1));
let content_secs = opts.duration_secs.or(natural);
let next_at = match (opts.looping, content_secs) {
(true, Some(secs)) => at + secs,
_ => f64::INFINITY,
};
Ok(Playback {
kind,
looping: opts.looping,
started_at: at,
content_secs,
next_at,
sequence: None,
clips,
})
}
}
}
pub fn loop_tick(&mut self, pb: &mut Playback, now: f64) -> Result<()> {
if !pb.looping || !pb.next_at.is_finite() {
return Ok(());
}
if now < pb.next_at - 0.25 {
return Ok(());
}
let start = pb.next_at;
match pb.kind {
PlayKind::Sequence => {
if let Some(sp) = &pb.sequence {
self.schedule_triggers(&sp.triggers, start)?;
self.schedule_control(&sp.control, start);
}
}
PlayKind::Arrangement => {
self.schedule_audio_clips(&pb.clips, start)?;
}
PlayKind::Sound => return Ok(()),
}
if let Some(secs) = pb.content_secs {
pb.started_at = start;
pb.next_at = start + secs;
}
Ok(())
}
}
async fn decode_audio(ctx: &web_sys::AudioContext, bytes: &[u8]) -> Result<web_sys::AudioBuffer> {
let array = js_sys::Uint8Array::new_with_length(bytes.len() as u32);
array.copy_from(bytes);
let promise = ctx
.decode_audio_data(&array.buffer())
.map_err(|e| anyhow::anyhow!("decodeAudioData: {e:?}"))?;
let value = JsFuture::from(promise)
.await
.map_err(|e| anyhow::anyhow!("decode await: {e:?}"))?;
value
.dyn_into::<web_sys::AudioBuffer>()
.map_err(|_| anyhow::anyhow!("decodeAudioData did not return an AudioBuffer"))
}
async fn compile_wasm(bytes: &[u8]) -> Result<js_sys::WebAssembly::Module> {
let array = js_sys::Uint8Array::new_with_length(bytes.len() as u32);
array.copy_from(bytes);
let value = JsFuture::from(js_sys::WebAssembly::compile(&array))
.await
.map_err(|e| anyhow::anyhow!("WebAssembly.compile: {e:?}"))?;
value
.dyn_into::<js_sys::WebAssembly::Module>()
.map_err(|_| anyhow::anyhow!("WebAssembly.compile did not return a Module"))
}