use crate::audio::SoundTriggerHandle;
use crate::audio::fft::{MAX_FREQ, fft_data_u8};
use crate::audio::sound_data::SoundData;
use crate::audio::sound_trigger::{SoundTrigger, SoundTriggerParams};
use crate::pipeline::constants::TEXTURE_SIZE;
use crate::pipeline::renderer_callback::RendererCallback;
use crate::storage::asset::scene::effect_state::OwnedTextureId;
use crate::ui::scoped_frame;
use crate::{WGPU_RENDER_STATE, wgpu_render_state};
use egui::load::SizedTexture;
use egui::{Color32, Frame, Image, Layout, Ui, UiBuilder};
use egui_knob::{Knob, KnobStyle, LabelPosition};
use emath::{Align, Pos2, Rect, Vec2, pos2, remap_clamp, vec2};
use epaint::{PathShape, PathStroke, Stroke};
use ndarray::Array1;
use serde::{Deserialize, Serialize};
use std::num::NonZeroU64;
use std::sync::Arc;
use wgpu::util::DeviceExt;
use wgpu::{
BindGroup, BindGroupDescriptor, BindGroupEntry, BindGroupLayoutDescriptor,
BindGroupLayoutEntry, BindingType, Buffer, BufferBindingType, BufferUsages,
CommandEncoderDescriptor, Extent3d, FragmentState, LoadOp, MultisampleState, Operations,
PipelineCompilationOptions, PipelineLayoutDescriptor, PrimitiveState,
RenderPassColorAttachment, RenderPassDescriptor, RenderPipeline, RenderPipelineDescriptor,
ShaderModuleDescriptor, ShaderSource, ShaderStages, StoreOp, TextureDescriptor,
TextureDimension, TextureFormat, TextureUsages, TextureView, TextureViewDescriptor,
VertexState,
};
#[derive(PartialEq, Debug)]
struct PreviewShader {
spectrum_pipeline: RenderPipeline,
spectrum_bind_group: BindGroup,
spectrum_texture_buffer: Buffer,
spectrum_texture_view: TextureView,
spectrum_texture_id: OwnedTextureId,
}
impl PreviewShader {
fn try_init() -> Option<Arc<Self>> {
let texture_desc = TextureDescriptor {
size: Extent3d {
width: TEXTURE_SIZE as u32,
height: TEXTURE_SIZE as u32,
depth_or_array_layers: 1,
},
mip_level_count: 1,
sample_count: 1,
dimension: TextureDimension::D2,
format: TextureFormat::Bgra8Unorm,
usage: TextureUsages::RENDER_ATTACHMENT | TextureUsages::TEXTURE_BINDING,
label: None,
view_formats: &[TextureFormat::Bgra8Unorm],
};
let wgpu_render_state = WGPU_RENDER_STATE.get()?.clone();
let device = wgpu_render_state.device;
let spectrum_texture = device.create_texture(&texture_desc);
let spectrum_texture_view = spectrum_texture.create_view(&TextureViewDescriptor::default());
let spectrum_vertex_shader = device.create_shader_module(ShaderModuleDescriptor {
label: Some("audio spectrum vertex shader"),
source: ShaderSource::Wgsl(include_str!("../shaders/vertex.wgsl").into()),
});
let spectrum_fragment_shader = device.create_shader_module(ShaderModuleDescriptor {
label: Some("audio spectrum fragment shader"),
source: ShaderSource::Wgsl(include_str!("../shaders/audio_spectrum.wgsl").into()),
});
let spectrum_bind_group_layout =
device.create_bind_group_layout(&BindGroupLayoutDescriptor {
label: Some("audio spectrum bind group layout"),
entries: &[BindGroupLayoutEntry {
binding: 0,
visibility: ShaderStages::FRAGMENT,
ty: BindingType::Buffer {
ty: BufferBindingType::Uniform,
has_dynamic_offset: false,
min_binding_size: NonZeroU64::new(1024),
},
count: None,
}],
});
let spectrum_pipeline_layout = device.create_pipeline_layout(&PipelineLayoutDescriptor {
label: Some("audio spectrum pipeline layout"),
bind_group_layouts: &[Some(&spectrum_bind_group_layout)],
immediate_size: 0,
});
let spectrum_pipeline = device.create_render_pipeline(&RenderPipelineDescriptor {
cache: None,
label: Some("audio spectrum pipeline"),
layout: Some(&spectrum_pipeline_layout),
vertex: VertexState {
module: &spectrum_vertex_shader,
entry_point: Some("vs_main"),
buffers: &[],
compilation_options: PipelineCompilationOptions::default(),
},
fragment: Some(FragmentState {
module: &spectrum_fragment_shader,
entry_point: Some("fs_main"),
targets: &[Some(TextureFormat::Bgra8Unorm.into())],
compilation_options: PipelineCompilationOptions::default(),
}),
primitive: PrimitiveState::default(),
depth_stencil: None,
multisample: MultisampleState::default(),
multiview_mask: None,
});
let contents = [0u8; 1024];
let spectrum_texture_buffer =
device.create_buffer_init(&wgpu::util::BufferInitDescriptor {
label: Some("audio spectrum uniform buffer"),
contents: &contents,
usage: BufferUsages::COPY_DST | BufferUsages::UNIFORM,
});
let spectrum_bind_group = device.create_bind_group(&BindGroupDescriptor {
label: Some("audio spectrum bind group"),
layout: &spectrum_bind_group_layout,
entries: &[BindGroupEntry {
binding: 0,
resource: spectrum_texture_buffer.as_entire_binding(),
}],
});
let spectrum_texture_id =
OwnedTextureId(wgpu_render_state.renderer.write().register_native_texture(
&device,
&spectrum_texture_view,
wgpu::FilterMode::Nearest,
));
Some(Arc::new(Self {
spectrum_pipeline,
spectrum_bind_group,
spectrum_texture_buffer,
spectrum_texture_view,
spectrum_texture_id,
}))
}
fn draw_spectrum_texture(&self, input_level: Array1<f32>) {
let wgpu_render_state = wgpu_render_state();
let device = wgpu_render_state.device;
if let Some(mut view) = wgpu_render_state.queue.write_buffer_with(
&self.spectrum_texture_buffer,
0,
NonZeroU64::new(1024).expect("Contents length is zero"),
) {
view.copy_from_slice(&fft_data_u8(input_level.to_vec()));
}
let mut encoder = device.create_command_encoder(&CommandEncoderDescriptor {
label: Some("Render animations for scene editor"),
});
{
let mut render_pass = encoder.begin_render_pass(&RenderPassDescriptor {
label: Some("Renderer Pass"),
color_attachments: &[Some(RenderPassColorAttachment {
view: &self.spectrum_texture_view,
resolve_target: None,
depth_slice: None,
ops: Operations {
load: LoadOp::Clear(wgpu::Color {
r: 0.0,
g: 0.0,
b: 0.0,
a: 1.0,
}),
store: StoreOp::Store,
},
})],
depth_stencil_attachment: None,
timestamp_writes: None,
occlusion_query_set: None,
multiview_mask: None,
});
render_pass.set_pipeline(&self.spectrum_pipeline);
render_pass.set_bind_group(0, &self.spectrum_bind_group, &[]);
render_pass.draw(0..3, 0..1);
}
RendererCallback::add(encoder.finish());
}
}
#[derive(PartialEq, Clone, Debug, Serialize, Deserialize)]
#[serde(from = "SoundTriggerEditorShell", into = "SoundTriggerEditorShell")]
pub struct SoundTriggerEditor {
pub sound_trigger_handle: Arc<SoundTriggerHandle>,
f_center: f32,
f_radius: f32,
preview_shader: Option<Arc<PreviewShader>>,
averaging_time: f32,
}
#[derive(Serialize, Deserialize)]
struct SoundTriggerEditorShell {
params: SoundTriggerParams,
f_center: f32,
f_radius: f32,
averaging_time: f32,
}
impl From<SoundTriggerEditorShell> for SoundTriggerEditor {
fn from(shell: SoundTriggerEditorShell) -> Self {
let mut sound_data = SoundData::default();
let sound_trigger_handle = sound_data.register_sound_trigger(shell.params);
sound_data.save();
Self {
sound_trigger_handle,
f_center: shell.f_center,
f_radius: shell.f_radius,
preview_shader: None,
averaging_time: shell.averaging_time,
}
}
}
impl From<SoundTriggerEditor> for SoundTriggerEditorShell {
fn from(editor: SoundTriggerEditor) -> Self {
let sound_data = SoundData::default();
let params = editor
.sound_trigger_handle
.get_params(&sound_data)
.copied()
.unwrap_or_default();
Self {
params,
f_center: editor.f_center,
f_radius: editor.f_radius,
averaging_time: editor.averaging_time,
}
}
}
impl Default for SoundTriggerEditor {
fn default() -> Self {
Self::new(SoundTriggerParams::default(), 8200., 990., 0.1)
}
}
impl SoundTriggerEditor {
fn new(
sound_trigger_params: SoundTriggerParams,
f_center: f32,
f_radius: f32,
averaging_time: f32,
) -> Self {
let mut data = SoundData::default();
let sound_trigger_handle = data.register_sound_trigger(sound_trigger_params);
data.save();
Self {
sound_trigger_handle,
f_center,
f_radius,
averaging_time,
preview_shader: None,
}
}
}
impl SoundTriggerEditor {
pub fn show(&mut self, ui: &mut Ui, sound_data: &mut SoundData) {
#[cfg(feature = "profiling")]
puffin::profile_function!("SoundTriggerEditor::show");
Frame::new().inner_margin(5.).show(ui, |ui| {
if let Some(trigger) = self.sound_trigger_handle.get_sound_trigger(sound_data) {
let mut spectrum = trigger.spectrum.clone();
spectrum.map_inplace(|x| {
*x = (*x * 10.0 + 10.0).log10() - 1.0;
});
let impulse = trigger.impulse.clamp(0.0, 1.0);
let output_level = trigger.current_level;
if self.preview_shader.is_none() {
self.preview_shader = PreviewShader::try_init();
}
if let Some(preview_shader) = &self.preview_shader {
preview_shader.draw_spectrum_texture(spectrum);
}
self.draw_spectrum(ui, sound_data);
ui.separator();
self.draw_adsr(ui, impulse, output_level, sound_data);
}
});
}
pub fn show_minified(&mut self, ui: &mut Ui, rect: Rect, sound_data: &SoundData) {
#[cfg(feature = "profiling")]
puffin::profile_function!("ADSREditor::show_minified");
let level = self.sound_trigger_handle.level(sound_data);
scoped_frame(
ui,
UiBuilder::new().max_rect(rect),
Frame::default(),
|ui| {
let to_screen = emath::RectTransform::from_to(
Rect::from_min_size(Pos2::ZERO, Vec2::new(4.0, 1.0)),
rect,
);
let painter = ui.painter().with_clip_rect(rect);
for i in 0..=4 {
let stroke = Stroke::new(
match i {
0 | 4 => 1.0,
_ => 0.5,
},
Color32::GRAY,
);
painter.add(PathShape::line(
vec![
to_screen.transform_pos(Pos2::new(i as f32, 0.0)),
to_screen.transform_pos(Pos2::new(i as f32, 1.0)),
],
stroke,
));
painter.add(PathShape::line(
vec![
to_screen.transform_pos(Pos2::new(0.0, i as f32 * 0.25)),
to_screen.transform_pos(Pos2::new(4.0, i as f32 * 0.25)),
],
stroke,
));
}
if let Some(sound_trigger_params) = self.sound_trigger_handle.get_params(sound_data)
{
self.draw_curve(ui, rect, level, sound_trigger_params);
}
},
);
}
fn draw_adsr(
&mut self,
ui: &mut Ui,
input_level: f32,
output_level: f32,
sound_data: &mut SoundData,
) {
let Some(sound_trigger_params) = &mut self.sound_trigger_handle.get_params_mut(sound_data)
else {
return;
};
let mut save_sound_data = false;
ui.with_layout(Layout::left_to_right(Align::Min), |ui| {
let meter_height = 200.0;
let (_, rect) = ui.allocate_space(Vec2::new(40.0, meter_height));
ui.painter().rect_filled(rect, 0., Color32::from_gray(100));
let mut value_rect = rect;
value_rect.min.y += meter_height * (1.0 - input_level);
ui.painter()
.rect_filled(value_rect, 0., Color32::LIGHT_GREEN);
let threshold_line_y =
rect.min.y + meter_height * (1.0 - sound_trigger_params.gate_activation_threshold);
ui.painter().line(
vec![
pos2(rect.min.x, threshold_line_y),
pos2(rect.max.x, threshold_line_y),
],
Stroke::new(2., Color32::BLUE),
);
let threshold_deac_line_y = rect.min.y
+ meter_height * (1.0 - sound_trigger_params.gate_deactivation_threshold);
ui.painter().line(
vec![
pos2(rect.min.x, threshold_deac_line_y),
pos2(rect.max.x, threshold_deac_line_y),
],
Stroke::new(2., Color32::BLUE),
);
let mut rect = ui.available_rect_before_wrap();
rect.set_height(200.);
let frame = Frame::new().inner_margin(5.0).fill(Color32::from_gray(60));
scoped_frame(ui, UiBuilder::new().max_rect(rect), frame, |ui| {
let rect = ui.available_rect_before_wrap();
self.draw_curve(ui, rect, output_level, sound_trigger_params);
});
});
scoped_frame(
ui,
UiBuilder::new()
.max_rect(Rect::from_min_size(
ui.cursor().min,
vec2(ui.available_width(), 100.),
))
.layout(Layout::left_to_right(Align::Max).with_cross_justify(true)),
Frame::default().inner_margin(5.0),
|ui| {
Frame::new().inner_margin(5.).show(ui, |ui| {
if ui
.add(
knob_default(&mut sound_trigger_params.attack_duration, 0.0, 2.0, 0.01)
.with_size(50.0)
.with_logarithmic_scaling()
.with_label("Attack", LabelPosition::Bottom),
)
.changed()
{
save_sound_data = true;
};
if ui
.add(
knob_default(
&mut sound_trigger_params.release_duration,
0.0,
10.0,
1.0,
)
.with_size(50.0)
.with_logarithmic_scaling()
.with_label("Release", LabelPosition::Bottom),
)
.changed()
{
save_sound_data = true;
};
if ui
.add(
knob_default(
&mut sound_trigger_params.gate_activation_threshold,
0.0,
1.0,
0.5,
)
.with_size(30.0)
.with_label("Threshold", LabelPosition::Bottom),
)
.changed()
{
save_sound_data = true;
}
});
},
);
if save_sound_data {
sound_data.save();
}
}
fn draw_spectrum(&mut self, ui: &mut Ui, sound_data: &mut SoundData) {
let Some(sound_trigger_params) = self.sound_trigger_handle.get_params_mut(sound_data)
else {
return;
};
let mut save_sound_data = false;
#[cfg(feature = "profiling")]
puffin::profile_function!("SoundTriggerEditor::draw_spectrum");
let size = vec2(ui.available_width(), 190.);
let mut spectrum_rect = Rect::from_min_size(ui.cursor().min, size);
if let Some(preview_shader) = &self.preview_shader {
let spectrum = Image::new(SizedTexture::new(
preview_shader.spectrum_texture_id.0,
size,
));
ui.add(spectrum);
}
let lower_f = self.f_center - self.f_radius;
let upper_f = self.f_center + self.f_radius;
let max_frequency_range = 0.0..=MAX_FREQ;
let ui_position_range = spectrum_rect.left()..=spectrum_rect.right();
let lower_x = remap_clamp(
lower_f,
max_frequency_range.clone(),
ui_position_range.clone(),
);
let upper_x = remap_clamp(
upper_f,
max_frequency_range.clone(),
ui_position_range.clone(),
);
spectrum_rect.min.x = lower_x;
spectrum_rect.max.x = upper_x;
ui.painter()
.rect_filled(spectrum_rect, 0., Color32::from_white_alpha(100));
scoped_frame(
ui,
UiBuilder::new()
.max_rect(Rect::from_min_size(
ui.cursor().min,
vec2(ui.available_width(), 100.),
))
.layout(Layout::left_to_right(Align::Max)),
Frame::default().inner_margin(5.0),
|ui| {
#[cfg(feature = "profiling")]
puffin::profile_function!("SoundTriggerEditor::draw_spectrum_knobs");
let mut filter_tune_changed = ui
.add(
knob_default(&mut self.f_center, 0., 24_000., 100.0)
.with_size(50.0)
.with_logarithmic_scaling()
.with_label("Frequency", LabelPosition::Bottom),
)
.changed();
filter_tune_changed |= ui
.add(
knob_default(&mut self.f_radius, 100., 24_000., 50.0)
.with_size(50.0)
.with_logarithmic_scaling()
.with_label("Range", LabelPosition::Bottom),
)
.changed();
if ui
.add(
knob_default(&mut sound_trigger_params.sensitivity, 0.01, 10.0, 1.0)
.with_size(50.0)
.with_logarithmic_scaling()
.with_label("Sensitivity", LabelPosition::Bottom),
)
.changed()
{
save_sound_data = true;
}
if filter_tune_changed {
sound_trigger_params.set_filter_tune(self.f_center, self.f_radius);
save_sound_data = true;
}
},
);
if save_sound_data {
sound_data.save();
}
}
fn draw_curve(&self, ui: &mut Ui, rect: Rect, output_level: f32, params: &SoundTriggerParams) {
let n = 100;
let to_screen =
emath::RectTransform::from_to(Rect::from_x_y_ranges(0.0..=1.0, 0.0..=1.0), rect);
let mut level_rect = rect;
level_rect.min.y += (1.0 - (output_level)) * rect.height();
ui.painter()
.rect_filled(level_rect, 0., Color32::LIGHT_GREEN);
let mut sound_trigger = SoundTrigger::new(*params, 100.);
let points: Vec<Pos2> = (0..=n)
.map(|i| {
let t = i as f32 / (n as f32);
let input = if 0.2 < t
&& t < (0.4
+ sound_trigger.params.decay_duration * 0.4
+ sound_trigger.params.attack_duration * 0.4)
{
1.0
} else {
0.0
};
let y = -sound_trigger.tick_adsr(input) + 1.;
to_screen * pos2(t, y)
})
.collect();
let thickness = 1.0;
let shape = epaint::Shape::line(points, PathStroke::new(thickness, Color32::WHITE));
ui.painter().add(shape);
}
}
fn knob_default<'a>(value: &'a mut f32, min: f32, max: f32, reset: f32) -> Knob<'a> {
Knob::new(value, min, max, KnobStyle::Wiper)
.with_font_size(12.0)
.with_colors(Color32::GRAY, Color32::WHITE, Color32::WHITE)
.with_stroke_width(3.0)
.with_double_click_reset(reset)
}