audio_visualizer/dynamic/window_top_btm/
mod.rs

1/*
2MIT License
3
4Copyright (c) 2021 Philipp Schuster
5
6Permission is hereby granted, free of charge, to any person obtaining a copy
7of this software and associated documentation files (the "Software"), to deal
8in the Software without restriction, including without limitation the rights
9to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
10copies of the Software, and to permit persons to whom the Software is
11furnished to do so, subject to the following conditions:
12
13The above copyright notice and this permission notice shall be included in all
14copies or substantial portions of the Software.
15
16THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
19AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
20LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
21OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
22SOFTWARE.
23*/
24//! This module provides the functionality to display a GUI window.
25//!
26//! In the window, the upper half shows the real-time recorded audio data
27//! whereas the lower half shows a diagram of transformed data, such as a
28//! lowpass filter or a frequency spectrum.
29//!
30//! It uses the [`minifb`] crate to display GUI windows.
31use crate::dynamic::live_input::{setup_audio_input_loop, AudioDevAndCfg};
32use crate::dynamic::window_top_btm::visualize_minifb::{
33    get_drawing_areas, setup_window, DEFAULT_H, DEFAULT_W,
34};
35use cpal::traits::StreamTrait;
36
37use minifb::Key;
38use plotters::chart::ChartContext;
39use plotters::coord::cartesian::Cartesian2d;
40use plotters::coord::types::RangedCoordf64;
41use plotters::prelude::BitMapBackend;
42use plotters::series::LineSeries;
43use plotters::style::{BLACK, CYAN};
44use plotters_bitmap::bitmap_pixel::BGRXPixel;
45use ringbuffer::{AllocRingBuffer, RingBuffer};
46use std::borrow::{Borrow, BorrowMut};
47use std::ops::Range;
48use std::sync::{Arc, Mutex};
49
50pub mod pixel_buf;
51pub mod visualize_minifb;
52
53/// Parameter type for [`open_window_connect_audio`]. Describes how the audio data shall
54/// be transformed, and thus, how it should be displayed in the lower part of the window.
55///
56/// The function is called every x milliseconds (refresh rate of window).
57///
58/// This works cross-platform (Windows, MacOS, Linux).
59#[allow(missing_debug_implementations)]
60pub enum TransformFn<'a> {
61    /// Synchronized x-axis with the original data. Useful for transformations on the
62    /// waveform, such as a (lowpass) filter.
63    ///
64    /// Functions takes amplitude values and transforms them to a new amplitude value.
65    /// It gets the sampling rate as second argument.
66    Basic(fn(&[f32], f32) -> Vec<f32>),
67    /// Use this, when the x-axis is different than for the original data. For example,
68    /// if you want to display a spectrum.
69    ///
70    /// Functions takes amplitude values (and their index) and transforms them to a new
71    /// (x,y)-pair. Takes a closure instead of a function, so that it can capture state.
72    /// It gets the sampling rate as second argument.
73    #[allow(clippy::complexity)]
74    Complex(&'a dyn Fn(&[f32], f32) -> Vec<(f64, f64)>),
75}
76
77/// Starts the audio recording via `cpal` on the given audio device (or the default input device),
78/// opens a GUI window and displays two graphs.
79///
80/// The upper graph is the latest audio input as
81/// wave form (live/real time). The lower graph can be customized, to show for example a
82/// spectrum or the lowpassed data.
83///
84/// This operation is blocking. It returns, when the GUI window is closed.
85///
86/// **This operation is expensive and will be very laggy in "Debug" builds!**
87///
88/// # Parameters
89/// - `name` Name of the GUI window
90/// - `preferred_height` Preferred height of GUI window. Default is [`DEFAULT_H`].
91/// - `preferred_width` Preferred height of GUI window. Default is [`DEFAULT_W`].
92/// - `preferred_x_range` Preferred range for the x-axis of the lower (=custom) diagram.
93///   If no value is present, the same value as for the upper diagram is used.
94/// - `preferred_y_range` Preferred range for the y-axis of the lower (=custom) diagram.
95///   If no value is present, the same value as for the upper diagram is used.
96/// - `x_desc` Description for the x-axis of the lower (=custom) diagram.
97/// - `y_desc` Description for the y-axis of the lower (=custom) diagram.
98/// - `preferred_input_dev` See [`AudioDevAndCfg`].
99/// - `audio_data_transform_fn` See [`open_window_connect_audio`].
100#[allow(clippy::too_many_arguments)]
101pub fn open_window_connect_audio(
102    name: &str,
103    preferred_height: Option<usize>,
104    preferred_width: Option<usize>,
105    preferred_x_range: Option<Range<f64>>,
106    preferred_y_range: Option<Range<f64>>,
107    x_desc: &str,
108    y_desc: &str,
109    input_dev_and_cfg: AudioDevAndCfg,
110    audio_data_transform_fn: TransformFn,
111) {
112    let sample_rate = input_dev_and_cfg.cfg().sample_rate.0 as f32;
113    let latest_audio_data = init_ringbuffer(sample_rate as usize);
114    let audio_buffer_len = latest_audio_data.lock().unwrap().len();
115    let stream = setup_audio_input_loop(latest_audio_data.clone(), input_dev_and_cfg);
116    // This will be 1/44100 or 1/48000; the two most common sampling rates.
117    let time_per_sample = 1.0 / sample_rate as f64;
118
119    // start recording; audio will be continuously stored in "latest_audio_data"
120    stream.play().unwrap();
121    let (mut window, top_cs, btm_cs, mut pixel_buf) = setup_window(
122        name,
123        preferred_height,
124        preferred_width,
125        preferred_x_range,
126        preferred_y_range,
127        x_desc,
128        y_desc,
129        audio_buffer_len,
130        time_per_sample,
131    );
132    window.set_target_fps(144);
133
134    // GUI refresh loop; CPU-limited by "window.limit_update_rate"
135    while window.is_open() {
136        if window.is_key_down(Key::Escape) {
137            break;
138        }
139
140        let (top_drawing_area, btm_drawing_area) = get_drawing_areas(
141            pixel_buf.borrow_mut(),
142            preferred_width.unwrap_or(DEFAULT_W),
143            preferred_height.unwrap_or(DEFAULT_H),
144        );
145
146        let top_chart = top_cs.clone().restore(&top_drawing_area);
147        let btm_chart = btm_cs.clone().restore(&btm_drawing_area);
148
149        // remove drawings from previous iteration (but keep axis etc)
150        top_chart.plotting_area().fill(&BLACK).borrow();
151        btm_chart.plotting_area().fill(&BLACK).borrow();
152
153        // lock released immediately after oneliner
154        let latest_audio_data = latest_audio_data.clone().lock().unwrap().to_vec();
155        fill_chart_waveform_over_time(
156            top_chart,
157            &latest_audio_data,
158            time_per_sample,
159            audio_buffer_len,
160        );
161        if let TransformFn::Basic(fnc) = audio_data_transform_fn {
162            let data = fnc(&latest_audio_data, sample_rate);
163            fill_chart_waveform_over_time(btm_chart, &data, time_per_sample, audio_buffer_len);
164        } else if let TransformFn::Complex(fnc) = audio_data_transform_fn {
165            let data = fnc(&latest_audio_data, sample_rate);
166            fill_chart_complex_fnc(btm_chart, data);
167        } else {
168            // required for compilation
169            drop(btm_chart);
170            panic!("invalid transform fn variant");
171        }
172
173        // make sure that "pixel_buf" is not borrowed longer
174        drop(top_drawing_area);
175        drop(btm_drawing_area);
176
177        // REQUIRED to call on of the .update*()-methods, otherwise mouse and keyboard events
178        // are not updated
179        //
180        // Update() also does the rate limiting/set the thread to sleep if not enough time
181        //  sine the last refresh happened
182        window
183            .update_with_buffer(
184                pixel_buf.borrow(),
185                preferred_width.unwrap_or(DEFAULT_W),
186                preferred_height.unwrap_or(DEFAULT_H),
187            )
188            .unwrap();
189    }
190    stream.pause().unwrap();
191}
192
193/// Inits a ringbuffer on the heap and fills it with zeroes.
194fn init_ringbuffer(sampling_rate: usize) -> Arc<Mutex<AllocRingBuffer<f32>>> {
195    // Must be a power (ringbuffer requirement).
196    let mut buf = AllocRingBuffer::new((5 * sampling_rate).next_power_of_two());
197    buf.fill(0.0);
198    Arc::new(Mutex::new(buf))
199}
200
201/// Fills the given chart with the waveform over time, from the past (left) to now/realtime (right).
202fn fill_chart_complex_fnc(
203    mut chart: ChartContext<BitMapBackend<BGRXPixel>, Cartesian2d<RangedCoordf64, RangedCoordf64>>,
204    audio_data: Vec<(f64, f64)>,
205) {
206    // dedicated function; otherwise lifetime problems/compiler errors
207    chart
208        .draw_series(LineSeries::new(audio_data, &CYAN))
209        .unwrap();
210}
211
212/// Fills the given chart with the waveform over time, from the past (left) to now/realtime (right).
213fn fill_chart_waveform_over_time(
214    mut chart: ChartContext<BitMapBackend<BGRXPixel>, Cartesian2d<RangedCoordf64, RangedCoordf64>>,
215    audio_data: &[f32],
216    time_per_sample: f64,
217    audio_history_buf_len: usize,
218) {
219    debug_assert_eq!(audio_data.len(), audio_history_buf_len);
220    let timeshift = audio_history_buf_len as f64 * time_per_sample;
221
222    // calculate timestamp of each index (x coordinate)
223    let data_iter = audio_data
224        .iter()
225        .enumerate()
226        // Important to reduce the calculation complexity by reducing the number of elements,
227        // because drawing tens of thousands of points into the diagram is very expensive.
228        //
229        // If we skip too many elements, animation becomes un-smooth.... 4 seems to be sensible
230        // due to tests by me.
231        .filter(|(i, _)| *i % 4 == 0)
232        .map(|(i, amplitude)| {
233            let timestamp = time_per_sample * (i as f64) - timeshift;
234            // Values for amplitude in interval [-1.0; 1.0]
235            (timestamp, (*amplitude) as f64)
236        });
237
238    // Draws all points as a line of connected points.
239    // LineSeries is reasonable efficient for the big workload, but still very expensive..
240    // (4-6ms in release mode on my intel i5 10th generation)
241    chart
242        .draw_series(LineSeries::new(data_iter, &CYAN))
243        .unwrap();
244}
245
246#[cfg(test)]
247mod tests {
248    use super::*;
249
250    #[ignore]
251    #[test]
252    fn test_record_live_audio_and_visualize() {
253        open_window_connect_audio(
254            "Test",
255            None,
256            None,
257            None,
258            None,
259            "x-axis",
260            "y-axis",
261            AudioDevAndCfg::new(None, None),
262            TransformFn::Basic(|vals, _| vals.to_vec()),
263        );
264    }
265}