voirs-spatial 0.1.0-rc.1

3D spatial audio and HRTF processing for VoiRS
Documentation
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
//! Audio analysis components for visual effect generation

use super::mapping::AudioVisualMapping;
use super::types::{ColorRGBA, FrequencyBand, VisualEvent, VisualEventType};
use crate::{types::AudioChannel, Result};
use std::time::{Duration, Instant};

/// Audio analysis for visual effect generation
pub(crate) struct VisualAudioAnalyzer {
    /// FFT analysis for frequency content
    fft_analyzer: FftAnalyzer,

    /// Onset detection for visual triggers
    onset_detector: OnsetDetector,

    /// Beat detection for rhythm visuals
    beat_detector: VisualBeatDetector,

    /// Spectral analysis for color mapping
    spectral_analyzer: SpectralAnalyzer,

    /// Amplitude tracking
    amplitude_tracker: AmplitudeTracker,
}

/// FFT analysis for frequency-based visuals
struct FftAnalyzer {
    /// FFT window size
    window_size: usize,

    /// Frequency bins
    frequency_bins: Vec<f32>,

    /// Previous frame for comparison
    previous_frame: Vec<f32>,

    /// Smoothing factor
    smoothing_factor: f32,
}

/// Audio onset detection for visual triggers
struct OnsetDetector {
    /// Spectral flux history
    flux_history: Vec<Vec<f32>>,

    /// Detection threshold
    threshold: f32,

    /// Peak picking parameters
    peak_picking: PeakPickingParams,

    /// Last onset time
    last_onset: Instant,
}

/// Peak picking parameters for onset detection
#[derive(Debug, Clone)]
struct PeakPickingParams {
    /// Minimum time between onsets
    min_interval: Duration,

    /// Threshold adaptation rate
    adaptation_rate: f32,

    /// Pre/post-roll for peak validation
    validation_window: usize,
}

/// Beat detection for rhythm-based visuals
struct VisualBeatDetector {
    /// Energy-based beat tracker
    energy_tracker: Vec<f32>,

    /// Tempo estimation
    tempo_estimator: TempoEstimator,

    /// Beat phase tracking
    phase_tracker: PhaseTracker,

    /// Confidence scoring
    confidence_tracker: ConfidenceTracker,
}

/// Tempo estimation for beat detection
#[derive(Debug)]
struct TempoEstimator {
    /// Autocorrelation buffer
    autocorr_buffer: Vec<f32>,

    /// Current tempo estimate (BPM)
    current_tempo: f32,

    /// Tempo stability measure
    stability: f32,

    /// Valid tempo range
    tempo_range: (f32, f32),
}

/// Phase tracking for beat alignment
#[derive(Debug)]
struct PhaseTracker {
    /// Current beat phase (0.0-1.0)
    current_phase: f32,

    /// Phase prediction
    predicted_phase: f32,

    /// Phase error tracking
    phase_error: f32,

    /// Phase correction factor
    correction_factor: f32,
}

/// Confidence tracking for beat detection
#[derive(Debug)]
struct ConfidenceTracker {
    /// Beat detection confidence
    beat_confidence: f32,

    /// Tempo confidence
    tempo_confidence: f32,

    /// Overall confidence
    overall_confidence: f32,

    /// Confidence history
    confidence_history: Vec<f32>,
}

/// Spectral analysis for advanced visual effects
struct SpectralAnalyzer {
    /// Spectral centroid tracking
    centroid_tracker: Vec<f32>,

    /// Spectral rolloff tracking
    rolloff_tracker: Vec<f32>,

    /// Spectral flux calculation
    flux_calculator: FluxCalculator,

    /// Harmonic analysis
    harmonic_analyzer: HarmonicAnalyzer,
}

/// Spectral flux calculation
#[derive(Debug)]
struct FluxCalculator {
    /// Previous spectrum
    previous_spectrum: Vec<f32>,

    /// Flux values
    flux_values: Vec<f32>,

    /// Smoothing window
    smoothing_window: usize,
}

/// Harmonic content analysis
#[derive(Debug)]
struct HarmonicAnalyzer {
    /// Fundamental frequency tracker
    f0_tracker: Vec<f32>,

    /// Harmonic strength
    harmonic_strength: Vec<f32>,

    /// Inharmonicity measure
    inharmonicity: f32,
}

/// Amplitude tracking for visual scaling
struct AmplitudeTracker {
    /// RMS amplitude history
    rms_history: Vec<f32>,

    /// Peak amplitude history
    peak_history: Vec<f32>,

    /// Dynamic range tracking
    dynamic_range: f32,

    /// Envelope following
    envelope_follower: EnvelopeFollower,
}

/// Envelope follower for smooth amplitude tracking
#[derive(Debug)]
struct EnvelopeFollower {
    /// Attack time constant
    attack_time: f32,

    /// Release time constant
    release_time: f32,

    /// Current envelope value
    current_value: f32,

    /// Sample rate
    sample_rate: f32,
}

/// Audio onset event
#[derive(Debug)]
pub(crate) struct OnsetEvent {
    pub(crate) strength: f32,
    pub(crate) flux_value: f32,
}

/// Beat detection event
#[derive(Debug)]
pub(crate) struct BeatEvent {
    pub(crate) strength: f32,
    pub(crate) is_downbeat: bool,
    pub(crate) confidence: f32,
}

impl VisualAudioAnalyzer {
    pub(crate) fn new() -> Self {
        Self {
            fft_analyzer: FftAnalyzer::new(1024),
            onset_detector: OnsetDetector::new(),
            beat_detector: VisualBeatDetector::new(),
            spectral_analyzer: SpectralAnalyzer::new(),
            amplitude_tracker: AmplitudeTracker::new(),
        }
    }

    pub(crate) fn analyze_frame(
        &mut self,
        audio_samples: &[f32],
        audio_channel_type: AudioChannel,
        mapping: &AudioVisualMapping,
    ) -> Result<Vec<VisualEvent>> {
        let mut events = Vec::new();

        // Perform FFT analysis
        self.fft_analyzer.analyze(audio_samples)?;

        // Detect onsets
        if let Some(onset) = self
            .onset_detector
            .detect(&self.fft_analyzer.frequency_bins)?
        {
            events.push(VisualEvent {
                source_id: format!("audio_{audio_channel_type:?}"),
                event_type: VisualEventType::Onset,
                intensity: onset.strength,
                color_hint: None,
                timestamp: Instant::now(),
            });
        }

        // Detect beats
        if let Some(beat) = self
            .beat_detector
            .detect(&self.fft_analyzer.frequency_bins)?
        {
            let event_type = if beat.is_downbeat {
                VisualEventType::Downbeat
            } else {
                VisualEventType::Beat
            };

            events.push(VisualEvent {
                source_id: format!("audio_{audio_channel_type:?}"),
                event_type,
                intensity: beat.strength,
                color_hint: None,
                timestamp: Instant::now(),
            });
        }

        // Analyze frequency bands
        self.analyze_frequency_bands(mapping, &audio_channel_type, &mut events)?;

        Ok(events)
    }

    fn analyze_frequency_bands(
        &self,
        mapping: &AudioVisualMapping,
        audio_channel_type: &AudioChannel,
        events: &mut Vec<VisualEvent>,
    ) -> Result<()> {
        let mappings = [
            (&mapping.low_freq_mapping, FrequencyBand::Low),
            (&mapping.mid_freq_mapping, FrequencyBand::Mid),
            (&mapping.high_freq_mapping, FrequencyBand::High),
        ];

        for (frequency_mapping, band) in mappings {
            let band_energy = self
                .fft_analyzer
                .calculate_band_energy(&frequency_mapping.frequency_range);

            if band_energy > 0.1 {
                // Threshold
                events.push(VisualEvent {
                    source_id: format!("audio_{audio_channel_type:?}"),
                    event_type: VisualEventType::FrequencyBand(band),
                    intensity: band_energy * frequency_mapping.intensity_scale,
                    color_hint: Some(frequency_mapping.base_color),
                    timestamp: Instant::now(),
                });
            }
        }

        Ok(())
    }
}

impl FftAnalyzer {
    fn new(window_size: usize) -> Self {
        Self {
            window_size,
            frequency_bins: vec![0.0; window_size / 2],
            previous_frame: vec![0.0; window_size / 2],
            smoothing_factor: 0.7,
        }
    }

    fn analyze(&mut self, samples: &[f32]) -> Result<()> {
        // Simplified FFT implementation (would use proper FFT library)
        let copy_len = samples.len().min(self.window_size);

        // Store previous frame
        self.previous_frame.copy_from_slice(&self.frequency_bins);

        // Calculate magnitude spectrum (simplified)
        for (i, bin) in self.frequency_bins.iter_mut().enumerate() {
            if i * 2 < copy_len {
                let new_value = samples[i * 2].abs();
                *bin = self.smoothing_factor * (*bin) + (1.0 - self.smoothing_factor) * new_value;
            }
        }

        Ok(())
    }

    fn calculate_band_energy(&self, frequency_range: &(f32, f32)) -> f32 {
        let start_bin = (frequency_range.0 / 20000.0 * self.frequency_bins.len() as f32) as usize;
        let end_bin = (frequency_range.1 / 20000.0 * self.frequency_bins.len() as f32) as usize;

        self.frequency_bins[start_bin..end_bin.min(self.frequency_bins.len())]
            .iter()
            .sum::<f32>()
            / (end_bin - start_bin) as f32
    }
}

impl OnsetDetector {
    fn new() -> Self {
        Self {
            flux_history: Vec::with_capacity(50),
            threshold: 0.3,
            peak_picking: PeakPickingParams {
                min_interval: Duration::from_millis(100),
                adaptation_rate: 0.9,
                validation_window: 3,
            },
            last_onset: Instant::now(),
        }
    }

    fn detect(&mut self, frequency_bins: &[f32]) -> Result<Option<OnsetEvent>> {
        // Calculate spectral flux
        if !self.flux_history.is_empty() {
            let last_frame = &self.flux_history[self.flux_history.len() - 1];
            let flux: f32 = frequency_bins
                .iter()
                .zip(last_frame.iter())
                .map(|(current, previous)| (current - previous).max(0.0))
                .sum();

            // Adaptive threshold - calculate average across all frames
            let total_bins: usize = self.flux_history.iter().map(|frame| frame.len()).sum();
            let total_flux: f32 = self.flux_history.iter().flatten().sum();
            let avg_flux = if total_bins > 0 {
                total_flux / total_bins as f32
            } else {
                0.0
            };
            let adaptive_threshold = self.threshold + avg_flux * 0.5;

            if flux > adaptive_threshold {
                let now = Instant::now();
                if now.duration_since(self.last_onset) >= self.peak_picking.min_interval {
                    self.last_onset = now;
                    return Ok(Some(OnsetEvent {
                        strength: flux.min(1.0),
                        flux_value: flux,
                    }));
                }
            }
        }

        // Store current frame for next comparison
        self.flux_history.push(frequency_bins.to_vec());
        if self.flux_history.len() > 50 {
            self.flux_history.remove(0);
        }

        Ok(None)
    }
}

impl VisualBeatDetector {
    fn new() -> Self {
        Self {
            energy_tracker: Vec::with_capacity(100),
            tempo_estimator: TempoEstimator {
                autocorr_buffer: vec![0.0; 200],
                current_tempo: 120.0,
                stability: 0.0,
                tempo_range: (60.0, 180.0),
            },
            phase_tracker: PhaseTracker {
                current_phase: 0.0,
                predicted_phase: 0.0,
                phase_error: 0.0,
                correction_factor: 0.1,
            },
            confidence_tracker: ConfidenceTracker {
                beat_confidence: 0.0,
                tempo_confidence: 0.0,
                overall_confidence: 0.0,
                confidence_history: Vec::with_capacity(50),
            },
        }
    }

    fn detect(&mut self, frequency_bins: &[f32]) -> Result<Option<BeatEvent>> {
        // Calculate energy
        let energy = frequency_bins.iter().sum::<f32>() / frequency_bins.len() as f32;
        self.energy_tracker.push(energy);

        if self.energy_tracker.len() > 100 {
            self.energy_tracker.remove(0);
        }

        // Simple beat detection based on energy peaks
        if self.energy_tracker.len() >= 5 {
            let len = self.energy_tracker.len();
            let current = self.energy_tracker[len - 1];
            let recent_avg = self.energy_tracker[len - 5..].iter().sum::<f32>() / 5.0;

            if current > recent_avg * 1.3 && current > 0.3 {
                return Ok(Some(BeatEvent {
                    strength: current,
                    is_downbeat: self.phase_tracker.current_phase < 0.25,
                    confidence: self.confidence_tracker.overall_confidence,
                }));
            }
        }

        Ok(None)
    }
}

impl SpectralAnalyzer {
    fn new() -> Self {
        Self {
            centroid_tracker: Vec::with_capacity(50),
            rolloff_tracker: Vec::with_capacity(50),
            flux_calculator: FluxCalculator {
                previous_spectrum: vec![0.0; 512],
                flux_values: Vec::with_capacity(50),
                smoothing_window: 5,
            },
            harmonic_analyzer: HarmonicAnalyzer {
                f0_tracker: Vec::with_capacity(50),
                harmonic_strength: vec![0.0; 10],
                inharmonicity: 0.0,
            },
        }
    }
}

impl AmplitudeTracker {
    fn new() -> Self {
        Self {
            rms_history: Vec::with_capacity(100),
            peak_history: Vec::with_capacity(100),
            dynamic_range: 0.0,
            envelope_follower: EnvelopeFollower {
                attack_time: 0.01,
                release_time: 0.1,
                current_value: 0.0,
                sample_rate: 44100.0,
            },
        }
    }
}