use super::mapping::AudioVisualMapping;
use super::types::{ColorRGBA, FrequencyBand, VisualEvent, VisualEventType};
use crate::{types::AudioChannel, Result};
use std::time::{Duration, Instant};
pub(crate) struct VisualAudioAnalyzer {
fft_analyzer: FftAnalyzer,
onset_detector: OnsetDetector,
beat_detector: VisualBeatDetector,
spectral_analyzer: SpectralAnalyzer,
amplitude_tracker: AmplitudeTracker,
}
struct FftAnalyzer {
window_size: usize,
frequency_bins: Vec<f32>,
previous_frame: Vec<f32>,
smoothing_factor: f32,
}
struct OnsetDetector {
flux_history: Vec<Vec<f32>>,
threshold: f32,
peak_picking: PeakPickingParams,
last_onset: Instant,
}
#[derive(Debug, Clone)]
struct PeakPickingParams {
min_interval: Duration,
adaptation_rate: f32,
validation_window: usize,
}
struct VisualBeatDetector {
energy_tracker: Vec<f32>,
tempo_estimator: TempoEstimator,
phase_tracker: PhaseTracker,
confidence_tracker: ConfidenceTracker,
}
#[derive(Debug)]
struct TempoEstimator {
autocorr_buffer: Vec<f32>,
current_tempo: f32,
stability: f32,
tempo_range: (f32, f32),
}
#[derive(Debug)]
struct PhaseTracker {
current_phase: f32,
predicted_phase: f32,
phase_error: f32,
correction_factor: f32,
}
#[derive(Debug)]
struct ConfidenceTracker {
beat_confidence: f32,
tempo_confidence: f32,
overall_confidence: f32,
confidence_history: Vec<f32>,
}
struct SpectralAnalyzer {
centroid_tracker: Vec<f32>,
rolloff_tracker: Vec<f32>,
flux_calculator: FluxCalculator,
harmonic_analyzer: HarmonicAnalyzer,
}
#[derive(Debug)]
struct FluxCalculator {
previous_spectrum: Vec<f32>,
flux_values: Vec<f32>,
smoothing_window: usize,
}
#[derive(Debug)]
struct HarmonicAnalyzer {
f0_tracker: Vec<f32>,
harmonic_strength: Vec<f32>,
inharmonicity: f32,
}
struct AmplitudeTracker {
rms_history: Vec<f32>,
peak_history: Vec<f32>,
dynamic_range: f32,
envelope_follower: EnvelopeFollower,
}
#[derive(Debug)]
struct EnvelopeFollower {
attack_time: f32,
release_time: f32,
current_value: f32,
sample_rate: f32,
}
#[derive(Debug)]
pub(crate) struct OnsetEvent {
pub(crate) strength: f32,
pub(crate) flux_value: f32,
}
#[derive(Debug)]
pub(crate) struct BeatEvent {
pub(crate) strength: f32,
pub(crate) is_downbeat: bool,
pub(crate) confidence: f32,
}
impl VisualAudioAnalyzer {
pub(crate) fn new() -> Self {
Self {
fft_analyzer: FftAnalyzer::new(1024),
onset_detector: OnsetDetector::new(),
beat_detector: VisualBeatDetector::new(),
spectral_analyzer: SpectralAnalyzer::new(),
amplitude_tracker: AmplitudeTracker::new(),
}
}
pub(crate) fn analyze_frame(
&mut self,
audio_samples: &[f32],
audio_channel_type: AudioChannel,
mapping: &AudioVisualMapping,
) -> Result<Vec<VisualEvent>> {
let mut events = Vec::new();
self.fft_analyzer.analyze(audio_samples)?;
if let Some(onset) = self
.onset_detector
.detect(&self.fft_analyzer.frequency_bins)?
{
events.push(VisualEvent {
source_id: format!("audio_{audio_channel_type:?}"),
event_type: VisualEventType::Onset,
intensity: onset.strength,
color_hint: None,
timestamp: Instant::now(),
});
}
if let Some(beat) = self
.beat_detector
.detect(&self.fft_analyzer.frequency_bins)?
{
let event_type = if beat.is_downbeat {
VisualEventType::Downbeat
} else {
VisualEventType::Beat
};
events.push(VisualEvent {
source_id: format!("audio_{audio_channel_type:?}"),
event_type,
intensity: beat.strength,
color_hint: None,
timestamp: Instant::now(),
});
}
self.analyze_frequency_bands(mapping, &audio_channel_type, &mut events)?;
Ok(events)
}
fn analyze_frequency_bands(
&self,
mapping: &AudioVisualMapping,
audio_channel_type: &AudioChannel,
events: &mut Vec<VisualEvent>,
) -> Result<()> {
let mappings = [
(&mapping.low_freq_mapping, FrequencyBand::Low),
(&mapping.mid_freq_mapping, FrequencyBand::Mid),
(&mapping.high_freq_mapping, FrequencyBand::High),
];
for (frequency_mapping, band) in mappings {
let band_energy = self
.fft_analyzer
.calculate_band_energy(&frequency_mapping.frequency_range);
if band_energy > 0.1 {
events.push(VisualEvent {
source_id: format!("audio_{audio_channel_type:?}"),
event_type: VisualEventType::FrequencyBand(band),
intensity: band_energy * frequency_mapping.intensity_scale,
color_hint: Some(frequency_mapping.base_color),
timestamp: Instant::now(),
});
}
}
Ok(())
}
}
impl FftAnalyzer {
fn new(window_size: usize) -> Self {
Self {
window_size,
frequency_bins: vec![0.0; window_size / 2],
previous_frame: vec![0.0; window_size / 2],
smoothing_factor: 0.7,
}
}
fn analyze(&mut self, samples: &[f32]) -> Result<()> {
let copy_len = samples.len().min(self.window_size);
self.previous_frame.copy_from_slice(&self.frequency_bins);
for (i, bin) in self.frequency_bins.iter_mut().enumerate() {
if i * 2 < copy_len {
let new_value = samples[i * 2].abs();
*bin = self.smoothing_factor * (*bin) + (1.0 - self.smoothing_factor) * new_value;
}
}
Ok(())
}
fn calculate_band_energy(&self, frequency_range: &(f32, f32)) -> f32 {
let start_bin = (frequency_range.0 / 20000.0 * self.frequency_bins.len() as f32) as usize;
let end_bin = (frequency_range.1 / 20000.0 * self.frequency_bins.len() as f32) as usize;
self.frequency_bins[start_bin..end_bin.min(self.frequency_bins.len())]
.iter()
.sum::<f32>()
/ (end_bin - start_bin) as f32
}
}
impl OnsetDetector {
fn new() -> Self {
Self {
flux_history: Vec::with_capacity(50),
threshold: 0.3,
peak_picking: PeakPickingParams {
min_interval: Duration::from_millis(100),
adaptation_rate: 0.9,
validation_window: 3,
},
last_onset: Instant::now(),
}
}
fn detect(&mut self, frequency_bins: &[f32]) -> Result<Option<OnsetEvent>> {
if !self.flux_history.is_empty() {
let last_frame = &self.flux_history[self.flux_history.len() - 1];
let flux: f32 = frequency_bins
.iter()
.zip(last_frame.iter())
.map(|(current, previous)| (current - previous).max(0.0))
.sum();
let total_bins: usize = self.flux_history.iter().map(|frame| frame.len()).sum();
let total_flux: f32 = self.flux_history.iter().flatten().sum();
let avg_flux = if total_bins > 0 {
total_flux / total_bins as f32
} else {
0.0
};
let adaptive_threshold = self.threshold + avg_flux * 0.5;
if flux > adaptive_threshold {
let now = Instant::now();
if now.duration_since(self.last_onset) >= self.peak_picking.min_interval {
self.last_onset = now;
return Ok(Some(OnsetEvent {
strength: flux.min(1.0),
flux_value: flux,
}));
}
}
}
self.flux_history.push(frequency_bins.to_vec());
if self.flux_history.len() > 50 {
self.flux_history.remove(0);
}
Ok(None)
}
}
impl VisualBeatDetector {
fn new() -> Self {
Self {
energy_tracker: Vec::with_capacity(100),
tempo_estimator: TempoEstimator {
autocorr_buffer: vec![0.0; 200],
current_tempo: 120.0,
stability: 0.0,
tempo_range: (60.0, 180.0),
},
phase_tracker: PhaseTracker {
current_phase: 0.0,
predicted_phase: 0.0,
phase_error: 0.0,
correction_factor: 0.1,
},
confidence_tracker: ConfidenceTracker {
beat_confidence: 0.0,
tempo_confidence: 0.0,
overall_confidence: 0.0,
confidence_history: Vec::with_capacity(50),
},
}
}
fn detect(&mut self, frequency_bins: &[f32]) -> Result<Option<BeatEvent>> {
let energy = frequency_bins.iter().sum::<f32>() / frequency_bins.len() as f32;
self.energy_tracker.push(energy);
if self.energy_tracker.len() > 100 {
self.energy_tracker.remove(0);
}
if self.energy_tracker.len() >= 5 {
let len = self.energy_tracker.len();
let current = self.energy_tracker[len - 1];
let recent_avg = self.energy_tracker[len - 5..].iter().sum::<f32>() / 5.0;
if current > recent_avg * 1.3 && current > 0.3 {
return Ok(Some(BeatEvent {
strength: current,
is_downbeat: self.phase_tracker.current_phase < 0.25,
confidence: self.confidence_tracker.overall_confidence,
}));
}
}
Ok(None)
}
}
impl SpectralAnalyzer {
fn new() -> Self {
Self {
centroid_tracker: Vec::with_capacity(50),
rolloff_tracker: Vec::with_capacity(50),
flux_calculator: FluxCalculator {
previous_spectrum: vec![0.0; 512],
flux_values: Vec::with_capacity(50),
smoothing_window: 5,
},
harmonic_analyzer: HarmonicAnalyzer {
f0_tracker: Vec::with_capacity(50),
harmonic_strength: vec![0.0; 10],
inharmonicity: 0.0,
},
}
}
}
impl AmplitudeTracker {
fn new() -> Self {
Self {
rms_history: Vec::with_capacity(100),
peak_history: Vec::with_capacity(100),
dynamic_range: 0.0,
envelope_follower: EnvelopeFollower {
attack_time: 0.01,
release_time: 0.1,
current_value: 0.0,
sample_rate: 44100.0,
},
}
}
}