1use std::{collections::HashMap, sync::Arc};
4
5use crate::{error::MediaError, media::MediaStore};
6
7pub const AUDIO_SAMPLE_RATE: u32 = 48_000;
8pub const AUDIO_CHANNELS: usize = 2;
9const CLIP_EDGE_FADE_SAMPLES: u64 = 384;
10
11#[derive(Debug, Clone, PartialEq)]
12pub struct AudioTimeline {
13 pub tracks: Vec<AudioTrack>,
14 pub clips: Vec<AudioClip>,
15}
16
17#[derive(Debug, Clone, PartialEq)]
18pub struct AudioTrack {
19 pub id: String,
20 pub name: String,
21 pub muted: bool,
22 pub solo: bool,
23 pub volume: f32,
24}
25
26#[derive(Debug, Clone, PartialEq)]
27pub struct AudioClip {
28 pub id: String,
29 pub source_id: String,
30 pub track_id: String,
31 pub name: String,
32 pub start_ms: u64,
33 pub duration_ms: u64,
34 pub source_start_ms: u64,
35 pub volume: f32,
36}
37
38#[derive(Debug, Clone, Copy, Default, PartialEq)]
39pub struct AudioMetadata {
40 pub sample_rate: u32,
41 pub channels: u16,
42 pub duration_samples: u64,
43}
44
45#[derive(Debug, Clone, PartialEq)]
46pub struct AudioBuffer {
47 channels: Vec<Vec<f32>>,
48 sample_rate: u32,
49}
50
51impl AudioBuffer {
52 pub fn silent(sample_rate: u32, channel_count: usize, frames: usize) -> Self {
53 Self {
54 channels: vec![vec![0.0; frames]; channel_count],
55 sample_rate,
56 }
57 }
58
59 pub fn from_channels(sample_rate: u32, channels: Vec<Vec<f32>>) -> Self {
60 Self {
61 sample_rate,
62 channels,
63 }
64 }
65
66 pub fn sample_rate(&self) -> u32 {
67 self.sample_rate
68 }
69
70 pub fn channel_count(&self) -> usize {
71 self.channels.len()
72 }
73
74 pub fn frames(&self) -> usize {
75 self.channels.first().map_or(0, Vec::len)
76 }
77
78 pub fn channels(&self) -> &[Vec<f32>] {
79 &self.channels
80 }
81
82 pub fn channels_mut(&mut self) -> &mut [Vec<f32>] {
83 &mut self.channels
84 }
85
86 pub fn interleaved_f32(&self) -> Vec<f32> {
87 let frames = self.frames();
88 let channels = self.channel_count();
89 let mut output = Vec::with_capacity(frames.saturating_mul(channels));
90 for frame in 0..frames {
91 for channel in 0..channels {
92 output.push(self.channels[channel][frame]);
93 }
94 }
95 output
96 }
97}
98
99pub trait AudioResolver: Send + Sync {
100 fn id(&self) -> &str;
101
102 fn metadata(&self) -> AudioMetadata;
103
104 fn resolve_range(
105 &self,
106 start_sample: u64,
107 frames: usize,
108 ) -> Result<Arc<AudioBuffer>, MediaError>;
109}
110
111pub struct AudioMixer<'a, P: MediaStore> {
112 timeline: &'a AudioTimeline,
113 provider: &'a P,
114}
115
116impl<'a, P: MediaStore> AudioMixer<'a, P> {
117 pub fn new(timeline: &'a AudioTimeline, provider: &'a P) -> Self {
118 Self { timeline, provider }
119 }
120
121 pub fn mix_range(&self, start_sample: u64, frames: usize) -> Result<AudioBuffer, MediaError> {
122 let mut output = AudioBuffer::silent(AUDIO_SAMPLE_RATE, AUDIO_CHANNELS, frames);
123 self.mix_into(start_sample, output.channels_mut())?;
124 Ok(output)
125 }
126
127 pub fn mix_into(&self, start_sample: u64, output: &mut [Vec<f32>]) -> Result<(), MediaError> {
128 let output_frames = output.first().map_or(0, Vec::len);
129 if output_frames == 0 || output.is_empty() {
130 return Ok(());
131 }
132
133 for channel in output.iter_mut() {
134 channel.fill(0.0);
135 }
136
137 let has_solo = self.timeline.tracks.iter().any(|track| track.solo);
138 let tracks = self
139 .timeline
140 .tracks
141 .iter()
142 .map(|track| (track.id.as_str(), track))
143 .collect::<HashMap<_, _>>();
144
145 for clip in &self.timeline.clips {
146 let Some(track) = tracks.get(clip.track_id.as_str()) else {
147 continue;
148 };
149 if track.muted || (has_solo && !track.solo) || track.volume <= 0.0 || clip.volume <= 0.0
150 {
151 continue;
152 }
153
154 let clip_start = ms_to_sample(clip.start_ms);
155 let clip_duration = ms_to_sample(clip.duration_ms).max(1);
156 let clip_end = clip_start.saturating_add(clip_duration);
157 let output_end = start_sample.saturating_add(output_frames as u64);
158 let overlap_start = start_sample.max(clip_start);
159 let overlap_end = output_end.min(clip_end);
160 if overlap_end <= overlap_start {
161 continue;
162 }
163
164 let frames_to_mix = usize::try_from(overlap_end - overlap_start).unwrap_or(0);
165 let output_offset = usize::try_from(overlap_start - start_sample).unwrap_or(0);
166 let clip_offset = overlap_start - clip_start;
167 let source_start = ms_to_sample(clip.source_start_ms).saturating_add(clip_offset);
168 let source = self
169 .provider
170 .get_audio_resolver(&clip.source_id)
171 .ok_or_else(|| MediaError::SourceNotFound {
172 media_source: clip.source_id.clone(),
173 })?;
174 let source_buffer = source.resolve_range(source_start, frames_to_mix)?;
175 let gain = track.volume * clip.volume;
176
177 for i in 0..frames_to_mix.min(source_buffer.frames()) {
178 let clip_sample = clip_offset.saturating_add(i as u64);
179 let envelope = clip_envelope(clip_sample, clip_duration);
180 let sample_gain = gain * envelope;
181 for (out_channel_index, output_channel) in output.iter_mut().enumerate() {
182 let source_sample =
183 sample_for_output_channel(&source_buffer, out_channel_index, i);
184 let Some(output_sample) = output_channel.get_mut(output_offset + i) else {
185 continue;
186 };
187 *output_sample += source_sample * sample_gain;
188 }
189 }
190 }
191
192 for channel in output {
193 for sample in channel {
194 *sample = sample.clamp(-1.0, 1.0);
195 }
196 }
197
198 Ok(())
199 }
200}
201
202pub fn ms_to_sample(ms: u64) -> u64 {
203 ((ms as u128 * u128::from(AUDIO_SAMPLE_RATE)) / 1_000) as u64
204}
205
206pub fn duration_samples(duration_frames: u32, fps: f32) -> u64 {
207 if fps <= 0.0 {
208 return 0;
209 }
210 ((duration_frames as f64 / fps as f64) * f64::from(AUDIO_SAMPLE_RATE)).round() as u64
211}
212
213fn sample_for_output_channel(buffer: &AudioBuffer, output_channel: usize, frame: usize) -> f32 {
214 match buffer.channel_count() {
215 0 => 0.0,
216 1 => buffer.channels()[0].get(frame).copied().unwrap_or(0.0),
217 count => buffer.channels()[output_channel.min(count - 1)]
218 .get(frame)
219 .copied()
220 .unwrap_or(0.0),
221 }
222}
223
224fn clip_envelope(clip_sample: u64, clip_duration: u64) -> f32 {
225 let fade = CLIP_EDGE_FADE_SAMPLES.min(clip_duration / 2);
226 if fade == 0 {
227 return 1.0;
228 }
229 if clip_sample < fade {
230 return clip_sample as f32 / fade as f32;
231 }
232 let remaining = clip_duration.saturating_sub(clip_sample);
233 if remaining < fade {
234 return remaining as f32 / fade as f32;
235 }
236 1.0
237}
238
239#[cfg(test)]
240mod tests {
241 use super::*;
242
243 #[derive(Debug)]
244 struct TestProvider {
245 sources: HashMap<String, Arc<AudioBuffer>>,
246 }
247
248 impl crate::media::MediaStore for TestProvider {
249 fn get_image_resolver(
250 &self,
251 _source: &str,
252 ) -> Option<Box<dyn crate::media::ImageResolver>> {
253 None
254 }
255
256 fn get_video_resolver(
257 &self,
258 _stream_id: &str,
259 ) -> Option<Box<dyn crate::media::VideoFrameResolver>> {
260 None
261 }
262
263 fn get_audio_resolver(&self, source_id: &str) -> Option<Box<dyn AudioResolver>> {
264 self.sources.get(source_id).cloned().map(|buffer| {
265 Box::new(TestResolver {
266 id: source_id.to_string(),
267 buffer,
268 }) as Box<dyn AudioResolver>
269 })
270 }
271 }
272
273 struct TestResolver {
274 id: String,
275 buffer: Arc<AudioBuffer>,
276 }
277
278 impl AudioResolver for TestResolver {
279 fn id(&self) -> &str {
280 &self.id
281 }
282
283 fn metadata(&self) -> AudioMetadata {
284 AudioMetadata {
285 sample_rate: self.buffer.sample_rate(),
286 channels: self.buffer.channel_count() as u16,
287 duration_samples: self.buffer.frames() as u64,
288 }
289 }
290
291 fn resolve_range(
292 &self,
293 start_sample: u64,
294 frames: usize,
295 ) -> Result<Arc<AudioBuffer>, MediaError> {
296 let start = usize::try_from(start_sample).unwrap_or(usize::MAX);
297 let end = start.saturating_add(frames).min(self.buffer.frames());
298 let mut channels = Vec::new();
299 for channel in self.buffer.channels() {
300 let mut out = vec![0.0; frames];
301 if start < channel.len() {
302 let src = &channel[start..end];
303 out[..src.len()].copy_from_slice(src);
304 }
305 channels.push(out);
306 }
307 Ok(Arc::new(AudioBuffer::from_channels(
308 self.buffer.sample_rate(),
309 channels,
310 )))
311 }
312 }
313
314 fn track(id: &str) -> AudioTrack {
315 AudioTrack {
316 id: id.to_string(),
317 name: id.to_string(),
318 muted: false,
319 solo: false,
320 volume: 1.0,
321 }
322 }
323
324 #[test]
325 fn mixes_single_clip_with_source_offset() {
326 let provider = TestProvider {
327 sources: HashMap::from([(
328 "tone".to_string(),
329 Arc::new(AudioBuffer::from_channels(
330 AUDIO_SAMPLE_RATE,
331 vec![vec![0.5; AUDIO_SAMPLE_RATE as usize]],
332 )),
333 )]),
334 };
335 let timeline = AudioTimeline {
336 tracks: vec![track("main")],
337 clips: vec![AudioClip {
338 id: "clip".to_string(),
339 source_id: "tone".to_string(),
340 track_id: "main".to_string(),
341 name: "tone".to_string(),
342 start_ms: 0,
343 duration_ms: 1_000,
344 source_start_ms: 0,
345 volume: 1.0,
346 }],
347 };
348
349 let mixed = AudioMixer::new(&timeline, &provider)
350 .mix_range(ms_to_sample(500), 5)
351 .unwrap();
352 assert_eq!(mixed.channel_count(), 2);
353 assert!(mixed.channels()[0][2] > 0.49);
354 assert_eq!(mixed.channels()[0], mixed.channels()[1]);
355 }
356
357 #[test]
358 fn applies_mute_solo_and_gain() {
359 let provider = TestProvider {
360 sources: HashMap::from([(
361 "tone".to_string(),
362 Arc::new(AudioBuffer::from_channels(
363 AUDIO_SAMPLE_RATE,
364 vec![
365 vec![1.0; AUDIO_SAMPLE_RATE as usize],
366 vec![0.5; AUDIO_SAMPLE_RATE as usize],
367 ],
368 )),
369 )]),
370 };
371 let mut solo = track("solo");
372 solo.solo = true;
373 solo.volume = 0.5;
374 let mut muted = track("muted");
375 muted.muted = true;
376 let timeline = AudioTimeline {
377 tracks: vec![solo, muted],
378 clips: vec![
379 AudioClip {
380 id: "a".to_string(),
381 source_id: "tone".to_string(),
382 track_id: "solo".to_string(),
383 name: "a".to_string(),
384 start_ms: 0,
385 duration_ms: 1_000,
386 source_start_ms: 0,
387 volume: 0.5,
388 },
389 AudioClip {
390 id: "b".to_string(),
391 source_id: "tone".to_string(),
392 track_id: "muted".to_string(),
393 name: "b".to_string(),
394 start_ms: 0,
395 duration_ms: 1_000,
396 source_start_ms: 0,
397 volume: 1.0,
398 },
399 ],
400 };
401
402 let mixed = AudioMixer::new(&timeline, &provider)
403 .mix_range(500, 4)
404 .unwrap();
405 assert_eq!(mixed.channels()[0], vec![0.25; 4]);
406 assert_eq!(mixed.channels()[1], vec![0.125; 4]);
407 }
408
409 #[test]
410 fn overlaps_clips_and_clamps_output() {
411 let provider = TestProvider {
412 sources: HashMap::from([(
413 "tone".to_string(),
414 Arc::new(AudioBuffer::from_channels(
415 AUDIO_SAMPLE_RATE,
416 vec![vec![0.75; AUDIO_SAMPLE_RATE as usize]],
417 )),
418 )]),
419 };
420 let timeline = AudioTimeline {
421 tracks: vec![track("main")],
422 clips: vec![
423 AudioClip {
424 id: "a".to_string(),
425 source_id: "tone".to_string(),
426 track_id: "main".to_string(),
427 name: "a".to_string(),
428 start_ms: 0,
429 duration_ms: 1_000,
430 source_start_ms: 0,
431 volume: 1.0,
432 },
433 AudioClip {
434 id: "b".to_string(),
435 source_id: "tone".to_string(),
436 track_id: "main".to_string(),
437 name: "b".to_string(),
438 start_ms: 0,
439 duration_ms: 1_000,
440 source_start_ms: 0,
441 volume: 1.0,
442 },
443 ],
444 };
445
446 let mixed = AudioMixer::new(&timeline, &provider)
447 .mix_range(500, 4)
448 .unwrap();
449 assert_eq!(mixed.channels()[0], vec![1.0; 4]);
450 assert_eq!(mixed.channels()[1], vec![1.0; 4]);
451 }
452}