1use serde::{Deserialize, Serialize};
6use serde_json::{Map, Value};
7
8use ffcine_core::anim::{Clock, Ease, Track as ParamTrack};
9use ffcine_core::{Error, Result};
10
11use crate::mix::ClipDefaults;
12use ffcine_timeline::{Clip, Effect, GenericEffect, HasMetadata, RationalTime, Track};
13
14pub const CLEAN: &str = "ffcine.audio.clean";
15pub const LEVEL: &str = "ffcine.audio.level";
16pub const SYNC: &str = "ffcine.audio.sync";
17pub const PAN: &str = "ffcine.audio.pan";
18pub const LOOP: &str = "ffcine.audio.loop";
19
20#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, Default)]
22#[serde(rename_all = "lowercase")]
23pub enum Interp {
24 #[default]
25 Linear,
26 Ease,
28 Hold,
29}
30
31impl Interp {
32 pub fn ease(self) -> Ease {
34 match self {
35 Self::Linear => Ease::Linear,
36 Self::Ease => Ease::Smoothstep,
37 Self::Hold => Ease::Hold,
38 }
39 }
40}
41
42#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
43pub struct Key {
44 pub t: RationalTime,
46 pub v: f64,
47 #[serde(default, alias = "ease")]
49 pub interp: Interp,
50}
51
52#[derive(Debug, Clone, PartialEq, Serialize)]
56#[serde(untagged)]
57pub enum Param {
58 Value(f64),
59 Keys(Vec<Key>),
60 Expr(ExprParam),
61}
62
63#[derive(Debug, Clone, PartialEq)]
65pub struct ExprParam {
66 src: Value,
67 track: ParamTrack<f64>,
68}
69
70impl ExprParam {
71 pub fn parse(v: &Value) -> Result<Self> {
73 let mut norm = v.clone();
74 if let Some(Value::Array(keys)) = norm.get_mut("keys") {
75 for k in keys.iter_mut().filter_map(Value::as_object_mut) {
76 for name in ["interp", "ease"] {
77 if k.get(name).and_then(Value::as_str) == Some("ease") {
78 k.insert(name.into(), "smoothstep".into());
79 }
80 }
81 }
82 }
83 Ok(Self {
84 track: ParamTrack::parse(&norm, Clock::Source, "audio param")?,
85 src: v.clone(),
86 })
87 }
88}
89
90impl Serialize for ExprParam {
91 fn serialize<S: serde::Serializer>(&self, s: S) -> std::result::Result<S::Ok, S::Error> {
92 self.src.serialize(s)
93 }
94}
95
96impl<'de> Deserialize<'de> for ExprParam {
97 fn deserialize<D: serde::Deserializer<'de>>(d: D) -> std::result::Result<Self, D::Error> {
98 let v = Value::deserialize(d)?;
99 Self::parse(&v).map_err(serde::de::Error::custom)
100 }
101}
102
103impl<'de> Deserialize<'de> for Param {
104 fn deserialize<D: serde::Deserializer<'de>>(d: D) -> std::result::Result<Self, D::Error> {
105 use serde::de::Error as _;
106 let v = Value::deserialize(d)?;
107 match &v {
108 Value::Number(n) => n
109 .as_f64()
110 .map(Param::Value)
111 .ok_or_else(|| D::Error::custom(format!("invalid number {n}"))),
112 Value::Array(a) if a.is_empty() || a.iter().any(Value::is_object) => {
113 Vec::<Key>::deserialize(v)
114 .map(Param::Keys)
115 .map_err(|e| D::Error::custom(format!("audio param keyframes: {e}")))
116 }
117 _ => ExprParam::parse(&v)
118 .map(Param::Expr)
119 .map_err(D::Error::custom),
120 }
121 }
122}
123
124impl Default for Param {
125 fn default() -> Self {
126 Param::Value(0.0)
127 }
128}
129
130impl From<f64> for Param {
131 fn from(v: f64) -> Self {
132 Param::Value(v)
133 }
134}
135
136impl Param {
137 pub fn is_constant(&self) -> bool {
138 matches!(self, Param::Value(_)) || matches!(self, Param::Keys(k) if k.len() < 2)
139 }
140
141 pub fn at(&self, t: f64) -> f64 {
143 let keys = match self {
144 Param::Value(v) => return *v,
145 Param::Keys(k) => k,
146 Param::Expr(e) => return e.track.at(Clock::Source, t, 0.0).unwrap_or(0.0),
147 };
148 let Some(first) = keys.first() else {
149 return 0.0;
150 };
151 let i = keys.partition_point(|k| k.t.to_seconds() <= t);
152 if i == 0 {
153 return first.v;
154 }
155 let a = &keys[i - 1];
156 let Some(b) = keys.get(i) else { return a.v };
157 let (ta, tb) = (a.t.to_seconds(), b.t.to_seconds());
158 let u = if tb > ta {
159 ((t - ta) / (tb - ta)).clamp(0.0, 1.0)
160 } else {
161 1.0
162 };
163 a.v + (b.v - a.v) * a.interp.ease().at(u)
164 }
165}
166
167#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, Default)]
169#[serde(default)]
170pub struct Pan {
171 pub pan: Param,
172}
173
174#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize, Default)]
176#[serde(rename_all = "lowercase")]
177pub enum Bus {
178 #[default]
179 Dialogue,
180 Music,
181 Ambience,
182 Sfx,
183}
184
185impl Bus {
186 pub const ALL: [Bus; 4] = [Bus::Dialogue, Bus::Music, Bus::Ambience, Bus::Sfx];
187
188 pub fn of_track(track: &Track) -> Self {
191 if let Some(Ok(b)) = track.ffcine_get::<Bus>("bus") {
192 return b;
193 }
194 let n = track.name.to_ascii_lowercase();
195 if n.contains("music") {
196 Bus::Music
197 } else if n.contains("amb") {
198 Bus::Ambience
199 } else if n.contains("sfx") || n.contains("fx") {
200 Bus::Sfx
201 } else {
202 Bus::Dialogue
203 }
204 }
205
206 pub fn from_name(name: &str) -> Option<Self> {
208 Self::ALL.into_iter().find(|b| b.name() == name)
209 }
210
211 pub fn name(self) -> &'static str {
212 match self {
213 Bus::Dialogue => "dialogue",
214 Bus::Music => "music",
215 Bus::Ambience => "ambience",
216 Bus::Sfx => "sfx",
217 }
218 }
219}
220
221#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, Default)]
223#[serde(rename_all = "lowercase")]
224pub enum Denoise {
225 Off,
226 #[default]
228 Afftdn,
229 Arnndn,
231}
232
233#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
237#[serde(default)]
238pub struct Clean {
239 #[serde(alias = "strength")]
242 pub amount: f64,
243 pub highpass: Option<f64>,
245 pub lowpass: Option<f64>,
247 pub wind: bool,
250 pub declick: bool,
251 pub denoise: Denoise,
252 pub model: Option<String>,
254 pub dialogue: Option<bool>,
257}
258
259impl Default for Clean {
260 fn default() -> Self {
261 Self {
262 amount: 0.6,
263 highpass: Some(80.0),
264 lowpass: None,
265 wind: false,
266 declick: false,
267 denoise: Denoise::Afftdn,
268 model: None,
269 dialogue: None,
270 }
271 }
272}
273
274impl Clean {
275 pub fn for_mode(mode: LevelMode) -> Self {
278 Self::for_mode_in(mode, &ClipDefaults::default())
279 }
280
281 pub fn for_mode_in(mode: LevelMode, d: &ClipDefaults) -> Self {
283 match mode {
284 LevelMode::Dialogue => Self {
285 amount: d.clean,
286 highpass: d.highpass,
287 ..Self::default()
288 },
289 _ => Self {
290 amount: if mode == LevelMode::Nat {
291 d.clean_nat
292 } else {
293 0.0
294 },
295 highpass: d.highpass,
296 dialogue: Some(false),
297 ..Self::default()
298 },
299 }
300 }
301
302 pub fn filter_spec(&self, mode: LevelMode) -> String {
305 let c = self.amount.clamp(0.0, 1.0);
306 let mut f: Vec<String> = Vec::new();
307 if let Some(hz) = self.highpass {
308 f.push(format!("highpass=f={hz}"));
309 }
310 if let Some(hz) = self.lowpass {
311 f.push(format!("lowpass=f={hz}"));
312 }
313 if self.wind {
314 f.push("highpass=f=150:p=2,highpass=f=150:p=2,lowshelf=f=300:g=-6".into());
315 }
316 if self.declick || self.wind {
317 f.push("adeclick".into());
318 }
319 let afftdn = format!(
320 "afftdn=nr={:.1}:nf={:.0}:tn=1",
321 6.0 + 10.0 * c,
322 -34.0 + 6.0 * c
323 );
324 if c > 0.0 {
325 match (&self.denoise, &self.model) {
326 (Denoise::Off, _) => {}
327 (Denoise::Arnndn, Some(m)) => {
328 f.push(format!("arnndn=m='{}':mix={c:.2}", m.replace('\'', "\\'")));
329 }
330 _ => f.push(afftdn),
331 }
332 }
333 if self.dialogue.unwrap_or(mode == LevelMode::Dialogue) {
334 if c > 0.0 {
335 f.push(format!("equalizer=f=3200:t=q:w=1.1:g={:.2}", 1.8 * c));
336 f.push(format!("deesser=i={:.2}:m=0.5:f=0.55", 0.35 * c));
337 }
338 f.push("acompressor=threshold=-24dB:ratio=2.5:attack=8:release=160:makeup=1".into());
339 }
340 f.join(",")
341 }
342}
343
344#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, Default)]
346#[serde(rename_all = "lowercase")]
347pub enum LevelMode {
348 #[default]
350 #[serde(alias = "orig")]
351 Dialogue,
352 Nat,
354 Music,
357 Ambience,
359 Sfx,
362 Off,
364}
365
366#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, Default)]
368#[serde(default)]
369pub struct Level {
370 pub mode: Option<LevelMode>,
372 pub lufs: Option<f64>,
374 pub gain_db: Param,
376 pub ref_dbfs: Option<f64>,
378 pub nat_db: Option<f64>,
380 pub duck_db: Option<f64>,
383 pub fade_in: Option<f64>,
385 pub fade_out: Option<f64>,
386}
387
388impl Level {
389 pub fn mode(&self, bus: Bus) -> LevelMode {
390 self.mode.unwrap_or(match bus {
391 Bus::Dialogue => LevelMode::Dialogue,
392 Bus::Music => LevelMode::Music,
393 Bus::Ambience => LevelMode::Ambience,
394 Bus::Sfx => LevelMode::Sfx,
395 })
396 }
397
398 pub fn audible(&self, bus: Bus) -> bool {
400 !(bus == Bus::Dialogue && self.mode(bus) == LevelMode::Music && self.nat_db.is_none())
401 }
402
403 pub fn target(&self, bus: Bus) -> Option<f64> {
405 self.target_in(bus, &ClipDefaults::default())
406 }
407
408 pub fn target_in(&self, bus: Bus, d: &ClipDefaults) -> Option<f64> {
410 if let Some(l) = self.lufs {
411 return Some(l);
412 }
413 match self.mode(bus) {
414 LevelMode::Dialogue => Some(d.dialogue_lufs),
415 LevelMode::Nat => Some(d.nat_lufs + self.nat_db.unwrap_or(0.0)),
416 LevelMode::Music if bus == Bus::Dialogue => {
417 Some(d.nat_lufs + self.nat_db.unwrap_or(0.0))
418 }
419 LevelMode::Music => Some(-20.0),
420 LevelMode::Ambience => Some(-36.0),
421 LevelMode::Sfx | LevelMode::Off => None,
422 }
423 }
424
425 pub fn duck(&self, bus: Bus) -> f64 {
427 self.duck_in(bus, &ClipDefaults::default())
428 }
429
430 pub fn duck_in(&self, bus: Bus, d: &ClipDefaults) -> f64 {
432 self.duck_db.unwrap_or(match (bus, self.mode(bus)) {
433 (Bus::Dialogue, LevelMode::Dialogue) => d.duck_db,
434 (Bus::Dialogue, LevelMode::Nat) => d.nat_duck_db,
435 _ => 0.0,
436 })
437 }
438}
439
440#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
443#[serde(default)]
444pub struct Loop {
445 pub r#in: Option<f64>,
446 pub out: Option<f64>,
447 pub crossfade: f64,
448}
449
450impl Default for Loop {
451 fn default() -> Self {
452 Self {
453 r#in: None,
454 out: None,
455 crossfade: 1.0,
456 }
457 }
458}
459
460#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, Default)]
463#[serde(default)]
464pub struct SyncInfo {
465 pub schema: String,
467 pub video: String,
469 pub offset: f64,
471 pub drift_ppm: f64,
473 pub confidence: f64,
475}
476
477pub fn clip_effect<T: for<'de> Deserialize<'de>>(clip: &Clip, name: &str) -> Result<Option<T>> {
480 for e in &clip.effects {
481 let Effect::Effect(g) = e else { continue };
482 if g.effect_name != name {
483 continue;
484 }
485 let Some(params) = g.ffcine() else {
486 return Ok(Some(serde_json::from_value(Value::Object(Map::new()))?));
487 };
488 if let Some(schema) = params.get("schema").and_then(Value::as_str)
489 && schema != format!("{name}/1")
490 {
491 return Err(Error::Invalid {
492 what: "audio effect schema",
493 value: schema.to_owned(),
494 });
495 }
496 return Ok(Some(serde_json::from_value(Value::Object(params.clone()))?));
497 }
498 Ok(None)
499}
500
501pub fn make_effect<T: Serialize>(label: &str, name: &str, params: &T) -> Result<Effect> {
503 let mut e = GenericEffect::new(label, name);
504 let Value::Object(mut m) = serde_json::to_value(params)? else {
505 return Err(Error::Invalid {
506 what: "effect parameters",
507 value: name.to_owned(),
508 });
509 };
510 m.insert("schema".into(), format!("{name}/1").into());
511 e.metadata
512 .insert(ffcine_timeline::FFCINE.into(), Value::Object(m));
513 Ok(Effect::Effect(e))
514}
515
516#[cfg(test)]
517mod tests {
518 use super::*;
519
520 #[test]
521 fn level_reads_resolve_keyframes() {
522 let v = serde_json::json!({
523 "schema": "ffcine.audio.level/1",
524 "gain_db": [
525 { "t": { "OTIO_SCHEMA": "RationalTime.1", "rate": 24.0, "value": 0.0 }, "v": -6.0 },
526 { "t": { "OTIO_SCHEMA": "RationalTime.1", "rate": 24.0, "value": 48.0 }, "v": 0.0, "interp": "hold" }
527 ],
528 "fade_in": 0.5
529 });
530 let l: Level = serde_json::from_value(v).expect("level");
531 assert!((l.gain_db.at(1.0) + 3.0).abs() < 1e-9);
532 assert!((l.gain_db.at(5.0)).abs() < 1e-9);
533 assert_eq!(l.fade_in, Some(0.5));
534 assert_eq!(l.target(Bus::Music), Some(-20.0));
535 let orig: Level =
536 serde_json::from_value(serde_json::json!({"mode": "orig"})).expect("orig");
537 assert_eq!(
538 (orig.target(Bus::Dialogue), orig.duck(Bus::Dialogue)),
539 (Some(-18.0), -14.0)
540 );
541 let muted: Level =
542 serde_json::from_value(serde_json::json!({"mode": "music"})).expect("music");
543 assert!(!muted.audible(Bus::Dialogue) && muted.audible(Bus::Music));
544 }
545
546 #[test]
547 fn clean_spec_matches_prototype() {
548 assert_eq!(
549 Clean::default().filter_spec(LevelMode::Dialogue),
550 "highpass=f=80,afftdn=nr=12.0:nf=-30:tn=1,equalizer=f=3200:t=q:w=1.1:g=1.08,\
551 deesser=i=0.21:m=0.5:f=0.55,\
552 acompressor=threshold=-24dB:ratio=2.5:attack=8:release=160:makeup=1"
553 );
554 assert_eq!(
555 Clean::for_mode(LevelMode::Nat).filter_spec(LevelMode::Nat),
556 "highpass=f=80"
557 );
558 }
559
560 #[test]
561 fn gain_takes_expressions() {
562 let v = serde_json::json!({"gain_db": "-6 + 6 * clip(st - 1, 0, 1)"});
563 let l: Level = serde_json::from_value(v.clone()).expect("level");
564 assert!(!l.gain_db.is_constant());
565 assert!((l.gain_db.at(0.5) + 6.0).abs() < 1e-9 && (l.gain_db.at(1.5) + 3.0).abs() < 1e-9);
566 let back = serde_json::to_value(&l).expect("ser");
568 assert_eq!(back["gain_db"], v["gain_db"]);
569 assert!(serde_json::from_value::<Level>(serde_json::json!({"gain_db": "st +"})).is_err());
570 assert!((Interp::Ease.ease().at(0.25) - 0.156_25).abs() < 1e-12);
572 }
573
574 #[test]
575 fn bad_keys_are_an_error_not_an_expression() {
576 let e = serde_json::from_value::<Level>(serde_json::json!({
577 "gain_db": [{"t": 1, "v": -6, "interp": "ease"}]
578 }))
579 .expect_err("bad key");
580 assert!(e.to_string().contains("audio param keyframes"), "{e}");
581 let l: Level = serde_json::from_value(serde_json::json!({"gain_db": []})).expect("[]");
582 assert_eq!(l.gain_db, Param::Keys(Vec::new()));
583 let l: Level = serde_json::from_value(serde_json::json!({"gain_db": {
585 "keys": [{"t": 0, "v": 0, "interp": "ease"}, {"t": 1, "v": 1}], "expr": "value"
586 }}))
587 .expect("keys expr");
588 assert!((l.gain_db.at(0.25) - 0.156_25).abs() < 1e-12);
589 }
590
591 #[test]
592 fn key_ease_is_an_alias_of_interp() {
593 let l: Level = serde_json::from_value(serde_json::json!({
594 "gain_db": [
595 {"t": {"OTIO_SCHEMA": "RationalTime.1", "rate": 24.0, "value": 0.0}, "v": 0.0, "ease": "ease"},
596 {"t": {"OTIO_SCHEMA": "RationalTime.1", "rate": 24.0, "value": 24.0}, "v": 1.0}
597 ]
598 }))
599 .expect("ease alias");
600 let Param::Keys(k) = &l.gain_db else {
601 panic!("keys: {:?}", l.gain_db)
602 };
603 assert_eq!(k[0].interp, Interp::Ease);
604 assert!((l.gain_db.at(0.25) - 0.156_25).abs() < 1e-12);
605 }
606}